forkeado de spla/fediverse
Totally recoded
This commit is contained in:
spla
pare
d82ad780d2
commit
c104d250d5
S'han modificat 6 arxius amb 433 adicions i 1384 eliminacions
22
README.md
22
README.md
|
|
@ -1,5 +1,5 @@
|
|||
# Fediverse Stats
|
||||
This code gets all peers from running Mastodon, Pleroma and Lemmy host servers and then all peers from host server's peers. Goal is to collect maximum number
|
||||
This code gets all peers from mastodon.social. Goal is to collect maximum number
|
||||
of alive fediverse's servers and then query their API to obtain their registered users (if their API provide such information).
|
||||
At the end it post the results to host server bot account.
|
||||
|
||||
|
|
@ -7,7 +7,7 @@ At the end it post the results to host server bot account.
|
|||
|
||||
- **Python 3**
|
||||
- Postgresql server
|
||||
- Mastodon or Pleroma running server.
|
||||
- Mastodon running server.
|
||||
|
||||
### Usage:
|
||||
|
||||
|
|
@ -15,19 +15,11 @@ Within Python Virtual Environment:
|
|||
|
||||
1. Run `pip install -r requirements.txt` to install needed libraries.
|
||||
|
||||
2. Run `python db-setup.py` to setup and create new Postgresql database and needed tables in it.
|
||||
2. Run `python fetchservers.py` to add servers to alive servers database.
|
||||
|
||||
3. Run `python setup.py` to get your bot's access token of your Mastodon or Pleroma server existing account. It will be saved to 'secrets/secrets.txt' for further use.
|
||||
3. Run `python fediverse.py` to query world alive servers API. It gets data from server's nodeinfo.
|
||||
|
||||
4. Run `python getpeers.py` to get all peers from your host and the whole world of fediverse's servers (or almost the whole world).
|
||||
|
||||
5. Run `python fetchservers.py` to add servers to alive servers database.
|
||||
|
||||
6. Run `python fediverse.py` to query world alive servers API. It gets data from server's nodeinfo.
|
||||
|
||||
7. Run `python uptime_setup.py` to get your Uptime bot's access token of your Mastodon or Pleroma server existing account. It will be saved to 'secrets/uptime_secrets.txt' for further use.
|
||||
|
||||
8. Use your favourite scheduling method to set `python fediverse.py` to run twice daily, `python fetchservers.py` one time daily, `python getworld.py` to run monthly and `python uptime.py` (choose your desired frequency) if you want to publish best fediverse's servers uptime.
|
||||
4. Use your favourite scheduling method to set `python fediverse.py` to run twice daily, `python fetchservers.py` one time daily.
|
||||
|
||||
18.2.2021 - New feature! Added [Lemmy project](https://join.lemmy.ml)
|
||||
12.5.2021 - New feature! Added Wordpress support. The code can now detect Wordpress instances with ActivityPub enabled plugin.
|
||||
|
|
@ -35,4 +27,6 @@ Within Python Virtual Environment:
|
|||
21.8.2021 - New feature! Added Best Fediverse's servers Uptime publishing bot.
|
||||
22.10.2021 - New feature! Added [Funkwhale](https://funkwhale.audio) support.
|
||||
26.10.2021 - New feature! Added [Socialhome](https://socialhome.network) support.
|
||||
2.3.2022 - Improved server nodeinfo detection
|
||||
2.3.2022 - Improved server nodeinfo detection.
|
||||
4.1.2023 - Refactored.
|
||||
4.1.2023 - Now all peers are obtained from mastodon.social's peers list.
|
||||
|
|
|
|||
762
fediverse.py
762
fediverse.py
|
|
@ -5,13 +5,33 @@ from datetime import datetime
|
|||
import urllib3
|
||||
import requests
|
||||
import socket
|
||||
from setup import Setup
|
||||
from database import Database
|
||||
from mastodon import Mastodon
|
||||
import psycopg2
|
||||
import matplotlib.pyplot as plt
|
||||
from matplotlib import pyplot as plt
|
||||
import matplotlib.dates as mdates
|
||||
from matplotlib.ticker import ScalarFormatter
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import ray
|
||||
import pdb
|
||||
|
||||
ray.init(num_cpus = 32) # Specify this system CPUs.
|
||||
SMALL_SIZE = 6
|
||||
MEDIUM_SIZE = 10
|
||||
BIGGER_SIZE = 12
|
||||
|
||||
plt.rc('font', size=MEDIUM_SIZE) # controls default text sizes
|
||||
plt.rc('axes', titlesize=MEDIUM_SIZE) # fontsize of the axes title
|
||||
plt.rc('axes', labelsize=MEDIUM_SIZE) # fontsize of the x and y labels
|
||||
plt.rc('xtick', labelsize=SMALL_SIZE) # fontsize of the tick labels
|
||||
plt.rc('ytick', labelsize=MEDIUM_SIZE) # fontsize of the tick labels
|
||||
plt.rc('legend', fontsize=MEDIUM_SIZE) # legend fontsize
|
||||
plt.rc('figure', titlesize=BIGGER_SIZE) # fontsize of the figure title
|
||||
|
||||
mdates.set_epoch('2000-01-01T00:00:00')
|
||||
y_formatter = ScalarFormatter(useOffset=False)
|
||||
|
||||
ray.init(num_cpus = 25) # Specify this system CPUs.
|
||||
|
||||
class Server:
|
||||
|
||||
|
|
@ -28,76 +48,35 @@ class Server:
|
|||
|
||||
downs = 0
|
||||
|
||||
try:
|
||||
|
||||
conn = None
|
||||
|
||||
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute("select alive, software, users_api, version, first_checked_at, downs from fediverse where server=(%s)", (self,))
|
||||
|
||||
row = cur.fetchone()
|
||||
|
||||
if row is not None:
|
||||
|
||||
was_alive = row[0]
|
||||
software = row[1]
|
||||
api = row[2]
|
||||
soft_version = row[3]
|
||||
first_checked_at = row[4]
|
||||
downs_qty = row[5]
|
||||
|
||||
cur.close()
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print(error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
was_alive, software, api, soft_version, first_checked_at, downs_qty = db.get_server_data(self)
|
||||
|
||||
alive = False
|
||||
|
||||
try:
|
||||
|
||||
user_agent = {'User-agent': "fediverse's servers stats (fediverse@mastodont.cat)"}
|
||||
data = requests.get('https://' + self + api, headers = setup.user_agent, timeout=3)
|
||||
|
||||
data = requests.get('https://' + self + api, headers = user_agent, timeout=3)
|
||||
nodeinfo_json = data.json()
|
||||
|
||||
try:
|
||||
|
||||
users = data.json()['usage']['users']['total']
|
||||
users = nodeinfo_json.get('usage').get('users').get('total') or '0'
|
||||
|
||||
if users == 0:
|
||||
|
||||
users = data.json()['usage']['users']['activeHalfyear']
|
||||
|
||||
if 'activeMonth' in data.json()['usage']['users'].keys():
|
||||
|
||||
mau = data.json()['usage']['users']['activeMonth']
|
||||
|
||||
else:
|
||||
|
||||
mau = 0
|
||||
mau = nodeinfo_json.get('usage').get('users').get('activeMonth') or '0'
|
||||
|
||||
if software == 'socialhome':
|
||||
|
||||
soft_version = data.json()['server']['version']
|
||||
soft_version = nodeinfo_json['server']['version']
|
||||
|
||||
else:
|
||||
|
||||
soft_version = data.json()['software']['version']
|
||||
soft_version = nodeinfo_json['software']['version']
|
||||
|
||||
if software == "wordpress" and "activitypub" in data.json()['protocols']:
|
||||
if software == "wordpress" and "activitypub" in nodeinfo_json['protocols']:
|
||||
|
||||
alive = True
|
||||
|
||||
elif software == "wordpress" and "activitypub" not in data.json()['protocols']:
|
||||
elif software == "wordpress" and "activitypub" not in nodeinfo_json['protocols']:
|
||||
|
||||
alive = False
|
||||
|
||||
|
|
@ -114,8 +93,8 @@ class Server:
|
|||
if api == '/api/v1/instance?':
|
||||
|
||||
try:
|
||||
users = data.json()['stats']['user_count']
|
||||
soft_version = data.json()['version']
|
||||
users = nodeinfo_json.get('stats').get('user_count') or '0'
|
||||
soft_version = nodeinfo_json['version']
|
||||
alive = True
|
||||
mau = 0
|
||||
except:
|
||||
|
|
@ -137,11 +116,11 @@ class Server:
|
|||
|
||||
if software != 'birdsitelive':
|
||||
|
||||
write_alive_server(self, software, soft_version, alive, api, users, downs, first_checked_at, mau)
|
||||
db.write_alive_server(self, software, soft_version, alive, api, users, downs, first_checked_at, mau)
|
||||
|
||||
else:
|
||||
|
||||
write_blocked_software(self, software, soft_version, alive, api, users, downs, first_checked_at)
|
||||
db.write_blocked_software(self, software, soft_version, alive, api, users, downs, first_checked_at)
|
||||
|
||||
except urllib3.exceptions.ProtocolError as protoerr:
|
||||
|
||||
|
|
@ -205,325 +184,19 @@ class Server:
|
|||
|
||||
downs = 1
|
||||
|
||||
write_not_alive_server(self, software, soft_version, alive, api, users, downs, first_checked_at)
|
||||
db.write_not_alive_server(self, software, soft_version, alive, api, users, downs, first_checked_at)
|
||||
|
||||
return (self, software, soft_version, alive, api, users, downs, first_checked_at, mau)
|
||||
|
||||
def write_alive_server(server, software, soft_version, alive, api, users, downs, first_checked_at, mau):
|
||||
|
||||
insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version, first_checked_at, last_checked_at, downs, mau) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
|
||||
|
||||
conn = None
|
||||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute(insert_sql, (server, users, now, software, alive, api, soft_version, now, now, downs, mau))
|
||||
|
||||
if first_checked_at != None:
|
||||
|
||||
cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), last_checked_at=(%s), downs=(%s), mau=(%s) where server=(%s)", (users, now, software, alive, api, soft_version, now, downs, mau, server))
|
||||
|
||||
else:
|
||||
|
||||
cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), first_checked_at=(%s), last_checked_at=(%s), downs=(%s), mau=(%s) where server=(%s)", (users, now, software, alive, api, soft_version, now, now, downs, mau, server))
|
||||
|
||||
cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
|
||||
|
||||
conn.commit()
|
||||
|
||||
cur.close()
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print(error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
def write_blocked_software(server, software, soft_version, alive, api, users, downs, first_checked_at):
|
||||
|
||||
insert_sql = "INSERT INTO blocked(server, users, updated_at, software, alive, users_api, version, first_checked_at, last_checked_at, downs) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
|
||||
|
||||
conn = None
|
||||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute(insert_sql, (server, users, now, software, alive, api, soft_version, now, now, downs))
|
||||
|
||||
if first_checked_at != None:
|
||||
|
||||
cur.execute("UPDATE blocked SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (users, now, software, alive, api, soft_version, now, downs, server))
|
||||
|
||||
else:
|
||||
|
||||
cur.execute("UPDATE blocked SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), first_checked_at=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (users, now, software, alive, api, soft_version, now, now, downs, server))
|
||||
|
||||
cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
|
||||
|
||||
conn.commit()
|
||||
|
||||
cur.close()
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print(error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
def write_not_alive_server(server, software, soft_version, alive, api, users, downs, first_checked_at):
|
||||
|
||||
conn = None
|
||||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
if first_checked_at != None:
|
||||
|
||||
cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), downs=(%s) where server=(%s)", (now, alive, downs, server))
|
||||
|
||||
else:
|
||||
|
||||
cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), first_checked_at=(%s), downs=(%s) where server=(%s)", (now, alive, now, downs, server))
|
||||
|
||||
cur.execute("UPDATE world SET checked='f' where server=(%s)", (server,))
|
||||
|
||||
conn.commit()
|
||||
|
||||
cur.close()
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print(error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
def delete_dead_servers():
|
||||
|
||||
conn = None
|
||||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute("select server from fediverse where downs > '14' and not alive and now() - first_checked_at > interval '7 days'")
|
||||
|
||||
rows = cur.fetchall()
|
||||
|
||||
for row in rows:
|
||||
|
||||
print(f'Deleting server {row[0]}...')
|
||||
|
||||
cur.execute("delete from fediverse where server=(%s)", (row[0],))
|
||||
|
||||
conn.commit()
|
||||
|
||||
cur.close()
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print(error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
def get_last_checked_servers():
|
||||
|
||||
############################################################################
|
||||
# get last checked servers from fediverse DB
|
||||
|
||||
alive_servers = []
|
||||
|
||||
try:
|
||||
|
||||
conn = None
|
||||
|
||||
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
# get world servers list
|
||||
|
||||
cur.execute("select server from world where server in (select server from fediverse where users_api != '')")
|
||||
|
||||
alive_servers = []
|
||||
|
||||
for row in cur:
|
||||
|
||||
alive_servers.append(row[0])
|
||||
|
||||
cur.close()
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print(error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
return alive_servers
|
||||
|
||||
def print_dead(server):
|
||||
|
||||
print(f'\nServer {server} is dead :-(')
|
||||
|
||||
def set_world_servers_check_to_false():
|
||||
|
||||
############################################################################
|
||||
# set all world servers's checked column to False
|
||||
|
||||
try:
|
||||
|
||||
conn = None
|
||||
|
||||
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute("UPDATE world SET checked='f'")
|
||||
|
||||
conn.commit()
|
||||
|
||||
cur.close()
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print(error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
def mastodon():
|
||||
|
||||
# Load secrets from secrets file
|
||||
secrets_filepath = "secrets/secrets.txt"
|
||||
uc_client_id = get_parameter("uc_client_id", secrets_filepath)
|
||||
uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
|
||||
uc_access_token = get_parameter("uc_access_token", secrets_filepath)
|
||||
|
||||
# Load configuration from config file
|
||||
config_filepath = "config/config.txt"
|
||||
mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
|
||||
|
||||
# Initialise Mastodon API
|
||||
mastodon = Mastodon(
|
||||
client_id=uc_client_id,
|
||||
client_secret=uc_client_secret,
|
||||
access_token=uc_access_token,
|
||||
api_base_url='https://' + mastodon_hostname,
|
||||
)
|
||||
|
||||
# Initialise access headers
|
||||
headers = {'Authorization': 'Bearer %s'%uc_access_token}
|
||||
|
||||
return (mastodon, mastodon_hostname)
|
||||
|
||||
def db_config():
|
||||
|
||||
# Load db configuration from config file
|
||||
config_filepath = "config/db_config.txt"
|
||||
fediverse_db = get_parameter("fediverse_db", config_filepath)
|
||||
fediverse_db_user = get_parameter("fediverse_db_user", config_filepath)
|
||||
|
||||
return (fediverse_db, fediverse_db_user)
|
||||
|
||||
def usage():
|
||||
|
||||
print('usage: python ' + sys.argv[0] + ' --multi' + ' (multiprocessing, fast)')
|
||||
print('usage: python ' + sys.argv[0] + ' --mono' + ' (one process, slow)')
|
||||
|
||||
def save_time(program, start, finish):
|
||||
|
||||
insert_sql = "INSERT INTO execution_time(program, start, finish) VALUES(%s,%s,%s) ON CONFLICT DO NOTHING"
|
||||
|
||||
conn = None
|
||||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute(insert_sql, (program, start, finish,))
|
||||
|
||||
cur.execute("UPDATE execution_time SET start=(%s), finish=(%s) where program=(%s)", (start, finish, program))
|
||||
|
||||
conn.commit()
|
||||
|
||||
cur.close()
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print(error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
def get_parameter(parameter, file_path):
|
||||
|
||||
# Check if secrets file exists
|
||||
if not os.path.isfile(file_path):
|
||||
print("File %s not found, exiting."%file_path)
|
||||
sys.exit(0)
|
||||
|
||||
# Find parameter in file
|
||||
with open(file_path) as f:
|
||||
for line in f:
|
||||
if line.startswith(parameter):
|
||||
return line.replace(parameter + ":", "").strip()
|
||||
|
||||
# Cannot find parameter, exit
|
||||
print(file_path + " Missing parameter %s "%parameter)
|
||||
sys.exit(0)
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
# usage modes
|
||||
db = Database()
|
||||
|
||||
if len(sys.argv) == 1:
|
||||
|
||||
usage()
|
||||
|
||||
elif len(sys.argv) == 2:
|
||||
|
||||
if sys.argv[1] == '--multi':
|
||||
|
||||
fediverse_db, fediverse_db_user = db_config()
|
||||
setup = Setup()
|
||||
|
||||
start = datetime.now()
|
||||
|
||||
|
|
@ -531,19 +204,20 @@ if __name__ == '__main__':
|
|||
|
||||
finish = start
|
||||
|
||||
save_time(program, start, finish)
|
||||
db.save_time(program, start, finish)
|
||||
|
||||
now = start
|
||||
|
||||
mastodon, mastodon_hostname = mastodon()
|
||||
mastodon = Mastodon(
|
||||
access_token = setup.mastodon_app_token,
|
||||
api_base_url= setup.mastodon_hostname
|
||||
)
|
||||
|
||||
total_servers = 0
|
||||
|
||||
total_users = 0
|
||||
|
||||
#set_world_servers_check_to_false()
|
||||
|
||||
alive_servers = get_last_checked_servers()
|
||||
alive_servers = db.get_last_checked_servers()
|
||||
|
||||
getservers = Server()
|
||||
|
||||
|
|
@ -555,276 +229,29 @@ if __name__ == '__main__':
|
|||
|
||||
print(f"duration = {time.time() - ray_start}.\nprocessed servers: {len(results)}")
|
||||
|
||||
###########################################################################
|
||||
# get current total servers and users, get users from every software
|
||||
|
||||
now = datetime.now()
|
||||
soft_total_project, soft_total_users, soft_total_mau, soft_total_servers, total_servers, total_users, total_mau = db.soft_totals()
|
||||
|
||||
gettotals_sql = "select count(server), sum(users), sum(mau) from fediverse where alive"
|
||||
get_soft_totals_sql = "select software, sum(users) as users, sum(mau) as mau, count(server) as servers from fediverse where users != 0 and mau is not null and alive group by software order by mau desc"
|
||||
|
||||
soft_total_project = []
|
||||
soft_total_users = []
|
||||
soft_total_mau = []
|
||||
soft_total_servers = []
|
||||
|
||||
try:
|
||||
|
||||
conn = None
|
||||
|
||||
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute(gettotals_sql)
|
||||
|
||||
row = cur.fetchone()
|
||||
|
||||
total_servers = row[0]
|
||||
|
||||
total_users = row[1]
|
||||
|
||||
total_mau = row[2]
|
||||
|
||||
cur.execute(get_soft_totals_sql)
|
||||
|
||||
rows = cur.fetchall()
|
||||
|
||||
for row in rows:
|
||||
|
||||
soft_total_project.append(row[0])
|
||||
soft_total_users.append(row[1])
|
||||
soft_total_mau.append(row[2])
|
||||
soft_total_servers.append(row[3])
|
||||
|
||||
cur.close()
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print(error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
###########################################################################
|
||||
# get last check values and write current total ones
|
||||
|
||||
select_sql = "select total_servers, total_users, total_mau from totals order by datetime desc limit 1"
|
||||
evo_servers, evo_users, evo_mau = db.last_values(total_servers, total_users, total_mau)
|
||||
|
||||
insert_sql = "INSERT INTO totals(datetime, total_servers, total_users, total_mau) VALUES(%s,%s,%s,%s)"
|
||||
|
||||
try:
|
||||
|
||||
conn = None
|
||||
|
||||
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute(select_sql)
|
||||
|
||||
row = cur.fetchone()
|
||||
|
||||
if row is not None:
|
||||
|
||||
servers_before = row[0]
|
||||
|
||||
users_before = row[1]
|
||||
|
||||
mau_before = row[2]
|
||||
|
||||
else:
|
||||
|
||||
servers_before = 0
|
||||
|
||||
users_before = 0
|
||||
|
||||
mau_before = 0
|
||||
|
||||
cur.execute(insert_sql, (now, total_servers, total_users, total_mau))
|
||||
|
||||
conn.commit()
|
||||
|
||||
cur.close()
|
||||
|
||||
evo_servers = total_servers - servers_before
|
||||
|
||||
evo_users = total_users - users_before
|
||||
|
||||
evo_mau = total_mau - mau_before
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print(error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
################################################################################
|
||||
# write evo values
|
||||
|
||||
insert_sql = "INSERT INTO evo(datetime, servers, users, mau) VALUES(%s,%s,%s,%s)"
|
||||
db.write_evo(evo_servers, evo_users, evo_mau)
|
||||
|
||||
conn = None
|
||||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute(insert_sql, (now, evo_servers, evo_users, evo_mau))
|
||||
|
||||
conn.commit()
|
||||
|
||||
cur.close()
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print(error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
##############################################################################
|
||||
# get world's last update datetime
|
||||
|
||||
conn = None
|
||||
last_update = db.last_world_datetime()
|
||||
|
||||
try:
|
||||
# get max servers and mau
|
||||
|
||||
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
|
||||
max_servers, max_mau = db.max()
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute("select updated_at from world order by updated_at desc limit 1")
|
||||
|
||||
row = cur.fetchone()
|
||||
|
||||
last_update = row[0]
|
||||
|
||||
last_update = last_update.strftime('%m/%d/%Y, %H:%M:%S')
|
||||
|
||||
cur.close()
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print(error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
##############################################################################
|
||||
# get max servers, users and mau
|
||||
|
||||
conn = None
|
||||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute("select MAX(total_servers) from totals")
|
||||
|
||||
row = cur.fetchone()
|
||||
|
||||
if row is not None:
|
||||
|
||||
max_servers = row[0]
|
||||
|
||||
else:
|
||||
|
||||
max_servers = 0
|
||||
|
||||
cur.execute("select MAX(total_users) from totals")
|
||||
|
||||
row = cur.fetchone()
|
||||
|
||||
if row is not None:
|
||||
|
||||
max_users = row[0]
|
||||
|
||||
else:
|
||||
|
||||
max_users = 0
|
||||
|
||||
cur.execute("select MAX(total_mau) from totals")
|
||||
|
||||
row = cur.fetchone()
|
||||
|
||||
if row is not None:
|
||||
|
||||
max_mau = row[0]
|
||||
|
||||
else:
|
||||
|
||||
max_mau = 0
|
||||
|
||||
cur.close()
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print(error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
###############################################################################
|
||||
# get plots
|
||||
|
||||
servers_plots = []
|
||||
|
||||
users_plots = []
|
||||
|
||||
mau_plots = []
|
||||
|
||||
conn = None
|
||||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute("select total_servers, total_users, total_mau from totals order by datetime desc limit 14")
|
||||
|
||||
rows = cur.fetchall()
|
||||
|
||||
for row in rows:
|
||||
|
||||
servers_plots.append(row[0])
|
||||
|
||||
users_plots.append(row[1])
|
||||
|
||||
mau_plots.append(row[2])
|
||||
|
||||
cur.close()
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print(error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
servers_plots, mau_plots, global_week, global_servers, global_users, global_mau = db.get_plots()
|
||||
|
||||
###############################################################################
|
||||
# generate graphs
|
||||
|
|
@ -845,22 +272,6 @@ if __name__ == '__main__':
|
|||
|
||||
plt.close()
|
||||
|
||||
plt.plot([-6, -5, -4, -3, -2, -1, 0], [users_plots[6], users_plots[5], users_plots[4], users_plots[3], users_plots[2], users_plots[1], users_plots[0]], marker='o', color='royalblue')
|
||||
|
||||
plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_users, max_users, max_users, max_users, max_users, max_users, max_users], color='red')
|
||||
|
||||
plt.title('fediverse: total registered users (max: ' + str(f"{max_users:,}" + ')'), loc='right', color='royalblue')
|
||||
|
||||
plt.legend(('users', 'max'), shadow=True, loc=(0.01, 0.80), handlelength=1.5, fontsize=10)
|
||||
|
||||
plt.xlabel('Last seven days')
|
||||
|
||||
plt.ylabel('Registered users')
|
||||
|
||||
plt.savefig('users.png')
|
||||
|
||||
plt.close()
|
||||
|
||||
plt.plot([-6, -5, -4, -3, -2, -1, 0], [mau_plots[6], mau_plots[5], mau_plots[4], mau_plots[3], mau_plots[2], mau_plots[1], mau_plots[0]], marker='o', color='royalblue')
|
||||
|
||||
plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_mau, max_mau, max_mau, max_mau, max_mau, max_mau, max_mau], color='red')
|
||||
|
|
@ -877,8 +288,39 @@ if __name__ == '__main__':
|
|||
|
||||
plt.close()
|
||||
|
||||
df = pd.DataFrame({'date': np.array(global_week),
|
||||
#'servers': np.array(global_servers),
|
||||
'users': np.array(global_users),
|
||||
'mau': np.array(global_mau)})
|
||||
|
||||
df['date'] = pd.to_datetime(df['date'])
|
||||
|
||||
fig, ax = plt.subplots()
|
||||
|
||||
ax.plot(df.date, df.users, label='Registered', color='orange')
|
||||
|
||||
ax.plot(df.date, df.mau, label='MAU', color='blue')
|
||||
|
||||
plt.tick_params(rotation=45)
|
||||
|
||||
ax.set_title("fediverse's registered and Monthly Active Users")
|
||||
|
||||
ax.set_xlabel('weeks')
|
||||
|
||||
ax.set_ylabel('users')
|
||||
|
||||
ax.grid(visible=True)
|
||||
|
||||
ax.legend(title='Users')
|
||||
|
||||
ax.yaxis.set_major_formatter(y_formatter)
|
||||
|
||||
plt.savefig('global.png')
|
||||
|
||||
plt.close()
|
||||
|
||||
###############################################################################
|
||||
# T O O T !
|
||||
# P O S T !
|
||||
|
||||
toot_text = "#fediverse alive servers stats" + " \n"
|
||||
|
||||
|
|
@ -896,18 +338,6 @@ if __name__ == '__main__':
|
|||
|
||||
toot_text += "max: " + str(f"{max_servers:,}") + "\n"
|
||||
|
||||
#if evo_users >= 0:
|
||||
|
||||
# toot_text += "total users: " + str(f"{total_users:,}") + " (+"+ str(f"{evo_users:,}") + ") \n"
|
||||
|
||||
# toot_text += "max: " + str(f"{max_users:,}") + "\n"
|
||||
|
||||
#elif evo_users < 0:
|
||||
|
||||
# toot_text += "total users: " + str(f"{total_users:,}") + " ("+ str(f"{evo_users:,}") + ") \n"
|
||||
|
||||
# toot_text += "max: " + str(f"{max_users:,}") + "\n"
|
||||
|
||||
if evo_mau >= 0:
|
||||
|
||||
toot_text += "total MAU: " + str(f"{total_mau:,}") + " (+"+ str(f"{evo_mau:,}") + ") \n"
|
||||
|
|
@ -920,7 +350,7 @@ if __name__ == '__main__':
|
|||
|
||||
toot_text += "max: " + str(f"{max_mau:,}") + "\n"
|
||||
|
||||
toot_text += "\ntop ten (soft MAU servers):\n\n"
|
||||
toot_text += "\ntop ten (MAU / servers):\n\n"
|
||||
|
||||
i = 0
|
||||
|
||||
|
|
@ -928,15 +358,17 @@ if __name__ == '__main__':
|
|||
|
||||
project_soft = soft_total_project[i]
|
||||
|
||||
#project_users = soft_total_users[i]
|
||||
|
||||
project_mau = soft_total_mau[i]
|
||||
|
||||
project_servers = soft_total_servers[i]
|
||||
|
||||
len_pr_soft = len(project_soft)
|
||||
|
||||
toot_text += f":{project_soft}: {project_mau:,} {project_servers:,}\n"
|
||||
if project_soft == 'ativity-relay':
|
||||
|
||||
project_soft = 'activityrelay'
|
||||
|
||||
toot_text += f":{project_soft}: {project_mau:,} / {project_servers:,}\n"
|
||||
|
||||
i += 1
|
||||
|
||||
|
|
@ -946,18 +378,14 @@ if __name__ == '__main__':
|
|||
|
||||
servers_image_id = mastodon.media_post('servers.png', "image/png", description='servers graph').id
|
||||
|
||||
#users_image_id = mastodon.media_post('users.png', "image/png", description='users graph').id
|
||||
|
||||
mau_image_id = mastodon.media_post('mau.png', "image/png", description='MAU graph').id
|
||||
|
||||
mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, mau_image_id})
|
||||
global_image_id = mastodon.media_post('global.png', "image/png", description='global graph').id
|
||||
|
||||
delete_dead_servers()
|
||||
mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, mau_image_id, global_image_id})
|
||||
|
||||
db.delete_dead_servers()
|
||||
|
||||
finish = datetime.now()
|
||||
|
||||
save_time(program, start, finish)
|
||||
|
||||
else:
|
||||
|
||||
usage()
|
||||
db.save_time(program, start, finish)
|
||||
|
|
|
|||
221
fetchservers.py
221
fetchservers.py
|
|
@ -4,13 +4,15 @@ import os
|
|||
import json
|
||||
import sys
|
||||
import os.path
|
||||
import psycopg2
|
||||
from setup import Setup
|
||||
from database import Database
|
||||
import requests
|
||||
import urllib3
|
||||
import socket
|
||||
import ray
|
||||
import pdb
|
||||
|
||||
ray.init(num_cpus = 32) # Specify this system CPUs.
|
||||
#ray.init(num_cpus = 25) # Specify this system CPUs.
|
||||
|
||||
from ray.exceptions import (
|
||||
RaySystemError,
|
||||
|
|
@ -19,18 +21,18 @@ from ray.exceptions import (
|
|||
ObjectStoreFullError,
|
||||
)
|
||||
|
||||
apis = ['/api/v1/instance?',
|
||||
'/api/v1/nodeinfo?',
|
||||
'/nodeinfo/2.0?',
|
||||
'/nodeinfo/2.0.json?',
|
||||
'/nodeinfo/2.1.json?',
|
||||
'/main/nodeinfo/2.0?',
|
||||
'/api/statusnet/config?',
|
||||
'/api/nodeinfo/2.0.json?',
|
||||
'/api/nodeinfo?',
|
||||
'/wp-json/nodeinfo/2.0?',
|
||||
'/api/v1/instance/nodeinfo/2.0?',
|
||||
'/.well-known/x-nodeinfo2?'
|
||||
apis = ['/api/v1/instance',
|
||||
'/api/v1/nodeinfo',
|
||||
'/nodeinfo/2.0',
|
||||
'/nodeinfo/2.0.json',
|
||||
'/nodeinfo/2.1.json',
|
||||
'/main/nodeinfo/2.0',
|
||||
'/api/statusnet/config',
|
||||
'/api/nodeinfo/2.0.json',
|
||||
'/api/nodeinfo',
|
||||
'/wp-json/nodeinfo/2.0',
|
||||
'/api/v1/instance/nodeinfo/2.0',
|
||||
'/.well-known/x-nodeinfo2'
|
||||
]
|
||||
|
||||
def is_json(myjson):
|
||||
|
|
@ -41,45 +43,6 @@ def is_json(myjson):
|
|||
return False
|
||||
return True
|
||||
|
||||
def write_api(server, software, users, alive, api, soft_version):
|
||||
|
||||
fediverse_db, fediverse_db_user = get_db_config()
|
||||
|
||||
insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
|
||||
|
||||
conn = None
|
||||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
print(f'Writing {server} nodeinfo data...')
|
||||
|
||||
cur.execute(insert_sql, (server, now, software, users, alive, api, soft_version))
|
||||
|
||||
cur.execute(
|
||||
"UPDATE fediverse SET updated_at=(%s), software=(%s), users=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)",
|
||||
(now, software, users, alive, api, soft_version, server)
|
||||
)
|
||||
|
||||
cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
|
||||
|
||||
conn.commit()
|
||||
|
||||
cur.close()
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print(error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
@ray.remote
|
||||
def getsoft(server):
|
||||
|
||||
|
|
@ -91,8 +54,6 @@ def getsoft(server):
|
|||
return
|
||||
if server.find(":") != -1:
|
||||
return
|
||||
if server == 'z.fedipen.xyz':
|
||||
return
|
||||
|
||||
soft = ''
|
||||
|
||||
|
|
@ -100,11 +61,9 @@ def getsoft(server):
|
|||
|
||||
url = 'https://' + server
|
||||
|
||||
user_agent = {'User-agent': "fediverse's stats (fediverse@mastodont.cat)"}
|
||||
|
||||
try:
|
||||
|
||||
response = requests.get(url + '/.well-known/nodeinfo', headers = user_agent, timeout=3)
|
||||
response = requests.get(url + '/.well-known/nodeinfo', headers = setup.user_agent, timeout=3)
|
||||
|
||||
if response.status_code == 200:
|
||||
|
||||
|
|
@ -112,11 +71,17 @@ def getsoft(server):
|
|||
|
||||
response_json = response.json()
|
||||
|
||||
if len(response_json['links']) == 1:
|
||||
|
||||
nodeinfo = response_json['links'][0]['href'].replace(f'https://{server}','')
|
||||
|
||||
elif len(response_json['links']) == 2:
|
||||
|
||||
nodeinfo = response_json['links'][1]['href'].replace(f'https://{server}','')
|
||||
|
||||
try:
|
||||
|
||||
nodeinfo_data = requests.get(url + nodeinfo, headers = user_agent, timeout=3)
|
||||
nodeinfo_data = requests.get(url + nodeinfo, headers = setup.user_agent, timeout=3)
|
||||
|
||||
if nodeinfo_data.status_code == 200:
|
||||
|
||||
|
|
@ -126,7 +91,7 @@ def getsoft(server):
|
|||
|
||||
else:
|
||||
|
||||
print(f"Server {server}'s nodeinfo not responding: error code {nodeinfo_data.status_code}")
|
||||
print(f"{nodeinfo} not responding: error code {nodeinfo_data.status_code}")
|
||||
|
||||
except:
|
||||
|
||||
|
|
@ -134,7 +99,7 @@ def getsoft(server):
|
|||
|
||||
except:
|
||||
|
||||
print(f'Server {server} not responding: error code {response.status_code}')
|
||||
print(f'{server} is not responding: error code {response.status_code}')
|
||||
print('*********************************************************************')
|
||||
|
||||
pass
|
||||
|
|
@ -144,7 +109,9 @@ def getsoft(server):
|
|||
|
||||
try:
|
||||
|
||||
response = requests.get(url + api, headers = user_agent, timeout=3)
|
||||
response = requests.get(url + api, headers = setup.user_agent, timeout=3)
|
||||
|
||||
if response.status_code == 200:
|
||||
|
||||
if is_json(response.text):
|
||||
|
||||
|
|
@ -233,12 +200,13 @@ def getsoft(server):
|
|||
soft = nodeinfo_json['software']['name']
|
||||
soft = soft.lower()
|
||||
soft_version = nodeinfo_json['software']['version']
|
||||
users = nodeinfo_json['usage']['users']['total']
|
||||
users = nodeinfo_json.get('usage').get('users').get('total') or '0'
|
||||
if users > 1000000:
|
||||
return
|
||||
|
||||
alive = True
|
||||
|
||||
write_api(server, soft, users, alive, nodeinfo, soft_version)
|
||||
db.write_api(server, soft, users, alive, nodeinfo, soft_version)
|
||||
|
||||
print(f"Server {server} ({soft} {soft_version}) is alive!")
|
||||
print('*********************************************************************')
|
||||
|
|
@ -259,11 +227,12 @@ def getsoft(server):
|
|||
users = nodeinfo_json['usage']['users']['total']
|
||||
if users > 1000000:
|
||||
return
|
||||
|
||||
alive = True
|
||||
|
||||
if soft == 'socialhome':
|
||||
|
||||
write_api(server, soft, users, alive, nodeinfo, soft_version)
|
||||
db.write_api(server, soft, users, alive, nodeinfo, soft_version)
|
||||
|
||||
print('*********************************************************************')
|
||||
print(f"Server {serve}r ({soft} {soft_version}) is alive!")
|
||||
|
|
@ -301,7 +270,7 @@ def getsoft(server):
|
|||
|
||||
alive = True
|
||||
|
||||
write_api(server, soft, users, alive, nodeinfo, soft_version)
|
||||
db.write_api(server, soft, users, alive, nodeinfo, soft_version)
|
||||
|
||||
print('*********************************************************************')
|
||||
print(f"Server {server} ({soft}) is alive!")
|
||||
|
|
@ -323,106 +292,6 @@ def getsoft(server):
|
|||
print(f'Server {server} is dead')
|
||||
print('*********************************************************************')
|
||||
|
||||
def get_world_servers():
|
||||
|
||||
world_servers = []
|
||||
|
||||
try:
|
||||
|
||||
conn = None
|
||||
|
||||
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
# get world servers list
|
||||
|
||||
cur.execute("select server from world where checked='f'")
|
||||
|
||||
rows = cur.fetchall()
|
||||
|
||||
for row in rows:
|
||||
|
||||
world_servers.append(row[0])
|
||||
|
||||
cur.close()
|
||||
|
||||
print("Remaining servers: " + str(len(world_servers)))
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print(error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
return world_servers
|
||||
|
||||
def save_time(program, start, finish):
|
||||
|
||||
insert_sql = "INSERT INTO execution_time(program, start, finish) VALUES(%s,%s,%s) ON CONFLICT DO NOTHING"
|
||||
|
||||
conn = None
|
||||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute(insert_sql, (program, start, finish,))
|
||||
|
||||
cur.execute("UPDATE execution_time SET start=(%s), finish=(%s) where program=(%s)", (start, finish, program))
|
||||
|
||||
conn.commit()
|
||||
|
||||
cur.close()
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print(error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
def get_parameter(parameter, file_path):
|
||||
|
||||
# Check if secrets file exists
|
||||
if not os.path.isfile(file_path):
|
||||
print("File %s not found, exiting." % file_path)
|
||||
sys.exit(0)
|
||||
|
||||
# Find parameter in file
|
||||
with open(file_path) as f:
|
||||
for line in f:
|
||||
if line.startswith(parameter):
|
||||
return line.replace(parameter + ":", "").strip()
|
||||
|
||||
# Cannot find parameter, exit
|
||||
print(file_path + " Missing parameter %s " % parameter)
|
||||
sys.exit(0)
|
||||
|
||||
def get_config():
|
||||
|
||||
# Load configuration from config file
|
||||
config_filepath = "config/config.txt"
|
||||
mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
|
||||
return mastodon_hostname
|
||||
|
||||
def get_db_config():
|
||||
|
||||
# Load database config from db_config file
|
||||
db_config_filepath = "config/db_config.txt"
|
||||
fediverse_db = get_parameter("fediverse_db", db_config_filepath)
|
||||
fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath)
|
||||
return (fediverse_db, fediverse_db_user)
|
||||
|
||||
###############################################################################
|
||||
# main
|
||||
|
||||
|
|
@ -430,7 +299,13 @@ if __name__ == '__main__':
|
|||
|
||||
## name: fetchservers.py
|
||||
|
||||
fediverse_db, fediverse_db_user = get_db_config()
|
||||
setup = Setup()
|
||||
|
||||
db = Database()
|
||||
|
||||
res = requests.get('https://' + 'mastodon.social' + setup.peers_api, headers = setup.user_agent, timeout=3)
|
||||
|
||||
hostname_peers = res.json()
|
||||
|
||||
start = datetime.now()
|
||||
|
||||
|
|
@ -438,19 +313,19 @@ if __name__ == '__main__':
|
|||
|
||||
finish = start
|
||||
|
||||
save_time(program, start, finish)
|
||||
db.save_time(program, start, finish)
|
||||
|
||||
now = start
|
||||
|
||||
mastodon_hostname = get_config()
|
||||
|
||||
world_servers = get_world_servers()
|
||||
#world_servers = db.get_world_servers()
|
||||
|
||||
ray_start = time.time()
|
||||
|
||||
try:
|
||||
|
||||
results = ray.get([getsoft.remote(server) for server in world_servers])
|
||||
#results = ray.get([getsoft.remote(server) for server in world_servers])
|
||||
results = ray.get([getsoft.remote(server) for server in hostname_peers])
|
||||
#[getsoft(server) for server in world_servers]
|
||||
|
||||
print(f"duration = {time.time() - ray_start}.\nprocessed servers: {len(results)}")
|
||||
|
||||
|
|
@ -460,4 +335,4 @@ if __name__ == '__main__':
|
|||
|
||||
finish = datetime.now()
|
||||
|
||||
save_time(program, start, finish)
|
||||
db.save_time(program, start, finish)
|
||||
|
|
|
|||
177
getpeers.py
177
getpeers.py
|
|
@ -1,177 +0,0 @@
|
|||
import os
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime
|
||||
import requests
|
||||
import json
|
||||
import psycopg2
|
||||
import ray
|
||||
import pdb
|
||||
|
||||
ray.init(num_cpus = 32) # Specify this system CPUs.
|
||||
|
||||
def write_server(server, federated_with):
|
||||
|
||||
insert_sql = "INSERT INTO world(server, federated_with, updated_at, saved_at, checked) VALUES(%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
|
||||
|
||||
conn = None
|
||||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute(insert_sql, (server, federated_with, now, now, 'f'))
|
||||
|
||||
print(f'writing {server} to world database')
|
||||
|
||||
conn.commit()
|
||||
|
||||
cur.close()
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print(error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
@ray.remote
|
||||
def get_peers(peer):
|
||||
|
||||
try:
|
||||
|
||||
user_agent = {'User-agent': "fediverse's stats (fediverse@mastodont.cat)"}
|
||||
|
||||
domain_res = requests.get('https://' + peer + '/api/v1/instance', headers = user_agent, timeout=3)
|
||||
|
||||
domain_res_json = domain_res.json()
|
||||
|
||||
if domain_res.status_code == 200:
|
||||
|
||||
domain_uri = domain_res_json['uri'].replace('https://', '')
|
||||
|
||||
if domain_uri != peer:
|
||||
|
||||
print(f'{peer} is an aliased domain of {domain_uri}!')
|
||||
|
||||
else:
|
||||
|
||||
response = requests.get('https://' + peer + peers_api, headers = user_agent, timeout=3)
|
||||
|
||||
response_json = response.json()
|
||||
|
||||
if response.status_code == 200:
|
||||
|
||||
try:
|
||||
|
||||
print(f"Server: {peer}, federated with {str(len(response_json))} servers")
|
||||
|
||||
for peer_peer in response_json:
|
||||
|
||||
write_server(peer_peer, peer)
|
||||
|
||||
except:
|
||||
|
||||
pass
|
||||
|
||||
except:
|
||||
|
||||
pass
|
||||
|
||||
def save_time(program, start, finish):
|
||||
|
||||
insert_sql = "INSERT INTO execution_time(program, start, finish) VALUES(%s,%s,%s) ON CONFLICT DO NOTHING"
|
||||
|
||||
conn = None
|
||||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute(insert_sql, (program, start, finish,))
|
||||
|
||||
cur.execute("UPDATE execution_time SET start=(%s), finish=(%s) where program=(%s)", (start, finish, program))
|
||||
|
||||
conn.commit()
|
||||
|
||||
cur.close()
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print(error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
def get_parameter( parameter, file_path ):
|
||||
|
||||
# Check if secrets file exists
|
||||
if not os.path.isfile(file_path):
|
||||
print("File %s not found, exiting."%file_path)
|
||||
sys.exit(0)
|
||||
|
||||
# Find parameter in file
|
||||
with open( file_path ) as f:
|
||||
for line in f:
|
||||
if line.startswith( parameter ):
|
||||
return line.replace(parameter + ":", "").strip()
|
||||
|
||||
# Cannot find parameter, exit
|
||||
print(file_path + " Missing parameter %s "%parameter)
|
||||
sys.exit(0)
|
||||
|
||||
###############################################################################
|
||||
# main
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
now = datetime.now()
|
||||
|
||||
peers_api = '/api/v1/instance/peers?'
|
||||
|
||||
# Load configuration from config file
|
||||
config_filepath = "config/config.txt"
|
||||
mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
|
||||
|
||||
# Load database config from db_config file
|
||||
db_config_filepath = "config/db_config.txt"
|
||||
fediverse_db = get_parameter("fediverse_db", db_config_filepath)
|
||||
fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath)
|
||||
|
||||
user_agent = {'User-agent': "fediverse's stats (fediverse@mastodont.cat)"}
|
||||
|
||||
res = requests.get('https://' + mastodon_hostname + peers_api, headers = user_agent, timeout=3)
|
||||
|
||||
hostname_peers = res.json()
|
||||
|
||||
start = datetime.now()
|
||||
|
||||
program = 'getpeers'
|
||||
|
||||
finish = start
|
||||
|
||||
save_time(program, start, finish)
|
||||
|
||||
for peer in hostname_peers:
|
||||
|
||||
write_server(peer, mastodon_hostname)
|
||||
|
||||
results = ray.get([get_peers.remote(server) for server in hostname_peers])
|
||||
|
||||
finish = datetime.now()
|
||||
|
||||
print(f"duration = {finish - start}.\nprocessed servers: {len(results)}")
|
||||
|
||||
save_time(program, start, finish)
|
||||
|
||||
|
||||
|
|
@ -1,9 +1,7 @@
|
|||
Mastodon.py>=1.5.1
|
||||
psycopg2-binary>=2.8.4
|
||||
aiohttp>=3.6.2
|
||||
aiodns>=2.0.0
|
||||
matplotlib>=3.3.4
|
||||
humanfriendly>=9.2
|
||||
urllib3>=1.26.8
|
||||
requests>=2.27.1
|
||||
ray>=1.11.0
|
||||
requests
|
||||
psycopg2-binary
|
||||
pytz
|
||||
ray
|
||||
Mastodon.py
|
||||
matplotlib
|
||||
pandas
|
||||
|
|
|
|||
275
setup.py
275
setup.py
|
|
@ -1,226 +1,157 @@
|
|||
import getpass
|
||||
from mastodon import Mastodon
|
||||
from mastodon.Mastodon import MastodonMalformedEventError, MastodonNetworkError, MastodonReadTimeout, MastodonAPIError, MastodonIllegalArgumentError
|
||||
import fileinput,re
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime
|
||||
import pytz
|
||||
from mastodon import Mastodon
|
||||
from mastodon.Mastodon import MastodonMalformedEventError, MastodonNetworkError, MastodonReadTimeout, MastodonAPIError, MastodonIllegalArgumentError
|
||||
import pdb
|
||||
|
||||
def create_dir():
|
||||
class Setup():
|
||||
|
||||
name = 'fediverse setup'
|
||||
|
||||
def __init__(self, config_file=None, mastodon_hostname=None, peers_api=None, user_agent=None, secrets_filepath=None, mastodon_app_token=None):
|
||||
|
||||
self.config_file = "config/config.txt"
|
||||
self.mastodon_hostname = self.__get_parameter("mastodon_hostname", self.config_file)
|
||||
self.peers_api = '/api/v1/instance/peers?'
|
||||
self.user_agent = {'User-agent': "fediverse's stats (fediverse@mastodont.cat)"}
|
||||
|
||||
self.secrets_filepath = 'secrets/secrets.txt'
|
||||
|
||||
is_setup = self.__check_mastodon_setup(self)
|
||||
|
||||
if is_setup:
|
||||
|
||||
self.mastodon_app_token = self.__get_mastodon_parameter("mastodon_app_token", self.secrets_filepath)
|
||||
|
||||
else:
|
||||
|
||||
self.mastodon_app_token = self.mastodon_setup(self)
|
||||
|
||||
@staticmethod
|
||||
def __check_mastodon_setup(self):
|
||||
|
||||
is_setup = False
|
||||
|
||||
if not os.path.isfile(self.secrets_filepath):
|
||||
print(f"File {self.secrets_filepath} not found, running setup.")
|
||||
else:
|
||||
is_setup = True
|
||||
|
||||
return is_setup
|
||||
|
||||
@staticmethod
|
||||
def mastodon_setup(self):
|
||||
|
||||
if not os.path.exists('secrets'):
|
||||
os.makedirs('secrets')
|
||||
|
||||
def create_file():
|
||||
self.mastodon_user = input("Mastodon user login? ")
|
||||
self.mastodon_password = input("Mastodon user password? ")
|
||||
self.app_name = 'fediverse'
|
||||
|
||||
if not os.path.exists('secrets/secrets.txt'):
|
||||
with open('secrets/secrets.txt', 'w'): pass
|
||||
print(secrets_filepath + " created!")
|
||||
self.mastodon_app_token = self.mastodon_log_in()
|
||||
|
||||
def create_config():
|
||||
if not os.path.exists(self.secrets_filepath):
|
||||
with open(self.secrets_filepath, 'w'): pass
|
||||
print(f"{self.secrets_filepath} created!")
|
||||
|
||||
if not os.path.exists('config'):
|
||||
os.makedirs('config')
|
||||
if not os.path.exists(config_filepath):
|
||||
print(config_filepath + " created!")
|
||||
with open('config/config.txt', 'w'): pass
|
||||
with open(self.secrets_filepath, 'a') as the_file:
|
||||
print("Writing Mastodon parameters to " + self.secrets_filepath)
|
||||
the_file.write(f'mastodon_app_token: {self.mastodon_app_token}')
|
||||
|
||||
def write_params():
|
||||
return self.mastodon_app_token
|
||||
|
||||
with open(secrets_filepath, 'a') as the_file:
|
||||
print("Writing secrets parameter names to " + secrets_filepath)
|
||||
the_file.write('uc_client_id: \n'+'uc_client_secret: \n'+'uc_access_token: \n')
|
||||
def mastodon_log_in(self):
|
||||
|
||||
def write_config():
|
||||
|
||||
with open(config_filepath, 'a') as the_file:
|
||||
the_file.write('mastodon_hostname: \n')
|
||||
print("adding parameter name 'mastodon_hostname' to "+ config_filepath)
|
||||
|
||||
def read_client_lines(self):
|
||||
|
||||
client_path = 'app_clientcred.txt'
|
||||
|
||||
with open(client_path) as fp:
|
||||
|
||||
line = fp.readline()
|
||||
cnt = 1
|
||||
while line:
|
||||
if cnt == 1:
|
||||
print("Writing client id to " + secrets_filepath)
|
||||
modify_file(secrets_filepath, "uc_client_id: ", value=line.rstrip())
|
||||
elif cnt == 2:
|
||||
print("Writing client secret to " + secrets_filepath)
|
||||
modify_file(secrets_filepath, "uc_client_secret: ", value=line.rstrip())
|
||||
line = fp.readline()
|
||||
cnt += 1
|
||||
|
||||
def read_token_line(self):
|
||||
|
||||
token_path = 'app_usercred.txt'
|
||||
|
||||
with open(token_path) as fp:
|
||||
|
||||
line = fp.readline()
|
||||
print("Writing access token to " + secrets_filepath)
|
||||
modify_file(secrets_filepath, "uc_access_token: ", value=line.rstrip())
|
||||
|
||||
def read_config_line():
|
||||
|
||||
with open(config_filepath) as fp:
|
||||
|
||||
line = fp.readline()
|
||||
modify_file(config_filepath, "mastodon_hostname: ", value=hostname)
|
||||
modify_file(config_filepath, "bot_username: ", value=bot_username)
|
||||
|
||||
def log_in():
|
||||
|
||||
error = 0
|
||||
token = ''
|
||||
|
||||
try:
|
||||
|
||||
global hostname
|
||||
hostname = input("Enter Mastodon hostname: ")
|
||||
user_name = input("User name, ex. user@" + hostname +"? ")
|
||||
user_password = getpass.getpass("User password? ")
|
||||
bot_username = input("Bot's username, ex. fediverse: ")
|
||||
app_name = input("This app name? ")
|
||||
Mastodon.create_app(
|
||||
app_name,
|
||||
response = Mastodon.create_app(
|
||||
self.app_name,
|
||||
scopes=["read","write"],
|
||||
to_file="app_clientcred.txt",
|
||||
api_base_url=hostname
|
||||
to_file=None,
|
||||
api_base_url=self.mastodon_hostname
|
||||
)
|
||||
mastodon = Mastodon(client_id = "app_clientcred.txt", api_base_url = hostname)
|
||||
mastodon.log_in(
|
||||
user_name,
|
||||
user_password,
|
||||
client_id = response[0]
|
||||
client_secret = response[1]
|
||||
|
||||
mastodon = Mastodon(client_id = client_id, client_secret = client_secret, api_base_url = self.mastodon_hostname)
|
||||
|
||||
token = mastodon.log_in(
|
||||
self.mastodon_user,
|
||||
self.mastodon_password,
|
||||
scopes = ["read", "write"],
|
||||
to_file = "app_usercred.txt"
|
||||
to_file = None
|
||||
)
|
||||
|
||||
print('Log in succesful!')
|
||||
|
||||
except MastodonIllegalArgumentError as i_error:
|
||||
|
||||
error = 1
|
||||
|
||||
if os.path.exists("secrets/secrets.txt"):
|
||||
print("Removing secrets/secrets.txt file..")
|
||||
os.remove("secrets/secrets.txt")
|
||||
if os.path.exists("app_clientcred.txt"):
|
||||
print("Removing app_clientcred.txt file..")
|
||||
os.remove("app_clientcred.txt")
|
||||
sys.exit(i_error)
|
||||
sys.stdout.write(f'\n{str(i_error)}\n')
|
||||
|
||||
except MastodonNetworkError as n_error:
|
||||
|
||||
error = 1
|
||||
|
||||
if os.path.exists("secrets/secrets.txt"):
|
||||
print("Removing secrets/secrets.txt file..")
|
||||
os.remove("secrets/secrets.txt")
|
||||
if os.path.exists("app_clientcred.txt"):
|
||||
print("Removing app_clientcred.txt file..")
|
||||
os.remove("app_clientcred.txt")
|
||||
sys.exit(n_error)
|
||||
sys.stdout.write(f'\n{str(n_error)}\n')
|
||||
|
||||
except MastodonReadTimeout as r_error:
|
||||
|
||||
error = 1
|
||||
|
||||
if os.path.exists("secrets/secrets.txt"):
|
||||
print("Removing secrets/secrets.txt file..")
|
||||
os.remove("secrets/secrets.txt")
|
||||
if os.path.exists("app_clientcred.txt"):
|
||||
print("Removing app_clientcred.txt file..")
|
||||
os.remove("app_clientcred.txt")
|
||||
sys.exit(r_error)
|
||||
sys.stdout.write(f'\n{str(r_error)}\n')
|
||||
|
||||
except MastodonAPIError as a_error:
|
||||
|
||||
error = 1
|
||||
|
||||
if os.path.exists("secrets/secrets.txt"):
|
||||
print("Removing secrets/secrets.txt file..")
|
||||
os.remove("secrets/secrets.txt")
|
||||
if os.path.exists("app_clientcred.txt"):
|
||||
print("Removing app_clientcred.txt file..")
|
||||
os.remove("app_clientcred.txt")
|
||||
sys.exit(a_error)
|
||||
sys.stdout.write(f'\n{str(a_error)}\n')
|
||||
|
||||
finally:
|
||||
|
||||
if error == 0:
|
||||
return token
|
||||
|
||||
create_dir()
|
||||
create_file()
|
||||
write_params()
|
||||
client_path = 'app_clientcred.txt'
|
||||
read_client_lines(client_path)
|
||||
token_path = 'app_usercred.txt'
|
||||
read_token_line(token_path)
|
||||
if os.path.exists("app_clientcred.txt"):
|
||||
print("Removing app_clientcred.txt temp file..")
|
||||
os.remove("app_clientcred.txt")
|
||||
if os.path.exists("app_usercred.txt"):
|
||||
print("Removing app_usercred.txt temp file..")
|
||||
os.remove("app_usercred.txt")
|
||||
print("Secrets setup done!\n")
|
||||
def __get_parameter(self, parameter, config_file):
|
||||
|
||||
def modify_file(file_name,pattern,value=""):
|
||||
if not os.path.isfile(config_file):
|
||||
print(f"File {config_file} not found..")
|
||||
|
||||
fh=fileinput.input(file_name,inplace=True)
|
||||
for line in fh:
|
||||
replacement=pattern + value
|
||||
line=re.sub(pattern,replacement,line)
|
||||
sys.stdout.write(line)
|
||||
fh.close()
|
||||
self.mastodon_hostname = input("\nMastodon hostname: ")
|
||||
|
||||
def get_parameter( parameter, file_path ):
|
||||
# Check if secrets file exists
|
||||
if not os.path.isfile(file_path):
|
||||
print("File %s not found, creating it."%file_path)
|
||||
log_in()
|
||||
self.__create_config(self)
|
||||
self.__write_config(self)
|
||||
|
||||
# Find parameter in file
|
||||
with open( file_path ) as f:
|
||||
with open( self.config_file ) as f:
|
||||
for line in f:
|
||||
if line.startswith( parameter ):
|
||||
return line.replace(parameter + ":", "").strip()
|
||||
|
||||
# Cannot find parameter, exit
|
||||
print(file_path + " Missing parameter %s "%parameter)
|
||||
sys.exit(0)
|
||||
def __get_mastodon_parameter(self, parameter, secrets_filepath):
|
||||
|
||||
def get_hostname( parameter, config_filepath ):
|
||||
# Check if secrets file exists
|
||||
if not os.path.isfile(config_filepath):
|
||||
print("File %s not found, creating it."%config_filepath)
|
||||
create_config()
|
||||
if not os.path.isfile(secrets_filepath):
|
||||
print(f"File {secrets_filepath} not found..")
|
||||
|
||||
# Find parameter in file
|
||||
with open( config_filepath ) as f:
|
||||
self.sign_in()
|
||||
|
||||
with open( self.secrets_filepath ) as f:
|
||||
for line in f:
|
||||
if line.startswith( parameter ):
|
||||
return line.replace(parameter + ":", "").strip()
|
||||
|
||||
# Cannot find parameter, exit
|
||||
print(config_filepath + " Missing parameter %s "%parameter)
|
||||
write_config()
|
||||
read_config_line()
|
||||
print("setup done!")
|
||||
sys.exit(0)
|
||||
@staticmethod
|
||||
def __create_config(self):
|
||||
|
||||
###############################################################################
|
||||
# main
|
||||
if not os.path.exists('config'):
|
||||
|
||||
if __name__ == '__main__':
|
||||
os.makedirs('config')
|
||||
|
||||
# Load secrets from secrets file
|
||||
secrets_filepath = "secrets/secrets.txt"
|
||||
uc_client_id = get_parameter("uc_client_id", secrets_filepath)
|
||||
uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
|
||||
uc_access_token = get_parameter("uc_access_token", secrets_filepath)
|
||||
if not os.path.exists(self.config_file):
|
||||
|
||||
# Load configuration from config file
|
||||
config_filepath = "config/config.txt"
|
||||
mastodon_hostname = get_hostname("mastodon_hostname", config_filepath)
|
||||
bot_username = get_parameter("bot_username", config_filepath)
|
||||
print(self.config_file + " created!")
|
||||
with open(self.config_file, 'w'): pass
|
||||
|
||||
@staticmethod
|
||||
def __write_config(self):
|
||||
|
||||
with open(self.config_file, 'a') as the_file:
|
||||
|
||||
the_file.write(f'mastodon_hostname: {self.mastodon_hostname}')
|
||||
print(f"adding parameters to {self.config_file}\n")
|
||||
|
|
|
|||
Loading…
Referencia en una nova incidència