import time import urllib3 from datetime import datetime from mastodon import Mastodon import os import json import sys import os.path import requests import psycopg2 from itertools import product import multiprocessing import aiohttp import asyncio import socket import matplotlib.pyplot as plt plt.style.use('seaborn') start_time = time.time() apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?', '/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?', '/wp-json/nodeinfo/2.0?'] client_exceptions = ( aiohttp.ClientResponseError, aiohttp.ClientConnectionError, aiohttp.ClientConnectorError, aiohttp.ClientError, asyncio.TimeoutError, socket.gaierror, ) def is_json(myjson): try: json_object = json.loads(myjson) except ValueError as e: print(e) return False return True def get_alive_servers(server): serv_api = '' serv_soft = '' soft_version = '' try: conn = None conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432") cur = conn.cursor() cur.execute("select alive, software, users_api, version, first_checked_at, downs from fediverse where server=(%s)", (server,)) row = cur.fetchone() if row is not None: was_alive = row[0] serv_soft = row[1] serv_api = row[2] soft_version = row[3] first_checked_at = row[4] downs_qty = row[5] cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() alive = False try: user_agent = {'User-agent': 'Mozilla/5.0'} data = requests.get('https://' + server + serv_api, headers = user_agent, timeout=3) if serv_soft == "mastodon": if serv_api == '/nodeinfo/2.0?': try: users = data.json()['usage']['users']['total'] soft_version = data.json()['software']['version'] alive = True except: users = 0 soft_version = "" if serv_api == '/nodeinfo/2.0.json?': try: users = data.json()['usage']['users']['total'] soft_version = data.json()['software']['version'] alive = True except: users = 0 soft_version = "" elif serv_api == '/api/v1/instance?': try: users = data.json()['stats']['user_count'] soft_version = data.json()['version'] alive = True except: users = 0 soft_version = "" if serv_soft == "pleroma" or serv_soft == "diaspora" or serv_soft == "peertube" or serv_soft == "pixelfed" or serv_soft == "hubzilla" or serv_soft == "writefreely" or serv_soft == "friendica": try: users = data.json()['usage']['users']['total'] soft_version = data.json()['software']['version'] alive = True except: users = 0 soft_version = "" if serv_soft == "gnusocialv2" or serv_soft == "gnusocial": try: users = data.json()['usage']['users']['total'] if users == 0: users = data.json()['usage']['users']['activeHalfyear'] soft_version = data.json()['software']['version'] alive = True except: users = 0 soft_version = "" if serv_soft == "plume" or serv_soft == 'red' or serv_soft == "misskey" or serv_soft == "zap" or serv_soft == "prismo" or serv_soft == "ravenvale" or serv_soft == "osada" or serv_soft == "groundpolis": try: users = data.json()['usage']['users']['total'] soft_version = data.json()['software']['version'] alive = True except: users = 0 soft_version = "" if serv_soft == "ganggo" or serv_soft == "squs" or serv_soft == "dolphin" or serv_soft == "lemmy" or serv_soft == "wordpress": try: users = data.json()['usage']['users']['total'] soft_version = data.json()['software']['version'] if serv_soft == "wordpress" and "activitypub" in data.json()['protocols']: alive = True elif serv_soft == "wordpress" and "activitypub" not in data.json()['protocols']: alive = False else: alive = True except: users = 0 soft_version = "" if alive: if downs_qty != None: downs = downs_qty else: downs = 0 if soft_version != "" and soft_version is not None: print("Server " + str(server) + " (" + serv_soft + " " + soft_version + ") is alive!") else: print("Server " + str(server) + " (" + serv_soft + ") is alive!") insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version, first_checked_at, last_checked_at, downs) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING" conn = None try: conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432") cur = conn.cursor() cur.execute(insert_sql, (server, users, now, serv_soft, alive, serv_api, soft_version, now, now, downs)) if first_checked_at != None: cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (users, now, serv_soft, alive, serv_api, soft_version, now, downs, server)) else: cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), first_checked_at=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (users, now, serv_soft, alive, serv_api, soft_version, now, now, downs, server)) cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,)) conn.commit() cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() except urllib3.exceptions.ProtocolError as protoerr: print(protoerr) print("Server " + server + " is dead :-(") alive = False pass except requests.exceptions.ChunkedEncodingError as chunkerr: print(chunkerr) print("Server " + server + " is dead :-(") alive = False pass except KeyError as e: print(e) print("Server " + server + " is dead :-(") alive = False pass except ValueError as verr: print(verr) print("Server " + server + " is dead :-(") alive = False pass except requests.exceptions.SSLError as errssl: print(errssl) print("Server " + server + " is dead :-(") alive = False pass except requests.exceptions.HTTPError as errh: print(errh) print("Server " + server + " is dead :-(") alive = False pass except requests.exceptions.ConnectionError as errc: print(errc) print("Server " + server + " is dead :-(") alive = False pass except requests.exceptions.Timeout as errt: print(errt) print("Server " + server + " is dead :-(") alive = False pass except requests.exceptions.RequestException as err: print(err) print("Server " + server + " is dead :-(") alive = False pass except socket.gaierror as gai_error: print(gai_error) pass if alive is False: if downs_qty != None: downs = downs_qty + 1 else: downs = 1 conn = None try: conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432") cur = conn.cursor() if first_checked_at != None: cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), first_checked_at=(%s), downs=(%s) where server=(%s)", (now, alive, NULL, downs, server)) else: cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), downs=(%s) where server=(%s)", (now, alive, downs, server)) cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,)) conn.commit() cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() def write_api(server, software, users, alive, api, soft_version): insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING" conn = None try: conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432") cur = conn.cursor() cur.execute(insert_sql, (server, now, software, users, alive, api, soft_version)) cur.execute("UPDATE fediverse SET updated_at=(%s), software=(%s), users=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)", (now, software, users, alive, api, soft_version, server)) cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,)) conn.commit() cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() async def getsoft(server): try: socket.gethostbyname(server) except socket.gaierror: pass return soft = '' url = 'https://' + server timeout = aiohttp.ClientTimeout(total=3) async with aiohttp.ClientSession(timeout=timeout) as session: for api in apis: try: async with session.get(url+api) as response: if response.status == 200: try: response_json = await response.json() except: pass except aiohttp.ClientConnectorError as err: print(err) pass else: if response.status == 200 and api != '/api/v1/instance?': try: soft = response_json['software']['name'] soft = soft.lower() soft_version = response_json['software']['version'] users = response_json['usage']['users']['total'] if users > 1000000: return alive = True write_api(server, soft, users, alive, api, soft_version) print("Server " + server + " (" + soft + " " + soft_version + ") is alive!") return except: pass if response.status == 200 and soft == '' and api == "/api/v1/instance?": soft = 'mastodon' users = response_json['stats']['user_count'] soft_version = response_json['version'] if users > 1000000: return alive = True write_api(server, soft, users, alive, api) print("Server " + server + " (" + soft + ") is alive!") def getserver(server, x): server = server[0].rstrip('.').lower() if server.find(".") == -1: return if server.find("@") != -1: return if server.find("/") != -1: return if server.find(":") != -1: return try: loop = asyncio.get_event_loop() coroutines = [getsoft(server)] soft = loop.run_until_complete(asyncio.gather(*coroutines, return_exceptions=True)) except: pass def set_world_servers_check_to_false(): ############################################################################ # set all world servers's checked column to False try: conn = None conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432") cur = conn.cursor() cur.execute("UPDATE world SET checked='f'") conn.commit() cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() def get_last_checked_servers(): ############################################################################ # get last checked servers from fediverse DB alive_servers = [] try: conn = None conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432") cur = conn.cursor() # get world servers list cur.execute("select server from world where server in (select server from fediverse where users_api != '')") alive_servers = [] for row in cur: alive_servers.append(row[0]) cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() return alive_servers def mastodon(): # Load secrets from secrets file secrets_filepath = "secrets/secrets.txt" uc_client_id = get_parameter("uc_client_id", secrets_filepath) uc_client_secret = get_parameter("uc_client_secret", secrets_filepath) uc_access_token = get_parameter("uc_access_token", secrets_filepath) # Load configuration from config file config_filepath = "config/config.txt" mastodon_hostname = get_parameter("mastodon_hostname", config_filepath) # Initialise Mastodon API mastodon = Mastodon( client_id=uc_client_id, client_secret=uc_client_secret, access_token=uc_access_token, api_base_url='https://' + mastodon_hostname, ) # Initialise access headers headers = {'Authorization': 'Bearer %s'%uc_access_token} return (mastodon, mastodon_hostname) def db_config(): # Load db configuration from config file config_filepath = "config/db_config.txt" fediverse_db = get_parameter("fediverse_db", config_filepath) fediverse_db_user = get_parameter("fediverse_db_user", config_filepath) return (fediverse_db, fediverse_db_user) def usage(): print('usage: python ' + sys.argv[0] + ' --multi' + ' (multiprocessing, fast)') print('usage: python ' + sys.argv[0] + ' --mono' + ' (one process, slow)') # Returns the parameter from the specified file def get_parameter(parameter, file_path): # Check if secrets file exists if not os.path.isfile(file_path): print("File %s not found, exiting."%file_path) sys.exit(0) # Find parameter in file with open(file_path) as f: for line in f: if line.startswith(parameter): return line.replace(parameter + ":", "").strip() # Cannot find parameter, exit print(file_path + " Missing parameter %s "%parameter) sys.exit(0) ############################################################################### # main if __name__ == '__main__': # usage modes if len(sys.argv) == 1: usage() elif len(sys.argv) == 2: if sys.argv[1] == '--multi': now = datetime.now() mastodon, mastodon_hostname = mastodon() fediverse_db, fediverse_db_user = db_config() total_servers = 0 total_users = 0 set_world_servers_check_to_false() alive_servers = get_last_checked_servers() ########################################################################### # multiprocessing! nprocs = multiprocessing.cpu_count() with multiprocessing.Pool(processes=64) as pool: results = pool.starmap(get_alive_servers, product(alive_servers)) elif sys.argv[1] == '--mono': now = datetime.now() mastodon, mastodon_hostname = mastodon() fediverse_db, fediverse_db_user = db_config() total_servers = 0 total_users = 0 set_world_servers_check_to_false() alive_servers = get_last_checked_servers() i = 0 while i < len(alive_servers): get_alive_servers(alive_servers[i]) i += 1 else: usage() ########################################################################### # get current total servers and users, get users from every software gettotals_sql = "select count(server), sum(users) from fediverse where alive" get_soft_totals_sql = "select software, sum(users) as users, count(server) as servers from fediverse where users != 0 and alive group by software order by users desc" soft_total_project = [] soft_total_users = [] soft_total_servers = [] try: conn = None conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432") cur = conn.cursor() cur.execute(gettotals_sql) row = cur.fetchone() total_servers = row[0] total_users = row[1] cur.execute(get_soft_totals_sql) rows = cur.fetchall() for row in rows: soft_total_project.append(row[0]) soft_total_users.append(row[1]) soft_total_servers.append(row[2]) cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() ########################################################################### # get last check values and write current total ones select_sql = "select total_servers, total_users from totals order by datetime desc limit 1" insert_sql = "INSERT INTO totals(datetime, total_servers, total_users) VALUES(%s,%s,%s)" try: conn = None conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432") cur = conn.cursor() cur.execute(select_sql) row = cur.fetchone() if row is not None: servers_before = row[0] users_before = row[1] else: servers_before = 0 users_before = 0 cur.execute(insert_sql, (now, total_servers, total_users)) conn.commit() cur.close() evo_servers = total_servers - servers_before evo_users = total_users - users_before except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() ################################################################################ # write evo values insert_sql = "INSERT INTO evo(datetime, servers, users) VALUES(%s,%s,%s)" conn = None try: conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432") cur = conn.cursor() cur.execute(insert_sql, (now, evo_servers, evo_users)) conn.commit() cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() ############################################################################## # get world's last update datetime conn = None try: conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432") cur = conn.cursor() cur.execute("select updated_at from world order by updated_at desc limit 1") row = cur.fetchone() last_update = row[0] last_update = last_update.strftime('%m/%d/%Y, %H:%M:%S') cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() ############################################################################## # get max servers and users conn = None try: conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432") cur = conn.cursor() cur.execute("select MAX(total_servers) from totals") row = cur.fetchone() if row is not None: max_servers = row[0] else: max_servers = 0 cur.execute("select MAX(total_users) from totals") row = cur.fetchone() if row is not None: max_users = row[0] else: max_users = 0 cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() ############################################################################### # get plots servers_plots = [] users_plots = [] conn = None try: conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432") cur = conn.cursor() cur.execute("select total_servers, total_users from totals order by datetime desc limit 14") rows = cur.fetchall() for row in rows: servers_plots.append(row[0]) users_plots.append(row[1]) cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() ############################################################################### # generate graphs plt.plot([-6, -5, -4, -3, -2, -1, 0], [servers_plots[6], servers_plots[5], servers_plots[4], servers_plots[3], servers_plots[2], servers_plots[1], servers_plots[0]], marker='o', color='mediumseagreen') plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_servers, max_servers, max_servers, max_servers, max_servers, max_servers, max_servers], color='red') plt.title('fediverse: total alive servers (max: ' + str(f"{max_servers:,}" + ')'), loc='right', color='blue') plt.xlabel('Last seven days') plt.ylabel('fediverse alive servers') plt.legend(('servers', 'max'), shadow=True, loc=(0.01, 1.00), handlelength=1.5, fontsize=10) plt.savefig('servers.png') plt.close() plt.plot([-6, -5, -4, -3, -2, -1, 0], [users_plots[6], users_plots[5], users_plots[4], users_plots[3], users_plots[2], users_plots[1], users_plots[0]], marker='o', color='royalblue') plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_users, max_users, max_users, max_users, max_users, max_users, max_users], color='red') plt.title('fediverse: total registered users (max: ' + str(f"{max_users:,}" + ')'), loc='right', color='royalblue') plt.legend(('users', 'max'), shadow=True, loc=(0.01, 0.80), handlelength=1.5, fontsize=10) plt.xlabel('Last seven days') plt.ylabel('Registered users') plt.savefig('users.png') plt.close() ############################################################################### # T O O T ! toot_text = "#fediverse alive servers stats" + " \n" toot_text += "\n" if evo_servers >= 0: toot_text += "alive servers: " + str(f"{total_servers:,}") + " (+"+ str(f"{evo_servers:,}") + ") \n" toot_text += "max: " + str(f"{max_servers:,}") + "\n" elif evo_servers < 0: toot_text += "alive servers: " + str(f"{total_servers:,}") + " ("+ str(f"{evo_servers:,}") + ") \n" toot_text += "max: " + str(f"{max_servers:,}") + "\n" if evo_users >= 0: toot_text += "total users: " + str(f"{total_users:,}") + " (+"+ str(f"{evo_users:,}") + ") \n" toot_text += "max: " + str(f"{max_users:,}") + "\n" elif evo_users < 0: toot_text += "total users: " + str(f"{total_users:,}") + " ("+ str(f"{evo_users:,}") + ") \n" toot_text += "max: " + str(f"{max_users:,}") + "\n" toot_text += "\n" toot_text += "top ten (soft users servers):" + " \n" toot_text += "\n" i = 0 while i < 10: project_soft = soft_total_project[i] project_users = soft_total_users[i] project_servers = soft_total_servers[i] len_pr_soft = len(project_soft) if project_soft == 'writefreely': str_len = 11 else: str_len = 13 toot_text += f"{':'+project_soft+':':<11}" + f"{project_users:>{str_len},}" + " " + f"{project_servers:>5,}" + "\n" i += 1 print("Tooting...") print(toot_text) servers_image_id = mastodon.media_post('servers.png', "image/png").id users_image_id = mastodon.media_post('users.png', "image/png").id mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, users_image_id})