diff --git a/README.md b/README.md index 5599d71..158cec6 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,7 @@ Within Python Virtual Environment: | prismo | nodeinfo/2.0.json | ['software']['name'] | | ravenvale | nodeinfo/2.0.json | ['software']['name'] | | squs | nodeinfo/2.0 | ['software']['name'] | +| wordpress | wp-json/nodeinfo/2.0 | ['software']['name'] | | writefreely | api/nodeinfo | ['software']['name'] | | zap | nodeinfo/2.0.json | ['software']['name'] | @@ -72,4 +73,6 @@ Within Python Virtual Environment: Note: install all needed packages with 'pip install package' or use 'pip install -r requirements.txt' to install them. -18.2.21 - New feature! Added [Lemmy project](https://join.lemmy.ml) +18.2.21 - New feature! Added [Lemmy project](https://join.lemmy.ml) +12.5.21 - New feature! Added Wordpress support. The code can now detect Wordpress instances with ActivityPub enabled plugin. +12.5.21 - New feature! New shinny creation of servers and users graphs. diff --git a/fediverse.py b/fediverse.py index 567277e..ce75b8a 100644 --- a/fediverse.py +++ b/fediverse.py @@ -1,8 +1,4 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - import time -start_time = time.time() import urllib3 from urllib3 import exceptions from datetime import datetime @@ -32,10 +28,17 @@ from asyncio import TimeoutError import socket from socket import gaierror, gethostbyname +import matplotlib +import matplotlib.pyplot as plt + +plt.style.use('seaborn') + +start_time = time.time() + from decimal import * getcontext().prec = 2 -apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?', '/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?'] +apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?', '/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?','/wp-json/nodeinfo/2.0?'] client_exceptions = ( aiohttp.ClientResponseError, @@ -46,12 +49,6 @@ client_exceptions = ( socket.gaierror, ) -now = datetime.now() - -############################################################################### -# INITIALISATION -############################################################################### - def is_json(myjson): try: json_object = json.loads(myjson) @@ -59,9 +56,11 @@ def is_json(myjson): return False return True -def alive_server(server, x): - - server = server[0].rstrip('.').lower() +def get_alive_servers(server): + + serv_api = '' + serv_soft = '' + soft_version = '' conn = None @@ -150,46 +149,61 @@ def alive_server(server, x): except: users = 0 soft_version = "" - if serv_soft == "ganggo" or serv_soft == "squs" or serv_soft == "dolphin": + if serv_soft == "ganggo" or serv_soft == "squs" or serv_soft == "dolphin" or serv_soft == "lemmy" or serv_soft == "wordpress": try: users = data.json()['usage']['users']['total'] soft_version = data.json()['software']['version'] - alive = True + if serv_soft == "wordpress" and "activitypub" in data.json()['protocols']: + + alive = True + + elif serv_soft == "wordpress" and "activitypub" not in data.json()['protocols']: + + alive = False + + else: + + alive = True + except: + users = 0 soft_version = "" - if soft_version != "" and soft_version != None: - print("Server " + str(server) + " (" + serv_soft + " " + soft_version + ") is alive!") - else: - print("Server " + str(server) + " (" + serv_soft + ") is alive!") + if alive: + + if soft_version != "" and soft_version != None: + print("Server " + str(server) + " (" + serv_soft + " " + soft_version + ") is alive!") + else: + print("Server " + str(server) + " (" + serv_soft + ") is alive!") - insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING" - conn = None + insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING" - try: + conn = None - conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") + try: - cur = conn.cursor() + conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") - cur.execute(insert_sql, (server, users, now, serv_soft, alive, serv_api, soft_version)) + cur = conn.cursor() - cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)", (users, now, serv_soft, alive, serv_api, soft_version, server)) + cur.execute(insert_sql, (server, users, now, serv_soft, alive, serv_api, soft_version)) - cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,)) + cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)", (users, now, serv_soft, alive, serv_api, soft_version, server)) - conn.commit() + cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,)) - cur.close() + conn.commit() - except (Exception, psycopg2.DatabaseError) as error: - print(error) + cur.close() - finally: + except (Exception, psycopg2.DatabaseError) as error: + print(error) - if conn is not None: - conn.close() + finally: + + if conn is not None: + conn.close() except urllib3.exceptions.ProtocolError as protoerr: @@ -245,6 +259,11 @@ def alive_server(server, x): alive = False pass + except socket.gaierror as gai_error: + + print(gai_error) + pass + if alive == False: conn = None @@ -311,6 +330,7 @@ async def getsoft(server): except socket.gaierror: + pass return soft = '' @@ -377,56 +397,7 @@ def getserver(server, x): pass -# Returns the parameter from the specified file -def get_parameter( parameter, file_path ): - # Check if secrets file exists - if not os.path.isfile(file_path): - print("File %s not found, exiting."%file_path) - sys.exit(0) - - # Find parameter in file - with open( file_path ) as f: - for line in f: - if line.startswith( parameter ): - return line.replace(parameter + ":", "").strip() - - # Cannot find parameter, exit - print(file_path + " Missing parameter %s "%parameter) - sys.exit(0) - -# Load secrets from secrets file -secrets_filepath = "secrets/secrets.txt" -uc_client_id = get_parameter("uc_client_id", secrets_filepath) -uc_client_secret = get_parameter("uc_client_secret", secrets_filepath) -uc_access_token = get_parameter("uc_access_token", secrets_filepath) - -# Load configuration from config file -config_filepath = "config/config.txt" -mastodon_hostname = get_parameter("mastodon_hostname", config_filepath) - -# Load database config from db_config file -db_config_filepath = "config/db_config.txt" -fediverse_db = get_parameter("fediverse_db", db_config_filepath) -fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath) - -# Initialise Mastodon API -mastodon = Mastodon( - client_id = uc_client_id, - client_secret = uc_client_secret, - access_token = uc_access_token, - api_base_url = 'https://' + mastodon_hostname, -) - -# Initialise access headers -headers={ 'Authorization': 'Bearer %s'%uc_access_token } - -############################################################################### -# main - -if __name__ == '__main__': - - total_servers = 0 - total_users = 0 +def set_world_servers_check_to_false(): ############################################################################ # set all world servers's checked column to False @@ -455,9 +426,10 @@ if __name__ == '__main__': conn.close() - ############################################################################ - # get last check servers from fediverse DB +def get_last_checked_servers(): + ############################################################################ + # get last checked servers from fediverse DB alive_servers = [] @@ -490,264 +462,438 @@ if __name__ == '__main__': conn.close() - ########################################################################### - # multiprocessing! + return alive_servers - m = Manager() - q = m.Queue() - z = zip(alive_servers) +def mastodon(): - serv_number = len(alive_servers) + # Load secrets from secrets file + secrets_filepath = "secrets/secrets.txt" + uc_client_id = get_parameter("uc_client_id", secrets_filepath) + uc_client_secret = get_parameter("uc_client_secret", secrets_filepath) + uc_access_token = get_parameter("uc_access_token", secrets_filepath) - pool_tuple = [(x, q) for x in z] - with Pool(processes=64) as pool: - pool.starmap(alive_server, pool_tuple) + # Load configuration from config file + config_filepath = "config/config.txt" + mastodon_hostname = get_parameter("mastodon_hostname", config_filepath) - ########################################################################### + # Initialise Mastodon API + mastodon = Mastodon( + client_id = uc_client_id, + client_secret = uc_client_secret, + access_token = uc_access_token, + api_base_url = 'https://' + mastodon_hostname, + ) - print("Getting the remaining servers from world") + # Initialise access headers + headers={ 'Authorization': 'Bearer %s'%uc_access_token } - world_servers = [] + return (mastodon, mastodon_hostname) - try: +def db_config(): - conn = None - conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") + # Load db configuration from config file + config_filepath = "config/db_config.txt" + fediverse_db = get_parameter("fediverse_db", config_filepath) + fediverse_db_user = get_parameter("fediverse_db_user", config_filepath) - cur = conn.cursor() + return (fediverse_db, fediverse_db_user) - ### get world servers list +def usage(): - cur.execute("select server from world where checked='f'") + print('usage: python ' + sys.argv[0] + ' --multi' + ' (multiprocessing, fast)') + print('usage: python ' + sys.argv[0] + ' --mono' + ' (one process, slow)') - for row in cur: +# Returns the parameter from the specified file +def get_parameter( parameter, file_path ): + # Check if secrets file exists + if not os.path.isfile(file_path): + print("File %s not found, exiting."%file_path) + sys.exit(0) - world_servers.append(row[0]) + # Find parameter in file + with open( file_path ) as f: + for line in f: + if line.startswith( parameter ): + return line.replace(parameter + ":", "").strip() - cur.close() + # Cannot find parameter, exit + print(file_path + " Missing parameter %s "%parameter) + sys.exit(0) - print("Remaining servers: " + str(len(world_servers))) +############################################################################### +# main - except (Exception, psycopg2.DatabaseError) as error: +if __name__ == '__main__': - print(error) + # usage modes - finally: + if len(sys.argv) == 1: - if conn is not None: + usage() - conn.close() + elif len(sys.argv) == 2: - ########################################################################### - # multiprocessing! + if sys.argv[1] == '--multi': + + now = datetime.now() - m = Manager() - q = m.Queue() - z = zip(world_servers) + mastodon, mastodon_hostname = mastodon() - serv_number = len(world_servers) + fediverse_db, fediverse_db_user = db_config() - pool_tuple = [(x, q) for x in z] - with Pool(processes=64) as pool: - pool.starmap(getserver, pool_tuple) + total_servers = 0 + total_users = 0 - ########################################################################### - # delete not alive servers from fediverse table + set_world_servers_check_to_false() - try: + alive_servers = get_last_checked_servers() - conn = None + ########################################################################### + # multiprocessing! - conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") + nprocs = multiprocessing.cpu_count() + with multiprocessing.Pool(processes=64) as pool: + results = pool.starmap(get_alive_servers, product(alive_servers)) - cur = conn.cursor() + elif sys.argv[1] == '--mono': - cur.execute("delete from fediverse where not alive") + now = datetime.now() - conn.commit() + mastodon, mastodon_hostname = mastodon() - cur.close() + fediverse_db, fediverse_db_user = db_config() - except (Exception, psycopg2.DatabaseError) as error: + total_servers = 0 + total_users = 0 - print(error) + set_world_servers_check_to_false() - finally: + alive_servers = get_last_checked_servers() - if conn is not None: + i = 0 - conn.close() + while i < len(alive_servers): - ########################################################################### - # get current total servers and users, get users from every software + get_alive_servers(alive_servers[i]) - gettotals_sql = "select count(server), sum(users) from fediverse where alive" - get_soft_totals_sql = "select software, sum(users) as users, count(server) as servers from fediverse where users != 0 and alive group by software order by users desc" - soft_total_project = [] - soft_total_users = [] - soft_total_servers = [] - - try: - - conn = None - conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") - - cur = conn.cursor() - - cur.execute(gettotals_sql) - - row = cur.fetchone() - - total_servers = row[0] - total_users = row[1] - - cur.execute(get_soft_totals_sql) - rows = cur.fetchall() - for row in rows: - soft_total_project.append(row[0]) - soft_total_users.append(row[1]) - soft_total_servers.append(row[2]) - cur.close() - - except (Exception, psycopg2.DatabaseError) as error: - - print(error) - - finally: - - if conn is not None: - - conn.close() - - ########################################################################### - # get last check values and write current total ones - - select_sql = "select total_servers, total_users from totals order by datetime desc limit 1" - insert_sql = "INSERT INTO totals(datetime, total_servers, total_users) VALUES(%s,%s,%s)" - - try: - - conn = None - conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") - - cur = conn.cursor() - - cur.execute(select_sql) - - row = cur.fetchone() - - if row != None: - - servers_before = row[0] - users_before = row[1] + i += 1 else: - servers_before = 0 - users_before = 0 + usage() - cur.execute(insert_sql, (now, total_servers, total_users)) + ########################################################################### + # get current total servers and users, get users from every software - conn.commit() + gettotals_sql = "select count(server), sum(users) from fediverse where alive" + get_soft_totals_sql = "select software, sum(users) as users, count(server) as servers from fediverse where users != 0 and alive group by software order by users desc" + soft_total_project = [] + soft_total_users = [] + soft_total_servers = [] - cur.close() + try: - evo_servers = total_servers - servers_before - evo_users = total_users - users_before + conn = None + conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") - except (Exception, psycopg2.DatabaseError) as error: + cur = conn.cursor() - print(error) + cur.execute(gettotals_sql) - finally: + row = cur.fetchone() - if conn is not None: + total_servers = row[0] + total_users = row[1] - conn.close() + cur.execute(get_soft_totals_sql) + rows = cur.fetchall() + for row in rows: + soft_total_project.append(row[0]) + soft_total_users.append(row[1]) + soft_total_servers.append(row[2]) + cur.close() - ################################################################################ - # write evo values + except (Exception, psycopg2.DatabaseError) as error: - insert_sql = "INSERT INTO evo(datetime, servers, users) VALUES(%s,%s,%s)" + print(error) - conn = None + finally: - try: + if conn is not None: - conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") + conn.close() - cur = conn.cursor() + ########################################################################### + # get last check values and write current total ones - cur.execute(insert_sql, (now, evo_servers, evo_users)) + select_sql = "select total_servers, total_users from totals order by datetime desc limit 1" + insert_sql = "INSERT INTO totals(datetime, total_servers, total_users) VALUES(%s,%s,%s)" - conn.commit() + try: - cur.close() + conn = None + conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") - except (Exception, psycopg2.DatabaseError) as error: + cur = conn.cursor() - print(error) + cur.execute(select_sql) - finally: + row = cur.fetchone() - if conn is not None: + if row != None: - conn.close() + servers_before = row[0] + users_before = row[1] - ############################################################################## - # get world's last update datetime + else: - conn = None + servers_before = 0 + users_before = 0 - try: + cur.execute(insert_sql, (now, total_servers, total_users)) - conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") + conn.commit() - cur = conn.cursor() + cur.close() - cur.execute("select updated_at from world order by updated_at desc limit 1") + evo_servers = total_servers - servers_before + evo_users = total_users - users_before - row = cur.fetchone() + except (Exception, psycopg2.DatabaseError) as error: - last_update = row[0] - last_update = last_update.strftime('%m/%d/%Y, %H:%M:%S') + print(error) - cur.close() + finally: - except (Exception, psycopg2.DatabaseError) as error: + if conn is not None: - print(error) + conn.close() - finally: + ################################################################################ + # write evo values - if conn is not None: + insert_sql = "INSERT INTO evo(datetime, servers, users) VALUES(%s,%s,%s)" - conn.close() + conn = None - ############################################################################### - # T O O T ! + try: - toot_text = "#fediverse alive servers stats" + " \n" - toot_text += "\n" - if evo_servers >= 0: - toot_text += "alive servers: " + str(total_servers) + " (+"+ str(evo_servers) + ") \n" - elif evo_servers < 0: - toot_text += "alive servers: " + str(total_servers) + " ("+ str(evo_servers) + ") \n" - if evo_users >= 0: - toot_text += "total users: " + str(total_users) + " (+"+ str(evo_users) + ") \n" - elif evo_users < 0: - toot_text += "total users: " + str(total_users) + " ("+ str(evo_users) + ") \n" - toot_text += "\n" - toot_text += "top five (soft: users servers):" + " \n" - toot_text += "\n" - toot_text += '{:<8}: {:>7} {:>4}'.format(str(soft_total_project[0]), str(soft_total_users[0]), str(soft_total_servers[0])) + " \n" - toot_text += '{:<8}: {:>7} {:>4}'.format(str(soft_total_project[1]), str(soft_total_users[1]), str(soft_total_servers[1])) + " \n" - toot_text += '{:<8}: {:>7} {:>4}'.format(str(soft_total_project[2]), str(soft_total_users[2]), str(soft_total_servers[2])) + " \n" - toot_text += '{:<8}: {:>7} {:>4}'.format(str(soft_total_project[3]), str(soft_total_users[3]), str(soft_total_servers[3])) + " \n" - toot_text += '{:<8}: {:>7} {:>4}'.format(str(soft_total_project[4]), str(soft_total_users[4]), str(soft_total_servers[4])) + " \n" - toot_text += "\n" - toot_text += "updated at " + str(last_update) + " \n" - print("Tooting...") - print(toot_text) + conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") - mastodon.status_post(toot_text, in_reply_to_id=None, ) + cur = conn.cursor() + + cur.execute(insert_sql, (now, evo_servers, evo_users)) + + conn.commit() + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + ############################################################################## + # get world's last update datetime + + conn = None + + try: + + conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") + + cur = conn.cursor() + + cur.execute("select updated_at from world order by updated_at desc limit 1") + + row = cur.fetchone() + + last_update = row[0] + last_update = last_update.strftime('%m/%d/%Y, %H:%M:%S') + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + ############################################################################## + # get max servers and users + + conn = None + + try: + + conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") + + cur = conn.cursor() + + cur.execute("select MAX(total_servers) from totals") + + row = cur.fetchone() + + if row != None: + + max_servers = row[0] + + else: + + max_servers = 0 + + cur.execute("select MAX(total_users) from totals") + + row = cur.fetchone() + + if row != None: + + max_users = row[0] + + else: + + max_users = 0 + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + ############################################################################### + # get plots + + servers_plots = [] + + users_plots = [] + + conn = None + + try: + + conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") + + cur = conn.cursor() + + cur.execute("select total_servers, total_users from totals order by datetime desc limit 14") + + rows = cur.fetchall() + + for row in rows: + + servers_plots.append(row[0]) + + users_plots.append(row[1]) + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + ############################################################################### + # generate graphs + + plt.plot([-6, -5, -4, -3, -2, -1, 0], [servers_plots[6], servers_plots[5],servers_plots[4],servers_plots[3],servers_plots[2],servers_plots[1],servers_plots[0]], marker='o', color='mediumseagreen') + + plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_servers, max_servers, max_servers, max_servers, max_servers, max_servers, max_servers], color='red') + + plt.title('fediverse: total alive servers (max: ' + str(f"{max_servers:,}" + ')') , loc='right', color='blue') + + plt.xlabel('Last seven days') + + plt.ylabel('fediverse alive servers') + + plt.legend(('servers', 'max'), shadow=True, loc=(0.01, 1.00), handlelength=1.5, fontsize=10) + + plt.savefig('servers.png') + + plt.close() + + plt.plot([-6, -5, -4, -3, -2, -1, 0], [users_plots[6], users_plots[5], users_plots[4], users_plots[3], users_plots[2], users_plots[1], users_plots[0]], marker='o', color='royalblue') + + plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_users, max_users, max_users, max_users, max_users, max_users, max_users], color='red') + + plt.title('fediverse: total registered users (max: ' + str(f"{max_users:,}" + ')') , loc='right', color='royalblue') + + plt.legend(('users', 'max'), shadow=True, loc=(0.01, 0.80), handlelength=1.5, fontsize=10) + + plt.xlabel('Last seven days') + + plt.ylabel('Registered users') + + plt.savefig('users.png') + + plt.close() + + ############################################################################### + # T O O T ! + + toot_text = "#fediverse alive servers stats" + " \n" + toot_text += "\n" + if evo_servers >= 0: + toot_text += "alive servers: " + str(f"{total_servers:,}") + " (+"+ str(f"{evo_servers:,}") + ") \n" + toot_text += "max: " + str(f"{max_servers:,}") + "\n" + elif evo_servers < 0: + toot_text += "alive servers: " + str(f"{total_servers:,}") + " ("+ str(f"{evo_servers:,}") + ") \n" + toot_text += "max: " + str(f"{max_servers:,}") + "\n" + if evo_users >= 0: + toot_text += "total users: " + str(f"{total_users:,}") + " (+"+ str(f"{evo_users:,}") + ") \n" + toot_text += "max: " + str(f"{max_users:,}") + "\n" + elif evo_users < 0: + toot_text += "total users: " + str(f"{total_users:,}") + " ("+ str(f"{evo_users:,}") + ") \n" + toot_text += "max: " + str(f"{max_users:,}") + "\n" + + toot_text += "\n" + toot_text += "top ten (soft users servers):" + " \n" + toot_text += "\n" + + i = 0 + while i < 10: + + project_soft = soft_total_project[i] + project_users = soft_total_users[i] + project_servers = soft_total_servers[i] + len_pr_soft = len(project_soft) + + if project_soft == 'writefreely': + + str_len = 11 + + else: + + str_len = 13 + + toot_text += f"{':'+project_soft+':':<11}" + f"{project_users:>{str_len},}" + " " + f"{project_servers:>5,}" + "\n" + + i += 1 + + print("Tooting...") + print(toot_text) + + servers_image_id = mastodon.media_post('servers.png', "image/png").id + + users_image_id = mastodon.media_post('users.png', "image/png").id + + mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, users_image_id})