From 0b52e0090d0fc39fb32867bb3ae0b0f3c2df8553 Mon Sep 17 00:00:00 2001 From: spla Date: Wed, 2 Mar 2022 22:26:05 +0100 Subject: [PATCH] Improved servers nodeinfo detection --- fediverse.py | 591 ++++++++++++++++++------------------------------ fetchservers.py | 309 ++++++++++++++++--------- 2 files changed, 417 insertions(+), 483 deletions(-) diff --git a/fediverse.py b/fediverse.py index 5024166..6300186 100644 --- a/fediverse.py +++ b/fediverse.py @@ -1,3 +1,5 @@ +from multiprocessing import set_start_method +from multiprocessing import get_context import time import urllib3 from datetime import datetime @@ -7,11 +9,10 @@ import json import sys import os.path import requests -import psycopg2 -from itertools import product -import multiprocessing import aiohttp import asyncio +import psycopg2 +from itertools import product import socket import matplotlib.pyplot as plt import pdb @@ -19,18 +20,6 @@ plt.style.use('seaborn') start_time = time.time() -apis = ['/nodeinfo/2.0?', - '/nodeinfo/2.0.json?', - '/main/nodeinfo/2.0?', - '/api/statusnet/config?', - '/api/nodeinfo/2.0.json?', - '/api/nodeinfo?', - '/api/v1/instance?', - '/wp-json/nodeinfo/2.0?', - '/api/v1/instance/nodeinfo/2.0?', - '/.well-known/x-nodeinfo2?' - ] - client_exceptions = ( aiohttp.ClientResponseError, aiohttp.ClientConnectionError, @@ -40,121 +29,91 @@ client_exceptions = ( socket.gaierror, ) -def is_json(myjson): +class Server: - try: - json_object = json.loads(myjson) - except ValueError as e: - print(e) - return False - return True + name = 'Server' -def get_alive_servers(server): + def __init_(self, server, software, users, alive, api, soft_version): - serv_api = '' - serv_soft = '' - soft_version = '' + self.server = server + self.software = software + self.users = users + self.alive = alive + self.api = api + self.version = self.soft_version - try: + def get_alive_servers(self, *args): - conn = None + fediverse_db, fediverse_db_user = db_config() - conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432") + now = datetime.now() - cur = conn.cursor() + if len(args) != 0: - cur.execute("select alive, software, users_api, version, first_checked_at, downs from fediverse where server=(%s)", (server,)) - - row = cur.fetchone() - - if row is not None: - - was_alive = row[0] - serv_soft = row[1] - serv_api = row[2] - soft_version = row[3] - first_checked_at = row[4] - downs_qty = row[5] - - cur.close() - - except (Exception, psycopg2.DatabaseError) as error: - - print(error) - - finally: - - if conn is not None: - conn.close() - - alive = False + self.server = args[0] try: - user_agent = {'User-agent': 'Mozilla/5.0'} + conn = None - data = requests.get('https://' + server + serv_api, headers = user_agent, timeout=3) + conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432") + + cur = conn.cursor() + + cur.execute("select alive, software, users_api, version, first_checked_at, downs from fediverse where server=(%s)", (self.server,)) + + row = cur.fetchone() + + if row is not None: + + was_alive = row[0] + self.software = row[1] + self.api = row[2] + self.soft_version = row[3] + first_checked_at = row[4] + downs_qty = row[5] + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + alive = False + + try: + + user_agent = {'User-agent': 'Mozilla/5.0'} + + data = requests.get('https://' + self.server + self.api, headers = user_agent, timeout=3) - if serv_soft == "mastodon": - if serv_api == '/nodeinfo/2.0?': - try: - users = data.json()['usage']['users']['total'] - soft_version = data.json()['software']['version'] - alive = True - except: - users = 0 - soft_version = "" - if serv_api == '/nodeinfo/2.0.json?': - try: - users = data.json()['usage']['users']['total'] - soft_version = data.json()['software']['version'] - alive = True - except: - users = 0 - soft_version = "" - elif serv_api == '/api/v1/instance?': - try: - users = data.json()['stats']['user_count'] - soft_version = data.json()['version'] - alive = True - except: - users = 0 - soft_version = "" - if serv_soft == "pleroma" or serv_soft == "diaspora" or serv_soft == "peertube" or serv_soft == "pixelfed" or serv_soft == "hubzilla" or serv_soft == "writefreely" or serv_soft == "friendica": try: - users = data.json()['usage']['users']['total'] - soft_version = data.json()['software']['version'] - alive = True - except: - users = 0 - soft_version = "" - if serv_soft == "gnusocialv2" or serv_soft == "gnusocial": - try: - users = data.json()['usage']['users']['total'] - if users == 0: - users = data.json()['usage']['users']['activeHalfyear'] - soft_version = data.json()['software']['version'] - alive = True - except: - users = 0 - soft_version = "" - if serv_soft == "plume" or serv_soft == 'red' or serv_soft == "misskey" or serv_soft == "zap" or serv_soft == "prismo" or serv_soft == "ravenvale" or serv_soft == "osada" or serv_soft == "groundpolis": - try: - users = data.json()['usage']['users']['total'] - soft_version = data.json()['software']['version'] - alive = True - except: - users = 0 - soft_version = "" - if serv_soft == "ganggo" or serv_soft == "squs" or serv_soft == "dolphin" or serv_soft == "lemmy" or serv_soft == "wordpress": - try: - users = data.json()['usage']['users']['total'] - soft_version = data.json()['software']['version'] - if serv_soft == "wordpress" and "activitypub" in data.json()['protocols']: + + self.users = data.json()['usage']['users']['total'] + + if self.users == 0: + + self.users = data.json()['usage']['users']['activeHalfyear'] + + if self.software == 'socialhome': + + self.soft_version = data.json()['server']['version'] + + else: + + self.soft_version = data.json()['software']['version'] + + if self.software == "wordpress" and "activitypub" in data.json()['protocols']: alive = True - elif serv_soft == "wordpress" and "activitypub" not in data.json()['protocols']: + elif self.software == "wordpress" and "activitypub" not in data.json()['protocols']: alive = False @@ -164,297 +123,173 @@ def get_alive_servers(server): except: - users = 0 - soft_version = "" + self.users = 0 - if serv_soft == 'funkwhale': + self.soft_version = "" - try: - users = data.json()['usage']['users']['total'] - soft_version = data.json()['software']['version'] - alive = True - except: - users = 0 - soft_version = "" + else: - if serv_soft == 'socialhome': + if self.api == '/api/v1/instance?': - try: - users = data.json()['usage']['users']['total'] - soft_version = data.json()['server']['version'] - alive = True - except: - users = 0 - soft_version = "" + try: + self.users = data.json()['stats']['user_count'] + self.soft_version = data.json()['version'] + alive = True + except: + self.users = 0 + self.soft_version = "" - if alive: + if alive: + + if downs_qty != None: + + downs = downs_qty + + else: + + downs = 0 + + if self.soft_version != "" and self.soft_version is not None: + + print(f'\n** Server {self.server} ({self.software} {self.soft_version}) is alive! **') + + else: + + print(f'\n** Server {self.server} ({self.software}) is alive! **') + + insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version, first_checked_at, last_checked_at, downs) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING" + + conn = None + + try: + + conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432") + + cur = conn.cursor() + + cur.execute(insert_sql, (self.server, self.users, now, self.software, alive, self.api, self.soft_version, now, now, downs)) + + if first_checked_at != None: + + cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (self.users, now, self.software, alive, self.api, self.soft_version, now, downs, self.server)) + + else: + + cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), first_checked_at=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (self.users, now, self.software, alive, self.api, self.soft_version, now, now, downs, self.server)) + + + cur.execute("UPDATE world SET checked='t' where server=(%s)", (self.server,)) + + conn.commit() + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + except urllib3.exceptions.ProtocolError as protoerr: + + print_dead(self.server) + pass + + except requests.exceptions.ChunkedEncodingError as chunkerr: + + print_dead(self.server) + pass + + except KeyError as e: + + print_dead(self.server) + pass + + except ValueError as verr: + + print_dead(self.server) + pass + + except requests.exceptions.SSLError as errssl: + + print_dead(self.server) + pass + + except requests.exceptions.HTTPError as errh: + + print_dead(self.server) + pass + + except requests.exceptions.ConnectionError as errc: + + print_dead(self.server) + pass + + except requests.exceptions.Timeout as errt: + + print_dead(self.server) + pass + + except requests.exceptions.RequestException as err: + + print_dead(self.server) + pass + + except socket.gaierror as gai_error: + + print_dead(self.server) + pass + + if not alive: if downs_qty != None: - downs = downs_qty + downs = downs_qty + 1 else: - downs = 0 - - if soft_version != "" and soft_version is not None: - print("Server " + str(server) + " (" + serv_soft + " " + soft_version + ") is alive!") - else: - print("Server " + str(server) + " (" + serv_soft + ") is alive!") - - insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version, first_checked_at, last_checked_at, downs) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING" + downs = 1 conn = None try: - conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432") + conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", + host="/var/run/postgresql", port="5432") cur = conn.cursor() - cur.execute(insert_sql, (server, users, now, serv_soft, alive, serv_api, soft_version, now, now, downs)) - if first_checked_at != None: - cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (users, now, serv_soft, alive, serv_api, soft_version, now, downs, server)) + cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), first_checked_at=(%s), downs=(%s) where server=(%s)", (now, alive, now, downs, self.server)) else: - cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), first_checked_at=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (users, now, serv_soft, alive, serv_api, soft_version, now, now, downs, server)) + cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), downs=(%s) where server=(%s)", (now, alive, downs, self.server)) - - cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,)) + cur.execute("UPDATE world SET checked='t' where server=(%s)", (self.server,)) conn.commit() cur.close() except (Exception, psycopg2.DatabaseError) as error: + print(error) finally: if conn is not None: + conn.close() - except urllib3.exceptions.ProtocolError as protoerr: +def print_dead(server): - print(protoerr) - print("Server " + server + " is dead :-(") - alive = False - pass - - except requests.exceptions.ChunkedEncodingError as chunkerr: - - print(chunkerr) - print("Server " + server + " is dead :-(") - alive = False - pass - - except KeyError as e: - - print(e) - print("Server " + server + " is dead :-(") - alive = False - pass - - except ValueError as verr: - - print(verr) - print("Server " + server + " is dead :-(") - alive = False - pass - - except requests.exceptions.SSLError as errssl: - - print(errssl) - print("Server " + server + " is dead :-(") - alive = False - pass - - except requests.exceptions.HTTPError as errh: - - print(errh) - print("Server " + server + " is dead :-(") - alive = False - pass - - except requests.exceptions.ConnectionError as errc: - - print(errc) - print("Server " + server + " is dead :-(") - alive = False - pass - - except requests.exceptions.Timeout as errt: - - print(errt) - print("Server " + server + " is dead :-(") - alive = False - pass - - except requests.exceptions.RequestException as err: - - print(err) - print("Server " + server + " is dead :-(") - alive = False - pass - - except socket.gaierror as gai_error: - - print(gai_error) - pass - - if alive is False: - - if downs_qty != None: - - downs = downs_qty + 1 - - else: - - downs = 1 - - conn = None - - try: - - conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", - host="/var/run/postgresql", port="5432") - - cur = conn.cursor() - - if first_checked_at != None: - - cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), first_checked_at=(%s), downs=(%s) where server=(%s)", (now, alive, now, downs, server)) - - else: - - cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), downs=(%s) where server=(%s)", (now, alive, downs, server)) - - cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,)) - - conn.commit() - - cur.close() - - except (Exception, psycopg2.DatabaseError) as error: - print(error) - - finally: - - if conn is not None: - conn.close() - -def write_api(server, software, users, alive, api, soft_version): - - insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING" - conn = None - - try: - - conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432") - - cur = conn.cursor() - - cur.execute(insert_sql, (server, now, software, users, alive, api, soft_version)) - - cur.execute("UPDATE fediverse SET updated_at=(%s), software=(%s), users=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)", (now, software, users, alive, api, soft_version, server)) - - cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,)) - - conn.commit() - - cur.close() - - except (Exception, psycopg2.DatabaseError) as error: - - print(error) - - finally: - - if conn is not None: - - conn.close() - -async def getsoft(server): - - try: - - socket.gethostbyname(server) - - except socket.gaierror: - - pass - return - - soft = '' - - url = 'https://' + server - - user_agent = {'User-agent': 'Mozilla/5.0'} - - timeout = aiohttp.ClientTimeout(total=3) - - async with aiohttp.ClientSession(headers=user_agent, timeout=timeout) as session: - for api in apis: - try: - async with session.get(url+api) as response: - if response.status == 200: - try: - response_json = await response.json() - except: - pass - except aiohttp.ClientConnectorError as err: - print(err) - pass - else: - if response.status == 200 and api != '/api/v1/instance?': - try: - soft = response_json['software']['name'] - soft = soft.lower() - soft_version = response_json['software']['version'] - users = response_json['usage']['users']['total'] - if users > 1000000: - return - alive = True - write_api(server, soft, users, alive, api, soft_version) - print("Server " + server + " (" + soft + " " + soft_version + ") is alive!") - return - except: - pass - if response.status == 200 and soft == '' and api == "/api/v1/instance?": - soft = 'mastodon' - users = response_json['stats']['user_count'] - soft_version = response_json['version'] - if users > 1000000: - return - alive = True - write_api(server, soft, users, alive, api) - print("Server " + server + " (" + soft + ") is alive!") - -def getserver(server, x): - - server = server[0].rstrip('.').lower() - - if server.find(".") == -1: - return - if server.find("@") != -1: - return - if server.find("/") != -1: - return - if server.find(":") != -1: - return - - try: - - loop = asyncio.get_event_loop() - coroutines = [getsoft(server)] - soft = loop.run_until_complete(asyncio.gather(*coroutines, return_exceptions=True)) - - except: - - pass + print(f'\nServer {server} is dead :-(') def set_world_servers_check_to_false(): @@ -562,7 +397,6 @@ def usage(): print('usage: python ' + sys.argv[0] + ' --multi' + ' (multiprocessing, fast)') print('usage: python ' + sys.argv[0] + ' --mono' + ' (one process, slow)') -# Returns the parameter from the specified file def get_parameter(parameter, file_path): # Check if secrets file exists @@ -594,8 +428,8 @@ if __name__ == '__main__': elif len(sys.argv) == 2: if sys.argv[1] == '--multi': - - now = datetime.now() + + set_start_method("spawn") mastodon, mastodon_hostname = mastodon() @@ -608,12 +442,16 @@ if __name__ == '__main__': alive_servers = get_last_checked_servers() + getservers = Server() + ########################################################################### # multiprocessing! - nprocs = multiprocessing.cpu_count() - with multiprocessing.Pool(processes=64) as pool: - results = pool.starmap(get_alive_servers, product(alive_servers)) + with get_context("spawn").Pool(processes=32) as pool: + + res = pool.starmap(getservers.get_alive_servers, product(alive_servers)) + pool.close() + pool.join() elif sys.argv[1] == '--mono': @@ -624,17 +462,22 @@ if __name__ == '__main__': fediverse_db, fediverse_db_user = db_config() total_servers = 0 + total_users = 0 set_world_servers_check_to_false() alive_servers = get_last_checked_servers() + getservers = Server() + i = 0 while i < len(alive_servers): - get_alive_servers(alive_servers[i]) + getservers.server = alive_servers[i] + + getservers.get_alive_servers() i += 1 @@ -645,6 +488,8 @@ if __name__ == '__main__': ########################################################################### # get current total servers and users, get users from every software + now = datetime.now() + gettotals_sql = "select count(server), sum(users) from fediverse where alive" get_soft_totals_sql = "select software, sum(users) as users, count(server) as servers from fediverse where users != 0 and alive group by software order by users desc" soft_total_project = [] @@ -948,4 +793,4 @@ if __name__ == '__main__': users_image_id = mastodon.media_post('users.png', "image/png", description='users graph').id - mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, users_image_id}) + #mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, users_image_id}) diff --git a/fetchservers.py b/fetchservers.py index 8e41d35..6751d13 100644 --- a/fetchservers.py +++ b/fetchservers.py @@ -1,3 +1,6 @@ +from multiprocessing import set_start_method +from multiprocessing import get_context +from itertools import product import time from datetime import datetime import os @@ -5,22 +8,10 @@ import json import sys import os.path import psycopg2 -from multiprocessing import Pool, Manager import aiohttp import asyncio import socket - -apis = ['/nodeinfo/2.0?', - '/nodeinfo/2.0.json?', - '/main/nodeinfo/2.0?', - '/api/statusnet/config?', - '/api/nodeinfo/2.0.json?', - '/api/nodeinfo?', - '/api/v1/instance?', - '/wp-json/nodeinfo/2.0?', - '/api/v1/instance/nodeinfo/2.0?', - '/.well-known/x-nodeinfo2?' - ] +import pdb client_exceptions = ( aiohttp.ClientResponseError, @@ -31,28 +22,24 @@ client_exceptions = ( socket.gaierror, ) - -def is_json(myjson): - - try: - json_object = json.loads(myjson) - except ValueError as e: - return False - return True - def write_api(server, software, users, alive, api, soft_version): + now = datetime.now() + + fediverse_db, fediverse_db_user = get_db_config() + insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING" conn = None try: - conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", - port="5432") + conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432") cur = conn.cursor() + print(f'Writing {server} nodeinfo data...') + cur.execute(insert_sql, (server, now, software, users, alive, api, soft_version)) cur.execute( @@ -73,16 +60,20 @@ def write_api(server, software, users, alive, api, soft_version): finally: if conn is not None: + conn.close() async def getsoft(server): + fediverse_db, fediverse_db_user = get_db_config() + try: socket.gethostbyname(server) - except socket.gaierror: + except socket.gaierror as g_error: + print(f'Server {server} error: {g_error}') pass return @@ -93,71 +84,134 @@ async def getsoft(server): user_agent = {'User-agent': 'Mozilla/5.0'} timeout = aiohttp.ClientTimeout(total=3) + async with aiohttp.ClientSession(timeout=timeout, headers=user_agent) as session: - for api in apis: - try: + try: + + async with session.get(url + '/.well-known/nodeinfo') as response: + + if response.status == 200: + + try: + + response_json = await response.json() + + nodeinfo = response_json['links'][0]['href'].replace(f'https://{server}','') + + except: + + pass + else: + + print(f'Server {server} not responding: {response.status}') + + pass + + async with session.get(url + nodeinfo) as nodeinfo_response: + + if nodeinfo_response.status == 200: - async with session.get(url + api) as response: - if response.status == 200: try: - response_json = await response.json() + + nodeinfo_json = await nodeinfo_response.json() + except: + pass - except aiohttp.ClientConnectorError as err: + else: - pass + print(f"Server {server}'s nodeinfo not responding: {response.status}") + + pass + + except aiohttp.ClientConnectorError as cc_err: + + pass + + except aiohttp.client_exceptions.ClientConnectorSSLError as ccssl_as: + + pass + + else: + + if nodeinfo_response.status == 200 and nodeinfo != '/api/v1/instance?': + + if nodeinfo != '/.well-known/x-nodeinfo2?': + + try: + + soft = nodeinfo_json['software']['name'] + soft = soft.lower() + soft_version = nodeinfo_json['software']['version'] + users = nodeinfo_json['usage']['users']['total'] + if users > 1000000: + return + alive = True + + write_api(server, soft, users, alive, nodeinfo, soft_version) + + print('*********************************************************************') + print("Server " + server + " (" + soft + " " + soft_version + ") is alive!") + print('*********************************************************************') + + return + + except: + + pass + + else: + + try: + + soft = nodeinfo_json['server']['software'] + soft = soft.lower() + soft_version = nodeinfo_json['server']['version'] + users = nodeinfo_json['usage']['users']['total'] + if users > 1000000: + return + alive = True + + if soft == 'socialhome': + + write_api(server, soft, users, alive, api, soft_version) + + print('*********************************************************************') + print("Server " + server + " (" + soft + " " + soft_version + ") is alive!") + print('*********************************************************************') + + return + + except: + + pass + + if nodeinfo_response.status == 200 and soft == '' and nodeinfo == "/api/v1/instance?": + + soft = 'mastodon' + users = nodeinfo_json['stats']['user_count'] + soft_version = nodeinfo_json['version'] + if users > 1000000: + return + alive = True + + write_api(server, soft, users, alive, api) + + print('*********************************************************************') + print("Server " + server + " (" + soft + ") is alive!") + print('*********************************************************************') else: - if response.status == 200 and api != '/api/v1/instance?': + print(f'Server {server} is dead') - if api != '/.well-known/x-nodeinfo2?': +def getserver(server, *args): - try: - soft = response_json['software']['name'] - soft = soft.lower() - soft_version = response_json['software']['version'] - users = response_json['usage']['users']['total'] - if users > 1000000: - return - alive = True - write_api(server, soft, users, alive, api, soft_version) - print("Server " + server + " (" + soft + " " + soft_version + ") is alive!") - return - except: - pass - else: + if len(args) != 0: - try: - soft = response_json['server']['software'] - soft = soft.lower() - soft_version = response_json['server']['version'] - users = response_json['usage']['users']['total'] - if users > 1000000: - return - alive = True - if soft == 'socialhome': - write_api(server, soft, users, alive, api, soft_version) - print("Server " + server + " (" + soft + " " + soft_version + ") is alive!") - return - except: - pass - - if response.status == 200 and soft == '' and api == "/api/v1/instance?": - soft = 'mastodon' - users = response_json['stats']['user_count'] - soft_version = response_json['version'] - if users > 1000000: - return - alive = True - write_api(server, soft, users, alive, api) - print("Server " + server + " (" + soft + ") is alive!") - -def getserver(server, x): - - server = server[0].rstrip('.').lower() + server = server[0].rstrip('.').lower() if server.find(".") == -1: return @@ -178,7 +232,46 @@ def getserver(server, x): pass +def get_world_servers(): + + world_servers = [] + + try: + + conn = None + + conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432") + + cur = conn.cursor() + + # get world servers list + + cur.execute("select server from world where checked='f'") + + rows = cur.fetchall() + + for row in rows: + + world_servers.append(row[0]) + + cur.close() + + print("Remaining servers: " + str(len(world_servers))) + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + return world_servers + def get_parameter(parameter, file_path): + # Check if secrets file exists if not os.path.isfile(file_path): print("File %s not found, exiting." % file_path) @@ -194,6 +287,10 @@ def get_parameter(parameter, file_path): print(file_path + " Missing parameter %s " % parameter) sys.exit(0) +def usage(): + + print('usage: python ' + sys.argv[0] + ' --multi' + ' (multiprocessing, fast)') + print('usage: python ' + sys.argv[0] + ' --mono' + ' (one process, slow)') def get_config(): @@ -214,59 +311,51 @@ def get_db_config(): # main if __name__ == '__main__': - - now = datetime.now() - start_time = time.time() - mastodon_hostname = get_config() + # usage modes - fediverse_db, fediverse_db_user = get_db_config() + if len(sys.argv) == 1: - world_servers = [] + usage() - try: + elif len(sys.argv) == 2: - conn = None + if sys.argv[1] == '--multi': - conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432") + now = datetime.now() - cur = conn.cursor() + start_time = time.time() - # get world servers list + mastodon_hostname = get_config() - cur.execute("select server from world where checked='f'") + fediverse_db, fediverse_db_user = get_db_config() - rows = cur.fetchall() - - for row in rows: - - world_servers.append(row[0]) + world_servers = get_world_servers() - cur.close() + with get_context("spawn").Pool(processes=32) as pool: - print("Remaining servers: " + str(len(world_servers))) + res = pool.starmap(getserver, product(world_servers)) - except (Exception, psycopg2.DatabaseError) as error: + pool.close() - print(error) + pool.join() - finally: + print('Done.') - if conn is not None: + elif sys.argv[1] == '--mono': - conn.close() + now = datetime.now() - ########################################################################### - # multiprocessing! + start_time = time.time() - m = Manager() - q = m.Queue() - z = zip(world_servers) + mastodon_hostname = get_config() - serv_number = len(world_servers) + fediverse_db, fediverse_db_user = get_db_config() - pool_tuple = [(x, q) for x in z] - with Pool(processes=64) as pool: - pool.starmap(getserver, pool_tuple) + world_servers = get_world_servers() - print('Done.') + for server in world_servers: + + getserver(server) + + print('Done.')