diff --git a/fediverse.py b/fediverse.py index dd9916b..7461558 100644 --- a/fediverse.py +++ b/fediverse.py @@ -25,583 +25,328 @@ from itertools import product from multiprocessing import Pool, Lock, Process, Queue, current_process, Manager import multiprocessing +import aiohttp +import aiodns +import asyncio +from aiohttp import ClientError, ClientSession, ClientConnectionError, ClientConnectorError, ClientSSLError, ClientConnectorSSLError, ServerTimeoutError +from asyncio import TimeoutError +import socket +from socket import gaierror, gethostbyname + from decimal import * getcontext().prec = 2 +apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?', '/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?'] + +client_exceptions = ( + aiohttp.ClientResponseError, + aiohttp.ClientConnectionError, + aiohttp.ClientConnectorError, + aiohttp.ClientError, + asyncio.TimeoutError, + socket.gaierror, +) + +now = datetime.now() + ############################################################################### # INITIALISATION ############################################################################### def is_json(myjson): - try: - json_object = json.loads(myjson) - except ValueError as e: - return False - return True + try: + json_object = json.loads(myjson) + except ValueError as e: + return False + return True -def getserver(server, x): +def alive_server(server, x): - server = server[0].rstrip('.').lower() + server = server[0].rstrip('.').lower() - if server.find(".") == -1: - print(server + " is Not a real domain!") - return - if server.find("@") != -1: - print(server + " is Not a real domain!") - return - if server.find("/") != -1: - print(server + " is Not a real domain!") - return - if server.find(":") != -1: - print(server + " is Not a real domain!") - return - - global mastodon - global pleroma - global gnusocial - global zap - global plume - global hubzilla - global misskey - global prismo - global osada - global groundpolis - global ganggo - global squs - global peertube - global friendica - global pixelfed - global writefreely - global ravenvale - global diaspora - global dolphin - - pl_users = 0 - global total_pl_users - mast_users = 0 - global total_mast_users - gs_users = 0 - global total_gs_users - zap_users = 0 - global total_zap_users - plume_users = 0 - global total_plume_users - hubzilla_users = 0 - global total_hubzilla_users - misskey_users = 0 - global total_misskey_users - prismo_users = 0 - global total_prismo_users - osada_users = 0 - global total_osada_users - gpolis_users = 0 - global total_gpolis_users - ggg_users = 0 - global total_ggg_users - squs_users = 0 - global total_squs_users - peertube_users = 0 - global total_peertube_users - diaspora_users = 0 - global total_diaspora_users - friendica_users = 0 - global total_friendica_users - pixelfed_users = 0 - global total_pixelfed_users - writefreely_users = 0 - global total_writefreely_users - ravenvale_users = 0 - global total_ravenvale_users - dolphin_users = 0 - global total_dolphin_users - - check_diaspora = False - check_peertube = False - check_zap = False - check_plume = False - check_hubzilla = False - check_misskey = False - check_prismo = False - check_osada = False - check_groundpolis = False - check_ganggo = False - check_squs = False - check_gnusocial = False - check_friendica = False - check_pixelfed = False - check_writefreely = False - check_raven = False - check_gnusocial2 = False - check_dolphin = False - - users = 0 - - try: - - res = requests.get('https://' + server + '/api/v1/instance?',timeout=3) - res_friendica = requests.get('https://' + server + '/nodeinfo/2.0?',timeout=3) - check_pix = "compatible; Pixelfed" in res.text - check_ravenvale = "ravenvale" in res.text - check_friendica = "friendica" in res_friendica.text - if check_friendica == False: - check_friendica = "Friendica" in res_friendica.text - - if (res.ok) and check_pix == False and check_ravenvale == False and check_friendica == False: - - if is_json(res.text) == True and server in res.text: - - nodeinfo = requests.get('https://' + server + '/nodeinfo/2.0.json?',timeout=3) - check_pleroma = "pleroma" in nodeinfo.text.lower() - - if nodeinfo.status_code == 200 and check_pleroma == True: - - print("Pleroma server: ") - soft = "pleroma" - pl_users = nodeinfo.json()['usage']['users']['total'] - total_pl_users = total_pl_users + pl_users - users = pl_users - print(server, pl_users) - - else: - - if type(res.json()) != int: - - if res.json().get('stats') != None: - print("Mastodon server: ") - soft = "mastodon" - mast_users = res.json()['stats']['user_count'] - else: - mast_users = 0 - soft = "mastodon" - else: - - mast_users = 0 - - if mast_users != None and mast_users < 999999: - total_mast_users = total_mast_users + mast_users - users = mast_users - print(server, mast_users) - - print("\n") - - if users > 1000000: - return - - insert_sql = "INSERT INTO fediverse(server, users, updated_at, software) VALUES(%s,%s,%s,%s) ON CONFLICT DO NOTHING" - conn = None - - try: - - conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") - - cur = conn.cursor() - - cur.execute(insert_sql, (server, users, now, soft)) - - cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s) where server=(%s)", (users, now, soft, server)) - - conn.commit() - - cur.close() - - except (Exception, psycopg2.DatabaseError) as error: - print(error) - - finally: - - if conn is not None: - conn.close() - return - - else: - - return - - except urllib3.exceptions.ProtocolError as protoerr: - - pass - - except requests.exceptions.ChunkedEncodingError as chunkerr: - - pass - - except KeyError as e: - - pass - - except ValueError as verr: - - pass - - except requests.exceptions.SSLError as errssl: - - pass - - except requests.exceptions.HTTPError as errh: - - pass - - except requests.exceptions.ConnectionError as errc: - - pass - - except requests.exceptions.Timeout as errt: - - pass - - except requests.exceptions.RequestException as err: - - pass - - else: + conn = None try: - soft = "" - others_nodeinfo = requests.get('https://' + server + '/nodeinfo/2.0?',timeout=3) - gs_nodeinfo = requests.get('https://' + server + '/main/nodeinfo/2.0?',timeout=3) - nodeinfo = requests.get('https://' + server + '/nodeinfo/2.0.json?',timeout=3) - statusnet = requests.get('https://' + server + '/api/statusnet/config?',timeout=3) - px_nodeinfo = requests.get('https://' + server + '/api/nodeinfo/2.0.json?',timeout=3) - wf_nodeinfo = requests.get('https://' + server + '/api/nodeinfo?',timeout=3) + conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") - try: + cur = conn.cursor() - if is_json(nodeinfo.text) == True: - if nodeinfo.json() != None: - soft = nodeinfo.json()['software']['name'] - if soft == "peertube": - check_peertube = True - elif soft == "prismo": - check_prismo = True - elif soft == "zap": - check_zap = True - elif soft == "osada": - check_osada = True - elif soft == "ravenvale": - check_raven = True - elif soft == "diaspora": - check_diaspora = True + cur.execute("select alive, software, users_api from fediverse where server=(%s)", (server,)) - if is_json(others_nodeinfo.text) == True: - soft = others_nodeinfo.json()['software']['name'] - if soft == "plume": - check_plume = True - elif soft == "hubzilla": - check_hubzilla = True - elif soft == "misskey": - check_misskey = True - elif soft == "groundpolis": - check_groundpolis = True - elif soft == "ganggo": - check_ganggo = True - elif soft == "squs": - check_squs = True - elif soft == "friendica" or soft == "Friendica": - check_friendica = True - elif soft == "dolphin": - check_dolphin = True + row = cur.fetchone() - if is_json(wf_nodeinfo.text) == True and soft == "": - #if wf_nodeinfo.json().get('error') != None and wf_nodeinfo.json().get('status') == None: - soft = wf_nodeinfo.json()['software']['name'] - if soft == "writefreely": - check_writefreely = True + if row != None: - if is_json(gs_nodeinfo.text) == True: - soft = gs_nodeinfo.json()['software']['name'] - if soft == "gnusocial": - check_gnusocial = True + was_alive = row[0] + serv_soft = row[1] + serv_api = row[2] - if is_json(statusnet.text) == True and soft == "": - if soft != "zap" and soft != "osada": - soft = statusnet.json()['site']['friendica']['FRIENDICA_PLATFORM'] - if soft == "Friendica": - check_friendica = True + cur.close() - if is_json(px_nodeinfo.text) == True and soft == "": - if soft != "Friendica" and soft != "hubzilla": - soft = px_nodeinfo.json()['software']['name'] - if soft == "pixelfed": - check_pixelfed = True - if soft == "gnusocial": - check_gnusocial2 = True + except (Exception, psycopg2.DatabaseError) as error: - except : + print(error) - pass + finally: - if check_diaspora == False and check_peertube == False and check_zap == False and check_plume == False and check_hubzilla == False and check_misskey == False and check_prismo == False and check_osada == False and check_groundpolis == False and check_ganggo == False and check_squs == False and check_gnusocial == False and check_friendica == False and check_pixelfed == False and check_writefreely == False and check_raven == False and check_gnusocial2 == False and check_dolphin == False: + if conn is not None: + conn.close() - api_reply = 0 - pass - - elif check_diaspora == True or check_peertube == True or check_zap == True or check_plume == True or check_hubzilla == True or check_misskey == True or check_prismo == True or check_osada == True or check_groundpolis == True or check_ganggo == True or check_squs == True or check_gnusocial == True or check_friendica == True or check_pixelfed == True or check_writefreely == True or check_raven == True or check_gnusocial2 == True or check_dolphin == True: - - api_reply = 1 - - if nodeinfo.ok == True and check_peertube == True: - - print("Peertube server") - soft = "peertube" - peertube_users = nodeinfo.json()['usage']['users']['total'] - total_peertube_users = total_peertube_users + peertube_users - users = peertube_users - print(server, peertube_users) - print("\n") - - if nodeinfo.ok == True and check_diaspora == True: - - print("Diaspora server") - soft = "diaspora" - diaspora_users = nodeinfo.json()['usage']['users']['total'] - total_diaspora_users = total_diaspora_users + diaspora_users - users = diaspora_users - print(server, diaspora_users) - print("\n") - - elif nodeinfo.ok == True and check_raven == True: - - print("Ravenvale server") - soft = "ravenvale" - ravenvale_users = nodeinfo.json()['usage']['users']['total'] - total_ravenvale_users = total_ravenvale_users + ravenvale_users - users = ravenvale_users - print(server, ravenvale_users) - print("\n") - - elif others_nodeinfo.ok == True and check_zap == True: - - print("Zap server") - soft = "zap" - zap_users = others_nodeinfo.json()['usage']['users']['total'] - total_zap_users = total_zap_users + zap_users - users = zap_users - print(server, zap_users) - print("\n") - - elif others_nodeinfo.ok == True and check_plume == True: - - print("Plume server") - soft = "plume" - plume_users = others_nodeinfo.json()['usage']['users']['total'] - total_plume_users = total_plume_users + plume_users - users = plume_users - print(server, plume_users) - print("\n") - - elif others_nodeinfo.ok == True and check_hubzilla == True: - - print("Hubzilla server") - soft = "hubzilla" - hubzilla_users = others_nodeinfo.json()['usage']['users']['total'] - total_hubzilla_users = total_hubzilla_users + hubzilla_users - users = hubzilla_users - print(server, hubzilla_users) - print("\n") - - elif others_nodeinfo.ok == True and check_misskey == True: - - print("Misskey server") - soft = "misskey" - misskey_users = others_nodeinfo.json()['usage']['users']['total'] - if misskey_users == 0: - misskey_users = others_nodeinfo.json()['usage']['users']['activeHalfyear'] - total_misskey_users = total_misskey_users + misskey_users - users = misskey_users - print(server, misskey_users) - print("\n") - - elif others_nodeinfo.ok == True and check_prismo == True: - - print("Prismo server") - soft = "prismo" - prismo_users = others_nodeinfo.json()['usage']['users']['total'] - total_prismo_users = total_prismo_users + prismo_users - users = prismo_users - print(server, prismo_users) - print("\n") - - elif others_nodeinfo.ok == True and check_osada == True: - - print("Osada server") - soft = "osada" - osada_users = others_nodeinfo.json()['usage']['users']['total'] - total_osada_users = total_osada_users + osada_users - users = osada_users - print(server, osada_users) - print("\n") - - elif others_nodeinfo.ok == True and check_groundpolis == True: - - print("Groundpolis server") - soft = "groundpolis" - gpolis_users = others_nodeinfo.json()['usage']['users']['total'] - total_gpolis_users = total_gpolis_users + gpolis_users - users = gpolis_users - print(server, gpolis_users) - print("\n") - - elif others_nodeinfo.ok == True and check_ganggo == True: - - print("Ganggo server") - soft = "ganggo" - ggg_users = others_nodeinfo.json()['usage']['users']['total'] - total_ggg_users = total_ggg_users + ggg_users - users = ggg_users - print(server, ggg_users) - print("\n") - - elif others_nodeinfo.ok == True and check_squs == True: - - print("Squs server") - soft = "squs" - squs_users = others_nodeinfo.json()['usage']['users']['total'] - total_squs_users = total_squs_users + squs_users - users = squs_users - print(server, squs_users) - print("\n") - - elif others_nodeinfo.ok == True and check_friendica == True: - - print("Friendica server") - soft = "friendica" - if others_nodeinfo.json()['usage'] != []: - friendica_users = others_nodeinfo.json()['usage']['users']['total'] - else: - friendica_users = 0 - total_friendica_users = total_friendica_users + friendica_users - users = friendica_users - print(server, friendica_users) - print("\n") - check_friendica = False - - elif others_nodeinfo.ok == True and check_dolphin == True: - - print("Dolphin server") - soft = "dolphin" - if others_nodeinfo.json()['usage'] != []: - dolphin_users = others_nodeinfo.json()['usage']['users']['total'] - else: - dolphin_users = 0 - total_dolphin_users = total_dolphin_users + dolphin_users - users = dolphin_users - print(server, dolphin_users) - print("\n") - check_dolphin = False - - elif gs_nodeinfo.ok == True and check_gnusocial == True: - - print("GNU Social server") - soft = "gnusocial" - gs_users = gs_nodeinfo.json()['usage']['users']['total'] - if gs_users == 0: - gs_users = gs_nodeinfo.json()['usage']['users']['activeHalfyear'] - total_gs_users = total_gs_users + gs_users - users = gs_users - print(server, gs_users) - print("\n") - - elif statusnet.ok == True and check_friendica == True: #statusnet.json()['site']['friendica']['FRIENDICA_PLATFORM'] == "Friendica": - - print("Friendica server") - soft = "friendica" - friendica_users = 0 - total_friendica_users = total_friendica_users + friendica_users - users = friendica_users - print(server, friendica_users) - print("\n") - - elif px_nodeinfo.ok == True and check_pixelfed == True: - - print("Pixelfed server") - soft = "pixelfed" - pixelfed_users = px_nodeinfo.json()['usage']['users']['total'] - total_pixelfed_users = total_pixelfed_users + pixelfed_users - users = pixelfed_users - print(server, pixelfed_users) - print("\n") - - elif px_nodeinfo.ok == True and check_gnusocial2 == True: - - print("GNU Social 2.x server") - soft = "gnusocialv2" - gs_users = px_nodeinfo.json()['usage']['users']['total'] - if gs_users == 0: - gs_users = px_nodeinfo.json()['usage']['users']['activeHalfyear'] - total_gs_users = total_gs_users + gs_users - users = gs_users - print(server, gs_users) - print("\n") - - elif wf_nodeinfo.ok == True and check_writefreely == True: - - print("WriteFreely server") - soft = "writefreely" - writefreely_users = wf_nodeinfo.json()['usage']['users']['total'] - total_writefreely_users = total_writefreely_users + writefreely_users - users = writefreely_users - print(server, writefreely_users) - print("\n") - - else: - - api_reply = 0 - - if api_reply == 1 and others_nodeinfo.ok or gs_nodeinfo.ok or nodeinfo.ok or statusnet.ok or px_nodeinfo.ok or wf_nodeinfo.ok: - - insert_sql = "INSERT INTO fediverse(server, users, updated_at, software) VALUES(%s,%s,%s,%s) ON CONFLICT DO NOTHING" - conn = None + alive = False try: - conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") + data = requests.get('https://' + server + serv_api, timeout=3) - cur = conn.cursor() + if serv_soft == "mastodon": + if serv_api == '/nodeinfo/2.0?': + try: + users = data.json()['usage']['users']['total'] + except: + users = 0 + if serv_api == '/nodeinfo/2.0.json?': + try: + users = data.json()['usage']['users']['total'] + except: + users = 0 + elif serv_api == '/api/v1/instance?': + try: + users = data.json()['stats']['user_count'] + except: + users = 0 + alive = True + if serv_soft == "pleroma" or serv_soft == "diaspora" or serv_soft == "peertube" or serv_soft == "pixelfed" or serv_soft == "hubzilla" or serv_soft == "writefreely" or serv_soft == "friendica": + try: + users = data.json()['usage']['users']['total'] + except: + users = 0 + alive = True + if serv_soft == "gnusocialv2" or serv_soft == "gnusocial": + try: + users = data.json()['usage']['users']['total'] + if users == 0: + users = data.json()['usage']['users']['activeHalfyear'] + except: + users = 0 + alive = True + if serv_soft == "plume" or serv_soft == 'red' or serv_soft == "misskey" or serv_soft == "zap" or serv_soft == "prismo" or serv_soft == "ravenvale" or serv_soft == "osada" or serv_soft == "groundpolis": + try: + users = data.json()['usage']['users']['total'] + except: + users = 0 + alive = True + if serv_soft == "ganggo" or serv_soft == "squs" or serv_soft == "dolphin": + try: + users = data.json()['usage']['users']['total'] + except: + users = 0 + alive = True - cur.execute(insert_sql, (server, users, now, soft)) + print("Server " + str(server) + " (" + serv_soft + ") is alive!") - cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s) where server=(%s)", (users, now, soft, server)) + insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api) VALUES(%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING" + conn = None - conn.commit() + try: - cur.close() - return + conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") - except (Exception, psycopg2.DatabaseError) as error: + cur = conn.cursor() - print(error) - return + cur.execute(insert_sql, (server, users, now, serv_soft, alive, serv_api)) - finally: + cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s) where server=(%s)", (users, now, serv_soft, alive, serv_api, server)) - if conn is not None: + cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,)) + conn.commit() + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + print(error) + + finally: + + if conn is not None: + conn.close() + + except urllib3.exceptions.ProtocolError as protoerr: + + print("Server " + server + " is dead :-(") + alive = False + pass + + except requests.exceptions.ChunkedEncodingError as chunkerr: + + print("Server " + server + " is dead :-(") + alive = False + pass + + except KeyError as e: + + print("Server " + server + " is dead :-(") + alive = False + pass + + except ValueError as verr: + + print("Server " + server + " is dead :-(") + alive = False + pass + + except requests.exceptions.SSLError as errssl: + + print("Server " + server + " is dead :-(") + alive = False + pass + + except requests.exceptions.HTTPError as errh: + + print("Server " + server + " is dead :-(") + alive = False + pass + + except requests.exceptions.ConnectionError as errc: + + print("Server " + server + " is dead :-(") + alive = False + pass + + except requests.exceptions.Timeout as errt: + + print("Server " + server + " is dead :-(") + alive = False + pass + + except requests.exceptions.RequestException as err: + + print("Server " + server + " is dead :-(") + alive = False + pass + + if alive == False: + + conn = None + + try: + + conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", + host="/var/run/postgresql", port="5432") + + cur = conn.cursor() + + cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s) where server=(%s)", (now, alive, server)) + + cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,)) + + conn.commit() + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + print(error) + + finally: + + if conn is not None: + conn.close() + +def write_api(server, software, users, alive, api): + + insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api) VALUES(%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING" + conn = None + + try: + + conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") + + cur = conn.cursor() + + cur.execute(insert_sql, (server, now, software, users, alive, api)) + + cur.execute("UPDATE fediverse SET updated_at=(%s), software=(%s), users=(%s), alive=(%s), users_api=(%s) where server=(%s)", (now, software, users, alive, api, server)) + + cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,)) + + conn.commit() + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + print(error) + + finally: + + if conn is not None: conn.close() + +async def getsoft(server): + + try: + + socket.gethostbyname(server) + + except socket.gaierror: + return - except KeyError as e: + soft = '' - pass + url = 'https://' + server - except ValueError as verr: + timeout = aiohttp.ClientTimeout(total=3) + async with aiohttp.ClientSession(timeout=timeout) as session: + for api in apis: + try: + async with session.get(url+api) as response: + if response.status == 200: + try: + response_json = await response.json() + except: + pass + except aiohttp.ClientConnectorError as err: + pass + else: + if response.status == 200 and api != '/api/v1/instance?': + try: + soft = response_json['software']['name'] + soft = soft.lower() + users = response_json['usage']['users']['total'] + alive = True + write_api(server, soft, users, alive, api) + print("Server " + server + " is alive!") + return + except: + pass + if response.status == 200 and soft == '' and api == "/api/v1/instance?": + soft = 'mastodon' + users = response_json['stats']['user_count'] + alive = True + write_api(server, soft, users, alive, api) + print("Server " + server + " is alive!") - pass +def getserver(server, x): - except requests.exceptions.ChunkedEncodingError as chunkerr: + server = server[0].rstrip('.').lower() - print(server) - pass + if server.find(".") == -1: + return + if server.find("@") != -1: + return + if server.find("/") != -1: + return + if server.find(":") != -1: + return - except requests.exceptions.ConnectionError as errc: + try: - pass + loop = asyncio.get_event_loop() + coroutines = [getsoft(server)] + soft = loop.run_until_complete(asyncio.gather(*coroutines, return_exceptions=True)) - except requests.exceptions.Timeout as errt: - - pass - - except requests.exceptions.ConnectionError as errc: + except: pass @@ -651,287 +396,329 @@ headers={ 'Authorization': 'Bearer %s'%uc_access_token } ############################################################################### # main -pl_users = 0 -total_pl_users = 0 -mast_users = 0 -total_mast_users = 0 -gs_users = 0 -total_gs_users = 0 -zap_users = 0 -total_zap_users = 0 -plume_users = 0 -total_plume_users = 0 -hubzilla_users = 0 -total_hubzilla_users = 0 -misskey_users = 0 -total_misskey_users = 0 -prismo_users = 0 -total_prismo_users = 0 -osada_users = 0 -total_osada_users = 0 -gpolis_users = 0 -total_gpolis_users = 0 -ggg_users = 0 -total_ggg_users = 0 -squs_users = 0 -total_squs_users = 0 -peertube_users = 0 -total_peertube_users = 0 -diaspora_users = 0 -total_diaspora_users = 0 -friendica_users = 0 -total_friendica_users = 0 -pixelfed_users = 0 -total_pixelfed_users = 0 -writefreely_users = 0 -total_writefreely_users = 0 -ravenvale_users = 0 -total_ravenvale_users = 0 +if __name__ == '__main__': -total_servers = 0 -total_users = 0 + total_servers = 0 + total_users = 0 -now = datetime.now() + ############################################################################ + # set all world servers's checked column to False -try: + try: - conn = None - conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") + conn = None - cur = conn.cursor() + conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") - ### get world servers list + cur = conn.cursor() - cur.execute("select server from world") + cur.execute("UPDATE world SET checked='f'") + + conn.commit() + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + ############################################################################ + # get last check servers from fediverse DB + + + alive_servers = [] + + try: + + conn = None + conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") + + cur = conn.cursor() + + ### get world servers list + + cur.execute("select server from world where server in (select server from fediverse where users_api != '')") + + alive_servers = [] + + for row in cur: + + alive_servers.append(row[0]) + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + ########################################################################### + # multiprocessing! + + m = Manager() + q = m.Queue() + z = zip(alive_servers) + + serv_number = len(alive_servers) + + pool_tuple = [(x, q) for x in z] + with Pool(processes=64) as pool: + pool.starmap(alive_server, pool_tuple) + + ########################################################################### + + print("Getting the remaining servers from world") world_servers = [] - for row in cur: + try: - world_servers.append(row[0]) + conn = None + conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") - cur.close() + cur = conn.cursor() -except (Exception, psycopg2.DatabaseError) as error: + ### get world servers list - print(error) + cur.execute("select server from world where checked='f'") -finally: + for row in cur: - if conn is not None: + world_servers.append(row[0]) - conn.close() + cur.close() -########################################################################### -# multiprocessing! + except (Exception, psycopg2.DatabaseError) as error: -m = Manager() -q = m.Queue() -z = zip(world_servers) + print(error) -serv_number = len(world_servers) + finally: -pool_tuple = [(x, q) for x in z] -with Pool(processes=32) as pool: - pool.starmap(getserver, pool_tuple) + if conn is not None: -########################################################################### -# delete not alive servers from fediverse table + conn.close() -try: + ########################################################################### + # multiprocessing! + + m = Manager() + q = m.Queue() + z = zip(world_servers) + + serv_number = len(world_servers) + + pool_tuple = [(x, q) for x in z] + with Pool(processes=64) as pool: + pool.starmap(getserver, pool_tuple) + + ########################################################################### + # delete not alive servers from fediverse table + + try: + + conn = None + + conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") + + cur = conn.cursor() + + cur.execute("delete from fediverse where not alive") + + conn.commit() + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + ########################################################################### + # get current total servers and users, get users from every software + + gettotals_sql = "select count(server), sum(users) from fediverse" + get_soft_totals_sql = "select software, sum(users) as users, count(server) as servers from fediverse where users != 0 group by software order by users desc" + soft_total_project = [] + soft_total_users = [] + soft_total_servers = [] + + try: + + conn = None + conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") + + cur = conn.cursor() + + cur.execute(gettotals_sql) + + row = cur.fetchone() + + total_servers = row[0] + total_users = row[1] + + cur.execute(get_soft_totals_sql) + rows = cur.fetchall() + for row in rows: + soft_total_project.append(row[0]) + soft_total_users.append(row[1]) + soft_total_servers.append(row[2]) + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + ########################################################################### + # get last check values and write current total ones + + select_sql = "select total_servers, total_users from totals order by datetime desc limit 1" + insert_sql = "INSERT INTO totals(datetime, total_servers, total_users) VALUES(%s,%s,%s)" + + try: + + conn = None + conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") + + cur = conn.cursor() + + cur.execute(select_sql) + + row = cur.fetchone() + + if row != None: + + servers_before = row[0] + users_before = row[1] + + else: + + servers_before = 0 + users_before = 0 + + cur.execute(insert_sql, (now, total_servers, total_users)) + + conn.commit() + + cur.close() + + evo_servers = total_servers - servers_before + evo_users = total_users - users_before + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + ################################################################################ + # write evo values + + insert_sql = "INSERT INTO evo(datetime, servers, users) VALUES(%s,%s,%s)" conn = None - conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") + try: - cur = conn.cursor() + conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") - cur.execute("delete from fediverse where updated_at != (%s)", (now,)) + cur = conn.cursor() - conn.commit() + cur.execute(insert_sql, (now, evo_servers, evo_users)) - cur.close() + conn.commit() -except (Exception, psycopg2.DatabaseError) as error: + cur.close() - print(error) + except (Exception, psycopg2.DatabaseError) as error: -finally: + print(error) - if conn is not None: + finally: - conn.close() + if conn is not None: -########################################################################### -# get current total servers and users, get users from every software + conn.close() -gettotals_sql = "select count(server), sum(users) from fediverse" -get_soft_totals_sql = "select software, sum(users) as users, count(server) as servers from fediverse where users != 0 group by software order by users desc" -soft_total_project = [] -soft_total_users = [] -soft_total_servers = [] - -try: + ############################################################################## + # get world's last update datetime conn = None - conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") - cur = conn.cursor() + try: - cur.execute(gettotals_sql) + conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") - row = cur.fetchone() + cur = conn.cursor() - total_servers = row[0] - total_users = row[1] + cur.execute("select updated_at from world order by updated_at desc limit 1") - cur.execute(get_soft_totals_sql) - rows = cur.fetchall() - for row in rows: - soft_total_project.append(row[0]) - soft_total_users.append(row[1]) - soft_total_servers.append(row[2]) - cur.close() + row = cur.fetchone() -except (Exception, psycopg2.DatabaseError) as error: + last_update = row[0] + last_update = last_update.strftime('%m/%d/%Y, %H:%M:%S') - print(error) + cur.close() -finally: + except (Exception, psycopg2.DatabaseError) as error: - if conn is not None: + print(error) - conn.close() + finally: -########################################################################### -# get last check values and write current total ones + if conn is not None: -select_sql = "select total_servers, total_users from totals order by datetime desc limit 1" -insert_sql = "INSERT INTO totals(datetime, total_servers, total_users) VALUES(%s,%s,%s)" + conn.close() -try: + ############################################################################### + # T O O T ! - conn = None - conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") + toot_text = "#fediverse alive servers stats" + " \n" + toot_text += "\n" + if evo_servers >= 0: + toot_text += "alive servers: " + str(total_servers) + " (+"+ str(evo_servers) + ") \n" + elif evo_servers < 0: + toot_text += "alive servers: " + str(total_servers) + " ("+ str(evo_servers) + ") \n" + if evo_users >= 0: + toot_text += "total users: " + str(total_users) + " (+"+ str(evo_users) + ") \n" + elif evo_users < 0: + toot_text += "total users: " + str(total_users) + " ("+ str(evo_users) + ") \n" + toot_text += "\n" + toot_text += "top five (soft: users servers):" + " \n" + toot_text += "\n" + toot_text += '{:>0}: {:>5} {:>5}'.format(str(soft_total_project[0]), str(soft_total_users[0]), str(soft_total_servers[0])) + " \n" + toot_text += '{:>0}: {:>11} {:>8}'.format(str(soft_total_project[1]), str(soft_total_users[1]), str(soft_total_servers[1])) + " \n" + toot_text += '{:>0}: {:>12} {:>7}'.format(str(soft_total_project[2]), str(soft_total_users[2]), str(soft_total_servers[2])) + " \n" + toot_text += '{:>0}: {:>13} {:>7}'.format(str(soft_total_project[3]), str(soft_total_users[3]), str(soft_total_servers[3])) + " \n" + toot_text += '{:>0}: {:>14} {:>7}'.format(str(soft_total_project[4]), str(soft_total_users[4]), str(soft_total_servers[4])) + " \n" + toot_text += "\n" + toot_text += "updated at " + str(last_update) + " \n" + print("Tooting...") + print(toot_text) - cur = conn.cursor() - - cur.execute(select_sql) - - row = cur.fetchone() - - if row != None: - - servers_before = row[0] - users_before = row[1] - - else: - - servers_before = 0 - users_before = 0 - - cur.execute(insert_sql, (now, total_servers, total_users)) - - conn.commit() - - cur.close() - - evo_servers = total_servers - servers_before - evo_users = total_users - users_before - -except (Exception, psycopg2.DatabaseError) as error: - - print(error) - -finally: - - if conn is not None: - - conn.close() - -################################################################################ -# write evo values - -insert_sql = "INSERT INTO evo(datetime, servers, users) VALUES(%s,%s,%s)" - -conn = None - -try: - - conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") - - cur = conn.cursor() - - cur.execute(insert_sql, (now, evo_servers, evo_users)) - - conn.commit() - - cur.close() - -except (Exception, psycopg2.DatabaseError) as error: - - print(error) - -finally: - - if conn is not None: - - conn.close() - -############################################################################## -# get world's last update datetime - -conn = None - -try: - - conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") - - cur = conn.cursor() - - cur.execute("select updated_at from world order by updated_at desc limit 1") - - row = cur.fetchone() - - last_update = row[0] - last_update = last_update.strftime('%m/%d/%Y, %H:%M:%S') - - cur.close() - -except (Exception, psycopg2.DatabaseError) as error: - - print(error) - -finally: - - if conn is not None: - - conn.close() - -############################################################################### -# T O O T ! - -toot_text = "#fediverse alive servers stats" + " \n" -toot_text += "\n" -if evo_servers >= 0: - toot_text += "alive servers: " + str(total_servers) + " (+"+ str(evo_servers) + ") \n" -elif evo_servers < 0: - toot_text += "alive servers: " + str(total_servers) + " (-"+ str(evo_servers) + ") \n" -if evo_users >= 0: - toot_text += "total users: " + str(total_users) + " (+"+ str(evo_users) + ") \n" -elif evo_users < 0: - toot_text += "total users: " + str(total_users) + " (-"+ str(evo_users) + ") \n" -toot_text += "\n" -toot_text += "top five (soft: users servers):" + " \n" -toot_text += "\n" -toot_text += '{:>0}: {:>5} {:>5}'.format(str(soft_total_project[0]), str(soft_total_users[0]), str(soft_total_servers[0])) + " \n" -toot_text += '{:>0}: {:>11} {:>8}'.format(str(soft_total_project[1]), str(soft_total_users[1]), str(soft_total_servers[1])) + " \n" -toot_text += '{:>0}: {:>12} {:>7}'.format(str(soft_total_project[2]), str(soft_total_users[2]), str(soft_total_servers[2])) + " \n" -toot_text += '{:>0}: {:>13} {:>7}'.format(str(soft_total_project[3]), str(soft_total_users[3]), str(soft_total_servers[3])) + " \n" -toot_text += '{:>0}: {:>14} {:>7}'.format(str(soft_total_project[4]), str(soft_total_users[4]), str(soft_total_servers[4])) + " \n" -toot_text += "\n" -toot_text += "updated at " + str(last_update) + " \n" -print("Tooting...") -print(toot_text) - -mastodon.status_post(toot_text, in_reply_to_id=None, ) + mastodon.status_post(toot_text, in_reply_to_id=None, )