#!/usr/bin/env python # -*- coding: utf-8 -*- import time start_time = time.time() import urllib3 from urllib3 import exceptions from datetime import datetime import pytz from subprocess import call from mastodon import Mastodon import threading import os import json import signal import sys import os.path import requests from requests import exceptions import operator import calendar import psycopg2 from itertools import product from multiprocessing import Pool, Lock, Process, Queue, current_process, Manager import multiprocessing from decimal import * getcontext().prec = 2 ############################################################################### # INITIALISATION ############################################################################### def is_json(myjson): try: json_object = json.loads(myjson) except ValueError as e: return False return True def getserver(server, x): server = server[0].rstrip('.').lower() if server.find(".") == -1: print(server + " is Not a real domain!") return if server.find("@") != -1: print(server + " is Not a real domain!") return if server.find("/") != -1: print(server + " is Not a real domain!") return if server.find(":") != -1: print(server + " is Not a real domain!") return global mastodon global pleroma global gnusocial global zap global plume global hubzilla global misskey global prismo global osada global groundpolis global ganggo global squs global peertube global friendica global pixelfed global writefreely global ravenvale global diaspora global dolphin pl_users = 0 global total_pl_users mast_users = 0 global total_mast_users gs_users = 0 global total_gs_users zap_users = 0 global total_zap_users plume_users = 0 global total_plume_users hubzilla_users = 0 global total_hubzilla_users misskey_users = 0 global total_misskey_users prismo_users = 0 global total_prismo_users osada_users = 0 global total_osada_users gpolis_users = 0 global total_gpolis_users ggg_users = 0 global total_ggg_users squs_users = 0 global total_squs_users peertube_users = 0 global total_peertube_users diaspora_users = 0 global total_diaspora_users friendica_users = 0 global total_friendica_users pixelfed_users = 0 global total_pixelfed_users writefreely_users = 0 global total_writefreely_users ravenvale_users = 0 global total_ravenvale_users dolphin_users = 0 global total_dolphin_users check_diaspora = False check_peertube = False check_zap = False check_plume = False check_hubzilla = False check_misskey = False check_prismo = False check_osada = False check_groundpolis = False check_ganggo = False check_squs = False check_gnusocial = False check_friendica = False check_pixelfed = False check_writefreely = False check_raven = False check_gnusocial2 = False check_dolphin = False users = 0 try: res = requests.get('https://' + server + '/api/v1/instance?',timeout=3) res_friendica = requests.get('https://' + server + '/nodeinfo/2.0?',timeout=3) check_pix = "compatible; Pixelfed" in res.text check_ravenvale = "ravenvale" in res.text check_friendica = "friendica" in res_friendica.text if check_friendica == False: check_friendica = "Friendica" in res_friendica.text if (res.ok) and check_pix == False and check_ravenvale == False and check_friendica == False: if is_json(res.text) == True and server in res.text: nodeinfo = requests.get('https://' + server + '/nodeinfo/2.0.json?',timeout=3) check_pleroma = "pleroma" in nodeinfo.text.lower() if nodeinfo.status_code == 200 and check_pleroma == True: print("Pleroma server: ") soft = "pleroma" pl_users = nodeinfo.json()['usage']['users']['total'] total_pl_users = total_pl_users + pl_users users = pl_users print(server, pl_users) else: if type(res.json()) != int: if res.json().get('stats') != None: print("Mastodon server: ") soft = "mastodon" mast_users = res.json()['stats']['user_count'] else: mast_users = 0 soft = "mastodon" else: mast_users = 0 if mast_users != None and mast_users < 999999: total_mast_users = total_mast_users + mast_users users = mast_users print(server, mast_users) print("\n") if users > 1000000: return insert_sql = "INSERT INTO fediverse(server, users, updated_at, software) VALUES(%s,%s,%s,%s) ON CONFLICT DO NOTHING" conn = None try: conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") cur = conn.cursor() cur.execute(insert_sql, (server, users, now, soft)) cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s) where server=(%s)", (users, now, soft, server)) conn.commit() cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() return else: return except urllib3.exceptions.ProtocolError as protoerr: pass except requests.exceptions.ChunkedEncodingError as chunkerr: pass except KeyError as e: pass except ValueError as verr: pass except requests.exceptions.SSLError as errssl: pass except requests.exceptions.HTTPError as errh: pass except requests.exceptions.ConnectionError as errc: pass except requests.exceptions.Timeout as errt: pass except requests.exceptions.RequestException as err: pass else: try: soft = "" others_nodeinfo = requests.get('https://' + server + '/nodeinfo/2.0?',timeout=3) gs_nodeinfo = requests.get('https://' + server + '/main/nodeinfo/2.0?',timeout=3) nodeinfo = requests.get('https://' + server + '/nodeinfo/2.0.json?',timeout=3) statusnet = requests.get('https://' + server + '/api/statusnet/config?',timeout=3) px_nodeinfo = requests.get('https://' + server + '/api/nodeinfo/2.0.json?',timeout=3) wf_nodeinfo = requests.get('https://' + server + '/api/nodeinfo?',timeout=3) try: if is_json(nodeinfo.text) == True: if nodeinfo.json() != None: soft = nodeinfo.json()['software']['name'] if soft == "peertube": check_peertube = True elif soft == "prismo": check_prismo = True elif soft == "zap": check_zap = True elif soft == "osada": check_osada = True elif soft == "ravenvale": check_raven = True elif soft == "diaspora": check_diaspora = True if is_json(others_nodeinfo.text) == True: soft = others_nodeinfo.json()['software']['name'] if soft == "plume": check_plume = True elif soft == "hubzilla": check_hubzilla = True elif soft == "misskey": check_misskey = True elif soft == "groundpolis": check_groundpolis = True elif soft == "ganggo": check_ganggo = True elif soft == "squs": check_squs = True elif soft == "friendica" or soft == "Friendica": check_friendica = True elif soft == "dolphin": check_dolphin = True if is_json(wf_nodeinfo.text) == True and soft == "": #if wf_nodeinfo.json().get('error') != None and wf_nodeinfo.json().get('status') == None: soft = wf_nodeinfo.json()['software']['name'] if soft == "writefreely": check_writefreely = True if is_json(gs_nodeinfo.text) == True: soft = gs_nodeinfo.json()['software']['name'] if soft == "gnusocial": check_gnusocial = True if is_json(statusnet.text) == True and soft == "": if soft != "zap" and soft != "osada": soft = statusnet.json()['site']['friendica']['FRIENDICA_PLATFORM'] if soft == "Friendica": check_friendica = True if is_json(px_nodeinfo.text) == True and soft == "": if soft != "Friendica" and soft != "hubzilla": soft = px_nodeinfo.json()['software']['name'] if soft == "pixelfed": check_pixelfed = True if soft == "gnusocial": check_gnusocial2 = True except : pass if check_diaspora == False and check_peertube == False and check_zap == False and check_plume == False and check_hubzilla == False and check_misskey == False and check_prismo == False and check_osada == False and check_groundpolis == False and check_ganggo == False and check_squs == False and check_gnusocial == False and check_friendica == False and check_pixelfed == False and check_writefreely == False and check_raven == False and check_gnusocial2 == False and check_dolphin == False: api_reply = 0 pass elif check_diaspora == True or check_peertube == True or check_zap == True or check_plume == True or check_hubzilla == True or check_misskey == True or check_prismo == True or check_osada == True or check_groundpolis == True or check_ganggo == True or check_squs == True or check_gnusocial == True or check_friendica == True or check_pixelfed == True or check_writefreely == True or check_raven == True or check_gnusocial2 == True or check_dolphin == True: api_reply = 1 if nodeinfo.ok == True and check_peertube == True: print("Peertube server") soft = "peertube" peertube_users = nodeinfo.json()['usage']['users']['total'] total_peertube_users = total_peertube_users + peertube_users users = peertube_users print(server, peertube_users) print("\n") if nodeinfo.ok == True and check_diaspora == True: print("Diaspora server") soft = "diaspora" diaspora_users = nodeinfo.json()['usage']['users']['total'] total_diaspora_users = total_diaspora_users + diaspora_users users = diaspora_users print(server, diaspora_users) print("\n") elif nodeinfo.ok == True and check_raven == True: print("Ravenvale server") soft = "ravenvale" ravenvale_users = nodeinfo.json()['usage']['users']['total'] total_ravenvale_users = total_ravenvale_users + ravenvale_users users = ravenvale_users print(server, ravenvale_users) print("\n") elif others_nodeinfo.ok == True and check_zap == True: print("Zap server") soft = "zap" zap_users = others_nodeinfo.json()['usage']['users']['total'] total_zap_users = total_zap_users + zap_users users = zap_users print(server, zap_users) print("\n") elif others_nodeinfo.ok == True and check_plume == True: print("Plume server") soft = "plume" plume_users = others_nodeinfo.json()['usage']['users']['total'] total_plume_users = total_plume_users + plume_users users = plume_users print(server, plume_users) print("\n") elif others_nodeinfo.ok == True and check_hubzilla == True: print("Hubzilla server") soft = "hubzilla" hubzilla_users = others_nodeinfo.json()['usage']['users']['total'] total_hubzilla_users = total_hubzilla_users + hubzilla_users users = hubzilla_users print(server, hubzilla_users) print("\n") elif others_nodeinfo.ok == True and check_misskey == True: print("Misskey server") soft = "misskey" misskey_users = others_nodeinfo.json()['usage']['users']['total'] if misskey_users == 0: misskey_users = others_nodeinfo.json()['usage']['users']['activeHalfyear'] total_misskey_users = total_misskey_users + misskey_users users = misskey_users print(server, misskey_users) print("\n") elif others_nodeinfo.ok == True and check_prismo == True: print("Prismo server") soft = "prismo" prismo_users = others_nodeinfo.json()['usage']['users']['total'] total_prismo_users = total_prismo_users + prismo_users users = prismo_users print(server, prismo_users) print("\n") elif others_nodeinfo.ok == True and check_osada == True: print("Osada server") soft = "osada" osada_users = others_nodeinfo.json()['usage']['users']['total'] total_osada_users = total_osada_users + osada_users users = osada_users print(server, osada_users) print("\n") elif others_nodeinfo.ok == True and check_groundpolis == True: print("Groundpolis server") soft = "groundpolis" gpolis_users = others_nodeinfo.json()['usage']['users']['total'] total_gpolis_users = total_gpolis_users + gpolis_users users = gpolis_users print(server, gpolis_users) print("\n") elif others_nodeinfo.ok == True and check_ganggo == True: print("Ganggo server") soft = "ganggo" ggg_users = others_nodeinfo.json()['usage']['users']['total'] total_ggg_users = total_ggg_users + ggg_users users = ggg_users print(server, ggg_users) print("\n") elif others_nodeinfo.ok == True and check_squs == True: print("Squs server") soft = "squs" squs_users = others_nodeinfo.json()['usage']['users']['total'] total_squs_users = total_squs_users + squs_users users = squs_users print(server, squs_users) print("\n") elif others_nodeinfo.ok == True and check_friendica == True: print("Friendica server") soft = "friendica" if others_nodeinfo.json()['usage'] != []: friendica_users = others_nodeinfo.json()['usage']['users']['total'] else: friendica_users = 0 total_friendica_users = total_friendica_users + friendica_users users = friendica_users print(server, friendica_users) print("\n") check_friendica = False elif others_nodeinfo.ok == True and check_dolphin == True: print("Dolphin server") soft = "dolphin" if others_nodeinfo.json()['usage'] != []: dolphin_users = others_nodeinfo.json()['usage']['users']['total'] else: dolphin_users = 0 total_dolphin_users = total_dolphin_users + dolphin_users users = dolphin_users print(server, dolphin_users) print("\n") check_dolphin = False elif gs_nodeinfo.ok == True and check_gnusocial == True: print("GNU Social server") soft = "gnusocial" gs_users = gs_nodeinfo.json()['usage']['users']['total'] if gs_users == 0: gs_users = gs_nodeinfo.json()['usage']['users']['activeHalfyear'] total_gs_users = total_gs_users + gs_users users = gs_users print(server, gs_users) print("\n") elif statusnet.ok == True and check_friendica == True: #statusnet.json()['site']['friendica']['FRIENDICA_PLATFORM'] == "Friendica": print("Friendica server") soft = "friendica" friendica_users = 0 total_friendica_users = total_friendica_users + friendica_users users = friendica_users print(server, friendica_users) print("\n") elif px_nodeinfo.ok == True and check_pixelfed == True: print("Pixelfed server") soft = "pixelfed" pixelfed_users = px_nodeinfo.json()['usage']['users']['total'] total_pixelfed_users = total_pixelfed_users + pixelfed_users users = pixelfed_users print(server, pixelfed_users) print("\n") elif px_nodeinfo.ok == True and check_gnusocial2 == True: print("GNU Social 2.x server") soft = "gnusocialv2" gs_users = px_nodeinfo.json()['usage']['users']['total'] if gs_users == 0: gs_users = px_nodeinfo.json()['usage']['users']['activeHalfyear'] total_gs_users = total_gs_users + gs_users users = gs_users print(server, gs_users) print("\n") elif wf_nodeinfo.ok == True and check_writefreely == True: print("WriteFreely server") soft = "writefreely" writefreely_users = wf_nodeinfo.json()['usage']['users']['total'] total_writefreely_users = total_writefreely_users + writefreely_users users = writefreely_users print(server, writefreely_users) print("\n") else: api_reply = 0 if api_reply == 1 and others_nodeinfo.ok or gs_nodeinfo.ok or nodeinfo.ok or statusnet.ok or px_nodeinfo.ok or wf_nodeinfo.ok: insert_sql = "INSERT INTO fediverse(server, users, updated_at, software) VALUES(%s,%s,%s,%s) ON CONFLICT DO NOTHING" conn = None try: conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") cur = conn.cursor() cur.execute(insert_sql, (server, users, now, soft)) cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s) where server=(%s)", (users, now, soft, server)) conn.commit() cur.close() return except (Exception, psycopg2.DatabaseError) as error: print(error) return finally: if conn is not None: conn.close() return except KeyError as e: pass except ValueError as verr: pass except requests.exceptions.ChunkedEncodingError as chunkerr: print(server) pass except requests.exceptions.ConnectionError as errc: pass except requests.exceptions.Timeout as errt: pass except requests.exceptions.ConnectionError as errc: pass # Returns the parameter from the specified file def get_parameter( parameter, file_path ): # Check if secrets file exists if not os.path.isfile(file_path): print("File %s not found, exiting."%file_path) sys.exit(0) # Find parameter in file with open( file_path ) as f: for line in f: if line.startswith( parameter ): return line.replace(parameter + ":", "").strip() # Cannot find parameter, exit print(file_path + " Missing parameter %s "%parameter) sys.exit(0) # Load secrets from secrets file secrets_filepath = "secrets/secrets.txt" uc_client_id = get_parameter("uc_client_id", secrets_filepath) uc_client_secret = get_parameter("uc_client_secret", secrets_filepath) uc_access_token = get_parameter("uc_access_token", secrets_filepath) # Load configuration from config file config_filepath = "config.txt" mastodon_hostname = get_parameter("mastodon_hostname", config_filepath) # Load database config from db_config file db_config_filepath = "config/db_config.txt" fediverse_db = get_parameter("fediverse_db", db_config_filepath) fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath) # Initialise Mastodon API mastodon = Mastodon( client_id = uc_client_id, client_secret = uc_client_secret, access_token = uc_access_token, api_base_url = 'https://' + mastodon_hostname, ) # Initialise access headers headers={ 'Authorization': 'Bearer %s'%uc_access_token } ############################################################################### # main pl_users = 0 total_pl_users = 0 mast_users = 0 total_mast_users = 0 gs_users = 0 total_gs_users = 0 zap_users = 0 total_zap_users = 0 plume_users = 0 total_plume_users = 0 hubzilla_users = 0 total_hubzilla_users = 0 misskey_users = 0 total_misskey_users = 0 prismo_users = 0 total_prismo_users = 0 osada_users = 0 total_osada_users = 0 gpolis_users = 0 total_gpolis_users = 0 ggg_users = 0 total_ggg_users = 0 squs_users = 0 total_squs_users = 0 peertube_users = 0 total_peertube_users = 0 diaspora_users = 0 total_diaspora_users = 0 friendica_users = 0 total_friendica_users = 0 pixelfed_users = 0 total_pixelfed_users = 0 writefreely_users = 0 total_writefreely_users = 0 ravenvale_users = 0 total_ravenvale_users = 0 total_servers = 0 total_users = 0 now = datetime.now() try: conn = None conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") cur = conn.cursor() ### get world servers list cur.execute("select server from world") world_servers = [] for row in cur: world_servers.append(row[0]) cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() ########################################################################### # multiprocessing! m = Manager() q = m.Queue() z = zip(world_servers) serv_number = len(world_servers) pool_tuple = [(x, q) for x in z] with Pool(processes=32) as pool: pool.starmap(getserver, pool_tuple) ########################################################################### # delete not alive servers from fediverse table try: conn = None conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") cur = conn.cursor() cur.execute("delete from fediverse where updated_at != (%s)", (now,)) conn.commit() cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() ########################################################################### # get current total servers and users, get users from every software gettotals_sql = "select count(server), sum(users) from fediverse" get_soft_totals_sql = "select software, sum(users) as users, count(server) as servers from fediverse where users != 0 group by software order by users desc" soft_total_project = [] soft_total_users = [] soft_total_servers = [] try: conn = None conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") cur = conn.cursor() cur.execute(gettotals_sql) row = cur.fetchone() total_servers = row[0] total_users = row[1] cur.execute(get_soft_totals_sql) rows = cur.fetchall() for row in rows: soft_total_project.append(row[0]) soft_total_users.append(row[1]) soft_total_servers.append(row[2]) cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() ########################################################################### # get last check values and write current total ones select_sql = "select total_servers, total_users from totals order by datetime desc limit 1" insert_sql = "INSERT INTO totals(datetime, total_servers, total_users) VALUES(%s,%s,%s)" try: conn = None conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") cur = conn.cursor() cur.execute(select_sql) row = cur.fetchone() if row != None: servers_before = row[0] users_before = row[1] else: servers_before = 0 users_before = 0 cur.execute(insert_sql, (now, total_servers, total_users)) conn.commit() cur.close() evo_servers = total_servers - servers_before evo_users = total_users - users_before except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() ################################################################################ # write evo values insert_sql = "INSERT INTO evo(datetime, servers, users) VALUES(%s,%s,%s)" conn = None try: conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") cur = conn.cursor() cur.execute(insert_sql, (now, evo_servers, evo_users)) conn.commit() cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() ############################################################################## # get world's last update datetime conn = None try: conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") cur = conn.cursor() cur.execute("select updated_at from world order by updated_at desc limit 1") row = cur.fetchone() last_update = row[0] last_update = last_update.strftime('%m/%d/%Y, %H:%M:%S') cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() ############################################################################### # T O O T ! toot_text = "#fediverse alive servers stats" + " \n" toot_text += "\n" if evo_servers >= 0: toot_text += "alive servers: " + str(total_servers) + " (+"+ str(evo_servers) + ") \n" elif evo_servers < 0: toot_text += "alive servers: " + str(total_servers) + " (-"+ str(evo_servers) + ") \n" if evo_users >= 0: toot_text += "total users: " + str(total_users) + " (+"+ str(evo_users) + ") \n" elif evo_users < 0: toot_text += "total users: " + str(total_users) + " (-"+ str(evo_users) + ") \n" toot_text += "\n" toot_text += "top five (soft: users servers):" + " \n" toot_text += "\n" toot_text += '{:>0}: {:>5} {:>5}'.format(str(soft_total_project[0]), str(soft_total_users[0]), str(soft_total_servers[0])) + " \n" toot_text += '{:>0}: {:>11} {:>8}'.format(str(soft_total_project[1]), str(soft_total_users[1]), str(soft_total_servers[1])) + " \n" toot_text += '{:>0}: {:>12} {:>7}'.format(str(soft_total_project[2]), str(soft_total_users[2]), str(soft_total_servers[2])) + " \n" toot_text += '{:>0}: {:>13} {:>7}'.format(str(soft_total_project[3]), str(soft_total_users[3]), str(soft_total_servers[3])) + " \n" toot_text += '{:>0}: {:>14} {:>7}'.format(str(soft_total_project[4]), str(soft_total_users[4]), str(soft_total_servers[4])) + " \n" toot_text += "\n" toot_text += "updated at " + str(last_update) + " \n" print("Tooting...") print(toot_text) mastodon.status_post(toot_text, in_reply_to_id=None, )