fediverse/fediverse.py

import time
import urllib3
from datetime import datetime
from mastodon import Mastodon
import os
import json
import sys
import os.path
import requests
import psycopg2
from itertools import product
import multiprocessing
import aiohttp
import asyncio
import socket
import matplotlib.pyplot as plt
import pdb
plt.style.use('seaborn')

start_time = time.time()

apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?', '/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?', '/wp-json/nodeinfo/2.0?']

client_exceptions = (
    aiohttp.ClientResponseError,
    aiohttp.ClientConnectionError,
    aiohttp.ClientConnectorError,
    aiohttp.ClientError,
    asyncio.TimeoutError,
    socket.gaierror,
)

def is_json(myjson):

    try:
        json_object = json.loads(myjson)
    except ValueError as e:
        print(e)
        return False
    return True

def get_alive_servers(server):

    serv_api = ''
    serv_soft = ''
    soft_version = ''

    try:

        conn = None

        conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")

        cur = conn.cursor()

        cur.execute("select alive, software, users_api, version, first_checked_at, downs from fediverse where server=(%s)", (server,))

        row = cur.fetchone()

        if row is not None:

            was_alive = row[0]
            serv_soft = row[1]
            serv_api = row[2]
            soft_version = row[3]
            first_checked_at = row[4]
            downs_qty = row[5]

        cur.close()

    except (Exception, psycopg2.DatabaseError) as error:

        print(error)

    finally:

        if conn is not None:
            conn.close()

        alive = False

        try:

            user_agent = {'User-agent': 'Mozilla/5.0'}

            data = requests.get('https://' + server + serv_api, headers = user_agent, timeout=3)

            if serv_soft == "mastodon":
                if serv_api == '/nodeinfo/2.0?':
                    try:
                        users = data.json()['usage']['users']['total']
                        soft_version = data.json()['software']['version']
                        alive = True
                    except:
                        users = 0
                        soft_version = ""
                if serv_api == '/nodeinfo/2.0.json?':
                    try:
                        users = data.json()['usage']['users']['total']
                        soft_version = data.json()['software']['version']
                        alive = True
                    except:
                        users = 0
                        soft_version = ""
                elif serv_api == '/api/v1/instance?':
                    try:
                        users = data.json()['stats']['user_count']
                        soft_version = data.json()['version']
                        alive = True
                    except:
                        users = 0
                        soft_version = ""
            if serv_soft == "pleroma" or serv_soft == "diaspora" or serv_soft == "peertube" or serv_soft == "pixelfed" or serv_soft == "hubzilla" or serv_soft == "writefreely" or serv_soft == "friendica":
                try:
                    users = data.json()['usage']['users']['total']
                    soft_version = data.json()['software']['version']
                    alive = True
                except:
                    users = 0
                    soft_version = ""
            if serv_soft == "gnusocialv2" or serv_soft == "gnusocial":
                try:
                    users = data.json()['usage']['users']['total']
                    if users == 0:
                        users = data.json()['usage']['users']['activeHalfyear']
                    soft_version = data.json()['software']['version']
                    alive = True
                except:
                    users = 0
                    soft_version = ""
            if serv_soft == "plume" or serv_soft == 'red' or serv_soft == "misskey" or serv_soft == "zap" or serv_soft == "prismo" or serv_soft == "ravenvale" or serv_soft == "osada" or serv_soft == "groundpolis":
                try:
                    users = data.json()['usage']['users']['total']
                    soft_version = data.json()['software']['version']
                    alive = True
                except:
                    users = 0
                    soft_version = ""
            if serv_soft == "ganggo" or serv_soft == "squs" or serv_soft == "dolphin" or serv_soft == "lemmy" or serv_soft == "wordpress":
                try:
                    users = data.json()['usage']['users']['total']
                    soft_version = data.json()['software']['version']
                    if serv_soft == "wordpress" and "activitypub" in data.json()['protocols']:

                        alive = True

                    elif serv_soft == "wordpress" and "activitypub" not in data.json()['protocols']:

                        alive = False

                    else:

                        alive = True

                except:
                    
                    users = 0
                    soft_version = ""

            if alive:
                
                if downs_qty != None:

                    downs = downs_qty

                else:

                    downs = 0

                if soft_version != "" and soft_version is not None:
                    print("Server " + str(server) + " (" + serv_soft + " " + soft_version + ") is alive!")
                else:
                    print("Server " + str(server) + " (" + serv_soft + ") is alive!")

                insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version, first_checked_at, last_checked_at, downs) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"

                conn = None

                try:

                    conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")

                    cur = conn.cursor()

                    cur.execute(insert_sql, (server, users, now, serv_soft, alive, serv_api, soft_version, now, now, downs))

                    if first_checked_at != None:

                        cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (users, now, serv_soft, alive, serv_api, soft_version, now, downs, server))

                    else:

                         cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), first_checked_at=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (users, now, serv_soft, alive, serv_api, soft_version, now, now, downs, server))


                    cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))

                    conn.commit()

                    cur.close()

                except (Exception, psycopg2.DatabaseError) as error:
                    print(error)

                finally:

                    if conn is not None:
                        conn.close()

        except urllib3.exceptions.ProtocolError as protoerr:

            print(protoerr)
            print("Server " + server + " is dead :-(")
            alive = False
            pass

        except requests.exceptions.ChunkedEncodingError as chunkerr:

            print(chunkerr)
            print("Server " + server + " is dead :-(")
            alive = False
            pass

        except KeyError as e:

            print(e)
            print("Server " + server + " is dead :-(")
            alive = False
            pass

        except ValueError as verr:

            print(verr)
            print("Server " + server + " is dead :-(")
            alive = False
            pass

        except requests.exceptions.SSLError as errssl:

            print(errssl)
            print("Server " + server + " is dead :-(")
            alive = False
            pass

        except requests.exceptions.HTTPError as errh:

            print(errh)
            print("Server " + server + " is dead :-(")
            alive = False
            pass

        except requests.exceptions.ConnectionError as errc:

            print(errc)
            print("Server " + server + " is dead :-(")
            alive = False
            pass

        except requests.exceptions.Timeout as errt:

            print(errt)
            print("Server " + server + " is dead :-(")
            alive = False
            pass

        except requests.exceptions.RequestException as err:

            print(err)
            print("Server " + server + " is dead :-(")
            alive = False
            pass

        except socket.gaierror as gai_error:

            print(gai_error)
            pass

        if alive is False:

            if downs_qty != None:

                downs = downs_qty + 1

            else:

                downs = 1

            conn = None

            try:

                conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="",
                                        host="/var/run/postgresql", port="5432")

                cur = conn.cursor()

                if first_checked_at != None:

                    cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), first_checked_at=(%s), downs=(%s) where server=(%s)", (now, alive, NULL, downs, server))

                else:

                    cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), downs=(%s) where server=(%s)", (now, alive, downs, server))

                cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))

                conn.commit()

                cur.close()

            except (Exception, psycopg2.DatabaseError) as error:
                print(error)

            finally:

                if conn is not None:
                    conn.close()

def write_api(server, software, users, alive, api, soft_version):

    insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
    conn = None

    try:

        conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")

        cur = conn.cursor()

        cur.execute(insert_sql, (server, now, software, users, alive, api, soft_version))

        cur.execute("UPDATE fediverse SET updated_at=(%s), software=(%s), users=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)", (now, software, users, alive, api, soft_version, server))

        cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))

        conn.commit()

        cur.close()

    except (Exception, psycopg2.DatabaseError) as error:

        print(error)

    finally:

        if conn is not None:

            conn.close()

async def getsoft(server):

    try:

        socket.gethostbyname(server)

    except socket.gaierror:

        pass
        return

    soft = ''

    url = 'https://' + server

    timeout = aiohttp.ClientTimeout(total=3)
    async with aiohttp.ClientSession(timeout=timeout) as session:
        for api in apis:
            try:
                async with session.get(url+api) as response:
                    if response.status == 200:
                        try:
                            response_json = await response.json()
                        except:
                            pass
            except aiohttp.ClientConnectorError as err:
                print(err)
                pass
            else:
                if response.status == 200 and api != '/api/v1/instance?':
                    try:
                        soft = response_json['software']['name']
                        soft = soft.lower()
                        soft_version = response_json['software']['version']
                        users = response_json['usage']['users']['total']
                        if users > 1000000:
                            return
                        alive = True
                        write_api(server, soft, users, alive, api, soft_version)
                        print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
                        return
                    except:
                        pass
                if response.status == 200 and soft == '' and api == "/api/v1/instance?":
                    soft = 'mastodon'
                    users = response_json['stats']['user_count']
                    soft_version = response_json['version']
                    if users > 1000000:
                        return
                    alive = True
                    write_api(server, soft, users, alive, api)
                    print("Server " + server + " (" + soft + ") is alive!")

def getserver(server, x):

    server = server[0].rstrip('.').lower()

    if server.find(".") == -1:
        return
    if server.find("@") != -1:
        return
    if server.find("/") != -1:
        return
    if server.find(":") != -1:
        return

    try:

        loop = asyncio.get_event_loop()
        coroutines = [getsoft(server)]
        soft = loop.run_until_complete(asyncio.gather(*coroutines, return_exceptions=True))

    except:

        pass

def set_world_servers_check_to_false():

    ############################################################################
    # set all world servers's checked column to False

    try:

        conn = None

        conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")

        cur = conn.cursor()

        cur.execute("UPDATE world SET checked='f'")

        conn.commit()

        cur.close()

    except (Exception, psycopg2.DatabaseError) as error:

        print(error)

    finally:

        if conn is not None:

            conn.close()

def get_last_checked_servers():

    ############################################################################
    # get last checked servers from fediverse DB

    alive_servers = []

    try:

        conn = None
        conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")

        cur = conn.cursor()

        # get world servers list

        cur.execute("select server from world where server in (select server from fediverse where users_api != '')")

        alive_servers = []

        for row in cur:

            alive_servers.append(row[0])

        cur.close()

    except (Exception, psycopg2.DatabaseError) as error:

        print(error)

    finally:

        if conn is not None:

            conn.close()

    return alive_servers

def mastodon():

    # Load secrets from secrets file
    secrets_filepath = "secrets/secrets.txt"
    uc_client_id = get_parameter("uc_client_id",     secrets_filepath)
    uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
    uc_access_token = get_parameter("uc_access_token",  secrets_filepath)

    # Load configuration from config file
    config_filepath = "config/config.txt"
    mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)

    # Initialise Mastodon API
    mastodon = Mastodon(
        client_id=uc_client_id,
        client_secret=uc_client_secret,
        access_token=uc_access_token,
        api_base_url='https://' + mastodon_hostname,
    )

    # Initialise access headers
    headers = {'Authorization': 'Bearer %s'%uc_access_token}

    return (mastodon, mastodon_hostname)

def db_config():

    # Load db configuration from config file
    config_filepath = "config/db_config.txt"
    fediverse_db = get_parameter("fediverse_db", config_filepath)
    fediverse_db_user = get_parameter("fediverse_db_user", config_filepath)

    return (fediverse_db, fediverse_db_user)

def usage():

    print('usage: python ' + sys.argv[0] + ' --multi' + ' (multiprocessing, fast)')
    print('usage: python ' + sys.argv[0] + ' --mono' + ' (one process, slow)')

# Returns the parameter from the specified file
def get_parameter(parameter, file_path):

    # Check if secrets file exists
    if not os.path.isfile(file_path):
        print("File %s not found, exiting."%file_path)
        sys.exit(0)

    # Find parameter in file
    with open(file_path) as f:
        for line in f:
            if line.startswith(parameter):
                return line.replace(parameter + ":", "").strip()

    # Cannot find parameter, exit
    print(file_path + "  Missing parameter %s "%parameter)
    sys.exit(0)

###############################################################################
# main

if __name__ == '__main__':

    # usage modes

    if len(sys.argv) == 1:

        usage()

    elif len(sys.argv) == 2:

        if sys.argv[1] == '--multi':
    
            now = datetime.now()

            mastodon, mastodon_hostname = mastodon()

            fediverse_db, fediverse_db_user = db_config()

            total_servers = 0
            total_users = 0

            set_world_servers_check_to_false()

            alive_servers = get_last_checked_servers()

            ###########################################################################
            # multiprocessing!

            nprocs = multiprocessing.cpu_count()
            with multiprocessing.Pool(processes=64) as pool:
                results = pool.starmap(get_alive_servers, product(alive_servers))

        elif sys.argv[1] == '--mono':

            now = datetime.now()

            mastodon, mastodon_hostname = mastodon()

            fediverse_db, fediverse_db_user = db_config()

            total_servers = 0
            total_users = 0

            set_world_servers_check_to_false()

            alive_servers = get_last_checked_servers()

            i = 0

            while i < len(alive_servers):

                get_alive_servers(alive_servers[i])

                i += 1

        else:

            usage()

        ###########################################################################
        # get current total servers and users, get users from every software

        gettotals_sql = "select count(server), sum(users) from fediverse where alive"
        get_soft_totals_sql = "select software, sum(users) as users, count(server) as servers from fediverse where users != 0 and alive group by software order by users desc"
        soft_total_project = []
        soft_total_users = []
        soft_total_servers = []

        try:

            conn = None
            conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")

            cur = conn.cursor()

            cur.execute(gettotals_sql)

            row = cur.fetchone()

            total_servers = row[0]
            total_users = row[1]

            cur.execute(get_soft_totals_sql)
            rows = cur.fetchall()
            for row in rows:
                soft_total_project.append(row[0])
                soft_total_users.append(row[1])
                soft_total_servers.append(row[2])
            cur.close()

        except (Exception, psycopg2.DatabaseError) as error:

            print(error)

        finally:

            if conn is not None:

                conn.close()

        ###########################################################################
        #  get last check values and write current total ones

        select_sql = "select total_servers, total_users from totals order by datetime desc limit 1"
        insert_sql = "INSERT INTO totals(datetime, total_servers, total_users) VALUES(%s,%s,%s)"

        try:

            conn = None
            conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")

            cur = conn.cursor()

            cur.execute(select_sql)

            row = cur.fetchone()

            if row is not None:

                servers_before = row[0]
                users_before = row[1]

            else:

                servers_before = 0
                users_before = 0

            cur.execute(insert_sql, (now, total_servers, total_users))

            conn.commit()

            cur.close()

            evo_servers = total_servers - servers_before
            evo_users = total_users - users_before

        except (Exception, psycopg2.DatabaseError) as error:

            print(error)

        finally:

            if conn is not None:

                conn.close()

        ################################################################################
        #  write evo values

        insert_sql = "INSERT INTO evo(datetime, servers, users) VALUES(%s,%s,%s)"

        conn = None

        try:

            conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")

            cur = conn.cursor()

            cur.execute(insert_sql, (now, evo_servers, evo_users))

            conn.commit()

            cur.close()

        except (Exception, psycopg2.DatabaseError) as error:

            print(error)

        finally:

            if conn is not None:

                conn.close()

        ##############################################################################
        # get world's last update datetime

        conn = None

        try:

            conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")

            cur = conn.cursor()

            cur.execute("select updated_at from world order by updated_at desc limit 1")

            row = cur.fetchone()

            last_update = row[0]
            last_update = last_update.strftime('%m/%d/%Y, %H:%M:%S')

            cur.close()

        except (Exception, psycopg2.DatabaseError) as error:

            print(error)

        finally:

            if conn is not None:

                conn.close()

        ##############################################################################
        # get max servers and users

        conn = None

        try:

            conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")

            cur = conn.cursor()

            cur.execute("select MAX(total_servers) from totals")

            row = cur.fetchone()

            if row is not None:

                max_servers = row[0]

            else:

                max_servers = 0

            cur.execute("select MAX(total_users) from totals")

            row = cur.fetchone()

            if row is not None:

                max_users = row[0]

            else:

                max_users = 0

            cur.close()

        except (Exception, psycopg2.DatabaseError) as error:

            print(error)

        finally:

            if conn is not None:

                conn.close()

        ###############################################################################
        # get plots

        servers_plots = []

        users_plots = []

        conn = None

        try:

            conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")

            cur = conn.cursor()

            cur.execute("select total_servers, total_users from totals order by datetime desc limit 14")

            rows = cur.fetchall()

            for row in rows:

                servers_plots.append(row[0])

                users_plots.append(row[1])

            cur.close()

        except (Exception, psycopg2.DatabaseError) as error:

            print(error)

        finally:

            if conn is not None:

                conn.close()

        ###############################################################################
        # generate graphs

        plt.plot([-6, -5, -4, -3, -2, -1, 0], [servers_plots[6], servers_plots[5], servers_plots[4], servers_plots[3], servers_plots[2], servers_plots[1], servers_plots[0]], marker='o', color='mediumseagreen')

        plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_servers, max_servers, max_servers, max_servers, max_servers, max_servers, max_servers], color='red')

        plt.title('fediverse: total alive servers (max: ' + str(f"{max_servers:,}" + ')'), loc='right', color='blue')

        plt.xlabel('Last seven days')

        plt.ylabel('fediverse alive servers')

        plt.legend(('servers', 'max'), shadow=True, loc=(0.01, 1.00), handlelength=1.5, fontsize=10)

        plt.savefig('servers.png')

        plt.close()

        plt.plot([-6, -5, -4, -3, -2, -1, 0], [users_plots[6], users_plots[5], users_plots[4], users_plots[3], users_plots[2], users_plots[1], users_plots[0]], marker='o', color='royalblue')

        plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_users, max_users, max_users, max_users, max_users, max_users, max_users], color='red')

        plt.title('fediverse: total registered users (max: ' + str(f"{max_users:,}" + ')'), loc='right', color='royalblue')

        plt.legend(('users', 'max'), shadow=True, loc=(0.01, 0.80), handlelength=1.5, fontsize=10)

        plt.xlabel('Last seven days')

        plt.ylabel('Registered users')

        plt.savefig('users.png')

        plt.close()

        ###############################################################################
        # T  O  O  T !

        toot_text = "#fediverse alive servers stats" + " \n"
        toot_text += "\n"
        if evo_servers >= 0:
            toot_text += "alive servers: " + str(f"{total_servers:,}") + " (+"+ str(f"{evo_servers:,}") + ") \n"
            toot_text += "max: " + str(f"{max_servers:,}") + "\n"
        elif evo_servers < 0:
            toot_text += "alive servers: " + str(f"{total_servers:,}") + " ("+ str(f"{evo_servers:,}") + ") \n"
            toot_text += "max: " + str(f"{max_servers:,}") + "\n"
        if evo_users >= 0:
            toot_text += "total users: " + str(f"{total_users:,}") + " (+"+ str(f"{evo_users:,}") + ") \n"
            toot_text += "max: " + str(f"{max_users:,}") + "\n"
        elif evo_users < 0:
            toot_text += "total users: " + str(f"{total_users:,}") + " ("+ str(f"{evo_users:,}") + ") \n"
            toot_text += "max: " + str(f"{max_users:,}") + "\n"

        toot_text += "\n"
        toot_text += "top ten (soft users servers):" + " \n"
        toot_text += "\n"

        i = 0
        while i < 10:

            project_soft = soft_total_project[i]
            project_users = soft_total_users[i]
            project_servers = soft_total_servers[i]
            len_pr_soft = len(project_soft)

            if project_soft == 'writefreely':

                str_len = 11

            else:

                str_len = 13

            toot_text += f"{':'+project_soft+':':<11}" + f"{project_users:>{str_len},}" + " " + f"{project_servers:>5,}" + "\n"

            i += 1

        print("Tooting...")
        print(toot_text)

        servers_image_id = mastodon.media_post('servers.png', "image/png", description='servers graph').id

        users_image_id = mastodon.media_post('users.png', "image/png", description='users graph').id

        mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, users_image_id})