fediverse/fetchservers.py

import time
from datetime import datetime
import os
import json
import sys
import os.path
import psycopg2
import requests
import urllib3
import socket
import ray

ray.init(num_cpus = 32) # Specify this system CPUs.

apis = ['/api/v1/instance?',
        '/nodeinfo/2.0?',
        '/nodeinfo/2.0.json?',
        '/main/nodeinfo/2.0?',
        '/api/statusnet/config?',
        '/api/nodeinfo/2.0.json?',
        '/api/nodeinfo?',
        '/wp-json/nodeinfo/2.0?',
        '/api/v1/instance/nodeinfo/2.0?',
        '/.well-known/x-nodeinfo2?'
        ]

def is_json(myjson):

    try:
        json_object = json.loads(myjson)
    except ValueError as e:
        return False
    return True

def write_api(server, software, users, alive, api, soft_version):

    fediverse_db, fediverse_db_user = get_db_config()

    insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"

    conn = None

    try:

        conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")

        cur = conn.cursor()

        print(f'Writing {server} nodeinfo data...')

        cur.execute(insert_sql, (server, now, software, users, alive, api, soft_version))

        cur.execute(
            "UPDATE fediverse SET updated_at=(%s), software=(%s), users=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)",
            (now, software, users, alive, api, soft_version, server)
            )

        cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))

        conn.commit()

        cur.close()

    except (Exception, psycopg2.DatabaseError) as error:

        print(error)

    finally:

        if conn is not None:

            conn.close()

@ray.remote
def getsoft(server):

    if server.find(".") == -1:
        return
    if server.find("@") != -1:
        return
    if server.find("/") != -1:
        return
    if server.find(":") != -1:
        return

    soft = ''

    is_nodeinfo = False

    url = 'https://' + server

    user_agent = {'User-agent': "fediverse's stats (fediverse@mastodont.cat)"}

    try:

        response = requests.get(url + '/.well-known/nodeinfo', headers = user_agent, timeout=3)

        if response.status_code == 200:

            try:

                response_json = response.json()

                nodeinfo = response_json['links'][0]['href'].replace(f'https://{server}','')

                try:

                    nodeinfo_data = requests.get(url + nodeinfo, headers = user_agent, timeout=3)

                    if nodeinfo_data.status_code == 200:

                        nodeinfo_json = nodeinfo_data.json()

                        is_nodeinfo = True

                    else:

                        print(f"Server {server}'s nodeinfo not responding: error code {nodeinfo_data.status_code}")

                except:

                    pass

            except:

                print(f'Server {server} not responding: error code {response.status_code}')
                print('*********************************************************************')

                pass
        else:

            for api in apis:

                try:

                    response = requests.get(url + api, headers = user_agent, timeout=3)

                    if is_json(response.text):

                        nodeinfo_json = response.json()

                        if 'software' in nodeinfo_json:

                            nodeinfo = api

                            is_nodeinfo = True

                            break

                        elif 'title' in nodeinfo_json:

                            if nodeinfo_json['title'] == 'Zap':

                                nodeinfo = api

                                is_nodeinfo = True

                                soft = 'zap'

                                break

                        elif 'version' in nodeinfo_json:

                            nodeinfo = api

                            is_nodeinfo = True

                            break

                except:

                    pass

    except requests.exceptions.SSLError as errssl:

        pass

    except requests.exceptions.HTTPError as errh:

        pass

    except requests.exceptions.ConnectionError as errc:

        pass

    except requests.exceptions.ReadTimeout as to_err:

        pass

    except requests.exceptions.TooManyRedirects as tmr_err:

        pass

    except urllib3.exceptions.LocationParseError as lp_err:

        pass

    except requests.exceptions.InvalidURL as iu_err:

        pass

    else:

        if is_nodeinfo:

            if nodeinfo != '/api/v1/instance?':

                if nodeinfo != '/.well-known/x-nodeinfo2?':

                    try:

                        soft = nodeinfo_json['software']['name']
                        soft = soft.lower()
                        soft_version = nodeinfo_json['software']['version']
                        users = nodeinfo_json['usage']['users']['total']
                        if users > 1000000:
                            return
                        alive = True

                        write_api(server, soft, users, alive, nodeinfo, soft_version)

                        print(f"Server {server} ({soft} {soft_version}) is alive!")
                        print('*********************************************************************')

                        return

                    except:

                        pass

                else:

                    try:

                        soft = nodeinfo_json['server']['software']
                        soft = soft.lower()
                        soft_version = nodeinfo_json['server']['version']
                        users = nodeinfo_json['usage']['users']['total']
                        if users > 1000000:
                            return
                        alive = True

                        if soft == 'socialhome':

                            write_api(server, soft, users, alive, nodeinfo, soft_version)

                            print('*********************************************************************')
                            print(f"Server {serve}r ({soft} {soft_version}) is alive!")
                            print('*********************************************************************')

                        return

                    except:

                        pass

            if soft == '' and nodeinfo == "/api/v1/instance?":

                soft = 'mastodon'

                try:

                    users = nodeinfo_json['stats']['user_count']

                    if users > 1000000:

                        return

                except:

                    users = 0

                try:

                    soft_version = nodeinfo_json['version']

                except:

                    soft_version = 'unknown'

                alive = True

                write_api(server, soft, users, alive, nodeinfo, soft_version)

                print('*********************************************************************')
                print(f"Server {server} ({soft}) is alive!")

            elif soft == 'zap' and nodeinfo == "/api/v1/instance?":

                soft = 'zap'
                users = nodeinfo_json['stats']['user_count']
                soft_version = nodeinfo_json['version']
                alive = True

                print(server, soft, users, alive, api)

                print('*********************************************************************')
                print(f"Server {server} ({soft}) is alive!")

        else:

            print(f'Server {server} is dead')
            print('*********************************************************************')

def get_world_servers():

    world_servers = []

    try:

        conn = None

        conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")

        cur = conn.cursor()

        # get world servers list

        cur.execute("select server from world")# where checked='f'")

        rows = cur.fetchall()

        for row in rows:

            world_servers.append(row[0])

        cur.close()

        print("Remaining servers: " + str(len(world_servers)))

    except (Exception, psycopg2.DatabaseError) as error:

        print(error)

    finally:

        if conn is not None:

            conn.close()

    return world_servers

def save_time(program, start, finish):

    insert_sql = "INSERT INTO execution_time(program, start, finish) VALUES(%s,%s,%s) ON CONFLICT DO NOTHING"

    conn = None

    try:

        conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")

        cur = conn.cursor()

        cur.execute(insert_sql, (program, start, finish,))

        cur.execute("UPDATE execution_time SET start=(%s), finish=(%s) where program=(%s)", (start, finish, program))

        conn.commit()

        cur.close()

    except (Exception, psycopg2.DatabaseError) as error:

        print(error)

    finally:

        if conn is not None:

            conn.close()

def get_parameter(parameter, file_path):

    # Check if secrets file exists
    if not os.path.isfile(file_path):
        print("File %s not found, exiting." % file_path)
        sys.exit(0)

    # Find parameter in file
    with open(file_path) as f:
        for line in f:
            if line.startswith(parameter):
                return line.replace(parameter + ":", "").strip()

    # Cannot find parameter, exit
    print(file_path + "  Missing parameter %s " % parameter)
    sys.exit(0)

def get_config():

    # Load configuration from config file
    config_filepath = "config/config.txt"
    mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
    return mastodon_hostname

def get_db_config():

    # Load database config from db_config file
    db_config_filepath = "config/db_config.txt"
    fediverse_db = get_parameter("fediverse_db", db_config_filepath)
    fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath)
    return (fediverse_db, fediverse_db_user)

###############################################################################
# main

if __name__ == '__main__':

    ## name: fetchservers.py

    fediverse_db, fediverse_db_user = get_db_config()

    start = datetime.now()

    program = 'fetchservers'

    finish = start

    save_time(program, start, finish)

    now = start

    mastodon_hostname = get_config()

    world_servers = get_world_servers()

    ray_start = time.time()

    results = ray.get([getsoft.remote(server) for server in world_servers])

    print(f"duration = {time.time() - ray_start}.\nprocessed servers: {len(results)}")

    finish = datetime.now()

    save_time(program, start, finish)