fediverse/fediverse.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

import time
start_time = time.time()
import urllib3
from urllib3 import exceptions
from datetime import datetime
from subprocess import call
from mastodon import Mastodon
import threading
import os
import json
import signal
import sys
import os.path
import requests
from requests import exceptions
import operator
import calendar
import psycopg2
from itertools import product

from multiprocessing import Pool, Lock, Process, Queue, current_process, Manager
import multiprocessing

import aiohttp
import aiodns
import asyncio
from aiohttp import ClientError, ClientSession, ClientConnectionError, ClientConnectorError, ClientSSLError, ClientConnectorSSLError, ServerTimeoutError
from asyncio import TimeoutError
import socket
from socket import gaierror, gethostbyname

from decimal import *
getcontext().prec = 2

apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?', '/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?']

client_exceptions = (
    aiohttp.ClientResponseError,
    aiohttp.ClientConnectionError,
    aiohttp.ClientConnectorError,
    aiohttp.ClientError,
    asyncio.TimeoutError,
    socket.gaierror,
)

now = datetime.now()

###############################################################################
# INITIALISATION
###############################################################################

def is_json(myjson):
    try:
        json_object = json.loads(myjson)
    except ValueError as e:
        return False
    return True

def alive_server(server, x):

    server = server[0].rstrip('.').lower()

    conn = None

    try:

        conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")

        cur = conn.cursor()

        cur.execute("select alive, software, users_api from fediverse where server=(%s)", (server,))

        row = cur.fetchone()

        if row != None:

            was_alive = row[0]
            serv_soft = row[1]
            serv_api = row[2]

        cur.close()

    except (Exception, psycopg2.DatabaseError) as error:

        print(error)

    finally:

        if conn is not None:
            conn.close()

        alive = False

        try:

            data = requests.get('https://' + server + serv_api, timeout=3)

            if serv_soft == "mastodon":
                if serv_api == '/nodeinfo/2.0?':
                    try:
                        users = data.json()['usage']['users']['total']
                        alive = True
                    except:
                        users = 0
                if serv_api == '/nodeinfo/2.0.json?':
                    try:
                        users = data.json()['usage']['users']['total']
                        alive = True
                    except:
                        users = 0
                elif serv_api == '/api/v1/instance?':
                    try:
                        users = data.json()['stats']['user_count']
                        alive = True
                    except:
                        users = 0
            if serv_soft == "pleroma" or serv_soft == "diaspora" or serv_soft == "peertube" or serv_soft == "pixelfed" or serv_soft == "hubzilla" or serv_soft == "writefreely" or serv_soft == "friendica":
                try:
                    users = data.json()['usage']['users']['total']
                    alive = True
                except:
                    users = 0
            if serv_soft == "gnusocialv2" or serv_soft == "gnusocial":
                try:
                    users = data.json()['usage']['users']['total']
                    if users == 0:
                        users = data.json()['usage']['users']['activeHalfyear']
                    alive = True
                except:
                    users = 0
            if serv_soft == "plume" or serv_soft == 'red' or serv_soft == "misskey" or serv_soft == "zap" or serv_soft == "prismo" or serv_soft == "ravenvale" or serv_soft == "osada" or serv_soft == "groundpolis":
                try:
                    users = data.json()['usage']['users']['total']
                    alive = True
                except:
                    users = 0
            if serv_soft == "ganggo" or serv_soft == "squs" or serv_soft == "dolphin":
                try:
                    users = data.json()['usage']['users']['total']
                    alive = True
                except:
                    users = 0

            print("Server " + str(server) + " (" + serv_soft + ") is alive!")

            insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api) VALUES(%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
            conn = None

            try:

                conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")

                cur = conn.cursor()

                cur.execute(insert_sql, (server, users, now, serv_soft, alive, serv_api))

                cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s) where server=(%s)", (users, now, serv_soft, alive, serv_api, server))

                cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))

                conn.commit()

                cur.close()

            except (Exception, psycopg2.DatabaseError) as error:
                print(error)

            finally:

                if conn is not None:
                    conn.close()

        except urllib3.exceptions.ProtocolError as protoerr:

            print("Server " + server + " is dead :-(")
            alive = False
            pass

        except requests.exceptions.ChunkedEncodingError as chunkerr:

            print("Server " + server + " is dead :-(")
            alive = False
            pass

        except KeyError as e:

            print("Server " + server + " is dead :-(")
            alive = False
            pass

        except ValueError as verr:

            print("Server " + server + " is dead :-(")
            alive = False
            pass

        except requests.exceptions.SSLError as errssl:

            print("Server " + server + " is dead :-(")
            alive = False
            pass

        except requests.exceptions.HTTPError as errh:

            print("Server " + server + " is dead :-(")
            alive = False
            pass

        except requests.exceptions.ConnectionError as errc:

            print("Server " + server + " is dead :-(")
            alive = False
            pass

        except requests.exceptions.Timeout as errt:

            print("Server " + server + " is dead :-(")
            alive = False
            pass

        except requests.exceptions.RequestException as err:

            print("Server " + server + " is dead :-(")
            alive = False
            pass

        if alive == False:

            conn = None

            try:

                conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="",
                                        host="/var/run/postgresql", port="5432")

                cur = conn.cursor()

                cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s) where server=(%s)", (now, alive, server))

                cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))

                conn.commit()

                cur.close()

            except (Exception, psycopg2.DatabaseError) as error:
                print(error)

            finally:

                if conn is not None:
                    conn.close()

def write_api(server, software, users, alive, api):

    insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api) VALUES(%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
    conn = None

    try:

        conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")

        cur = conn.cursor()

        cur.execute(insert_sql, (server, now, software, users, alive, api))

        cur.execute("UPDATE fediverse SET updated_at=(%s), software=(%s), users=(%s), alive=(%s), users_api=(%s) where server=(%s)", (now, software, users, alive, api, server))

        cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))

        conn.commit()

        cur.close()

    except (Exception, psycopg2.DatabaseError) as error:

        print(error)

    finally:

        if conn is not None:

            conn.close()

async def getsoft(server):

    try:

        socket.gethostbyname(server)

    except socket.gaierror:

        return

    soft = ''

    url = 'https://' + server

    timeout = aiohttp.ClientTimeout(total=3)
    async with aiohttp.ClientSession(timeout=timeout) as session:
        for api in apis:
            try:
                async with session.get(url+api) as response:
                    if response.status == 200:
                        try:
                            response_json = await response.json()
                        except:
                            pass
            except aiohttp.ClientConnectorError as err:
                pass
            else:
               if response.status == 200 and api != '/api/v1/instance?':
                   try:
                       soft = response_json['software']['name']
                       soft = soft.lower()
                       users = response_json['usage']['users']['total']
                       alive = True
                       write_api(server, soft, users, alive, api)
                       print("Server " + server + " is alive!")
                       return
                   except:
                       pass
               if response.status == 200 and soft == '' and api == "/api/v1/instance?":
                   soft = 'mastodon'
                   users = response_json['stats']['user_count']
                   alive = True
                   write_api(server, soft, users, alive, api)
                   print("Server " + server + " is alive!")

def getserver(server, x):

    server = server[0].rstrip('.').lower()

    if server.find(".") == -1:
        return
    if server.find("@") != -1:
       return
    if server.find("/") != -1:
       return
    if server.find(":") != -1:
       return

    try:

        loop = asyncio.get_event_loop()
        coroutines = [getsoft(server)]
        soft = loop.run_until_complete(asyncio.gather(*coroutines, return_exceptions=True))

    except:

        pass

# Returns the parameter from the specified file
def get_parameter( parameter, file_path ):
    # Check if secrets file exists
    if not os.path.isfile(file_path):
        print("File %s not found, exiting."%file_path)
        sys.exit(0)

    # Find parameter in file
    with open( file_path ) as f:
        for line in f:
            if line.startswith( parameter ):
                return line.replace(parameter + ":", "").strip()

    # Cannot find parameter, exit
    print(file_path + "  Missing parameter %s "%parameter)
    sys.exit(0)

# Load secrets from secrets file
secrets_filepath = "secrets/secrets.txt"
uc_client_id     = get_parameter("uc_client_id",     secrets_filepath)
uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
uc_access_token  = get_parameter("uc_access_token",  secrets_filepath)

# Load configuration from config file
config_filepath = "config/config.txt"
mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)

# Load database config from db_config file
db_config_filepath = "config/db_config.txt"
fediverse_db = get_parameter("fediverse_db", db_config_filepath)
fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath)

# Initialise Mastodon API
mastodon = Mastodon(
    client_id = uc_client_id,
    client_secret = uc_client_secret,
    access_token = uc_access_token,
    api_base_url = 'https://' + mastodon_hostname,
)

# Initialise access headers
headers={ 'Authorization': 'Bearer %s'%uc_access_token }

###############################################################################
# main

if __name__ == '__main__':

    total_servers = 0
    total_users = 0

    ############################################################################
    # set all world servers's checked column to False

    try:

        conn = None

        conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")

        cur = conn.cursor()

        cur.execute("UPDATE world SET checked='f'")

        conn.commit()

        cur.close()

    except (Exception, psycopg2.DatabaseError) as error:

        print(error)

    finally:

        if conn is not None:

            conn.close()

    ############################################################################
    # get last check servers from fediverse DB


    alive_servers = []

    try:

        conn = None
        conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")

        cur = conn.cursor()

        ### get world servers list

        cur.execute("select server from world where server in (select server from fediverse where users_api != '')")

        alive_servers = []

        for row in cur:

            alive_servers.append(row[0])

        cur.close()

    except (Exception, psycopg2.DatabaseError) as error:

        print(error)

    finally:

        if conn is not None:

            conn.close()

    ###########################################################################
    # multiprocessing!

    m = Manager()
    q = m.Queue()
    z = zip(alive_servers)

    serv_number = len(alive_servers)

    pool_tuple = [(x, q) for x in z]
    with Pool(processes=64) as pool:
        pool.starmap(alive_server, pool_tuple)

    ###########################################################################

    print("Getting the remaining servers from world")

    world_servers = []

    try:

        conn = None
        conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")

        cur = conn.cursor()

        ### get world servers list

        cur.execute("select server from world where checked='f'")

        for row in cur:

            world_servers.append(row[0])

        cur.close()

    except (Exception, psycopg2.DatabaseError) as error:

        print(error)

    finally:

        if conn is not None:

            conn.close()

    ###########################################################################
    # multiprocessing!

    m = Manager()
    q = m.Queue()
    z = zip(world_servers)

    serv_number = len(world_servers)

    pool_tuple = [(x, q) for x in z]
    with Pool(processes=64) as pool:
        pool.starmap(getserver, pool_tuple)

    ###########################################################################
    # delete not alive servers from fediverse table

    try:

        conn = None

        conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")

        cur = conn.cursor()

        cur.execute("delete from fediverse where not alive")

        conn.commit()

        cur.close()

    except (Exception, psycopg2.DatabaseError) as error:

        print(error)

    finally:

        if conn is not None:

            conn.close()

    ###########################################################################
    # get current total servers and users, get users from every software

    gettotals_sql = "select count(server), sum(users) from fediverse where alive"
    get_soft_totals_sql = "select software, sum(users) as users, count(server) as servers from fediverse where users != 0 and alive group by software order by users desc"
    soft_total_project = []
    soft_total_users = []
    soft_total_servers = []

    try:

        conn = None
        conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")

        cur = conn.cursor()

        cur.execute(gettotals_sql)

        row = cur.fetchone()

        total_servers = row[0]
        total_users = row[1]

        cur.execute(get_soft_totals_sql)
        rows = cur.fetchall()
        for row in rows:
            soft_total_project.append(row[0])
            soft_total_users.append(row[1])
            soft_total_servers.append(row[2])
        cur.close()

    except (Exception, psycopg2.DatabaseError) as error:

        print(error)

    finally:

        if conn is not None:

            conn.close()

    ###########################################################################
    #  get last check values and write current total ones

    select_sql = "select total_servers, total_users from totals order by datetime desc limit 1"
    insert_sql = "INSERT INTO totals(datetime, total_servers, total_users) VALUES(%s,%s,%s)"

    try:

        conn = None
        conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")

        cur = conn.cursor()

        cur.execute(select_sql)

        row = cur.fetchone()

        if row != None:

            servers_before = row[0]
            users_before = row[1]

        else:

            servers_before = 0
            users_before = 0

        cur.execute(insert_sql, (now, total_servers, total_users))

        conn.commit()

        cur.close()

        evo_servers = total_servers - servers_before
        evo_users = total_users - users_before

    except (Exception, psycopg2.DatabaseError) as error:

        print(error)

    finally:

        if conn is not None:

            conn.close()

    ################################################################################
    #  write evo values

    insert_sql = "INSERT INTO evo(datetime, servers, users) VALUES(%s,%s,%s)"

    conn = None

    try:

        conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")

        cur = conn.cursor()

        cur.execute(insert_sql, (now, evo_servers, evo_users))

        conn.commit()

        cur.close()

    except (Exception, psycopg2.DatabaseError) as error:

        print(error)

    finally:

        if conn is not None:

            conn.close()

    ##############################################################################
    # get world's last update datetime

    conn = None

    try:

        conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")

        cur = conn.cursor()

        cur.execute("select updated_at from world order by updated_at desc limit 1")

        row = cur.fetchone()

        last_update = row[0]
        last_update = last_update.strftime('%m/%d/%Y, %H:%M:%S')

        cur.close()

    except (Exception, psycopg2.DatabaseError) as error:

        print(error)

    finally:

        if conn is not None:

            conn.close()

    ###############################################################################
    # T  O  O  T !

    toot_text = "#fediverse alive servers stats" + " \n"
    toot_text += "\n"
    if evo_servers >= 0:
        toot_text += "alive servers: " + str(total_servers) + " (+"+ str(evo_servers) + ") \n"
    elif evo_servers < 0:
        toot_text += "alive servers: " + str(total_servers) + " ("+ str(evo_servers) + ") \n"
    if evo_users >= 0:
        toot_text += "total users: " + str(total_users) + " (+"+ str(evo_users) + ") \n"
    elif evo_users < 0:
        toot_text += "total users: " + str(total_users) + " ("+ str(evo_users) + ") \n"
    toot_text += "\n"
    toot_text += "top five (soft: users servers):"  + " \n"
    toot_text += "\n"
    toot_text += '{:>0}:  {:>5}   {:>5}'.format(str(soft_total_project[0]), str(soft_total_users[0]), str(soft_total_servers[0])) + " \n"
    toot_text += '{:>0}:  {:>11}  {:>8}'.format(str(soft_total_project[1]), str(soft_total_users[1]), str(soft_total_servers[1])) + " \n"
    toot_text += '{:>0}:  {:>12}  {:>7}'.format(str(soft_total_project[2]), str(soft_total_users[2]), str(soft_total_servers[2])) + " \n"
    toot_text += '{:>0}:  {:>13}  {:>7}'.format(str(soft_total_project[3]), str(soft_total_users[3]), str(soft_total_servers[3])) + " \n"
    toot_text += '{:>0}:  {:>14}  {:>7}'.format(str(soft_total_project[4]), str(soft_total_users[4]), str(soft_total_servers[4])) + " \n"
    toot_text += "\n"
    toot_text += "updated at " + str(last_update) + " \n"
    print("Tooting...")
    print(toot_text)

    mastodon.status_post(toot_text, in_reply_to_id=None, )