fediverse/fediverse.py

import sys
import os
import time
from datetime import datetime
import urllib3
import requests
import socket
from setup import Setup
from database import Database
from mastodon import Mastodon
from matplotlib import pyplot as plt
import matplotlib.dates as mdates
from matplotlib.ticker import ScalarFormatter
import numpy as np
import pandas as pd
import ray
import pdb

SMALL_SIZE = 6
MEDIUM_SIZE = 10
BIGGER_SIZE = 12

plt.rc('font', size=MEDIUM_SIZE)          # controls default text sizes
plt.rc('axes', titlesize=MEDIUM_SIZE)     # fontsize of the axes title
plt.rc('axes', labelsize=MEDIUM_SIZE)    # fontsize of the x and y labels
plt.rc('xtick', labelsize=SMALL_SIZE)    # fontsize of the tick labels
plt.rc('ytick', labelsize=MEDIUM_SIZE)    # fontsize of the tick labels
plt.rc('legend', fontsize=MEDIUM_SIZE)    # legend fontsize
plt.rc('figure', titlesize=BIGGER_SIZE)  # fontsize of the figure title

mdates.set_epoch('2000-01-01T00:00:00')
y_formatter = ScalarFormatter(useOffset=False)

ray.init(num_cpus = 25) # Specify this system CPUs.

class Server:

    name = 'Server'

    def __init_(self, server=None):

        self.server = server

    @ray.remote
    def get_alive_servers(self):

        users = 0

        downs = 0

        was_alive, software, api, soft_version, first_checked_at, downs_qty = db.get_server_data(self)

        alive = False

        try:

            data = requests.get('https://' + self + api, headers = setup.user_agent, timeout=3)

            nodeinfo_json = data.json()

            try:

                users = nodeinfo_json.get('usage').get('users').get('total') or '0'
                if users > 1000000:

                    users = 1

                mau = nodeinfo_json.get('usage').get('users').get('activeMonth') or '0'

                if software == 'socialhome':

                    soft_version = nodeinfo_json['server']['version']

                else:

                    soft_version = nodeinfo_json['software']['version']

                if software == "wordpress" and "activitypub" in nodeinfo_json['protocols']:

                    alive = True

                elif software == "wordpress" and "activitypub" not in nodeinfo_json['protocols']:

                    alive = False

                else:

                    alive = True

            except:

                soft_version = ""

            else: 

                if api == '/api/v1/instance?':

                    try:
                        users = nodeinfo_json.get('stats').get('user_count') or '0'
                        if users > 1000000:

                            users = 1

                        soft_version = nodeinfo_json['version']
                        alive = True
                        mau = 0
                    except:
                        soft_version = ""

            if alive:

                if downs_qty != None:

                    downs = downs_qty

                if soft_version != "" and soft_version is not None:

                    print(f'\n** Server {self} ({software} {soft_version}) is alive! **')

                else:

                    print(f'\n** Server {self} ({software}) is alive! **')

                if software != 'birdsitelive':

                    db.write_alive_server(self, software, soft_version, alive, api, users, downs, first_checked_at, mau)

                else:

                    db.write_blocked_software(self, software, soft_version, alive, api, users, downs, first_checked_at)

        except urllib3.exceptions.ProtocolError as protoerr:

            print_dead(self)
            pass

        except requests.exceptions.ChunkedEncodingError as chunkerr:

            print_dead(self)
            pass

        except KeyError as e:

            print_dead(self)
            pass

        except ValueError as verr:

            print_dead(self)
            pass

        except requests.exceptions.SSLError as errssl:

            print_dead(self)
            pass

        except requests.exceptions.HTTPError as errh:

            print_dead(self)
            pass

        except requests.exceptions.ConnectionError as errc:

            print_dead(self)
            pass

        except requests.exceptions.Timeout as errt:

            print_dead(self)
            pass

        except requests.exceptions.RequestException as err:

            print_dead(self)
            pass

        except socket.gaierror as gai_error:

            print_dead(self)
            pass

        if not alive:

            mau = 0

            if downs_qty != None:

                downs = downs_qty + 1

            else:

                downs = 1

            db.write_not_alive_server(self, software, soft_version, alive, api, users, downs, first_checked_at)

        return (self, software, soft_version, alive, api, users, downs, first_checked_at, mau)

def print_dead(server):

    print(f'\nServer {server} is dead :-(')

if __name__ == '__main__':

    db = Database()

    setup = Setup()

    start = datetime.now()

    program = 'fediverse'

    finish = start

    db.save_time(program, start, finish)

    now = start

    mastodon = Mastodon(
            access_token = setup.mastodon_app_token,
            api_base_url= setup.mastodon_hostname
            )

    total_servers = 0

    total_users = 0

    alive_servers = db.get_last_checked_servers()

    getservers = Server()

    getservers.now = now

    ray_start = time.time()

    results = ray.get([getservers.get_alive_servers.remote(server) for server in alive_servers])

    print(f"duration = {time.time() - ray_start}.\nprocessed servers: {len(results)}")

    # get current total servers and users, get users from every software

    soft_total_project, soft_total_users, soft_total_mau, soft_total_servers, total_servers, total_users, total_mau = db.soft_totals()

    # get last check values and write current total ones

    evo_servers, evo_users, evo_mau = db.last_values(total_servers, total_users, total_mau)

    # write evo values

    db.write_evo(evo_servers, evo_users, evo_mau)

    # get world's last update datetime

    last_update = db.last_world_datetime()

    # get max servers and mau

    max_servers, max_mau = db.max()

    # get plots

    servers_plots, mau_plots, global_week, global_servers, global_users, global_mau = db.get_plots()

    ###############################################################################
    # generate graphs

    plt.plot([-6, -5, -4, -3, -2, -1, 0], [servers_plots[6], servers_plots[5], servers_plots[4], servers_plots[3], servers_plots[2], servers_plots[1], servers_plots[0]], marker='o', color='mediumseagreen')

    plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_servers, max_servers, max_servers, max_servers, max_servers, max_servers, max_servers], color='red')

    plt.title('fediverse: total alive servers (max: ' + str(f"{max_servers:,}" + ')'), loc='right', color='blue')

    plt.xlabel('Last seven days')

    plt.ylabel('fediverse alive servers')

    plt.legend(('servers', 'max'), shadow=True, loc=(0.01, 1.00), handlelength=1.5, fontsize=10)

    plt.savefig('servers.png')

    plt.close()

    plt.plot([-6, -5, -4, -3, -2, -1, 0], [mau_plots[6], mau_plots[5], mau_plots[4], mau_plots[3], mau_plots[2], mau_plots[1], mau_plots[0]], marker='o', color='royalblue')

    plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_mau, max_mau, max_mau, max_mau, max_mau, max_mau, max_mau], color='red')

    plt.title('fediverse: total MAU (max: ' + str(f"{max_mau:,}" + ')'), loc='right', color='royalblue')

    plt.legend(('mau', 'max'), shadow=True, loc=(0.01, 0.80), handlelength=1.5, fontsize=10)

    plt.xlabel('Last seven days')

    plt.ylabel('MAU')

    plt.savefig('mau.png')

    plt.close()

    df = pd.DataFrame({'date': np.array(global_week),
        #'servers': np.array(global_servers),
        'users': np.array(global_users),
        'mau': np.array(global_mau)})

    df['date'] = pd.to_datetime(df['date'])

    fig, ax = plt.subplots()

    ax.plot(df.date, df.users, label='Registered', color='orange')

    ax.plot(df.date, df.mau, label='MAU', color='blue')

    plt.tick_params(rotation=45)

    ax.set_title("fediverse's registered and Monthly Active Users")

    ax.set_xlabel('weeks')

    ax.set_ylabel('users')

    ax.grid(visible=True)

    ax.legend(title='Users')

    ax.yaxis.set_major_formatter(y_formatter)

    plt.savefig('global.png')

    plt.close()

    ###############################################################################
    # P O S T !

    toot_text = "#fediverse alive servers stats" + " \n"

    toot_text += "\n"

    if evo_servers >= 0:

        toot_text += "alive servers: " + str(f"{total_servers:,}") + " (+"+ str(f"{evo_servers:,}") + ") \n"

        toot_text += "max: " + str(f"{max_servers:,}") + "\n"

    elif evo_servers < 0:

        toot_text += "alive servers: " + str(f"{total_servers:,}") + " ("+ str(f"{evo_servers:,}") + ") \n"

        toot_text += "max: " + str(f"{max_servers:,}") + "\n"

    if evo_mau >= 0:

        toot_text += "total MAU: " + str(f"{total_mau:,}") + " (+"+ str(f"{evo_mau:,}") + ") \n"

        toot_text += "max: " + str(f"{max_mau:,}") + "\n"

    elif evo_mau < 0:

        toot_text += "total MAU: " + str(f"{total_mau:,}") + " ("+ str(f"{evo_mau:,}") + ") \n"

        toot_text += "max: " + str(f"{max_mau:,}") + "\n"

    toot_text += "\ntop ten (MAU / servers):\n\n"

    i = 0

    while i < 10:

        project_soft = soft_total_project[i]

        project_mau = soft_total_mau[i]

        project_servers = soft_total_servers[i]

        len_pr_soft = len(project_soft)

        if project_soft == 'ativity-relay':

            project_soft = 'activityrelay'

        toot_text += f":{project_soft}: {project_mau:,} / {project_servers:,}\n"

        i += 1

    print("Tooting...")

    print(toot_text)

    servers_image_id = mastodon.media_post('servers.png', "image/png", description='servers graph').id

    mau_image_id = mastodon.media_post('mau.png', "image/png", description='MAU graph').id

    global_image_id = mastodon.media_post('global.png', "image/png", description='global graph').id

    mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, mau_image_id, global_image_id})

    db.delete_dead_servers()

    finish = datetime.now()

    db.save_time(program, start, finish)