parametrize db connection, with tests

.gitignore

@@ -0,0 +1,75 @@
+secrets/secrets.txt
+config/config.txt
+config/db_config.txt
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+.noseids
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Ipython Notebook
+.ipynb_checkpoints
+
+# vim swap files
+.*sw?
+
+# IDEs
+.vscode/
+*.code-workspace
+

README.md

@@ -1,5 +1,5 @@
 # Fediverse Stats
-This code gets all peers from running Mastodon, Pleroma and Lemmy host servers and then all peers from host server's peers. Goal is to collect maximum number
+This code gets all peers from mastodon.social. Goal is to collect maximum number
 of alive fediverse's servers and then query their API to obtain their registered users (if their API provide such information).
 At the end it post the results to host server bot account.
 
@@ -7,7 +7,7 @@ At the end it post the results to host server bot account.
 
 -   **Python 3**
 -   Postgresql server
--   Mastodon or Pleroma running server.
+-   Mastodon running server.
 
 ### Usage:
 
@@ -15,19 +15,11 @@ Within Python Virtual Environment:
 
 1. Run `pip install -r requirements.txt` to install needed libraries.
 
-2. Run `python db-setup.py` to setup and create new Postgresql database and needed tables in it.
+2. Run `python fetchservers.py` to add servers to alive servers database.
 
-3. Run `python setup.py` to get your bot's access token of your Mastodon or Pleroma server existing account. It will be saved to 'secrets/secrets.txt' for further use.
+3. Run `python fediverse.py` to query world alive servers API. It gets data from server's nodeinfo.
 
-4. Run `python getpeers.py` to get all peers from your host and the whole world of fediverse's servers (or almost the whole world).
-
-5. Run `python fetchservers.py` to add servers to alive servers database.
-
-6. Run `python fediverse.py` to query world alive servers API. It gets data from server's nodeinfo.
-
-7. Run `python uptime_setup.py` to get your Uptime bot's access token of your Mastodon or Pleroma server existing account. It will be saved to 'secrets/uptime_secrets.txt' for further use.
-
-8. Use your favourite scheduling method to set `python fediverse.py` to run twice daily, `python fetchservers.py` one time daily, `python getworld.py` to run monthly and `python uptime.py` (choose your desired frequency) if you want to publish best fediverse's servers uptime.   
+4. Use your favourite scheduling method to set `python fediverse.py` to run twice daily, `python fetchservers.py` one time daily, `python fediquery.py` to run every minute and `python uptime.py' every minute to publish best fediverse uptime.
 
 18.2.2021 - New feature! Added [Lemmy project](https://join.lemmy.ml)  
 12.5.2021 - New feature! Added Wordpress support. The code can now detect Wordpress instances with ActivityPub enabled plugin.  
@@ -35,4 +27,9 @@ Within Python Virtual Environment:
 21.8.2021 - New feature! Added Best Fediverse's servers Uptime publishing bot.  
 22.10.2021 - New feature! Added [Funkwhale](https://funkwhale.audio) support.  
 26.10.2021 - New feature! Added [Socialhome](https://socialhome.network) support.  
-2.3.2022 - Improved server nodeinfo detection
+2.3.2022 - Improved server nodeinfo detection.  
+4.1.2023 - Refactored.  
+4.1.2023 - Now peers are obtained from mastodon.social's peers list.  
+5.1.2023 - Added fediquery.py. Allow ask the main bot fediverse about `soft` and `server` and it replies to the asking user with its data if any.  
+7.1.2023 - When querying a not found server, we will be added to database if it's alive.  
+8.1.2023 - Save MAU to database.

config/test_db_config.txt

@@ -0,0 +1,3 @@
+fediverse_db: fediverse_test
+fediverse_db_user: tester
+fediverse_db_user_password: tester

db-setup.py

@@ -1,173 +0,0 @@
-import getpass
-import os
-import sys
-from mastodon import Mastodon
-from mastodon.Mastodon import MastodonMalformedEventError, MastodonNetworkError, MastodonReadTimeout, MastodonAPIError
-import psycopg2
-from psycopg2 import sql
-from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
-
-def get_parameter( parameter, file_path ):
-    # Check if secrets file exists
-    if not os.path.isfile(file_path):
-        print("File %s not found, asking."%file_path)
-        write_parameter( parameter, file_path )
-        #sys.exit(0)
-
-    # Find parameter in file
-    with open( file_path ) as f:
-        for line in f:
-            if line.startswith( parameter ):
-                return line.replace(parameter + ":", "").strip()
-
-    # Cannot find parameter, exit
-    print(file_path + "  Missing parameter %s "%parameter)
-    sys.exit(0)
-
-def write_parameter( parameter, file_path ):
-    
-    if not os.path.exists('config'):
-        os.makedirs('config')
-    print("Setting up fediverse parameters...")
-    print("\n")
-    fediverse_db = input("fediverse db name: ")
-    fediverse_db_user = input("fediverse db user: ")
-    mastodon_db = input("Mastodon db name: ")
-    mastodon_db_user = input("Mastodon db user: ")
-  
-    with open(file_path, "w") as text_file:
-
-        print("fediverse_db: {}".format(fediverse_db), file=text_file)
-        print("fediverse_db_user: {}".format(fediverse_db_user), file=text_file)
-        print("mastodon_db: {}".format(mastodon_db), file=text_file)
-        print("mastodon_db_user: {}".format(mastodon_db_user), file=text_file)
-
-def create_table(db, db_user, table, sql):
-
-    conn = None
-
-    try:
-
-        conn = psycopg2.connect(database = db, user = db_user, password = "", host = "/var/run/postgresql", port = "5432")
-        cur = conn.cursor()
-
-        print("Creating table.. "+table)
-        # Create the table in PostgreSQL database
-        cur.execute(sql)
-
-        conn.commit()
-
-        print("Table "+table+" created!")
-        print("\n")
-
-    except (Exception, psycopg2.DatabaseError) as error:
-
-        print(error)
-
-    finally:
-
-        if conn is not None:
-
-            conn.close()
-
-###############################################################################
-# main
-
-if __name__ == '__main__':
-
-    # Load configuration from config file
-    config_filepath = "config/db_config.txt"
-    fediverse_db = get_parameter("fediverse_db", config_filepath)
-    fediverse_db_user = get_parameter("fediverse_db_user", config_filepath)
-    mastodon_db = get_parameter("mastodon_db", config_filepath)
-    mastodon_db_user = get_parameter("mastodon_db_user", config_filepath)
-
-    ############################################################
-    # create database
-    ############################################################
-
-    conn = None
-
-    try:
-
-        conn = psycopg2.connect(dbname='postgres',
-            user=fediverse_db_user, host='',
-            password='')
-
-        conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
-
-        cur = conn.cursor()
-
-        print("Creating database " + fediverse_db + ". Please wait...")
-
-        cur.execute(sql.SQL("CREATE DATABASE {}").format(
-              sql.Identifier(fediverse_db))
-            )
-        print("Database " + fediverse_db + " created!")
-
-    except (Exception, psycopg2.DatabaseError) as error:
-
-        print(error)
-
-    finally:
-
-        if conn is not None:
-
-            conn.close()
-
-    #############################################################################################
-
-    try:
-
-        conn = None
-        conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
-
-    except (Exception, psycopg2.DatabaseError) as error:
-
-        print(error)
-        # Load configuration from config file
-        os.remove("db_config.txt")
-        print("Exiting. Run db-setup again with right parameters")
-        sys.exit(0)
-
-    if conn is not None:
-
-        print("\n")
-        print("fediverse parameters saved to db-config.txt!")
-        print("\n")
-
-    ############################################################
-    # Create needed tables
-    ############################################################
-
-    print("Creating table...")
-
-    ########################################
-
-    db = fediverse_db
-    db_user = fediverse_db_user
-    table = "world"
-    sql = "create table "+table+" (server varchar(200) PRIMARY KEY, federated_with varchar(200), updated_at timestamptz, saved_at timestamptz, checked boolean)"
-    create_table(db, db_user, table, sql)
-
-    table = "fediverse"
-    sql = "create table "+table+" (server varchar(200) PRIMARY KEY, users INT, updated_at timestamptz, software varchar(50), version varchar(100), first_checked_at timestamptz, last_checked_at timestamptz, downs int)"
-    create_table(db, db_user, table, sql)
-
-    table = "totals"
-    sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, total_servers INT, total_users INT)"
-    create_table(db, db_user, table, sql)
-
-    table = "evo"
-    sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, servers INT, users INT)"
-    create_table(db, db_user, table, sql)
-
-    table = "closed"
-    sql = "create table "+table+" (server varchar(200) PRIMARY KEY, software varchar(10), closed boolean)"
-    create_table(db, db_user, table, sql)
-
-    #####################################
-
-    print("Done!")
-    print("Now you can run setup.py!")
-    print("\n")

federation/__init__.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python3
+
+from .database import Database

federation/setup.py

@@ -0,0 +1,156 @@
+import os
+import sys
+from datetime import datetime
+import pytz
+from mastodon import Mastodon
+from mastodon.Mastodon import MastodonMalformedEventError, MastodonNetworkError, MastodonReadTimeout, MastodonAPIError, MastodonIllegalArgumentError
+
+class Setup():
+
+    name = 'fediverse setup'
+
+    def __init__(self, config_file=None, mastodon_hostname=None, peers_api=None, user_agent=None, secrets_filepath=None, mastodon_app_token=None):
+
+        self.config_file = "config/config.txt"
+        self.mastodon_hostname = self.__get_parameter("mastodon_hostname", self.config_file)
+        self.peers_api = '/api/v1/instance/peers?'
+        self.user_agent = {'User-agent': "fediverse's stats (fediverse@mastodont.cat)"}
+
+        self.secrets_filepath = 'secrets/secrets.txt'
+
+        is_setup = self.__check_mastodon_setup(self)
+
+        if is_setup:
+
+            self.mastodon_app_token = self.__get_mastodon_parameter("mastodon_app_token", self.secrets_filepath)
+
+        else:
+
+            self.mastodon_app_token = self.mastodon_setup(self)
+
+    @staticmethod
+    def __check_mastodon_setup(self):
+
+        is_setup = False
+
+        if not os.path.isfile(self.secrets_filepath):
+            print(f"File {self.secrets_filepath} not found, running setup.")
+        else:
+            is_setup = True
+
+        return is_setup
+
+    @staticmethod
+    def mastodon_setup(self):
+
+        if not os.path.exists('secrets'):
+            os.makedirs('secrets')
+
+        self.mastodon_user = input("Mastodon user login? ")
+        self.mastodon_password = input("Mastodon user password? ")
+        self.app_name = 'fediverse'
+
+        self.mastodon_app_token = self.mastodon_log_in()
+
+        if not os.path.exists(self.secrets_filepath):
+            with open(self.secrets_filepath, 'w'): pass
+            print(f"{self.secrets_filepath} created!")
+
+        with open(self.secrets_filepath, 'a') as the_file:
+            print("Writing Mastodon parameters to " + self.secrets_filepath)
+            the_file.write(f'mastodon_app_token: {self.mastodon_app_token}')
+
+        return self.mastodon_app_token
+
+    def mastodon_log_in(self):
+
+        token = ''
+
+        try:
+
+            response = Mastodon.create_app(
+                self.app_name,
+                scopes=["read","write"],
+                to_file=None,
+                api_base_url=self.mastodon_hostname
+                )
+            client_id = response[0]
+            client_secret = response[1]
+
+            mastodon = Mastodon(client_id = client_id, client_secret = client_secret, api_base_url = self.mastodon_hostname)
+
+            token = mastodon.log_in(
+                self.mastodon_user,
+                self.mastodon_password,
+                scopes = ["read", "write"],
+                to_file = None
+            )
+
+            print('Log in succesful!')
+
+        except MastodonIllegalArgumentError as i_error:
+
+            sys.stdout.write(f'\n{str(i_error)}\n')
+
+        except MastodonNetworkError as n_error:
+
+            sys.stdout.write(f'\n{str(n_error)}\n')
+
+        except MastodonReadTimeout as r_error:
+
+            sys.stdout.write(f'\n{str(r_error)}\n')
+
+        except MastodonAPIError as a_error:
+
+            sys.stdout.write(f'\n{str(a_error)}\n')
+
+        finally:
+
+            return token
+
+    def __get_parameter(self, parameter, config_file):
+
+        if not os.path.isfile(config_file):
+            print(f"File {config_file} not found..")
+
+            self.mastodon_hostname = input("\nMastodon hostname: ")
+
+            self.__create_config(self)
+            self.__write_config(self)
+
+        with open( self.config_file ) as f:
+            for line in f:
+                if line.startswith( parameter ):
+                    return line.replace(parameter + ":", "").strip()
+
+    def __get_mastodon_parameter(self, parameter, secrets_filepath):
+
+        if not os.path.isfile(secrets_filepath):
+            print(f"File {secrets_filepath} not found..")
+
+            self.sign_in()
+
+        with open( self.secrets_filepath ) as f:
+            for line in f:
+                if line.startswith( parameter ):
+                    return line.replace(parameter + ":", "").strip()
+
+    @staticmethod
+    def __create_config(self):
+
+        if not os.path.exists('config'):
+
+            os.makedirs('config')
+
+        if not os.path.exists(self.config_file):
+
+            print(self.config_file + " created!")
+            with open(self.config_file, 'w'): pass
+
+    @staticmethod
+    def __write_config(self):
+
+        with open(self.config_file, 'a') as the_file:
+
+            the_file.write(f'mastodon_hostname: {self.mastodon_hostname}')
+            print(f"adding parameters to {self.config_file}\n")

fediverse.py

@@ -1,843 +0,0 @@
-import sys
-import os
-import time
-from datetime import datetime
-import urllib3
-import requests
-import socket
-from mastodon import Mastodon
-import psycopg2
-import matplotlib.pyplot as plt
-import ray
-
-ray.init(num_cpus = 32) # Specify this system CPUs.
-
-class Server:
-
-    name = 'Server'
-
-    def __init_(self, server, software, users, alive, api, soft_version, now):
-
-        self.server = server
-        self.software = software
-        self.users = users
-        self.alive = alive
-        self.api = api
-        self.version = self.soft_version
-        self.now = now
-
-    @ray.remote
-    def get_alive_servers(server):
-
-        users = 0
-
-        downs = 0
-
-        fediverse_db, fediverse_db_user = db_config()
-
-        try:
-
-            conn = None
-
-            conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
-
-            cur = conn.cursor()
-
-            cur.execute("select alive, software, users_api, version, first_checked_at, downs from fediverse where server=(%s)", (server,))
-
-            row = cur.fetchone()
-
-            if row is not None:
-
-                was_alive = row[0]
-                software = row[1]
-                api = row[2]
-                soft_version = row[3]
-                first_checked_at = row[4]
-                downs_qty = row[5]
-
-            cur.close()
-
-        except (Exception, psycopg2.DatabaseError) as error:
-
-            print(error)
-
-        finally:
-
-            if conn is not None:
-
-                conn.close()
-
-            alive = False
-
-            try:
-
-                user_agent = {'User-agent': 'Mozilla/5.0'}
-
-                data = requests.get('https://' + server + api, headers = user_agent, timeout=3)
-
-                try:
-
-                    users = data.json()['usage']['users']['total']
-
-                    if users == 0:
-
-                        users = data.json()['usage']['users']['activeHalfyear']
-
-                    if software == 'socialhome':
-
-                        soft_version = data.json()['server']['version']
-
-                    else:
-
-                        soft_version = data.json()['software']['version']
-
-                    if software == "wordpress" and "activitypub" in data.json()['protocols']:
-
-                        alive = True
-
-                    elif software == "wordpress" and "activitypub" not in data.json()['protocols']:
-
-                        alive = False
-
-                    else:
-
-                        alive = True
-
-                except:
-
-                    soft_version = ""
-
-                else: 
-
-                    if api == '/api/v1/instance?':
-
-                        try:
-                            users = data.json()['stats']['user_count']
-                            soft_version = data.json()['version']
-                            alive = True
-                        except:
-                            soft_version = ""
-
-                if alive:
-
-                    if downs_qty != None:
-
-                        downs = downs_qty
-
-                    if soft_version != "" and soft_version is not None:
-
-                        print(f'\n** Server {server} ({software} {soft_version}) is alive! **')
-
-                    else:
-
-                        print(f'\n** Server {server} ({software}) is alive! **')
-
-                    write_alive_server(server, software, soft_version, alive, api, users, downs, first_checked_at)
-
-            except urllib3.exceptions.ProtocolError as protoerr:
-
-                print_dead(server)
-                pass
-
-            except requests.exceptions.ChunkedEncodingError as chunkerr:
-
-                print_dead(server)
-                pass
-
-            except KeyError as e:
-
-                print_dead(server)
-                pass
-
-            except ValueError as verr:
-
-                print_dead(server)
-                pass
-
-            except requests.exceptions.SSLError as errssl:
-
-                print_dead(server)
-                pass
-
-            except requests.exceptions.HTTPError as errh:
-
-                print_dead(server)
-                pass
-
-            except requests.exceptions.ConnectionError as errc:
-
-                print_dead(server)
-                pass
-
-            except requests.exceptions.Timeout as errt:
-
-                print_dead(server)
-                pass
-
-            except requests.exceptions.RequestException as err:
-
-                print_dead(server)
-                pass
-
-            except socket.gaierror as gai_error:
-
-                print_dead(server)
-                pass
-
-            if not alive:
-
-                if downs_qty != None:
-
-                    downs = downs_qty + 1
-
-                else:
-
-                    downs = 1
-
-                write_not_alive_server(server, software, soft_version, alive, api, users, downs, first_checked_at)
-
-        return (server, software, soft_version, alive, api, users, downs, first_checked_at)
-
-def write_alive_server(server, software, soft_version, alive, api, users, downs, first_checked_at):
-
-    insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version, first_checked_at, last_checked_at, downs) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
-
-    conn = None
-
-    try:
-
-        conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
-
-        cur = conn.cursor()
-
-        cur.execute(insert_sql, (server, users, now, software, alive, api, soft_version, now, now, downs))
-
-        if first_checked_at != None:
-
-            cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (users, now, software, alive, api, soft_version, now, downs, server))
-
-        else:
-
-            cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), first_checked_at=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (users, now, software, alive, api, soft_version, now, now, downs, server))
-
-            cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
-
-            conn.commit()
-
-            cur.close()
-
-    except (Exception, psycopg2.DatabaseError) as error:
-
-        print(error)
-
-    finally:
-
-        if conn is not None:
-
-            conn.close()
-
-def write_not_alive_server(server, software, soft_version, alive, api, users, downs, first_checked_at):
-
-    conn = None
-
-    try:
-
-        conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
-
-        cur = conn.cursor()
-
-        if first_checked_at != None:
-
-            cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), downs=(%s) where server=(%s)", (now, alive, downs, server))
-
-        else:
-
-            cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), first_checked_at=(%s), downs=(%s) where server=(%s)", (now, alive, now, downs, server))
-
-        cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
-
-        conn.commit()
-
-        cur.close()
-
-    except (Exception, psycopg2.DatabaseError) as error:
-
-        print(error)
-
-    finally:
-
-        if conn is not None:
-
-            conn.close()
-
-def delete_dead_servers():
-
-    conn = None
-
-    try:
-
-        conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
-
-        cur = conn.cursor()
-
-        cur.execute("select server from fediverse where downs > '14' and not alive and now() - first_checked_at > interval '7 days'")
-
-        rows = cur.fetchall()
-
-        for row in rows:
-
-            print(f'Deleting server {row[0]}...')
-
-            cur.execute("delete from fediverse where server=(%s)", (row[0],))
-
-        conn.commit()
-
-        cur.close()
-
-    except (Exception, psycopg2.DatabaseError) as error:
-
-        print(error)
-
-    finally:
-
-        if conn is not None:
-
-            conn.close()
-
-def get_last_checked_servers():
-
-    ############################################################################
-    # get last checked servers from fediverse DB
-
-    alive_servers = []
-
-    try:
-
-        conn = None
-
-        conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
-
-        cur = conn.cursor()
-
-        # get world servers list
-
-        cur.execute("select server from world where server in (select server from fediverse where users_api != '')")
-
-        alive_servers = []
-
-        for row in cur:
-
-            alive_servers.append(row[0])
-
-        cur.close()
-
-    except (Exception, psycopg2.DatabaseError) as error:
-
-        print(error)
-
-    finally:
-
-        if conn is not None:
-
-            conn.close()
-
-    return alive_servers
-
-def print_dead(server):
-
-    print(f'\nServer {server} is dead :-(')
-
-def set_world_servers_check_to_false():
-
-    ############################################################################
-    # set all world servers's checked column to False
-
-    try:
-
-        conn = None
-
-        conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
-
-        cur = conn.cursor()
-
-        cur.execute("UPDATE world SET checked='f'")
-
-        conn.commit()
-
-        cur.close()
-
-    except (Exception, psycopg2.DatabaseError) as error:
-
-        print(error)
-
-    finally:
-
-        if conn is not None:
-
-            conn.close()
-
-def set_world_servers_check_to_false():
-
-    ############################################################################
-    # set all world servers's checked column to False
-
-    try:
-
-        conn = None
-
-        conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
-
-        cur = conn.cursor()
-
-        cur.execute("UPDATE world SET checked='f'")
-
-        conn.commit()
-
-        cur.close()
-
-    except (Exception, psycopg2.DatabaseError) as error:
-
-        print(error)
-
-    finally:
-
-        if conn is not None:
-
-            conn.close()
-
-def mastodon():
-
-    # Load secrets from secrets file
-    secrets_filepath = "secrets/secrets.txt"
-    uc_client_id = get_parameter("uc_client_id",     secrets_filepath)
-    uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
-    uc_access_token = get_parameter("uc_access_token",  secrets_filepath)
-
-    # Load configuration from config file
-    config_filepath = "config/config.txt"
-    mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
-
-    # Initialise Mastodon API
-    mastodon = Mastodon(
-        client_id=uc_client_id,
-        client_secret=uc_client_secret,
-        access_token=uc_access_token,
-        api_base_url='https://' + mastodon_hostname,
-    )
-
-    # Initialise access headers
-    headers = {'Authorization': 'Bearer %s'%uc_access_token}
-
-    return (mastodon, mastodon_hostname)
-
-def db_config():
-
-    # Load db configuration from config file
-    config_filepath = "config/db_config.txt"
-    fediverse_db = get_parameter("fediverse_db", config_filepath)
-    fediverse_db_user = get_parameter("fediverse_db_user", config_filepath)
-
-    return (fediverse_db, fediverse_db_user)
-
-def usage():
-
-    print('usage: python ' + sys.argv[0] + ' --multi' + ' (multiprocessing, fast)')
-    print('usage: python ' + sys.argv[0] + ' --mono' + ' (one process, slow)')
-
-def get_parameter(parameter, file_path):
-
-    # Check if secrets file exists
-    if not os.path.isfile(file_path):
-        print("File %s not found, exiting."%file_path)
-        sys.exit(0)
-
-    # Find parameter in file
-    with open(file_path) as f:
-        for line in f:
-            if line.startswith(parameter):
-                return line.replace(parameter + ":", "").strip()
-
-    # Cannot find parameter, exit
-    print(file_path + "  Missing parameter %s "%parameter)
-    sys.exit(0)
-
-if __name__ == '__main__':
-
-    # usage modes
-
-    if len(sys.argv) == 1:
-
-        usage()
-
-    elif len(sys.argv) == 2:
-
-        if sys.argv[1] == '--multi':
-
-            now = datetime.now()
-
-            mastodon, mastodon_hostname = mastodon()
-
-            fediverse_db, fediverse_db_user = db_config()
-
-            total_servers = 0
-
-            total_users = 0
-
-            set_world_servers_check_to_false()
-
-            alive_servers = get_last_checked_servers()
-
-            getservers = Server()
-
-            getservers.now = datetime.now()
-
-            start = time.time()
-
-            results = ray.get([getservers.get_alive_servers.remote(server) for server in alive_servers])
-
-            print(f"duration = {time.time() - start}.\nprocessed servers: {len(results)}")
-
-            ###########################################################################
-            # get current total servers and users, get users from every software
-
-            now = datetime.now()
-
-            gettotals_sql = "select count(server), sum(users) from fediverse where alive"
-            get_soft_totals_sql = "select software, sum(users) as users, count(server) as servers from fediverse where users != 0 and alive group by software order by users desc"
-            soft_total_project = []
-            soft_total_users = []
-            soft_total_servers = []
-
-            try:
-
-                conn = None
-
-                conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
-
-                cur = conn.cursor()
-
-                cur.execute(gettotals_sql)
-
-                row = cur.fetchone()
-
-                total_servers = row[0]
-
-                total_users = row[1]
-
-                cur.execute(get_soft_totals_sql)
-
-                rows = cur.fetchall()
-
-                for row in rows:
-
-                    soft_total_project.append(row[0])
-                    soft_total_users.append(row[1])
-                    soft_total_servers.append(row[2])
-
-                cur.close()
-
-            except (Exception, psycopg2.DatabaseError) as error:
-
-                print(error)
-
-            finally:
-
-                if conn is not None:
-
-                    conn.close()
-
-            ###########################################################################
-            #  get last check values and write current total ones
-
-            select_sql = "select total_servers, total_users from totals order by datetime desc limit 1"
-
-            insert_sql = "INSERT INTO totals(datetime, total_servers, total_users) VALUES(%s,%s,%s)"
-
-            try:
-
-                conn = None
-
-                conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
-
-                cur = conn.cursor()
-
-                cur.execute(select_sql)
-
-                row = cur.fetchone()
-
-                if row is not None:
-
-                    servers_before = row[0]
-
-                    users_before = row[1]
-
-                else:
-
-                    servers_before = 0
-
-                    users_before = 0
-
-                cur.execute(insert_sql, (now, total_servers, total_users))
-
-                conn.commit()
-
-                cur.close()
-
-                evo_servers = total_servers - servers_before
-
-                evo_users = total_users - users_before
-
-            except (Exception, psycopg2.DatabaseError) as error:
-
-                print(error)
-
-            finally:
-
-                if conn is not None:
-
-                    conn.close()
-
-            ################################################################################
-            #  write evo values
-
-            insert_sql = "INSERT INTO evo(datetime, servers, users) VALUES(%s,%s,%s)"
-
-            conn = None
-
-            try:
-
-                conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
-
-                cur = conn.cursor()
-
-                cur.execute(insert_sql, (now, evo_servers, evo_users))
-
-                conn.commit()
-
-                cur.close()
-
-            except (Exception, psycopg2.DatabaseError) as error:
-
-                print(error)
-
-            finally:
-
-                if conn is not None:
-
-                    conn.close()
-
-            ##############################################################################
-            # get world's last update datetime
-
-            conn = None
-
-            try:
-
-                conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
-
-                cur = conn.cursor()
-
-                cur.execute("select updated_at from world order by updated_at desc limit 1")
-
-                row = cur.fetchone()
-
-                last_update = row[0]
-
-                last_update = last_update.strftime('%m/%d/%Y, %H:%M:%S')
-
-                cur.close()
-
-            except (Exception, psycopg2.DatabaseError) as error:
-
-                print(error)
-
-            finally:
-
-                if conn is not None:
-
-                    conn.close()
-
-            ##############################################################################
-            # get max servers and users
-
-            conn = None
-
-            try:
-
-                conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
-
-                cur = conn.cursor()
-
-                cur.execute("select MAX(total_servers) from totals")
-
-                row = cur.fetchone()
-
-                if row is not None:
-
-                    max_servers = row[0]
-
-                else:
-
-                    max_servers = 0
-
-                cur.execute("select MAX(total_users) from totals")
-
-                row = cur.fetchone()
-
-                if row is not None:
-
-                    max_users = row[0]
-
-                else:
-
-                    max_users = 0
-
-                cur.close()
-
-            except (Exception, psycopg2.DatabaseError) as error:
-
-                print(error)
-
-            finally:
-
-                if conn is not None:
-
-                    conn.close()
-
-            ###############################################################################
-            # get plots
-
-            servers_plots = []
-
-            users_plots = []
-
-            conn = None
-
-            try:
-
-                conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
-
-                cur = conn.cursor()
-
-                cur.execute("select total_servers, total_users from totals order by datetime desc limit 14")
-
-                rows = cur.fetchall()
-
-                for row in rows:
-
-                    servers_plots.append(row[0])
-
-                    users_plots.append(row[1])
-
-                cur.close()
-
-            except (Exception, psycopg2.DatabaseError) as error:
-
-                print(error)
-
-            finally:
-
-                if conn is not None:
-
-                    conn.close()
-
-            ###############################################################################
-            # generate graphs
-
-            plt.plot([-6, -5, -4, -3, -2, -1, 0], [servers_plots[6], servers_plots[5], servers_plots[4], servers_plots[3], servers_plots[2], servers_plots[1], servers_plots[0]], marker='o', color='mediumseagreen')
-
-            plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_servers, max_servers, max_servers, max_servers, max_servers, max_servers, max_servers], color='red')
-
-            plt.title('fediverse: total alive servers (max: ' + str(f"{max_servers:,}" + ')'), loc='right', color='blue')
-
-            plt.xlabel('Last seven days')
-
-            plt.ylabel('fediverse alive servers')
-
-            plt.legend(('servers', 'max'), shadow=True, loc=(0.01, 1.00), handlelength=1.5, fontsize=10)
-
-            plt.savefig('servers.png')
-
-            plt.close()
-
-            plt.plot([-6, -5, -4, -3, -2, -1, 0], [users_plots[6], users_plots[5], users_plots[4], users_plots[3], users_plots[2], users_plots[1], users_plots[0]], marker='o', color='royalblue')
-
-            plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_users, max_users, max_users, max_users, max_users, max_users, max_users], color='red')
-
-            plt.title('fediverse: total registered users (max: ' + str(f"{max_users:,}" + ')'), loc='right', color='royalblue')
-
-            plt.legend(('users', 'max'), shadow=True, loc=(0.01, 0.80), handlelength=1.5, fontsize=10)
-
-            plt.xlabel('Last seven days')
-
-            plt.ylabel('Registered users')
-
-            plt.savefig('users.png')
-
-            plt.close()
-
-            ###############################################################################
-            # T  O  O  T !
-
-            toot_text = "#fediverse alive servers stats" + " \n"
-
-            toot_text += "\n"
-
-            if evo_servers >= 0:
-
-                toot_text += "alive servers: " + str(f"{total_servers:,}") + " (+"+ str(f"{evo_servers:,}") + ") \n"
-
-                toot_text += "max: " + str(f"{max_servers:,}") + "\n"
-
-            elif evo_servers < 0:
-
-                toot_text += "alive servers: " + str(f"{total_servers:,}") + " ("+ str(f"{evo_servers:,}") + ") \n"
-
-                toot_text += "max: " + str(f"{max_servers:,}") + "\n"
-
-            if evo_users >= 0:
-
-                toot_text += "total users: " + str(f"{total_users:,}") + " (+"+ str(f"{evo_users:,}") + ") \n"
-
-                toot_text += "max: " + str(f"{max_users:,}") + "\n"
-
-            elif evo_users < 0:
-
-                toot_text += "total users: " + str(f"{total_users:,}") + " ("+ str(f"{evo_users:,}") + ") \n"
-
-                toot_text += "max: " + str(f"{max_users:,}") + "\n"
-
-            toot_text += "\ntop ten (soft users servers):\n\n"
-
-            i = 0
-
-            while i < 10:
-
-                project_soft = soft_total_project[i]
-
-                project_users = soft_total_users[i]
-
-                project_servers = soft_total_servers[i]
-
-                len_pr_soft = len(project_soft)
-
-                toot_text += f":{project_soft}: {project_users:,} {project_servers:,}\n"
-
-                i += 1
-
-            print("Tooting...")
-
-            print(toot_text)
-
-            servers_image_id = mastodon.media_post('servers.png', "image/png", description='servers graph').id
-
-            users_image_id = mastodon.media_post('users.png', "image/png", description='users graph').id
-
-            mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, users_image_id})
-
-            delete_dead_servers()
-
-    else:
-
-        usage()

fetchservers.py

@@ -1,361 +0,0 @@
-from multiprocessing import set_start_method
-from multiprocessing import get_context
-from itertools import product
-import time
-from datetime import datetime
-import os
-import json
-import sys
-import os.path
-import psycopg2
-import aiohttp
-import asyncio
-import socket
-import pdb
-
-client_exceptions = (
-    aiohttp.ClientResponseError,
-    aiohttp.ClientConnectionError,
-    aiohttp.ClientConnectorError,
-    aiohttp.ClientError,
-    asyncio.TimeoutError,
-    socket.gaierror,
-)
-
-def write_api(server, software, users, alive, api, soft_version):
-
-    now = datetime.now()
-
-    fediverse_db, fediverse_db_user = get_db_config()
-
-    insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
-
-    conn = None
-
-    try:
-
-        conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
-
-        cur = conn.cursor()
-
-        print(f'Writing {server} nodeinfo data...')
-
-        cur.execute(insert_sql, (server, now, software, users, alive, api, soft_version))
-
-        cur.execute(
-            "UPDATE fediverse SET updated_at=(%s), software=(%s), users=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)",
-            (now, software, users, alive, api, soft_version, server)
-            )
-
-        cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
-
-        conn.commit()
-
-        cur.close()
-
-    except (Exception, psycopg2.DatabaseError) as error:
-
-        print(error)
-
-    finally:
-
-        if conn is not None:
-
-            conn.close()
-
-async def getsoft(server):
-
-    fediverse_db, fediverse_db_user = get_db_config()
-
-    try:
-
-        socket.gethostbyname(server)
-
-    except socket.gaierror as g_error:
-
-        print(f'Server {server} error: {g_error}')
-        pass
-        return
-
-    soft = ''
-
-    url = 'https://' + server
-
-    user_agent = {'User-agent': 'Mozilla/5.0'}
-
-    timeout = aiohttp.ClientTimeout(total=3)
-
-    async with aiohttp.ClientSession(timeout=timeout, headers=user_agent) as session:
-
-        try:
-
-            async with session.get(url + '/.well-known/nodeinfo') as response:
-
-                if response.status == 200:
-
-                    try:
-
-                        response_json = await response.json()
-
-                        nodeinfo = response_json['links'][0]['href'].replace(f'https://{server}','')
-
-                    except:
-
-                        pass
-                else:
-
-                    print(f'Server {server} not responding: {response.status}')
-
-                    pass
-
-                async with session.get(url + nodeinfo) as nodeinfo_response:
-
-                    if nodeinfo_response.status == 200:
-
-                        try:
-
-                            nodeinfo_json = await nodeinfo_response.json()
-
-                        except:
-
-                            pass
-
-                    else:
-
-                        print(f"Server {server}'s nodeinfo not responding: {response.status}")
-
-                        pass
-
-        except aiohttp.ClientConnectorError as cc_err:
-
-            pass
-
-        except aiohttp.client_exceptions.ClientConnectorSSLError as ccssl_as:
-
-            pass
-
-        else:
-
-            if nodeinfo_response.status == 200 and nodeinfo != '/api/v1/instance?':
-
-                if nodeinfo != '/.well-known/x-nodeinfo2?':
-
-                    try:
-
-                        soft = nodeinfo_json['software']['name']
-                        soft = soft.lower()
-                        soft_version = nodeinfo_json['software']['version']
-                        users = nodeinfo_json['usage']['users']['total']
-                        if users > 1000000:
-                            return
-                        alive = True
-
-                        write_api(server, soft, users, alive, nodeinfo, soft_version)
-
-                        print('*********************************************************************')
-                        print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
-                        print('*********************************************************************')
-
-                        return
-
-                    except:
-
-                        pass
-
-                else:
-
-                    try:
-
-                        soft = nodeinfo_json['server']['software']
-                        soft = soft.lower()
-                        soft_version = nodeinfo_json['server']['version']
-                        users = nodeinfo_json['usage']['users']['total']
-                        if users > 1000000:
-                            return
-                        alive = True
-
-                        if soft == 'socialhome':
-
-                            write_api(server, soft, users, alive, api, soft_version)
-
-                            print('*********************************************************************')
-                            print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
-                            print('*********************************************************************')
-
-                        return
-
-                    except:
-
-                        pass
-
-            if nodeinfo_response.status == 200 and soft == '' and nodeinfo == "/api/v1/instance?":
-
-                soft = 'mastodon'
-                users = nodeinfo_json['stats']['user_count']
-                soft_version = nodeinfo_json['version']
-                if users > 1000000:
-                    return
-                alive = True
-
-                write_api(server, soft, users, alive, api)
-
-                print('*********************************************************************')
-                print("Server " + server + " (" + soft + ") is alive!")
-                print('*********************************************************************')
-
-            else:
-
-                print(f'Server {server} is dead')
-
-def getserver(server, *args):
-
-    if len(args) != 0:
-
-        server = server[0].rstrip('.').lower()
-
-    if server.find(".") == -1:
-        return
-    if server.find("@") != -1:
-        return
-    if server.find("/") != -1:
-        return
-    if server.find(":") != -1:
-        return
-
-    try:
-
-        loop = asyncio.get_event_loop()
-        coroutines = [getsoft(server)]
-        soft = loop.run_until_complete(asyncio.gather(*coroutines, return_exceptions=True))
-
-    except:
-
-        pass
-
-def get_world_servers():
-
-    world_servers = []
-
-    try:
-
-        conn = None
-
-        conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
-
-        cur = conn.cursor()
-
-        # get world servers list
-
-        cur.execute("select server from world where checked='f'")
-
-        rows = cur.fetchall()
-
-        for row in rows:
-
-            world_servers.append(row[0])
-
-        cur.close()
-
-        print("Remaining servers: " + str(len(world_servers)))
-
-    except (Exception, psycopg2.DatabaseError) as error:
-
-        print(error)
-
-    finally:
-
-        if conn is not None:
-
-            conn.close()
-
-    return world_servers
-
-def get_parameter(parameter, file_path):
-
-    # Check if secrets file exists
-    if not os.path.isfile(file_path):
-        print("File %s not found, exiting." % file_path)
-        sys.exit(0)
-
-    # Find parameter in file
-    with open(file_path) as f:
-        for line in f:
-            if line.startswith(parameter):
-                return line.replace(parameter + ":", "").strip()
-
-    # Cannot find parameter, exit
-    print(file_path + "  Missing parameter %s " % parameter)
-    sys.exit(0)
-
-def usage():
-
-    print('usage: python ' + sys.argv[0] + ' --multi' + ' (multiprocessing, fast)')
-    print('usage: python ' + sys.argv[0] + ' --mono' + ' (one process, slow)')
-
-def get_config():
-
-    # Load configuration from config file
-    config_filepath = "config/config.txt"
-    mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
-    return mastodon_hostname
-
-def get_db_config():
-
-    # Load database config from db_config file
-    db_config_filepath = "config/db_config.txt"
-    fediverse_db = get_parameter("fediverse_db", db_config_filepath)
-    fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath)
-    return (fediverse_db, fediverse_db_user)
-
-###############################################################################
-# main
-
-if __name__ == '__main__':
-
-    # usage modes
-
-    if len(sys.argv) == 1:
-
-        usage()
-
-    elif len(sys.argv) == 2:
-
-        if sys.argv[1] == '--multi':
-
-            now = datetime.now()
-
-            start_time = time.time()
-
-            mastodon_hostname = get_config()
-
-            fediverse_db, fediverse_db_user = get_db_config()
-
-            world_servers = get_world_servers()
-
-            with get_context("spawn").Pool(processes=32) as pool:
-
-                res = pool.starmap(getserver, product(world_servers))
-
-                pool.close()
-
-                pool.join()
-
-            print('Done.')
-
-        elif sys.argv[1] == '--mono':
-
-            now = datetime.now()
-
-            start_time = time.time()
-
-            mastodon_hostname = get_config()
-
-            fediverse_db, fediverse_db_user = get_db_config()
-
-            world_servers = get_world_servers()
-
-            for server in world_servers:
-
-                getserver(server)
-
-            print('Done.')

getpeers.py

@@ -1,156 +0,0 @@
-import os
-import sys
-import time
-from datetime import datetime
-import requests
-import threading
-import json
-import psycopg2
-import pdb
-                
-def write_servers(world_servers):
-
-    i = 0
-
-    while i < len(world_servers):
-
-        saved_at = datetime.now()
-
-        insert_sql = "INSERT INTO world(server, federated_with, updated_at, saved_at) VALUES(%s,%s,%s,%s) ON CONFLICT DO NOTHING"
-
-        conn = None
-
-        try:
-
-            conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
-
-            cur = conn.cursor()
-
-            cur.execute(insert_sql, (world_servers[i], peer, updated_at, saved_at,))
-
-            print(f'writing to database {str(i)} of {str(len(world_servers))}: {world_servers[i]}')
-
-            conn.commit()
-
-            cur.close()
-
-        except (Exception, psycopg2.DatabaseError) as error:
-
-            print(error)
-
-        finally:
-
-            if conn is not None:
-
-                conn.close()
-
-        i += 1
-
-def get_peers(peer):
-
-    try:
-
-        user_agent = {'User-agent': 'Mozilla/5.0'}
-
-        response = requests.get('https://' + peer + peers_api, headers = user_agent, timeout=3)
-
-        response_json = response.json()
-
-        if response.status_code == 200:
-
-            try:
-
-                print(f"Server: {peer}, federated with {str(len(response_json))} servers")
-
-                for peer_peer in response_json:
-
-                    exist_count = world_peers.count(peer_peer)
-
-                    if exist_count == 0:
-
-                        world_peers.append(peer_peer)
-            except:
-
-                pass
-    except:
-
-        pass
-
-class Peerthreads(threading.Thread):
-    def __init__(self, peer, peers_semaphore):
-        threading.Thread.__init__(self)
-        self.sql_conn = None 
-        self.peer = peer
-        self.peers_semaphore = peers_semaphore
-
-    def run(self):
-        self.peers_semaphore.acquire()
-        get_peers(self.peer)
-        self.peers_semaphore.release()
-
-# Returns the parameter from the specified file
-def get_parameter( parameter, file_path ):
-    # Check if secrets file exists
-    if not os.path.isfile(file_path):
-        print("File %s not found, exiting."%file_path)
-        sys.exit(0)
-
-    # Find parameter in file
-    with open( file_path ) as f:
-        for line in f:
-            if line.startswith( parameter ):
-                return line.replace(parameter + ":", "").strip()
-
-    # Cannot find parameter, exit
-    print(file_path + "  Missing parameter %s "%parameter)
-    sys.exit(0)
-
-###############################################################################
-# main
-
-if __name__ == '__main__':
-
-    start_time = time.time()
-
-    updated_at = datetime.now()
-
-    peers_api = '/api/v1/instance/peers?'
-
-    # Load configuration from config file
-    config_filepath = "config/config.txt"
-    mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
-
-    # Load database config from db_config file
-    db_config_filepath = "config/db_config.txt"
-    fediverse_db = get_parameter("fediverse_db", db_config_filepath)
-    fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath)
-
-    world_peers = []
-
-    user_agent = {'User-agent': 'Mozilla/5.0'}
-
-    res = requests.get('https://' + mastodon_hostname + peers_api, headers = user_agent, timeout=3)
-
-    hostname_peers = res.json()
-
-    for peer in hostname_peers:
-
-        exist_count = world_peers.count(peer)
-
-        if exist_count == 0:
-
-            world_peers.append(peer)
-
-    peers_semaphore = threading.Semaphore(100)
-    peer_threads = []
-    for peer in hostname_peers:
-        p_thread = Peerthreads(peer, peers_semaphore)
-        peer_threads.append(p_thread)
-        p_thread.start()
-    for p_thread in peer_threads:
-        p_thread.join()
-
-    write_servers(world_peers)
-
-    exec_time = str(round((time.time() - start_time), 2))
-    print(exec_time)

requirements-dev.txt

@@ -0,0 +1,10 @@
+requests
+psycopg2-binary
+pytz
+ray
+Mastodon.py
+matplotlib
+pandas
+humanfriendly
+pytest
+ipdb

requirements.txt

@@ -1,9 +1,8 @@
-Mastodon.py>=1.5.1
-psycopg2-binary>=2.8.4
-aiohttp>=3.6.2
-aiodns>=2.0.0
-matplotlib>=3.3.4
-humanfriendly>=9.2
-urllib3>=1.26.8
-requests>=2.27.1
-ray>=1.11.0
+requests
+psycopg2-binary
+pytz
+ray
+Mastodon.py
+matplotlib
+pandas
+humanfriendly

scripts/fediquery.py

@@ -0,0 +1,196 @@
+import sys
+import os
+import os.path
+import re
+from datetime import datetime, timedelta
+from setup import Setup
+from mastodon import Mastodon
+from database import Database
+from query import Query
+import pdb
+
+def cleanhtml(raw_html):
+    cleanr = re.compile('<.*?>')
+    cleantext = re.sub(cleanr, '', raw_html)
+    return cleantext
+
+def unescape(s):
+    s = s.replace("&apos;", "'")
+    return s
+
+def replying():
+
+    reply = False
+
+    content = cleanhtml(text)
+    content = unescape(content)
+
+    try:
+
+        start = content.index("@")
+        end = content.index(" ")
+        if len(content) > end:
+
+            content = content[0: start:] + content[end +1::]
+
+        neteja = content.count('@')
+
+        i = 0
+        while i < neteja :
+
+            start = content.rfind("@")
+            end = len(content)
+            content = content[0: start:] + content[end +1::]
+            i += 1
+
+        question = content.lower()
+
+        query_word = question
+        query_word_length = len(query_word)
+
+        if query_word[:4]  == 'soft':
+
+            reply = True
+
+        if query_word[:6]  == 'server':
+
+            reply = True
+
+        if query_word[:3]  == 'mau':
+
+            reply = True
+
+        return (reply, query_word)
+
+    except ValueError as v_error:
+
+        print(v_error)
+
+        query_word = ''
+
+        return (reply, query_word)
+
+# main
+
+if __name__ == '__main__':
+
+    setup = Setup()
+
+    mastodon = Mastodon(
+            access_token = setup.mastodon_app_token,
+            api_base_url= setup.mastodon_hostname
+            )
+
+    db = Database()
+
+    query = Query()
+
+    now = datetime.now()
+
+    bot_id = mastodon.me().id
+
+    notifications = mastodon.notifications()
+
+    if len(notifications) == 0:
+
+        print('No mentions')
+        sys.exit(0)
+
+    for notif in notifications:
+
+        notification_id = notif.id
+
+        if notif.type != 'mention':
+
+            print(f'dismissing notification {notification_id}')
+
+            mastodon.notifications_dismiss(notification_id)
+
+            continue
+
+        account_id = notif.account.id
+
+        username = notif.account.acct
+
+        status_id = notif.status.id
+
+        text  = notif.status.content
+
+        visibility = notif.status.visibility
+
+        reply, query_word = replying()
+
+        if reply == True:
+
+            if query_word[:4] == 'soft':
+
+                key_word = query_word[:4]
+
+                search_soft = query_word[5:]
+
+                if search_soft != '':
+
+                    servers, users, mau = db.get_soft_data(search_soft)
+
+                    toot_text = f'@{username}, my data for {search_soft} software:\n\n'
+
+                    if servers != 0:
+
+                        toot_text += f'software :{search_soft}:\nservers: {servers:,}\nusers: {users:,}\nMAU: {mau:,}'
+
+                    else:
+
+                        toot_text += 'software not found!'
+
+                    mastodon.status_post(toot_text, in_reply_to_id=status_id,visibility=visibility)
+
+                    print(f'Notification {notification_id} replied')
+
+                    mastodon.notifications_dismiss(notification_id)
+
+            if query_word[:6] == 'server':
+
+                key_word = query_word[:6]
+
+                search_server = query_word[7:]
+
+                if search_server != '':
+
+                    server, software, version, users, mau, alive = db.fediquery_server_data(search_server)
+
+                    toot_text = f'@{username}, my data for {search_server}:\n\n'
+
+                    if server == '' or server != '' and not alive:
+
+                        server, software, version, users, mau, alive = query.getsoft(search_server)
+
+                        if server != '' and alive:
+
+                            toot_text += f"\nServer not found but it's alive. Added!\n\n"
+
+                    if alive:
+
+                        toot_text += f'server: {server}\nsoftware: :{software}:\nversion: {version}\nMAU: {int(mau):,}\nusers: {int(users):,}\nalive: {alive}'
+
+                    else:
+
+                        toot_text += 'server not found!'
+
+                    mastodon.status_post(toot_text, in_reply_to_id=status_id,visibility=visibility)
+
+                    print(f'Notification {notification_id} replied')
+
+                    mastodon.notifications_dismiss(notification_id)
+        else:
+
+            try:
+
+                print(f'Dismissing notification {notification_id}')
+
+                mastodon.notifications_dismiss(notification_id)
+
+            except MastodonNotFoundError as notfound_error:
+
+                print(f'{notfound_error}')
+
+                continue

scripts/fediverse.py

@@ -0,0 +1,439 @@
+import sys
+import os
+import time
+from datetime import datetime
+import urllib3
+import requests
+import socket
+from setup import Setup
+from federation import Database
+from mastodon import Mastodon
+from matplotlib import pyplot as plt
+import matplotlib.dates as mdates
+from matplotlib.ticker import ScalarFormatter
+import numpy as np
+import pandas as pd
+import ray
+import pdb
+
+SMALL_SIZE = 6
+MEDIUM_SIZE = 10
+BIGGER_SIZE = 12
+
+plt.rc('font', size=MEDIUM_SIZE)          # controls default text sizes
+plt.rc('axes', titlesize=MEDIUM_SIZE)     # fontsize of the axes title
+plt.rc('axes', labelsize=MEDIUM_SIZE)    # fontsize of the x and y labels
+plt.rc('xtick', labelsize=SMALL_SIZE)    # fontsize of the tick labels
+plt.rc('ytick', labelsize=MEDIUM_SIZE)    # fontsize of the tick labels
+plt.rc('legend', fontsize=MEDIUM_SIZE)    # legend fontsize
+plt.rc('figure', titlesize=BIGGER_SIZE)  # fontsize of the figure title
+
+mdates.set_epoch('2000-01-01T00:00:00')
+y_formatter = ScalarFormatter(useOffset=False)
+
+ray.init(num_cpus=25)  # Specify this system CPUs.
+
+
+class Server:
+
+    name = 'Server'
+
+    def __init_(self, server=None):
+
+        self.server = server
+
+    @ray.remote
+    def get_alive_servers(self):
+
+        users = 0
+
+        downs = 0
+
+        was_alive, software, api, soft_version, first_checked_at, downs_qty, mau = db.get_server_data(self)
+
+        alive = False
+
+        try:
+
+            data = requests.get('https://' + self + api,
+                                headers=setup.user_agent, timeout=3)
+
+            nodeinfo_json = data.json()
+
+            try:
+
+                users = nodeinfo_json.get('usage').get('users').get('total') or '0'
+                if int(users) > 1000000:
+
+                    users = 1
+
+                mau = nodeinfo_json.get('usage').get(
+                    'users').get('activeMonth') or '0'
+
+                if software == 'socialhome':
+
+                    soft_version = nodeinfo_json['server']['version']
+
+                else:
+
+                    soft_version = nodeinfo_json['software']['version']
+
+                if software == "wordpress" and "activitypub" in nodeinfo_json['protocols']:
+
+                    alive = True
+
+                elif software == "wordpress" and "activitypub" not in nodeinfo_json['protocols']:
+
+                    alive = False
+
+                else:
+
+                    alive = True
+
+            except:
+
+                soft_version = ""
+
+            else:
+
+                if api == '/api/v1/instance':
+
+                    try:
+                        users = nodeinfo_json.get('stats').get('user_count') or '0'
+                        if int(users) > 1000000:
+
+                            users = 1
+
+                        soft_version = nodeinfo_json['version']
+                        alive = True
+                        mau = 0
+                    except:
+                        soft_version = ""
+
+            if alive:
+
+                if downs_qty != None:
+
+                    downs = downs_qty
+
+                if soft_version != "" and soft_version is not None:
+
+                    print(
+                        f'\n** Server {self} ({software} {soft_version}) is alive! **')
+
+                else:
+
+                    print(f'\n** Server {self} ({software}) is alive! **')
+
+                if software != 'birdsitelive':
+
+                    db.write_alive_server(
+                        self, software, soft_version, alive, api, users, downs, first_checked_at, mau)
+
+                else:
+
+                    db.write_blocked_software(
+                        self, software, soft_version, alive, api, users, downs, first_checked_at)
+
+        except urllib3.exceptions.ProtocolError as protoerr:
+
+            print_dead(self)
+            pass
+
+        except requests.exceptions.ChunkedEncodingError as chunkerr:
+
+            print_dead(self)
+            pass
+
+        except KeyError as e:
+
+            print_dead(self)
+            pass
+
+        except ValueError as verr:
+
+            print_dead(self)
+            pass
+
+        except requests.exceptions.SSLError as errssl:
+
+            print_dead(self)
+            pass
+
+        except requests.exceptions.HTTPError as errh:
+
+            print_dead(self)
+            pass
+
+        except requests.exceptions.ConnectionError as errc:
+
+            print_dead(self)
+            pass
+
+        except requests.exceptions.Timeout as errt:
+
+            print_dead(self)
+            pass
+
+        except requests.exceptions.RequestException as err:
+
+            print_dead(self)
+            pass
+
+        except socket.gaierror as gai_error:
+
+            print_dead(self)
+            pass
+
+        if not alive:
+
+            if downs_qty != None:
+
+                downs = downs_qty + 1
+
+            else:
+
+                downs = 1
+
+            if downs_qty > 60 and not was_alive:
+
+                db.write_not_alive_server(self, software, soft_version, alive, api, users, downs, first_checked_at, mau)
+
+            elif downs_qty < 60 and not was_alive:
+
+                db.write_not_alive_server(self, software, soft_version, alive, api, users, downs, first_checked_at, mau)
+
+            elif downs_qty < 60 and was_alive:
+
+                alive = True
+
+                db.write_alive_server(self, software, soft_version, alive, api, users, downs, first_checked_at, mau)
+
+        return (self, software, soft_version, alive, api, users, downs, first_checked_at, mau)
+
+
+def print_dead(server):
+
+    print(f'\nServer {server} is dead :-(')
+
+
+if __name__ == '__main__':
+
+    db = Database()
+
+    setup = Setup()
+
+    start = datetime.now()
+
+    program = 'fediverse'
+
+    finish = start
+
+    db.save_time(program, start, finish)
+
+    now = start
+
+    mastodon = Mastodon(
+        access_token=setup.mastodon_app_token,
+        api_base_url=setup.mastodon_hostname
+    )
+
+    total_servers = 0
+
+    total_users = 0
+
+    alive_servers = db.get_last_checked_servers()
+
+    getservers = Server()
+
+    getservers.now = now
+
+    ray_start = time.time()
+
+    results = ray.get([getservers.get_alive_servers.remote(server)
+                      for server in alive_servers])
+
+    print(
+        f"duration = {time.time() - ray_start}.\nprocessed servers: {len(results)}")
+
+    # get current total servers and users, get users from every software
+
+    soft_total_project, soft_total_users, soft_total_mau, soft_total_servers, total_servers, total_users, total_mau = db.soft_totals()
+
+    # get last check values and write current total ones
+
+    evo_servers, evo_users, evo_mau = db.last_values(
+        total_servers, total_users, total_mau)
+
+    # write evo values
+
+    db.write_evo(evo_servers, evo_users, evo_mau)
+
+    # get world's last update datetime
+
+    last_update = db.last_world_datetime()
+
+    # get max servers and mau
+
+    max_servers, max_mau = db.max()
+
+    # get plots
+
+    servers_plots, mau_plots, global_week, global_servers, global_users, global_mau = db.get_plots()
+
+    ###############################################################################
+    # generate graphs
+
+    plt.plot([-6, -5, -4, -3, -2, -1, 0], [servers_plots[6], servers_plots[5], servers_plots[4], servers_plots[3],
+             servers_plots[2], servers_plots[1], servers_plots[0]], marker='o', color='mediumseagreen')
+
+    plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_servers, max_servers, max_servers,
+             max_servers, max_servers, max_servers, max_servers], color='red')
+
+    plt.title('fediverse: total alive servers (max: ' +
+              str(f"{max_servers:,}" + ')'), loc='right', color='blue')
+
+    plt.xlabel('Last seven days')
+
+    plt.ylabel('fediverse alive servers')
+
+    plt.grid(visible=True)
+
+    plt.legend(('servers', 'max'), shadow=True, loc=(0.01, 1.00), handlelength=1.5, fontsize=10)
+
+    plt.savefig('servers.png')
+
+    plt.close()
+
+    plt.plot([-6, -5, -4, -3, -2, -1, 0], [mau_plots[6], mau_plots[5], mau_plots[4],
+             mau_plots[3], mau_plots[2], mau_plots[1], mau_plots[0]], marker='o', color='royalblue')
+
+    plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_mau, max_mau, max_mau,
+             max_mau, max_mau, max_mau, max_mau], color='red')
+
+    plt.title('fediverse: total MAU (max: ' +
+              str(f"{max_mau:,}" + ')'), loc='right', color='royalblue')
+
+    plt.legend(('mau', 'max'), shadow=True, loc=(
+        0.01, 0.80), handlelength=1.5, fontsize=10)
+
+    plt.xlabel('Last seven days')
+
+    plt.ylabel('MAU')
+
+    plt.grid(visible=True)
+
+    plt.savefig('mau.png')
+
+    plt.close()
+
+    df = pd.DataFrame({'date': np.array(global_week),
+                       #'servers': np.array(global_servers),
+                       'users': np.array(global_users),
+                       'mau': np.array(global_mau)})
+
+    df['date'] = pd.to_datetime(df['date'])
+
+    fig, ax = plt.subplots()
+
+    ax.plot(df.date, df.users, label='Registered', color='orange')
+
+    ax.plot(df.date, df.mau, label='MAU', color='blue')
+
+    plt.tick_params(rotation=45)
+
+    ax.set_title("fediverse's registered and Monthly Active Users")
+
+    ax.set_xlabel('weeks')
+
+    ax.set_ylabel('users')
+
+    ax.grid(visible=True)
+
+    ax.legend(title='Users')
+
+    ax.yaxis.set_major_formatter(y_formatter)
+
+    plt.savefig('global.png')
+
+    plt.close()
+
+    ###############################################################################
+    # P O S T !
+
+    toot_text = "#fediverse alive servers stats" + " \n"
+
+    toot_text += "\n"
+
+    if evo_servers >= 0:
+
+        toot_text += "alive servers: " + \
+            str(f"{total_servers:,}") + \
+            " (+" + str(f"{evo_servers:,}") + ") \n"
+
+        toot_text += "max: " + str(f"{max_servers:,}") + "\n"
+
+    elif evo_servers < 0:
+
+        toot_text += "alive servers: " + \
+            str(f"{total_servers:,}") + " (" + str(f"{evo_servers:,}") + ") \n"
+
+        toot_text += "max: " + str(f"{max_servers:,}") + "\n"
+
+    if evo_mau >= 0:
+
+        toot_text += "total MAU: " + \
+            str(f"{total_mau:,}") + " (+" + str(f"{evo_mau:,}") + ") \n"
+
+        toot_text += "max: " + str(f"{max_mau:,}") + "\n"
+
+    elif evo_mau < 0:
+
+        toot_text += "total MAU: " + \
+            str(f"{total_mau:,}") + " (" + str(f"{evo_mau:,}") + ") \n"
+
+        toot_text += "max: " + str(f"{max_mau:,}") + "\n"
+
+    toot_text += "\ntop ten (MAU / servers):\n\n"
+
+    i = 0
+
+    while i < 10:
+
+        project_soft = soft_total_project[i]
+
+        project_mau = soft_total_mau[i]
+
+        project_servers = soft_total_servers[i]
+
+        len_pr_soft = len(project_soft)
+
+        if project_soft == 'ativity-relay':
+
+            project_soft = 'activityrelay'
+
+        toot_text += f":{project_soft}: {project_mau:,} / {project_servers:,}\n"
+
+        i += 1
+
+    print("Tooting...")
+
+    print(toot_text)
+
+    servers_image_id = mastodon.media_post(
+        'servers.png', "image/png", description='servers graph').id
+
+    mau_image_id = mastodon.media_post(
+        'mau.png', "image/png", description='MAU graph').id
+
+    global_image_id = mastodon.media_post(
+        'global.png', "image/png", description='global graph').id
+
+    mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={
+                         servers_image_id, mau_image_id, global_image_id})
+
+    db.delete_dead_servers()
+
+    finish = datetime.now()
+
+    db.save_time(program, start, finish)

scripts/fetchservers.py

@@ -0,0 +1,335 @@
+import time
+from datetime import datetime
+import os
+import json
+import sys
+import os.path
+from setup import Setup
+from database import Database
+import requests
+import urllib3
+import socket
+import ray
+import pdb
+
+ray.init(num_cpus = 25) # Specify this system CPUs.
+
+from ray.exceptions import (
+    RaySystemError,
+    RayError,
+    RayTaskError,
+    ObjectStoreFullError,
+)
+
+apis = ['/api/v1/instance',
+        '/api/v1/nodeinfo',
+        '/nodeinfo/2.0',
+        '/nodeinfo/2.0.json',
+        '/nodeinfo/2.1.json',
+        '/main/nodeinfo/2.0',
+        '/api/statusnet/config',
+        '/api/nodeinfo/2.0.json',
+        '/api/nodeinfo',
+        '/wp-json/nodeinfo/2.0',
+        '/api/v1/instance/nodeinfo/2.0',
+        '/.well-known/x-nodeinfo2'
+        ]
+
+def is_json(myjson):
+
+    try:
+        json_object = json.loads(myjson)
+    except ValueError as e:
+        return False
+    return True
+
+@ray.remote
+def getsoft(server):
+
+    if server.find(".") == -1:
+        return
+    if server.find("@") != -1:
+        return
+    if server.find("/") != -1:
+        return
+    if server.find(":") != -1:
+        return
+
+    soft = ''
+
+    is_nodeinfo = False
+
+    url = 'https://' + server
+
+    try:
+
+        response = requests.get(url + '/.well-known/nodeinfo', headers = setup.user_agent, timeout=3)
+
+        if response.status_code == 200:
+
+            try:
+
+                response_json = response.json()
+
+                if len(response_json['links']) == 1:
+
+                    nodeinfo = response_json['links'][0]['href'].replace(f'https://{server}','')
+
+                elif len(response_json['links']) == 2:
+
+                    nodeinfo = response_json['links'][1]['href'].replace(f'https://{server}','')
+
+                try:
+
+                    nodeinfo_data = requests.get(url + nodeinfo, headers = setup.user_agent, timeout=3)
+
+                    if nodeinfo_data.status_code == 200:
+
+                        nodeinfo_json = nodeinfo_data.json()
+
+                        is_nodeinfo = True
+
+                    else:
+
+                        print(f"{nodeinfo} not responding: error code {nodeinfo_data.status_code}")
+
+                except:
+
+                    pass
+
+            except:
+
+                print(f'{server} is not responding: error code {response.status_code}')
+                print('*********************************************************************')
+
+                pass
+        else:
+
+            for api in apis:
+
+                try:
+
+                    response = requests.get(url + api, headers = setup.user_agent, timeout=3)
+
+                    if response.status_code == 200:
+
+                        if is_json(response.text):
+
+                            nodeinfo_json = response.json()
+
+                            if 'software' in nodeinfo_json:
+
+                                nodeinfo = api
+
+                                is_nodeinfo = True
+
+                                break
+
+                            elif 'title' in nodeinfo_json:
+
+                                if nodeinfo_json['title'] == 'Zap':
+
+                                    nodeinfo = api
+
+                                    is_nodeinfo = True
+
+                                    soft = 'zap'
+
+                                    break
+
+                            elif 'version' in nodeinfo_json:
+
+                                nodeinfo = api
+
+                                is_nodeinfo = True
+
+                                break
+
+                except:
+
+                    pass
+
+    except requests.exceptions.SSLError as errssl:
+
+        pass
+
+    except requests.exceptions.HTTPError as errh:
+
+        pass
+
+    except requests.exceptions.ConnectionError as errc:
+
+        pass
+
+    except requests.exceptions.ReadTimeout as to_err:
+
+        pass
+
+    except requests.exceptions.TooManyRedirects as tmr_err:
+
+        pass
+
+    except urllib3.exceptions.LocationParseError as lp_err:
+
+        pass
+
+    except requests.exceptions.InvalidURL as iu_err:
+
+        pass
+
+    except requests.exceptions.ChunkedEncodingError as chunk_err:
+
+        print(f'ChunkedEncodingError! {server}')
+        pass
+
+    except ray.exceptions.RaySystemError as ray_sys_error:
+
+        print(ray_sys_error) 
+        pass
+
+    else:
+
+        if is_nodeinfo:
+
+            if nodeinfo != '/api/v1/instance?':
+
+                if nodeinfo != '/.well-known/x-nodeinfo2?':
+
+                    try:
+
+                        soft = nodeinfo_json['software']['name']
+                        soft = soft.lower()
+                        soft_version = nodeinfo_json['software']['version']
+                        users = nodeinfo_json.get('usage').get('users').get('total') or '0'
+                        if int(users) > 1000000:
+                            return
+                        self.mau = nodeinfo_json.get('usage').get('users').get('activeMonth') or 0
+                        alive = True
+
+                        db.write_api(server, soft, users, mau, alive, nodeinfo, soft_version)
+
+                        print(f"Server {server} ({soft} {soft_version}) is alive!")
+                        print('*********************************************************************')
+
+                        return
+
+                    except:
+
+                        pass
+
+                else:
+
+                    try:
+
+                        soft = nodeinfo_json['server']['software']
+                        soft = soft.lower()
+                        soft_version = nodeinfo_json['server']['version']
+                        users = nodeinfo_json['usage']['users']['total']
+                        if int(users) > 1000000:
+                            return
+                        self.mau = nodeinfo_json.get('usage').get('users').get('activeMonth') or 0
+                        alive = True
+
+                        if soft == 'socialhome':
+
+                            db.write_api(server, soft, users, mau, alive, nodeinfo, soft_version)
+
+                            print('*********************************************************************')
+                            print(f"Server {serve}r ({soft} {soft_version}) is alive!")
+                            print('*********************************************************************')
+
+                        return
+
+                    except:
+
+                        pass
+
+            if soft == '' and nodeinfo == "/api/v1/instance?":
+
+                soft = 'mastodon'
+
+                try:
+
+                    users = nodeinfo_json['stats']['user_count']
+
+                    if int(users) > 1000000:
+
+                        return
+
+                except:
+
+                    users = 0
+
+                try:
+
+                    soft_version = nodeinfo_json['version']
+
+                except:
+
+                    soft_version = 'unknown'
+
+                mau = 0
+                alive = True
+
+                db.write_api(server, soft, users, mau, alive, nodeinfo, soft_version)
+
+                print('*********************************************************************')
+                print(f"Server {server} ({soft}) is alive!")
+
+            elif soft == 'zap' and nodeinfo == "/api/v1/instance?":
+
+                soft = 'zap'
+                users = nodeinfo_json['stats']['user_count']
+                soft_version = nodeinfo_json['version']
+                alive = True
+
+                print(server, soft, users, alive, api)
+
+                print('*********************************************************************')
+                print(f"Server {server} ({soft}) is alive!")
+
+        else:
+
+            print(f'Server {server} is dead')
+            print('*********************************************************************')
+
+###############################################################################
+# main
+
+if __name__ == '__main__':
+
+    ## name: fetchservers.py
+
+    setup = Setup()
+
+    db = Database()
+
+    res = requests.get('https://' + 'mastodon.social' + setup.peers_api, headers = setup.user_agent, timeout=3)
+
+    hostname_peers = res.json()
+
+    start = datetime.now()
+
+    program = 'fetchservers'
+
+    finish = start
+
+    db.save_time(program, start, finish)
+
+    now = start
+
+    ray_start = time.time()
+
+    try:
+
+        results = ray.get([getsoft.remote(server) for server in hostname_peers])
+
+        print(f"duration = {time.time() - ray_start}.\nprocessed servers: {len(results)}")
+
+    except:
+
+        pass
+
+    finish = datetime.now()
+
+    db.save_time(program, start, finish)

scripts/query.py

@@ -0,0 +1,300 @@
+import time
+from datetime import datetime
+import os
+import json
+import sys
+import os.path
+import requests
+import urllib3
+import socket
+from database import Database
+from setup import Setup
+import pdb
+
+apis = ['/api/v1/instance?',
+        '/api/v1/nodeinfo?',
+        '/nodeinfo/2.0?',
+        '/nodeinfo/2.0.json?',
+        '/nodeinfo/2.1.json?',
+        '/main/nodeinfo/2.0?',
+        '/api/statusnet/config?',
+        '/api/nodeinfo/2.0.json?',
+        '/api/nodeinfo?',
+        '/wp-json/nodeinfo/2.0?',
+        '/api/v1/instance/nodeinfo/2.0?',
+        '/.well-known/x-nodeinfo2?'
+        ]
+
+def is_json(myjson):
+
+    try:
+        json_object = json.loads(myjson)
+    except ValueError as e:
+        return False
+    return True
+
+class Query():
+
+    name = "Query server data"
+
+    def __init__(self, server=None, db=None, setup=None, soft=None, soft_version=None, version=None, users=None, mau=None, alive=False):
+
+        self.server = server
+        self.db = Database()
+        self.setup = Setup()
+        self.soft = ''
+        self.soft_version = ''
+        self.users = 0
+        self.mau = 0
+        self.alive = alive
+
+    def getsoft(self, server):
+
+        if server.find(".") == -1:
+            return
+        if server.find("@") != -1:
+            return
+        if server.find("/") != -1:
+            return
+        if server.find(":") != -1:
+            return
+
+        is_nodeinfo = False
+
+        url = 'https://' + server
+
+        try:
+
+            response = requests.get(url + '/.well-known/nodeinfo', headers = self.setup.user_agent, timeout=3)
+
+            if response.status_code == 200:
+
+                try:
+
+                    response_json = response.json()
+
+                    if len(response_json['links']) == 1:
+
+                        nodeinfo = response_json['links'][0]['href'].replace(f'https://{server}','')
+
+                    elif len(response_json['links']) == 2:
+
+                        nodeinfo = response_json['links'][1]['href'].replace(f'https://{server}','')
+
+                    try:
+
+                        nodeinfo_data = requests.get(url + nodeinfo, headers = self.setup.user_agent, timeout=3)
+
+                        if nodeinfo_data.status_code == 200:
+
+                            nodeinfo_json = nodeinfo_data.json()
+
+                            is_nodeinfo = True
+
+                        else:
+
+                            print(f"{nodeinfo} not responding: error code {nodeinfo_data.status_code}")
+
+                    except:
+
+                        pass
+
+                except:
+
+                    print(f'{server} is not responding: error code {response.status_code}')
+                    print('*********************************************************************')
+
+                    pass
+            else:
+
+                for api in apis:
+
+                    try:
+
+                        response = requests.get(url + api, headers = self.setup.user_agent, timeout=3)
+
+                        if response.status_code == 200:
+
+                            if is_json(response.text):
+
+                                nodeinfo_json = response.json()
+
+                                if 'software' in nodeinfo_json:
+
+                                    nodeinfo = api
+
+                                    is_nodeinfo = True
+
+                                    break
+
+                                elif 'title' in nodeinfo_json:
+
+                                    if nodeinfo_json['title'] == 'Zap':
+
+                                        nodeinfo = api
+
+                                        is_nodeinfo = True
+
+                                        soft = 'zap'
+
+                                        break
+
+                                elif 'version' in nodeinfo_json:
+
+                                    nodeinfo = api
+
+                                    is_nodeinfo = True
+
+                                    break
+
+                    except:
+
+                        pass
+
+        except requests.exceptions.SSLError as errssl:
+
+            pass
+
+        except requests.exceptions.HTTPError as errh:
+
+            pass
+
+        except requests.exceptions.ConnectionError as errc:
+
+            pass
+
+        except requests.exceptions.ReadTimeout as to_err:
+
+            pass
+
+        except requests.exceptions.TooManyRedirects as tmr_err:
+
+            pass
+
+        except urllib3.exceptions.LocationParseError as lp_err:
+
+            pass
+
+        except requests.exceptions.InvalidURL as iu_err:
+
+            pass
+
+        except requests.exceptions.ChunkedEncodingError as chunk_err:
+
+            print(f'ChunkedEncodingError! {server}')
+            pass
+
+        except ray.exceptions.RaySystemError as ray_sys_error:
+
+            print(ray_sys_error) 
+            pass
+
+        else:
+
+            if is_nodeinfo:
+
+                if nodeinfo != '/api/v1/instance?':
+
+                    if nodeinfo != '/.well-known/x-nodeinfo2?':
+
+                        try:
+
+                            self.soft = nodeinfo_json['software']['name']
+                            self.soft = self.soft.lower()
+                            self.soft_version = nodeinfo_json['software']['version']
+                            self.users = nodeinfo_json.get('usage').get('users').get('total') or '0'
+                            if int(self.users) > 1000000:
+                                return
+                            self.mau = nodeinfo_json.get('usage').get('users').get('activeMonth') or 0
+
+                            self.alive = True
+
+                            self.db.write_api(server, self.soft, self.users, self.mau, self.alive, nodeinfo, self.soft_version)
+
+                            print(f"Server {server} ({self.soft} {self.soft_version}) is alive!")
+                            print('*********************************************************************')
+
+                            return (server, self.soft, self.soft_version, self.users, self.mau, self.alive)
+
+                        except:
+
+                            pass
+
+                    else:
+
+                        try:
+
+                            self.soft = nodeinfo_json['server']['software']
+                            self.soft = soft.lower()
+                            self.soft_version = nodeinfo_json['server']['version']
+                            self.users = nodeinfo_json['usage']['users']['total']
+                            if int(self.users) > 1000000:
+                                return
+                            self.mau = nodeinfo_json.get('usage').get('users').get('activeMonth') or 0
+
+                            self.alive = True
+
+                            if self.soft == 'socialhome':
+
+                                self.db.write_api(server, soft, users, alive, nodeinfo, soft_version)
+
+                                print('*********************************************************************')
+                                print(f"Server {server} ({self.soft} {self.soft_version}) is alive!")
+                                print('*********************************************************************')
+
+                            return
+
+                        except:
+
+                            pass
+
+                if self.soft == '' and nodeinfo == "/api/v1/instance?":
+
+                    self.soft = 'mastodon'
+
+                    try:
+
+                        self.users = nodeinfo_json['stats']['user_count']
+
+                        if int(self.users) > 1000000:
+
+                            return
+
+                    except:
+
+                        self.users = 0
+
+                    try:
+
+                        self.soft_version = nodeinfo_json['version']
+
+                    except:
+
+                        self.soft_version = 'unknown'
+
+                    alive = True
+
+                    self.db.write_api(server, self.soft, self.users, self.alive, nodeinfo, self.soft_version)
+
+                    print('*********************************************************************')
+                    print(f"Server {server} ({self.soft}) is alive!")
+
+                elif self.soft == 'zap' and nodeinfo == "/api/v1/instance?":
+
+                    self.soft = 'zap'
+                    self.users = nodeinfo_json['stats']['user_count']
+                    self.soft_version = nodeinfo_json['version']
+                    self.alive = True
+
+                    print(server, self.soft, self.users, self.alive, api)
+
+                    print('*********************************************************************')
+                    print(f"Server {server} ({self.soft}) is alive!")
+
+            else:
+
+                print(f'Server {server} is dead')
+                print('*********************************************************************')
+
+        return (server, self.soft, self.soft_version, self.users, self.mau, self.alive)
+

scripts/uptime.py

@@ -0,0 +1,44 @@
+import time
+import os
+from datetime import datetime, timedelta
+import humanfriendly
+from setup import Setup
+from database import Database
+from mastodon import Mastodon
+
+# main
+
+if __name__ == '__main__':
+
+    setup = Setup()
+
+    mastodon = Mastodon(
+            access_token = setup.mastodon_app_token,
+            api_base_url= setup.mastodon_hostname
+            )
+
+    db = Database()
+
+    alive_servers, max_uptime, best_servers, software_lst, servers_lst = db.get_uptime()
+    
+    toot_text = '\nAlive servers: ' + str(alive_servers)
+    toot_text += '\n\n'
+    toot_text += f"Best #fediverse's server uptime:\n{humanfriendly.format_timespan(max_uptime)}"
+    toot_text += '\n'
+    toot_text += f'\nBest uptime servers:\n{str(best_servers)} ({str(round((best_servers*100)/alive_servers,2))}%)'
+    toot_text += '\n\n'
+    toot_text += 'Best uptime softwares & servers:\n'
+    i = 0
+    while i < len(software_lst):
+
+        soft_percent = db.get_percentage(servers_lst[i], software_lst[i])
+        toot_text += ':' + str(software_lst[i]) + ': ' + str(servers_lst[i]) + ' (' + str(soft_percent) + '%)\n'
+        
+        if len(toot_text) > 470:
+            break
+        i +=1
+
+    print("Tooting...")
+    print(toot_text)
+
+    mastodon.status_post(toot_text, in_reply_to_id=None)

setup.py

@@ -1,226 +0,0 @@
-import getpass
-from mastodon import Mastodon
-from mastodon.Mastodon import MastodonMalformedEventError, MastodonNetworkError, MastodonReadTimeout, MastodonAPIError, MastodonIllegalArgumentError
-import fileinput,re
-import os
-import sys
-
-def create_dir():
-
-    if not os.path.exists('secrets'):
-        os.makedirs('secrets')
-
-def create_file():
-
-    if not os.path.exists('secrets/secrets.txt'):
-        with open('secrets/secrets.txt', 'w'): pass
-        print(secrets_filepath + " created!")
-
-def create_config():
-
-    if not os.path.exists('config'):
-        os.makedirs('config')
-    if not os.path.exists(config_filepath):
-        print(config_filepath + " created!")
-        with open('config/config.txt', 'w'): pass
-
-def write_params():
-
-    with open(secrets_filepath, 'a') as the_file:
-        print("Writing secrets parameter names to " + secrets_filepath)
-        the_file.write('uc_client_id: \n'+'uc_client_secret: \n'+'uc_access_token: \n')
-
-def write_config():
-
-    with open(config_filepath, 'a') as the_file:
-        the_file.write('mastodon_hostname: \n')
-        print("adding parameter name 'mastodon_hostname' to "+ config_filepath)
-
-def read_client_lines(self):
-
-    client_path = 'app_clientcred.txt'
-
-    with open(client_path) as fp:
-
-        line = fp.readline()
-        cnt = 1
-        while line:
-           if cnt == 1:
-               print("Writing client id to " + secrets_filepath)
-               modify_file(secrets_filepath, "uc_client_id: ", value=line.rstrip())
-           elif cnt == 2:
-               print("Writing client secret to " + secrets_filepath)
-               modify_file(secrets_filepath, "uc_client_secret: ", value=line.rstrip())
-           line = fp.readline()
-           cnt += 1
-
-def read_token_line(self):
-
-    token_path = 'app_usercred.txt'
-
-    with open(token_path) as fp:
-
-        line = fp.readline()
-        print("Writing access token to " + secrets_filepath)
-        modify_file(secrets_filepath, "uc_access_token: ", value=line.rstrip())
-
-def read_config_line():
-
-    with open(config_filepath) as fp:
-
-        line = fp.readline()
-        modify_file(config_filepath, "mastodon_hostname: ", value=hostname)
-        modify_file(config_filepath, "bot_username: ", value=bot_username)
-
-def log_in():
-
-    error = 0
-
-    try:
-
-        global hostname
-        hostname = input("Enter Mastodon hostname: ")
-        user_name = input("User name, ex. user@" + hostname +"? ")
-        user_password = getpass.getpass("User password? ")
-        bot_username = input("Bot's username, ex. fediverse: ")
-        app_name = input("This app name? ")
-        Mastodon.create_app(
-            app_name,
-            scopes=["read","write"],
-            to_file="app_clientcred.txt",
-            api_base_url=hostname
-            )
-        mastodon = Mastodon(client_id = "app_clientcred.txt", api_base_url = hostname)
-        mastodon.log_in(
-            user_name,
-            user_password,
-            scopes = ["read", "write"],
-            to_file = "app_usercred.txt"
-            )
-
-    except MastodonIllegalArgumentError as i_error:
-
-        error = 1
-
-        if os.path.exists("secrets/secrets.txt"):
-            print("Removing secrets/secrets.txt file..")
-            os.remove("secrets/secrets.txt")
-        if os.path.exists("app_clientcred.txt"):
-            print("Removing app_clientcred.txt file..")
-            os.remove("app_clientcred.txt")
-        sys.exit(i_error)
-
-    except MastodonNetworkError as n_error:
-
-        error = 1
-
-        if os.path.exists("secrets/secrets.txt"):
-            print("Removing secrets/secrets.txt file..")
-            os.remove("secrets/secrets.txt")
-        if os.path.exists("app_clientcred.txt"):
-            print("Removing app_clientcred.txt file..")
-            os.remove("app_clientcred.txt")
-        sys.exit(n_error)
-
-    except MastodonReadTimeout as r_error:
-
-        error = 1
-
-        if os.path.exists("secrets/secrets.txt"):
-            print("Removing secrets/secrets.txt file..")
-            os.remove("secrets/secrets.txt")
-        if os.path.exists("app_clientcred.txt"):
-            print("Removing app_clientcred.txt file..")
-            os.remove("app_clientcred.txt")
-        sys.exit(r_error)
-
-    except MastodonAPIError as a_error:
-
-        error = 1
-
-        if os.path.exists("secrets/secrets.txt"):
-            print("Removing secrets/secrets.txt file..")
-            os.remove("secrets/secrets.txt")
-        if os.path.exists("app_clientcred.txt"):
-            print("Removing app_clientcred.txt file..")
-            os.remove("app_clientcred.txt")
-        sys.exit(a_error)
-
-    finally:
-
-        if error == 0:
-
-            create_dir()
-            create_file()
-            write_params()
-            client_path = 'app_clientcred.txt'
-            read_client_lines(client_path)
-            token_path = 'app_usercred.txt'
-            read_token_line(token_path)
-            if os.path.exists("app_clientcred.txt"):
-                print("Removing app_clientcred.txt temp file..")
-                os.remove("app_clientcred.txt")
-            if os.path.exists("app_usercred.txt"):
-                print("Removing app_usercred.txt temp file..")
-                os.remove("app_usercred.txt")
-            print("Secrets setup done!\n")
-
-def modify_file(file_name,pattern,value=""):
-
-    fh=fileinput.input(file_name,inplace=True)
-    for line in fh:
-        replacement=pattern + value
-        line=re.sub(pattern,replacement,line)
-        sys.stdout.write(line)
-    fh.close()
-
-def get_parameter( parameter, file_path ):
-    # Check if secrets file exists
-    if not os.path.isfile(file_path):
-        print("File %s not found, creating it."%file_path)
-        log_in()
-
-    # Find parameter in file
-    with open( file_path ) as f:
-        for line in f:
-            if line.startswith( parameter ):
-                return line.replace(parameter + ":", "").strip()
-
-    # Cannot find parameter, exit
-    print(file_path + " Missing parameter %s "%parameter)
-    sys.exit(0)
-
-def get_hostname( parameter, config_filepath ):
-    # Check if secrets file exists
-    if not os.path.isfile(config_filepath):
-        print("File %s not found, creating it."%config_filepath)
-        create_config()
-
-    # Find parameter in file
-    with open( config_filepath ) as f:
-        for line in f:
-            if line.startswith( parameter ):
-                return line.replace(parameter + ":", "").strip()
-
-    # Cannot find parameter, exit
-    print(config_filepath + "  Missing parameter %s "%parameter)
-    write_config()
-    read_config_line()
-    print("setup done!")
-    sys.exit(0)
-
-###############################################################################
-# main
-
-if __name__ == '__main__':
-
-    # Load secrets from secrets file
-    secrets_filepath = "secrets/secrets.txt"
-    uc_client_id     = get_parameter("uc_client_id",     secrets_filepath)
-    uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
-    uc_access_token  = get_parameter("uc_access_token",  secrets_filepath)
-
-    # Load configuration from config file
-    config_filepath = "config/config.txt"
-    mastodon_hostname = get_hostname("mastodon_hostname", config_filepath)
-    bot_username = get_parameter("bot_username", config_filepath)

tests/__init__.py

@@ -0,0 +1 @@
+#!/usr/bin/env python3

tests/test_database.py

@@ -0,0 +1,15 @@
+from federation import Database
+
+def test__database_connection():
+    db = Database(config_file='config/test_db_config.txt', fediverse_db_host='localhost', fediverse_db_port='5432')
+
+    # noexcept
+    db.check_connection()
+
+
+def test__database_connection__noservers():
+    db = Database(config_file='config/test_db_config.txt', fediverse_db_host='localhost', fediverse_db_port='5432')
+
+    checked_server = db.get_last_checked_servers()
+
+    assert checked_server == []

uptime.py

@@ -1,182 +0,0 @@
-import time
-import os
-from datetime import datetime, timedelta
-import humanfriendly
-from mastodon import Mastodon
-import psycopg2
-
-def get_uptime():
-    
-    try:
-
-        conn = None
-
-        conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
-
-        cur = conn.cursor()
-
-        cur.execute("select count(server) from fediverse where alive")
-
-        row = cur.fetchone()
-
-        alive_servers = row[0]
-
-        cur.execute("select MAX(last_checked_at - first_checked_at) from fediverse")
-
-        row = cur.fetchone()
-
-        max_uptime = row[0]
-
-        cur.execute("select count(server) from fediverse where last_checked_at-first_checked_at=(%s)", (max_uptime,))
-
-        row = cur.fetchone()
-
-        if row is not None:
-
-            best_servers = row[0]
-
-        cur.execute("select software, count(*) as servers from fediverse where last_checked_at-first_checked_at=(%s) group by software order by servers desc", (max_uptime,))
-
-        rows = cur.fetchall()
-
-        software_lst = []
-
-        servers_lst = []
-
-        for row in rows:
-
-            software_lst.append(row[0])
-
-            servers_lst.append(row[1])
-
-        cur.close()
-
-        return (alive_servers, max_uptime, best_servers, software_lst, servers_lst)
-
-    except (Exception, psycopg2.DatabaseError) as error:
-
-        print(error)
-
-    finally:
-
-        if conn is not None:
-
-            conn.close()
-
-def get_percentage(uptime_servers, software):
-
-    try:
-
-        conn = None
-
-        conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
-
-        cur = conn.cursor()
-
-        cur.execute("select count(server) from fediverse where alive and software=(%s)", (software,))
-
-        row = cur.fetchone()
-
-        if row is not None:
-
-            soft_total_servers = row[0]
-
-        cur.close()
-
-        soft_uptime_percent = round((uptime_servers * 100) / soft_total_servers, 1)
-
-        return (soft_uptime_percent)
-
-    except (Exception, psycopg2.DatabaseError) as error:
-
-        print(error)
-
-    finally:
-
-        if conn is not None:
-
-            conn.close()
-
-def mastodon():
-
-    # Load secrets from secrets file
-    secrets_filepath = "secrets/uptime_secrets.txt"
-    uc_client_id = get_parameter("uc_client_id",     secrets_filepath)
-    uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
-    uc_access_token = get_parameter("uc_access_token",  secrets_filepath)
-
-    # Load configuration from config file
-    config_filepath = "config/config.txt"
-    mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
-
-    # Initialise Mastodon API
-    mastodon = Mastodon(
-        client_id=uc_client_id,
-        client_secret=uc_client_secret,
-        access_token=uc_access_token,
-        api_base_url='https://' + mastodon_hostname,
-    )
-
-    # Initialise access headers
-    headers = {'Authorization': 'Bearer %s'%uc_access_token}
-
-    return (mastodon, mastodon_hostname)
-
-def db_config():
-
-    # Load db configuration from config file
-    config_filepath = "config/db_config.txt"
-    fediverse_db = get_parameter("fediverse_db", config_filepath)
-    fediverse_db_user = get_parameter("fediverse_db_user", config_filepath)
-
-    return (fediverse_db, fediverse_db_user)
-
-# Returns the parameter from the specified file
-def get_parameter(parameter, file_path):
-
-    # Check if secrets file exists
-    if not os.path.isfile(file_path):
-        print("File %s not found, exiting."%file_path)
-        sys.exit(0)
-
-    # Find parameter in file
-    with open(file_path) as f:
-        for line in f:
-            if line.startswith(parameter):
-                return line.replace(parameter + ":", "").strip()
-
-    # Cannot find parameter, exit
-    print(file_path + "  Missing parameter %s "%parameter)
-    sys.exit(0)
-
-# main
-
-if __name__ == '__main__':
-
-    mastodon, mastodon_hostname = mastodon()
-
-    fediverse_db, fediverse_db_user = db_config()
-    
-    alive_servers, max_uptime, best_servers, software_lst, servers_lst = get_uptime()
-    
-    toot_text = '\nAlive servers: ' + str(alive_servers)
-    toot_text += '\n\n'
-    toot_text += f"Best #fediverse's server uptime:\n{humanfriendly.format_timespan(max_uptime)}"
-    toot_text += '\n'
-    toot_text += f'\nBest uptime servers:\n{str(best_servers)} ({str(round((best_servers*100)/alive_servers,2))}%)'
-    toot_text += '\n\n'
-    toot_text += 'Best uptime softwares & servers:\n'
-    i = 0
-    while i < len(software_lst):
-
-        soft_percent = get_percentage(servers_lst[i], software_lst[i])
-        toot_text += ':' + str(software_lst[i]) + ': ' + str(servers_lst[i]) + ' (' + str(soft_percent) + '%)\n'
-        
-        if len(toot_text) > 470:
-            break
-        i +=1
-
-    print("Tooting...")
-    print(toot_text)
-
-    mastodon.status_post(toot_text, in_reply_to_id=None)

uptime_setup.py

@@ -1,188 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import getpass
-from mastodon import Mastodon
-from mastodon.Mastodon import MastodonMalformedEventError, MastodonNetworkError, MastodonReadTimeout, MastodonAPIError, MastodonIllegalArgumentError
-import fileinput,re
-import os
-import sys
-
-def create_dir():
-  if not os.path.exists('secrets'):
-    os.makedirs('secrets')
-
-def create_file():
-  if not os.path.exists('secrets/uptime_secrets.txt'):
-    with open('secrets/uptime_secrets.txt', 'w'): pass
-    print(secrets_filepath + " created!")
-
-def create_config():
-  if not os.path.exists('config'):
-    os.makedirs('config')
-  if not os.path.exists(config_filepath):
-    print(config_filepath + " created!")
-    with open('config/config.txt', 'w'): pass
-
-def write_params():
-  with open(secrets_filepath, 'a') as the_file:
-    print("Writing secrets parameter names to " + secrets_filepath)
-    the_file.write('uc_client_id: \n'+'uc_client_secret: \n'+'uc_access_token: \n')
-
-def write_config():
-  with open(config_filepath, 'a') as the_file:
-    the_file.write('mastodon_hostname: \n')
-    print("adding parameter name 'mastodon_hostname' to "+ config_filepath)
-
-def read_client_lines(self):
-  client_path = 'app_clientcred.txt'
-  with open(client_path) as fp:
-    line = fp.readline()
-    cnt = 1
-    while line:
-       if cnt == 1:
-         print("Writing client id to " + secrets_filepath)
-         modify_file(secrets_filepath, "uc_client_id: ", value=line.rstrip())
-       elif cnt == 2:
-         print("Writing client secret to " + secrets_filepath)
-         modify_file(secrets_filepath, "uc_client_secret: ", value=line.rstrip())
-       line = fp.readline()
-       cnt += 1
-
-def read_token_line(self):
-  token_path = 'app_usercred.txt'
-  with open(token_path) as fp:
-    line = fp.readline()
-    print("Writing access token to " + secrets_filepath)
-    modify_file(secrets_filepath, "uc_access_token: ", value=line.rstrip())
-
-def read_config_line():
-  with open(config_filepath) as fp:
-    line = fp.readline()
-    modify_file(config_filepath, "mastodon_hostname: ", value=hostname)
-
-def log_in():
-  error = 0
-  try:
-    global hostname
-    hostname = input("Enter Mastodon hostname: ")
-    user_name = input("User name, ex. user@" + hostname +"? ")
-    user_password = getpass.getpass("User password? ")
-    app_name = input("This app name? ")
-    Mastodon.create_app(app_name, scopes=["read","write"],
-      to_file="app_clientcred.txt", api_base_url=hostname)
-    mastodon = Mastodon(client_id = "app_clientcred.txt", api_base_url = hostname)
-    mastodon.log_in(
-       user_name,
-       user_password,
-       scopes = ["read", "write"],
-       to_file = "app_usercred.txt"
-    )
-  except MastodonIllegalArgumentError as i_error:
-    error = 1
-    if os.path.exists("secrets/uptime_secrets.txt"):
-      print("Removing secrets/uptime_secrets.txt file..")
-      os.remove("secrets/uptime_secrets.txt")
-    if os.path.exists("app_clientcred.txt"):
-      print("Removing app_clientcred.txt file..")
-      os.remove("app_clientcred.txt")
-    sys.exit(i_error)
-  except MastodonNetworkError as n_error:
-    error = 1
-    if os.path.exists("secrets/uptime_secrets.txt"):
-      print("Removing secrets/uptime_secrets.txt file..")
-      os.remove("secrets/uptime_secrets.txt")
-    if os.path.exists("app_clientcred.txt"):
-      print("Removing app_clientcred.txt file..")
-      os.remove("app_clientcred.txt")
-    sys.exit(n_error)
-  except MastodonReadTimeout as r_error:
-    error = 1
-    if os.path.exists("secrets/uptime_secrets.txt"):
-      print("Removing secrets/uptime_secrets.txt file..")
-      os.remove("secrets/uptime_secrets.txt")
-    if os.path.exists("app_clientcred.txt"):
-      print("Removing app_clientcred.txt file..")
-      os.remove("app_clientcred.txt")
-    sys.exit(r_error)
-  except MastodonAPIError as a_error:
-    error = 1
-    if os.path.exists("secrets/uptime_secrets.txt"):
-      print("Removing secrets/uptime_secrets.txt file..")
-      os.remove("secrets/uptime_secrets.txt")
-    if os.path.exists("app_clientcred.txt"):
-      print("Removing app_clientcred.txt file..")
-      os.remove("app_clientcred.txt")
-    sys.exit(a_error)
-  finally:
-    if error == 0:
-
-      create_dir()
-      create_file()
-      write_params()
-      client_path = 'app_clientcred.txt'
-      read_client_lines(client_path)
-      token_path = 'app_usercred.txt'
-      read_token_line(token_path)
-      if os.path.exists("app_clientcred.txt"):
-        print("Removing app_clientcred.txt temp file..")
-        os.remove("app_clientcred.txt")
-      if os.path.exists("app_usercred.txt"):
-        print("Removing app_usercred.txt temp file..")
-        os.remove("app_usercred.txt")
-      print("uptime secrets setup done!\n")
-
-def modify_file(file_name,pattern,value=""):
-    fh=fileinput.input(file_name,inplace=True)
-    for line in fh:
-        replacement=pattern + value
-        line=re.sub(pattern,replacement,line)
-        sys.stdout.write(line)
-    fh.close()
-
-# Returns the parameter from the specified file
-def get_parameter( parameter, file_path ):
-    # Check if secrets file exists
-    if not os.path.isfile(file_path):
-        print("File %s not found, creating it."%file_path)
-        log_in()
-
-    # Find parameter in file
-    with open( file_path ) as f:
-        for line in f:
-            if line.startswith( parameter ):
-                return line.replace(parameter + ":", "").strip()
-
-    # Cannot find parameter, exit
-    print(file_path + " Missing parameter %s "%parameter)
-    sys.exit(0)
-
-# Returns the parameter from the specified file
-def get_hostname( parameter, config_filepath ):
-    # Check if secrets file exists
-    if not os.path.isfile(config_filepath):
-        print("File %s not found, creating it."%config_filepath)
-        create_config()
-
-    # Find parameter in file
-    with open( config_filepath ) as f:
-        for line in f:
-            if line.startswith( parameter ):
-                return line.replace(parameter + ":", "").strip()
-
-    # Cannot find parameter, exit
-    print(config_filepath + "  Missing parameter %s "%parameter)
-    write_config()
-    read_config_line()
-    print("setup done!")
-    sys.exit(0)
-
-# Load secrets from secrets file
-secrets_filepath = "secrets/uptime_secrets.txt"
-uc_client_id     = get_parameter("uc_client_id",     secrets_filepath)
-uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
-uc_access_token  = get_parameter("uc_access_token",  secrets_filepath)
-
-# Load configuration from config file
-#config_filepath = "config/config.txt"
-#mastodon_hostname = get_hostname("mastodon_hostname", config_filepath)