README.md

@@ -1,5 +1,5 @@
 # Fediverse Stats
-This code gets all peers from mastodon.social. Goal is to collect maximum number
+This code gets all peers from running Mastodon, Pleroma and Lemmy host servers and then all peers from host server's peers. Goal is to collect maximum number
 of alive fediverse's servers and then query their API to obtain their registered users (if their API provide such information).
 At the end it post the results to host server bot account.
 
@@ -7,7 +7,7 @@ At the end it post the results to host server bot account.
 
 -   **Python 3**
 -   Postgresql server
--   Mastodon running server.
+-   Mastodon or Pleroma running server.
 
 ### Usage:
 
@@ -15,11 +15,19 @@ Within Python Virtual Environment:
 
 1. Run `pip install -r requirements.txt` to install needed libraries.
 
-2. Run `python fetchservers.py` to add servers to alive servers database.
+2. Run `python db-setup.py` to setup and create new Postgresql database and needed tables in it.
 
-3. Run `python fediverse.py` to query world alive servers API. It gets data from server's nodeinfo.
+3. Run `python setup.py` to get your bot's access token of your Mastodon or Pleroma server existing account. It will be saved to 'secrets/secrets.txt' for further use.
 
-4. Use your favourite scheduling method to set `python fediverse.py` to run twice daily, `python fetchservers.py` one time daily.
+4. Run `python getpeers.py` to get all peers from your host and the whole world of fediverse's servers (or almost the whole world).
+
+5. Run `python fetchservers.py` to add servers to alive servers database.
+
+6. Run `python fediverse.py` to query world alive servers API. It gets data from server's nodeinfo.
+
+7. Run `python uptime_setup.py` to get your Uptime bot's access token of your Mastodon or Pleroma server existing account. It will be saved to 'secrets/uptime_secrets.txt' for further use.
+
+8. Use your favourite scheduling method to set `python fediverse.py` to run twice daily, `python fetchservers.py` one time daily, `python getworld.py` to run monthly and `python uptime.py` (choose your desired frequency) if you want to publish best fediverse's servers uptime.   
 
 18.2.2021 - New feature! Added [Lemmy project](https://join.lemmy.ml)  
 12.5.2021 - New feature! Added Wordpress support. The code can now detect Wordpress instances with ActivityPub enabled plugin.  
@@ -27,6 +35,4 @@ Within Python Virtual Environment:
 21.8.2021 - New feature! Added Best Fediverse's servers Uptime publishing bot.  
 22.10.2021 - New feature! Added [Funkwhale](https://funkwhale.audio) support.  
 26.10.2021 - New feature! Added [Socialhome](https://socialhome.network) support.  
-2.3.2022 - Improved server nodeinfo detection.
+2.3.2022 - Improved server nodeinfo detection
-4.1.2023 - Refactored.
-4.1.2023 - Now all peers are obtained from mastodon.social's peers list.

fediverse.py

@@ -5,33 +5,13 @@ from datetime import datetime
 import urllib3
 import requests
 import socket
-from setup import Setup
-from database import Database
 from mastodon import Mastodon
-from matplotlib import pyplot as plt
+import psycopg2
-import matplotlib.dates as mdates
+import matplotlib.pyplot as plt
-from matplotlib.ticker import ScalarFormatter
-import numpy as np
-import pandas as pd
 import ray
 import pdb
 
-SMALL_SIZE = 6
+ray.init(num_cpus = 32) # Specify this system CPUs.
-MEDIUM_SIZE = 10
-BIGGER_SIZE = 12
-
-plt.rc('font', size=MEDIUM_SIZE)          # controls default text sizes
-plt.rc('axes', titlesize=MEDIUM_SIZE)     # fontsize of the axes title
-plt.rc('axes', labelsize=MEDIUM_SIZE)    # fontsize of the x and y labels
-plt.rc('xtick', labelsize=SMALL_SIZE)    # fontsize of the tick labels
-plt.rc('ytick', labelsize=MEDIUM_SIZE)    # fontsize of the tick labels
-plt.rc('legend', fontsize=MEDIUM_SIZE)    # legend fontsize
-plt.rc('figure', titlesize=BIGGER_SIZE)  # fontsize of the figure title
-
-mdates.set_epoch('2000-01-01T00:00:00')
-y_formatter = ScalarFormatter(useOffset=False)
-
-ray.init(num_cpus = 25) # Specify this system CPUs.
 
 class Server:
 
@@ -48,35 +28,76 @@ class Server:
 
         downs = 0
 
-        was_alive, software, api, soft_version, first_checked_at, downs_qty = db.get_server_data(self)
+        try:
+
+            conn = None
+
+            conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
+
+            cur = conn.cursor()
+
+            cur.execute("select alive, software, users_api, version, first_checked_at, downs from fediverse where server=(%s)", (self,))
+
+            row = cur.fetchone()
+
+            if row is not None:
+
+                was_alive = row[0]
+                software = row[1]
+                api = row[2]
+                soft_version = row[3]
+                first_checked_at = row[4]
+                downs_qty = row[5]
+
+            cur.close()
+
+        except (Exception, psycopg2.DatabaseError) as error:
+
+            print(error)
+
+        finally:
+
+            if conn is not None:
+
+                conn.close()
 
             alive = False
 
             try:
 
-            data = requests.get('https://' + self + api, headers = setup.user_agent, timeout=3)
+                user_agent = {'User-agent': "fediverse's servers stats (fediverse@mastodont.cat)"}
 
-            nodeinfo_json = data.json()
+                data = requests.get('https://' + self + api, headers = user_agent, timeout=3)
 
                 try:
 
-                users = nodeinfo_json.get('usage').get('users').get('total') or '0'
+                    users = data.json()['usage']['users']['total']
 
-                mau = nodeinfo_json.get('usage').get('users').get('activeMonth') or '0'
+                    if users == 0:
 
-                if software == 'socialhome':
+                        users = data.json()['usage']['users']['activeHalfyear']
 
-                    soft_version = nodeinfo_json['server']['version']
+                    if 'activeMonth' in data.json()['usage']['users'].keys():
+
+                        mau = data.json()['usage']['users']['activeMonth']
 
                     else:
 
-                    soft_version = nodeinfo_json['software']['version']
+                        mau = 0
 
-                if software == "wordpress" and "activitypub" in nodeinfo_json['protocols']:
+                    if software == 'socialhome':
+
+                        soft_version = data.json()['server']['version']
+
+                    else:
+
+                        soft_version = data.json()['software']['version']
+
+                    if software == "wordpress" and "activitypub" in data.json()['protocols']:
 
                         alive = True
 
-                elif software == "wordpress" and "activitypub" not in nodeinfo_json['protocols']:
+                    elif software == "wordpress" and "activitypub" not in data.json()['protocols']:
 
                         alive = False
 
@@ -93,8 +114,8 @@ class Server:
                     if api == '/api/v1/instance?':
 
                         try:
-                        users = nodeinfo_json.get('stats').get('user_count') or '0'
+                            users = data.json()['stats']['user_count']
-                        soft_version = nodeinfo_json['version']
+                            soft_version = data.json()['version']
                             alive = True
                             mau = 0
                         except:
@@ -116,11 +137,11 @@ class Server:
 
                     if software != 'birdsitelive':
 
-                    db.write_alive_server(self, software, soft_version, alive, api, users, downs, first_checked_at, mau)
+                        write_alive_server(self, software, soft_version, alive, api, users, downs, first_checked_at, mau)
 
                     else:
 
-                    db.write_blocked_software(self, software, soft_version, alive, api, users, downs, first_checked_at)
+                        write_blocked_software(self, software, soft_version, alive, api, users, downs, first_checked_at)
 
             except urllib3.exceptions.ProtocolError as protoerr:
 
@@ -184,19 +205,325 @@ class Server:
 
                     downs = 1
 
-            db.write_not_alive_server(self, software, soft_version, alive, api, users, downs, first_checked_at)
+                write_not_alive_server(self, software, soft_version, alive, api, users, downs, first_checked_at)
 
         return (self, software, soft_version, alive, api, users, downs, first_checked_at, mau)
 
+def write_alive_server(server, software, soft_version, alive, api, users, downs, first_checked_at, mau):
+
+    insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version, first_checked_at, last_checked_at, downs, mau) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
+
+    conn = None
+
+    try:
+
+        conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
+
+        cur = conn.cursor()
+
+        cur.execute(insert_sql, (server, users, now, software, alive, api, soft_version, now, now, downs, mau))
+
+        if first_checked_at != None:
+
+            cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), last_checked_at=(%s), downs=(%s), mau=(%s) where server=(%s)", (users, now, software, alive, api, soft_version, now, downs, mau, server))
+
+        else:
+
+            cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), first_checked_at=(%s), last_checked_at=(%s), downs=(%s), mau=(%s) where server=(%s)", (users, now, software, alive, api, soft_version, now, now, downs, mau, server))
+
+            cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
+
+        conn.commit()
+
+        cur.close()
+
+    except (Exception, psycopg2.DatabaseError) as error:
+
+        print(error)
+
+    finally:
+
+        if conn is not None:
+
+            conn.close()
+
+def write_blocked_software(server, software, soft_version, alive, api, users, downs, first_checked_at):
+
+    insert_sql = "INSERT INTO blocked(server, users, updated_at, software, alive, users_api, version, first_checked_at, last_checked_at, downs) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
+
+    conn = None
+
+    try:
+
+        conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
+
+        cur = conn.cursor()
+
+        cur.execute(insert_sql, (server, users, now, software, alive, api, soft_version, now, now, downs))
+
+        if first_checked_at != None:
+
+            cur.execute("UPDATE blocked SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (users, now, software, alive, api, soft_version, now, downs, server))
+
+        else:
+
+            cur.execute("UPDATE blocked SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), first_checked_at=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (users, now, software, alive, api, soft_version, now, now, downs, server))
+
+            cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
+
+        conn.commit()
+
+        cur.close()
+
+    except (Exception, psycopg2.DatabaseError) as error:
+
+        print(error)
+
+    finally:
+
+        if conn is not None:
+
+            conn.close()
+
+def write_not_alive_server(server, software, soft_version, alive, api, users, downs, first_checked_at):
+
+    conn = None
+
+    try:
+
+        conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
+
+        cur = conn.cursor()
+
+        if first_checked_at != None:
+
+            cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), downs=(%s) where server=(%s)", (now, alive, downs, server))
+
+        else:
+
+            cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), first_checked_at=(%s), downs=(%s) where server=(%s)", (now, alive, now, downs, server))
+
+        cur.execute("UPDATE world SET checked='f' where server=(%s)", (server,))
+
+        conn.commit()
+
+        cur.close()
+
+    except (Exception, psycopg2.DatabaseError) as error:
+
+        print(error)
+
+    finally:
+
+        if conn is not None:
+
+            conn.close()
+
+def delete_dead_servers():
+
+    conn = None
+
+    try:
+
+        conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
+
+        cur = conn.cursor()
+
+        cur.execute("select server from fediverse where downs > '14' and not alive and now() - first_checked_at > interval '7 days'")
+
+        rows = cur.fetchall()
+
+        for row in rows:
+
+            print(f'Deleting server {row[0]}...')
+
+            cur.execute("delete from fediverse where server=(%s)", (row[0],))
+
+        conn.commit()
+
+        cur.close()
+
+    except (Exception, psycopg2.DatabaseError) as error:
+
+        print(error)
+
+    finally:
+
+        if conn is not None:
+
+            conn.close()
+
+def get_last_checked_servers():
+
+    ############################################################################
+    # get last checked servers from fediverse DB
+
+    alive_servers = []
+
+    try:
+
+        conn = None
+
+        conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
+
+        cur = conn.cursor()
+
+        # get world servers list
+
+        cur.execute("select server from world where server in (select server from fediverse where users_api != '')")
+
+        alive_servers = []
+
+        for row in cur:
+
+            alive_servers.append(row[0])
+
+        cur.close()
+
+    except (Exception, psycopg2.DatabaseError) as error:
+
+        print(error)
+
+    finally:
+
+        if conn is not None:
+
+            conn.close()
+
+    return alive_servers
+
 def print_dead(server):
 
     print(f'\nServer {server} is dead :-(')
 
+def set_world_servers_check_to_false():
+
+    ############################################################################
+    # set all world servers's checked column to False
+
+    try:
+
+        conn = None
+
+        conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
+
+        cur = conn.cursor()
+
+        cur.execute("UPDATE world SET checked='f'")
+
+        conn.commit()
+
+        cur.close()
+
+    except (Exception, psycopg2.DatabaseError) as error:
+
+        print(error)
+
+    finally:
+
+        if conn is not None:
+
+            conn.close()
+
+def mastodon():
+
+    # Load secrets from secrets file
+    secrets_filepath = "secrets/secrets.txt"
+    uc_client_id = get_parameter("uc_client_id",     secrets_filepath)
+    uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
+    uc_access_token = get_parameter("uc_access_token",  secrets_filepath)
+
+    # Load configuration from config file
+    config_filepath = "config/config.txt"
+    mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
+
+    # Initialise Mastodon API
+    mastodon = Mastodon(
+        client_id=uc_client_id,
+        client_secret=uc_client_secret,
+        access_token=uc_access_token,
+        api_base_url='https://' + mastodon_hostname,
+    )
+
+    # Initialise access headers
+    headers = {'Authorization': 'Bearer %s'%uc_access_token}
+
+    return (mastodon, mastodon_hostname)
+
+def db_config():
+
+    # Load db configuration from config file
+    config_filepath = "config/db_config.txt"
+    fediverse_db = get_parameter("fediverse_db", config_filepath)
+    fediverse_db_user = get_parameter("fediverse_db_user", config_filepath)
+
+    return (fediverse_db, fediverse_db_user)
+
+def usage():
+
+    print('usage: python ' + sys.argv[0] + ' --multi' + ' (multiprocessing, fast)')
+    print('usage: python ' + sys.argv[0] + ' --mono' + ' (one process, slow)')
+
+def save_time(program, start, finish):
+
+    insert_sql = "INSERT INTO execution_time(program, start, finish) VALUES(%s,%s,%s) ON CONFLICT DO NOTHING"
+
+    conn = None
+
+    try:
+
+        conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
+
+        cur = conn.cursor()
+
+        cur.execute(insert_sql, (program, start, finish,))
+
+        cur.execute("UPDATE execution_time SET start=(%s), finish=(%s) where program=(%s)", (start, finish, program))
+
+        conn.commit()
+
+        cur.close()
+
+    except (Exception, psycopg2.DatabaseError) as error:
+
+        print(error)
+
+    finally:
+
+        if conn is not None:
+
+            conn.close()
+
+def get_parameter(parameter, file_path):
+
+    # Check if secrets file exists
+    if not os.path.isfile(file_path):
+        print("File %s not found, exiting."%file_path)
+        sys.exit(0)
+
+    # Find parameter in file
+    with open(file_path) as f:
+        for line in f:
+            if line.startswith(parameter):
+                return line.replace(parameter + ":", "").strip()
+
+    # Cannot find parameter, exit
+    print(file_path + "  Missing parameter %s "%parameter)
+    sys.exit(0)
+
 if __name__ == '__main__':
 
-    db = Database()
+    # usage modes
 
-    setup = Setup()
+    if len(sys.argv) == 1:
+
+        usage()
+
+    elif len(sys.argv) == 2:
+
+        if sys.argv[1] == '--multi':
+
+            fediverse_db, fediverse_db_user = db_config()
 
             start = datetime.now()
 
@@ -204,20 +531,19 @@ if __name__ == '__main__':
 
             finish = start
 
-    db.save_time(program, start, finish)
+            save_time(program, start, finish)
 
             now = start
 
-    mastodon = Mastodon(
+            mastodon, mastodon_hostname = mastodon()
-            access_token = setup.mastodon_app_token,
-            api_base_url= setup.mastodon_hostname
-            )
 
             total_servers = 0
 
             total_users = 0
 
-    alive_servers = db.get_last_checked_servers()
+            #set_world_servers_check_to_false()
+
+            alive_servers = get_last_checked_servers()
 
             getservers = Server()
 
@@ -229,29 +555,276 @@ if __name__ == '__main__':
 
             print(f"duration = {time.time() - ray_start}.\nprocessed servers: {len(results)}")
 
+            ###########################################################################
             # get current total servers and users, get users from every software
 
-    soft_total_project, soft_total_users, soft_total_mau, soft_total_servers, total_servers, total_users, total_mau = db.soft_totals()
+            now = datetime.now()
 
+            gettotals_sql = "select count(server), sum(users), sum(mau) from fediverse where alive"
+            get_soft_totals_sql = "select software, sum(users) as users, sum(mau) as mau, count(server) as servers from fediverse where users != 0 and mau is not null and alive group by software order by mau desc"
+
+            soft_total_project = []
+            soft_total_users = []
+            soft_total_mau = []
+            soft_total_servers = []
+
+            try:
+
+                conn = None
+
+                conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
+
+                cur = conn.cursor()
+
+                cur.execute(gettotals_sql)
+
+                row = cur.fetchone()
+
+                total_servers = row[0]
+
+                total_users = row[1]
+
+                total_mau = row[2]
+
+                cur.execute(get_soft_totals_sql)
+
+                rows = cur.fetchall()
+
+                for row in rows:
+
+                    soft_total_project.append(row[0])
+                    soft_total_users.append(row[1])
+                    soft_total_mau.append(row[2])
+                    soft_total_servers.append(row[3])
+
+                cur.close()
+
+            except (Exception, psycopg2.DatabaseError) as error:
+
+                print(error)
+
+            finally:
+
+                if conn is not None:
+
+                    conn.close()
+
+            ###########################################################################
             #  get last check values and write current total ones
 
-    evo_servers, evo_users, evo_mau = db.last_values(total_servers, total_users, total_mau)
+            select_sql = "select total_servers, total_users, total_mau from totals order by datetime desc limit 1"
 
+            insert_sql = "INSERT INTO totals(datetime, total_servers, total_users, total_mau) VALUES(%s,%s,%s,%s)"
+
+            try:
+
+                conn = None
+
+                conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
+
+                cur = conn.cursor()
+
+                cur.execute(select_sql)
+
+                row = cur.fetchone()
+
+                if row is not None:
+
+                    servers_before = row[0]
+
+                    users_before = row[1]
+
+                    mau_before = row[2]
+
+                else:
+
+                    servers_before = 0
+
+                    users_before = 0
+
+                    mau_before = 0
+
+                cur.execute(insert_sql, (now, total_servers, total_users, total_mau))
+
+                conn.commit()
+
+                cur.close()
+
+                evo_servers = total_servers - servers_before
+
+                evo_users = total_users - users_before
+
+                evo_mau = total_mau - mau_before
+
+            except (Exception, psycopg2.DatabaseError) as error:
+
+                print(error)
+
+            finally:
+
+                if conn is not None:
+
+                    conn.close()
+
+            ################################################################################
             #  write evo values
 
-    db.write_evo(evo_servers, evo_users, evo_mau)
+            insert_sql = "INSERT INTO evo(datetime, servers, users, mau) VALUES(%s,%s,%s,%s)"
 
+            conn = None
+
+            try:
+
+                conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
+
+                cur = conn.cursor()
+
+                cur.execute(insert_sql, (now, evo_servers, evo_users, evo_mau))
+
+                conn.commit()
+
+                cur.close()
+
+            except (Exception, psycopg2.DatabaseError) as error:
+
+                print(error)
+
+            finally:
+
+                if conn is not None:
+
+                    conn.close()
+
+            ##############################################################################
             # get world's last update datetime
 
-    last_update = db.last_world_datetime()
+            conn = None
 
-    # get max servers and mau
+            try:
 
-    max_servers, max_mau = db.max()
+                conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
 
+                cur = conn.cursor()
+
+                cur.execute("select updated_at from world order by updated_at desc limit 1")
+
+                row = cur.fetchone()
+
+                last_update = row[0]
+
+                last_update = last_update.strftime('%m/%d/%Y, %H:%M:%S')
+
+                cur.close()
+
+            except (Exception, psycopg2.DatabaseError) as error:
+
+                print(error)
+
+            finally:
+
+                if conn is not None:
+
+                    conn.close()
+
+            ##############################################################################
+            # get max servers, users and mau
+
+            conn = None
+
+            try:
+
+                conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
+
+                cur = conn.cursor()
+
+                cur.execute("select MAX(total_servers) from totals")
+
+                row = cur.fetchone()
+
+                if row is not None:
+
+                    max_servers = row[0]
+
+                else:
+
+                    max_servers = 0
+
+                cur.execute("select MAX(total_users) from totals")
+
+                row = cur.fetchone()
+
+                if row is not None:
+
+                    max_users = row[0]
+
+                else:
+
+                    max_users = 0
+
+                cur.execute("select MAX(total_mau) from totals")
+
+                row = cur.fetchone()
+
+                if row is not None:
+
+                    max_mau = row[0]
+
+                else:
+
+                    max_mau = 0
+
+                cur.close()
+
+            except (Exception, psycopg2.DatabaseError) as error:
+
+                print(error)
+
+            finally:
+
+                if conn is not None:
+
+                    conn.close()
+
+            ###############################################################################
             # get plots
 
-    servers_plots, mau_plots, global_week, global_servers, global_users, global_mau = db.get_plots()
+            servers_plots = []
+
+            users_plots = []
+
+            mau_plots = []
+
+            conn = None
+
+            try:
+
+                conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
+
+                cur = conn.cursor()
+
+                cur.execute("select total_servers, total_users, total_mau from totals order by datetime desc limit 14")
+
+                rows = cur.fetchall()
+
+                for row in rows:
+
+                    servers_plots.append(row[0])
+
+                    users_plots.append(row[1])
+
+                    mau_plots.append(row[2])
+
+                cur.close()
+
+            except (Exception, psycopg2.DatabaseError) as error:
+
+                print(error)
+
+            finally:
+
+                if conn is not None:
+
+                    conn.close()
 
             ###############################################################################
             # generate graphs
@@ -272,6 +845,22 @@ if __name__ == '__main__':
 
             plt.close()
 
+            plt.plot([-6, -5, -4, -3, -2, -1, 0], [users_plots[6], users_plots[5], users_plots[4], users_plots[3], users_plots[2], users_plots[1], users_plots[0]], marker='o', color='royalblue')
+
+            plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_users, max_users, max_users, max_users, max_users, max_users, max_users], color='red')
+
+            plt.title('fediverse: total registered users (max: ' + str(f"{max_users:,}" + ')'), loc='right', color='royalblue')
+
+            plt.legend(('users', 'max'), shadow=True, loc=(0.01, 0.80), handlelength=1.5, fontsize=10)
+
+            plt.xlabel('Last seven days')
+
+            plt.ylabel('Registered users')
+
+            plt.savefig('users.png')
+
+            plt.close()
+
             plt.plot([-6, -5, -4, -3, -2, -1, 0], [mau_plots[6], mau_plots[5], mau_plots[4], mau_plots[3], mau_plots[2], mau_plots[1], mau_plots[0]], marker='o', color='royalblue')
 
             plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_mau, max_mau, max_mau, max_mau, max_mau, max_mau, max_mau], color='red')
@@ -288,39 +877,8 @@ if __name__ == '__main__':
 
             plt.close()
 
-    df = pd.DataFrame({'date': np.array(global_week),
-        #'servers': np.array(global_servers),
-        'users': np.array(global_users),
-        'mau': np.array(global_mau)})
-
-    df['date'] = pd.to_datetime(df['date'])
-
-    fig, ax = plt.subplots()
-
-    ax.plot(df.date, df.users, label='Registered', color='orange')
-
-    ax.plot(df.date, df.mau, label='MAU', color='blue')
-
-    plt.tick_params(rotation=45)
-
-    ax.set_title("fediverse's registered and Monthly Active Users")
-
-    ax.set_xlabel('weeks')
-
-    ax.set_ylabel('users')
-
-    ax.grid(visible=True)
-
-    ax.legend(title='Users')
-
-    ax.yaxis.set_major_formatter(y_formatter)
-
-    plt.savefig('global.png')
-
-    plt.close()
-
             ###############################################################################
-    # P O S T !
+            # T  O  O  T !
 
             toot_text = "#fediverse alive servers stats" + " \n"
 
@@ -338,6 +896,18 @@ if __name__ == '__main__':
 
                 toot_text += "max: " + str(f"{max_servers:,}") + "\n"
 
+            #if evo_users >= 0:
+
+            #    toot_text += "total users: " + str(f"{total_users:,}") + " (+"+ str(f"{evo_users:,}") + ") \n"
+
+            #    toot_text += "max: " + str(f"{max_users:,}") + "\n"
+
+            #elif evo_users < 0:
+
+            #    toot_text += "total users: " + str(f"{total_users:,}") + " ("+ str(f"{evo_users:,}") + ") \n"
+
+            #    toot_text += "max: " + str(f"{max_users:,}") + "\n"
+
             if evo_mau >= 0:
 
                 toot_text += "total MAU: " + str(f"{total_mau:,}") + " (+"+ str(f"{evo_mau:,}") + ") \n"
@@ -350,7 +920,7 @@ if __name__ == '__main__':
 
                 toot_text += "max: " + str(f"{max_mau:,}") + "\n"
 
-    toot_text += "\ntop ten (MAU / servers):\n\n"
+            toot_text += "\ntop ten (soft MAU servers):\n\n"
 
             i = 0
 
@@ -358,17 +928,15 @@ if __name__ == '__main__':
 
                 project_soft = soft_total_project[i]
 
+                #project_users = soft_total_users[i]
+
                 project_mau = soft_total_mau[i]
 
                 project_servers = soft_total_servers[i]
 
                 len_pr_soft = len(project_soft)
 
-        if project_soft == 'ativity-relay':
+                toot_text += f":{project_soft}: {project_mau:,} {project_servers:,}\n"
-
-            project_soft = 'activityrelay'
-
-        toot_text += f":{project_soft}: {project_mau:,} / {project_servers:,}\n"
 
                 i += 1
 
@@ -378,14 +946,18 @@ if __name__ == '__main__':
 
             servers_image_id = mastodon.media_post('servers.png', "image/png", description='servers graph').id
 
+            #users_image_id = mastodon.media_post('users.png', "image/png", description='users graph').id
+
             mau_image_id = mastodon.media_post('mau.png', "image/png", description='MAU graph').id
 
-    global_image_id = mastodon.media_post('global.png', "image/png", description='global graph').id
+            mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, mau_image_id})
 
-    mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, mau_image_id, global_image_id})
+            delete_dead_servers()
-
-    db.delete_dead_servers()
 
             finish = datetime.now()
 
-    db.save_time(program, start, finish)
+            save_time(program, start, finish)
+
+    else:
+
+        usage()

fetchservers.py

@@ -4,15 +4,13 @@ import os
 import json
 import sys
 import os.path
-from setup import Setup
+import psycopg2
-from database import Database
 import requests
 import urllib3
 import socket
 import ray
-import pdb
 
-#ray.init(num_cpus = 25) # Specify this system CPUs.
+ray.init(num_cpus = 32) # Specify this system CPUs.
 
 from ray.exceptions import (
     RaySystemError,
@@ -21,18 +19,18 @@ from ray.exceptions import (
     ObjectStoreFullError,
 )
 
-apis = ['/api/v1/instance',
+apis = ['/api/v1/instance?',
-        '/api/v1/nodeinfo',
+        '/api/v1/nodeinfo?',
-        '/nodeinfo/2.0',
+        '/nodeinfo/2.0?',
-        '/nodeinfo/2.0.json',
+        '/nodeinfo/2.0.json?',
-        '/nodeinfo/2.1.json',
+        '/nodeinfo/2.1.json?',
-        '/main/nodeinfo/2.0',
+        '/main/nodeinfo/2.0?',
-        '/api/statusnet/config',
+        '/api/statusnet/config?',
-        '/api/nodeinfo/2.0.json',
+        '/api/nodeinfo/2.0.json?',
-        '/api/nodeinfo',
+        '/api/nodeinfo?',
-        '/wp-json/nodeinfo/2.0',
+        '/wp-json/nodeinfo/2.0?',
-        '/api/v1/instance/nodeinfo/2.0',
+        '/api/v1/instance/nodeinfo/2.0?',
-        '/.well-known/x-nodeinfo2'
+        '/.well-known/x-nodeinfo2?'
         ]
 
 def is_json(myjson):
@@ -43,6 +41,45 @@ def is_json(myjson):
         return False
     return True
 
+def write_api(server, software, users, alive, api, soft_version):
+
+    fediverse_db, fediverse_db_user = get_db_config()
+
+    insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
+
+    conn = None
+
+    try:
+
+        conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
+
+        cur = conn.cursor()
+
+        print(f'Writing {server} nodeinfo data...')
+
+        cur.execute(insert_sql, (server, now, software, users, alive, api, soft_version))
+
+        cur.execute(
+            "UPDATE fediverse SET updated_at=(%s), software=(%s), users=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)",
+            (now, software, users, alive, api, soft_version, server)
+            )
+
+        cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
+
+        conn.commit()
+
+        cur.close()
+
+    except (Exception, psycopg2.DatabaseError) as error:
+
+        print(error)
+
+    finally:
+
+        if conn is not None:
+
+            conn.close()
+
 @ray.remote
 def getsoft(server):
 
@@ -61,9 +98,11 @@ def getsoft(server):
 
     url = 'https://' + server
 
+    user_agent = {'User-agent': "fediverse's stats (fediverse@mastodont.cat)"}
+
     try:
 
-        response = requests.get(url + '/.well-known/nodeinfo', headers = setup.user_agent, timeout=3)
+        response = requests.get(url + '/.well-known/nodeinfo', headers = user_agent, timeout=3)
 
         if response.status_code == 200:
 
@@ -71,17 +110,11 @@ def getsoft(server):
 
                 response_json = response.json()
 
-                if len(response_json['links']) == 1:
-
                 nodeinfo = response_json['links'][0]['href'].replace(f'https://{server}','')
 
-                elif len(response_json['links']) == 2:
-
-                    nodeinfo = response_json['links'][1]['href'].replace(f'https://{server}','')
-
                 try:
 
-                    nodeinfo_data = requests.get(url + nodeinfo, headers = setup.user_agent, timeout=3)
+                    nodeinfo_data = requests.get(nodeinfo, headers = user_agent, timeout=3)
 
                     if nodeinfo_data.status_code == 200:
 
@@ -91,7 +124,7 @@ def getsoft(server):
 
                     else:
 
-                        print(f"{nodeinfo} not responding: error code {nodeinfo_data.status_code}")
+                        print(f"Server {server}'s nodeinfo not responding: error code {nodeinfo_data.status_code}")
 
                 except:
 
@@ -99,7 +132,7 @@ def getsoft(server):
 
             except:
 
-                print(f'{server} is not responding: error code {response.status_code}')
+                print(f'Server {server} not responding: error code {response.status_code}')
                 print('*********************************************************************')
 
                 pass
@@ -109,9 +142,7 @@ def getsoft(server):
 
                 try:
 
-                    response = requests.get(url + api, headers = setup.user_agent, timeout=3)
+                    response = requests.get(url + api, headers = user_agent, timeout=3)
-
-                    if response.status_code == 200:
 
                     if is_json(response.text):
 
@@ -200,13 +231,12 @@ def getsoft(server):
                         soft = nodeinfo_json['software']['name']
                         soft = soft.lower()
                         soft_version = nodeinfo_json['software']['version']
-                        users = nodeinfo_json.get('usage').get('users').get('total') or '0'
+                        users = nodeinfo_json['usage']['users']['total']
                         if users > 1000000:
                             return
-
                         alive = True
 
-                        db.write_api(server, soft, users, alive, nodeinfo, soft_version)
+                        write_api(server, soft, users, alive, nodeinfo, soft_version)
 
                         print(f"Server {server} ({soft} {soft_version}) is alive!")
                         print('*********************************************************************')
@@ -227,12 +257,11 @@ def getsoft(server):
                         users = nodeinfo_json['usage']['users']['total']
                         if users > 1000000:
                             return
-
                         alive = True
 
                         if soft == 'socialhome':
 
-                            db.write_api(server, soft, users, alive, nodeinfo, soft_version)
+                            write_api(server, soft, users, alive, nodeinfo, soft_version)
 
                             print('*********************************************************************')
                             print(f"Server {serve}r ({soft} {soft_version}) is alive!")
@@ -270,7 +299,7 @@ def getsoft(server):
 
                 alive = True
 
-                db.write_api(server, soft, users, alive, nodeinfo, soft_version)
+                write_api(server, soft, users, alive, nodeinfo, soft_version)
 
                 print('*********************************************************************')
                 print(f"Server {server} ({soft}) is alive!")
@@ -292,6 +321,106 @@ def getsoft(server):
             print(f'Server {server} is dead')
             print('*********************************************************************')
 
+def get_world_servers():
+
+    world_servers = []
+
+    try:
+
+        conn = None
+
+        conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
+
+        cur = conn.cursor()
+
+        # get world servers list
+
+        cur.execute("select server from world where checked='f'")
+
+        rows = cur.fetchall()
+
+        for row in rows:
+
+            world_servers.append(row[0])
+
+        cur.close()
+
+        print("Remaining servers: " + str(len(world_servers)))
+
+    except (Exception, psycopg2.DatabaseError) as error:
+
+        print(error)
+
+    finally:
+
+        if conn is not None:
+
+            conn.close()
+
+    return world_servers
+
+def save_time(program, start, finish):
+
+    insert_sql = "INSERT INTO execution_time(program, start, finish) VALUES(%s,%s,%s) ON CONFLICT DO NOTHING"
+
+    conn = None
+
+    try:
+
+        conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
+
+        cur = conn.cursor()
+
+        cur.execute(insert_sql, (program, start, finish,))
+
+        cur.execute("UPDATE execution_time SET start=(%s), finish=(%s) where program=(%s)", (start, finish, program))
+
+        conn.commit()
+
+        cur.close()
+
+    except (Exception, psycopg2.DatabaseError) as error:
+
+        print(error)
+
+    finally:
+
+        if conn is not None:
+
+            conn.close()
+
+def get_parameter(parameter, file_path):
+
+    # Check if secrets file exists
+    if not os.path.isfile(file_path):
+        print("File %s not found, exiting." % file_path)
+        sys.exit(0)
+
+    # Find parameter in file
+    with open(file_path) as f:
+        for line in f:
+            if line.startswith(parameter):
+                return line.replace(parameter + ":", "").strip()
+
+    # Cannot find parameter, exit
+    print(file_path + "  Missing parameter %s " % parameter)
+    sys.exit(0)
+
+def get_config():
+
+    # Load configuration from config file
+    config_filepath = "config/config.txt"
+    mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
+    return mastodon_hostname
+
+def get_db_config():
+
+    # Load database config from db_config file
+    db_config_filepath = "config/db_config.txt"
+    fediverse_db = get_parameter("fediverse_db", db_config_filepath)
+    fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath)
+    return (fediverse_db, fediverse_db_user)
+
 ###############################################################################
 # main
 
@@ -299,13 +428,7 @@ if __name__ == '__main__':
 
     ## name: fetchservers.py
 
-    setup = Setup()
+    fediverse_db, fediverse_db_user = get_db_config()
-
-    db = Database()
-
-    res = requests.get('https://' + 'mastodon.social' + setup.peers_api, headers = setup.user_agent, timeout=3)
-
-    hostname_peers = res.json()
 
     start = datetime.now()
 
@@ -313,19 +436,19 @@ if __name__ == '__main__':
 
     finish = start
 
-    db.save_time(program, start, finish)
+    save_time(program, start, finish)
 
     now = start
 
-    #world_servers = db.get_world_servers()
+    mastodon_hostname = get_config()
+
+    world_servers = get_world_servers()
 
     ray_start = time.time()
 
     try:
 
-        #results = ray.get([getsoft.remote(server) for server in world_servers])
+        results = ray.get([getsoft.remote(server) for server in world_servers])
-        results = ray.get([getsoft.remote(server) for server in hostname_peers])
-        #[getsoft(server) for server in world_servers]
 
         print(f"duration = {time.time() - ray_start}.\nprocessed servers: {len(results)}")
 
@@ -335,4 +458,4 @@ if __name__ == '__main__':
 
     finish = datetime.now()
 
-    db.save_time(program, start, finish)
+    save_time(program, start, finish)

getpeers.py

@@ -0,0 +1,177 @@
+import os
+import sys
+import time
+from datetime import datetime
+import requests
+import json
+import psycopg2
+import ray
+import pdb
+
+ray.init(num_cpus = 32) # Specify this system CPUs.
+
+def write_server(server, federated_with):
+
+    insert_sql = "INSERT INTO world(server, federated_with, updated_at, saved_at, checked) VALUES(%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
+
+    conn = None
+
+    try:
+
+        conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
+
+        cur = conn.cursor()
+
+        cur.execute(insert_sql, (server, federated_with, now, now, 'f'))
+
+        print(f'writing {server} to world database')
+
+        conn.commit()
+
+        cur.close()
+
+    except (Exception, psycopg2.DatabaseError) as error:
+
+        print(error)
+
+    finally:
+
+        if conn is not None:
+
+            conn.close()
+
+@ray.remote
+def get_peers(peer):
+
+    try:
+
+        user_agent = {'User-agent': "fediverse's stats (fediverse@mastodont.cat)"}
+
+        domain_res = requests.get('https://' + peer + '/api/v1/instance', headers = user_agent, timeout=3)
+
+        domain_res_json = domain_res.json()
+
+        if domain_res.status_code == 200:
+
+            domain_uri = domain_res_json['uri'].replace('https://', '')
+
+            if domain_uri != peer:
+
+                print(f'{peer} is an aliased domain of {domain_uri}!')
+
+            else:
+
+                response = requests.get('https://' + peer + peers_api, headers = user_agent, timeout=3)
+
+                response_json = response.json()
+
+                if response.status_code == 200:
+
+                    try:
+
+                        print(f"Server: {peer}, federated with {str(len(response_json))} servers")
+
+                        for peer_peer in response_json:
+
+                            write_server(peer_peer, peer)
+
+                    except:
+
+                        pass
+
+    except:
+
+        pass
+
+def save_time(program, start, finish):
+
+    insert_sql = "INSERT INTO execution_time(program, start, finish) VALUES(%s,%s,%s) ON CONFLICT DO NOTHING"
+
+    conn = None
+
+    try:
+
+        conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
+
+        cur = conn.cursor()
+
+        cur.execute(insert_sql, (program, start, finish,))
+
+        cur.execute("UPDATE execution_time SET start=(%s), finish=(%s) where program=(%s)", (start, finish, program))
+
+        conn.commit()
+
+        cur.close()
+
+    except (Exception, psycopg2.DatabaseError) as error:
+
+        print(error)
+
+    finally:
+
+        if conn is not None:
+
+            conn.close()
+
+def get_parameter( parameter, file_path ):
+
+    # Check if secrets file exists
+    if not os.path.isfile(file_path):
+        print("File %s not found, exiting."%file_path)
+        sys.exit(0)
+
+    # Find parameter in file
+    with open( file_path ) as f:
+        for line in f:
+            if line.startswith( parameter ):
+                return line.replace(parameter + ":", "").strip()
+
+    # Cannot find parameter, exit
+    print(file_path + "  Missing parameter %s "%parameter)
+    sys.exit(0)
+
+###############################################################################
+# main
+
+if __name__ == '__main__':
+
+    now = datetime.now()
+
+    peers_api = '/api/v1/instance/peers?'
+
+    # Load configuration from config file
+    config_filepath = "config/config.txt"
+    mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
+
+    # Load database config from db_config file
+    db_config_filepath = "config/db_config.txt"
+    fediverse_db = get_parameter("fediverse_db", db_config_filepath)
+    fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath)
+
+    user_agent = {'User-agent': "fediverse's stats (fediverse@mastodont.cat)"}
+
+    res = requests.get('https://' + mastodon_hostname + peers_api, headers = user_agent, timeout=3)
+
+    hostname_peers = res.json()
+
+    start = datetime.now()
+
+    program = 'getpeers'
+
+    finish = start
+
+    save_time(program, start, finish)
+
+    for peer in hostname_peers:
+
+        write_server(peer, mastodon_hostname)
+
+    results = ray.get([get_peers.remote(server) for server in hostname_peers])
+
+    finish = datetime.now()
+
+    print(f"duration = {finish - start}.\nprocessed servers: {len(results)}")
+
+    save_time(program, start, finish)
+
+

requirements.txt

@@ -1,7 +1,9 @@
-requests
+Mastodon.py>=1.5.1
-psycopg2-binary
+psycopg2-binary>=2.8.4
-pytz
+aiohttp>=3.6.2
-ray
+aiodns>=2.0.0
-Mastodon.py
+matplotlib>=3.3.4
-matplotlib
+humanfriendly>=9.2
-pandas
+urllib3>=1.26.8
+requests>=2.27.1
+ray>=1.11.0

setup.py

@@ -1,157 +1,226 @@
-import os
+import getpass
-import sys
-from datetime import datetime
-import pytz
 from mastodon import Mastodon
 from mastodon.Mastodon import MastodonMalformedEventError, MastodonNetworkError, MastodonReadTimeout, MastodonAPIError, MastodonIllegalArgumentError
-import pdb
+import fileinput,re
+import os
+import sys
 
-class Setup():
+def create_dir():
-
-    name = 'fediverse setup'
-
-    def __init__(self, config_file=None, mastodon_hostname=None, peers_api=None, user_agent=None, secrets_filepath=None, mastodon_app_token=None):
-
-        self.config_file = "config/config.txt"
-        self.mastodon_hostname = self.__get_parameter("mastodon_hostname", self.config_file)
-        self.peers_api = '/api/v1/instance/peers?'
-        self.user_agent = {'User-agent': "fediverse's stats (fediverse@mastodont.cat)"}
-
-        self.secrets_filepath = 'secrets/secrets.txt'
-
-        is_setup = self.__check_mastodon_setup(self)
-
-        if is_setup:
-
-            self.mastodon_app_token = self.__get_mastodon_parameter("mastodon_app_token", self.secrets_filepath)
-
-        else:
-
-            self.mastodon_app_token = self.mastodon_setup(self)
-
-    @staticmethod
-    def __check_mastodon_setup(self):
-
-        is_setup = False
-
-        if not os.path.isfile(self.secrets_filepath):
-            print(f"File {self.secrets_filepath} not found, running setup.")
-        else:
-            is_setup = True
-
-        return is_setup
-
-    @staticmethod
-    def mastodon_setup(self):
 
     if not os.path.exists('secrets'):
         os.makedirs('secrets')
 
-        self.mastodon_user = input("Mastodon user login? ")
+def create_file():
-        self.mastodon_password = input("Mastodon user password? ")
-        self.app_name = 'fediverse'
 
-        self.mastodon_app_token = self.mastodon_log_in()
+    if not os.path.exists('secrets/secrets.txt'):
+        with open('secrets/secrets.txt', 'w'): pass
+        print(secrets_filepath + " created!")
 
-        if not os.path.exists(self.secrets_filepath):
+def create_config():
-            with open(self.secrets_filepath, 'w'): pass
-            print(f"{self.secrets_filepath} created!")
 
-        with open(self.secrets_filepath, 'a') as the_file:
+    if not os.path.exists('config'):
-            print("Writing Mastodon parameters to " + self.secrets_filepath)
+        os.makedirs('config')
-            the_file.write(f'mastodon_app_token: {self.mastodon_app_token}')
+    if not os.path.exists(config_filepath):
+        print(config_filepath + " created!")
+        with open('config/config.txt', 'w'): pass
 
-        return self.mastodon_app_token
+def write_params():
 
-    def mastodon_log_in(self):
+    with open(secrets_filepath, 'a') as the_file:
+        print("Writing secrets parameter names to " + secrets_filepath)
+        the_file.write('uc_client_id: \n'+'uc_client_secret: \n'+'uc_access_token: \n')
 
-        token = ''
+def write_config():
+
+    with open(config_filepath, 'a') as the_file:
+        the_file.write('mastodon_hostname: \n')
+        print("adding parameter name 'mastodon_hostname' to "+ config_filepath)
+
+def read_client_lines(self):
+
+    client_path = 'app_clientcred.txt'
+
+    with open(client_path) as fp:
+
+        line = fp.readline()
+        cnt = 1
+        while line:
+           if cnt == 1:
+               print("Writing client id to " + secrets_filepath)
+               modify_file(secrets_filepath, "uc_client_id: ", value=line.rstrip())
+           elif cnt == 2:
+               print("Writing client secret to " + secrets_filepath)
+               modify_file(secrets_filepath, "uc_client_secret: ", value=line.rstrip())
+           line = fp.readline()
+           cnt += 1
+
+def read_token_line(self):
+
+    token_path = 'app_usercred.txt'
+
+    with open(token_path) as fp:
+
+        line = fp.readline()
+        print("Writing access token to " + secrets_filepath)
+        modify_file(secrets_filepath, "uc_access_token: ", value=line.rstrip())
+
+def read_config_line():
+
+    with open(config_filepath) as fp:
+
+        line = fp.readline()
+        modify_file(config_filepath, "mastodon_hostname: ", value=hostname)
+        modify_file(config_filepath, "bot_username: ", value=bot_username)
+
+def log_in():
+
+    error = 0
 
     try:
 
-            response = Mastodon.create_app(
+        global hostname
-                self.app_name,
+        hostname = input("Enter Mastodon hostname: ")
+        user_name = input("User name, ex. user@" + hostname +"? ")
+        user_password = getpass.getpass("User password? ")
+        bot_username = input("Bot's username, ex. fediverse: ")
+        app_name = input("This app name? ")
+        Mastodon.create_app(
+            app_name,
             scopes=["read","write"],
-                to_file=None,
+            to_file="app_clientcred.txt",
-                api_base_url=self.mastodon_hostname
+            api_base_url=hostname
             )
-            client_id = response[0]
+        mastodon = Mastodon(client_id = "app_clientcred.txt", api_base_url = hostname)
-            client_secret = response[1]
+        mastodon.log_in(
-
+            user_name,
-            mastodon = Mastodon(client_id = client_id, client_secret = client_secret, api_base_url = self.mastodon_hostname)
+            user_password,
-        
-            token = mastodon.log_in(
-                self.mastodon_user,
-                self.mastodon_password,
             scopes = ["read", "write"],
-                to_file = None
+            to_file = "app_usercred.txt"
             )
 
-            print('Log in succesful!')
-
     except MastodonIllegalArgumentError as i_error:
 
-            sys.stdout.write(f'\n{str(i_error)}\n')
+        error = 1
+
+        if os.path.exists("secrets/secrets.txt"):
+            print("Removing secrets/secrets.txt file..")
+            os.remove("secrets/secrets.txt")
+        if os.path.exists("app_clientcred.txt"):
+            print("Removing app_clientcred.txt file..")
+            os.remove("app_clientcred.txt")
+        sys.exit(i_error)
 
     except MastodonNetworkError as n_error:
 
-            sys.stdout.write(f'\n{str(n_error)}\n')
+        error = 1
+
+        if os.path.exists("secrets/secrets.txt"):
+            print("Removing secrets/secrets.txt file..")
+            os.remove("secrets/secrets.txt")
+        if os.path.exists("app_clientcred.txt"):
+            print("Removing app_clientcred.txt file..")
+            os.remove("app_clientcred.txt")
+        sys.exit(n_error)
 
     except MastodonReadTimeout as r_error:
 
-            sys.stdout.write(f'\n{str(r_error)}\n')
+        error = 1
+
+        if os.path.exists("secrets/secrets.txt"):
+            print("Removing secrets/secrets.txt file..")
+            os.remove("secrets/secrets.txt")
+        if os.path.exists("app_clientcred.txt"):
+            print("Removing app_clientcred.txt file..")
+            os.remove("app_clientcred.txt")
+        sys.exit(r_error)
 
     except MastodonAPIError as a_error:
 
-            sys.stdout.write(f'\n{str(a_error)}\n')
+        error = 1
+
+        if os.path.exists("secrets/secrets.txt"):
+            print("Removing secrets/secrets.txt file..")
+            os.remove("secrets/secrets.txt")
+        if os.path.exists("app_clientcred.txt"):
+            print("Removing app_clientcred.txt file..")
+            os.remove("app_clientcred.txt")
+        sys.exit(a_error)
 
     finally:
 
-            return token
+        if error == 0:
 
-    def __get_parameter(self, parameter, config_file):
+            create_dir()
+            create_file()
+            write_params()
+            client_path = 'app_clientcred.txt'
+            read_client_lines(client_path)
+            token_path = 'app_usercred.txt'
+            read_token_line(token_path)
+            if os.path.exists("app_clientcred.txt"):
+                print("Removing app_clientcred.txt temp file..")
+                os.remove("app_clientcred.txt")
+            if os.path.exists("app_usercred.txt"):
+                print("Removing app_usercred.txt temp file..")
+                os.remove("app_usercred.txt")
+            print("Secrets setup done!\n")
 
-        if not os.path.isfile(config_file):
+def modify_file(file_name,pattern,value=""):
-            print(f"File {config_file} not found..")
 
-            self.mastodon_hostname = input("\nMastodon hostname: ")
+    fh=fileinput.input(file_name,inplace=True)
+    for line in fh:
+        replacement=pattern + value
+        line=re.sub(pattern,replacement,line)
+        sys.stdout.write(line)
+    fh.close()
 
-            self.__create_config(self)
+def get_parameter( parameter, file_path ):
-            self.__write_config(self)
+    # Check if secrets file exists
+    if not os.path.isfile(file_path):
+        print("File %s not found, creating it."%file_path)
+        log_in()
 
-        with open( self.config_file ) as f:
+    # Find parameter in file
+    with open( file_path ) as f:
         for line in f:
             if line.startswith( parameter ):
                 return line.replace(parameter + ":", "").strip()
 
-    def __get_mastodon_parameter(self, parameter, secrets_filepath):
+    # Cannot find parameter, exit
+    print(file_path + " Missing parameter %s "%parameter)
+    sys.exit(0)
 
-        if not os.path.isfile(secrets_filepath):
+def get_hostname( parameter, config_filepath ):
-            print(f"File {secrets_filepath} not found..")
+    # Check if secrets file exists
+    if not os.path.isfile(config_filepath):
+        print("File %s not found, creating it."%config_filepath)
+        create_config()
 
-            self.sign_in()
+    # Find parameter in file
-
+    with open( config_filepath ) as f:
-        with open( self.secrets_filepath ) as f:
         for line in f:
             if line.startswith( parameter ):
                 return line.replace(parameter + ":", "").strip()
 
-    @staticmethod
+    # Cannot find parameter, exit
-    def __create_config(self):
+    print(config_filepath + "  Missing parameter %s "%parameter)
+    write_config()
+    read_config_line()
+    print("setup done!")
+    sys.exit(0)
 
-        if not os.path.exists('config'):
+###############################################################################
+# main
 
-            os.makedirs('config')
+if __name__ == '__main__':
 
-        if not os.path.exists(self.config_file):
+    # Load secrets from secrets file
+    secrets_filepath = "secrets/secrets.txt"
+    uc_client_id     = get_parameter("uc_client_id",     secrets_filepath)
+    uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
+    uc_access_token  = get_parameter("uc_access_token",  secrets_filepath)
 
-            print(self.config_file + " created!")
+    # Load configuration from config file
-            with open(self.config_file, 'w'): pass
+    config_filepath = "config/config.txt"
-
+    mastodon_hostname = get_hostname("mastodon_hostname", config_filepath)
-    @staticmethod
+    bot_username = get_parameter("bot_username", config_filepath)
-    def __write_config(self):
-
-        with open(self.config_file, 'a') as the_file:
-
-            the_file.write(f'mastodon_hostname: {self.mastodon_hostname}')
-            print(f"adding parameters to {self.config_file}\n")