Added Funkwhale and Socialhome software detections

README.md

@@ -73,8 +73,13 @@ Within Python Virtual Environment:
 | writefreely | api/nodeinfo          | ['software']['name'] |
 | zap         | nodeinfo/2.0.json     | ['software']['name'] |
 
-5. Use your favourite scheduling method to set `python fediverse.py` to run twice daily, `python fetchservers.py` one time daily and `python getworld.py` to run monthly.   
+7. Run `python uptime_setup.py` to get your Uptime bot's access token of your Mastodon or Pleroma server existing account. It will be saved to 'secrets/uptime_secrets.txt' for further use.
 
-18.2.21 - New feature! Added [Lemmy project](https://join.lemmy.ml)  
+8. Use your favourite scheduling method to set `python fediverse.py` to run twice daily, `python fetchservers.py` one time daily, `python getworld.py` to run monthly and `python uptime.py` (choose your desired frequency) if you want to publish best fediverse's servers uptime.   
-12.5.21 - New feature! Added Wordpress support. The code can now detect Wordpress instances with ActivityPub enabled plugin.  
+
-12.5.21 - New feature! New shinny creation of servers and users graphs.  
+18.2.2021 - New feature! Added [Lemmy project](https://join.lemmy.ml)  
+12.5.2021 - New feature! Added Wordpress support. The code can now detect Wordpress instances with ActivityPub enabled plugin.  
+12.5.2021 - New feature! New shinny creation of servers and users graphs.  
+21.8.2021 - New feature! Added Best Fediverse's servers Uptime publishing bot.  
+22.10.2021 - New feature! Added [Funkwhale](https://funkwhale.audio) support.  
+26.10.2021 - New feature! Added [Socialhome](https://socialhome.network) support.  

db-setup.py

@@ -1,6 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
 import getpass
 import os
 import sys
@@ -10,7 +7,6 @@ import psycopg2
 from psycopg2 import sql
 from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
 
-# Returns the parameter from the specified file
 def get_parameter( parameter, file_path ):
     # Check if secrets file exists
     if not os.path.isfile(file_path):
@@ -29,132 +25,153 @@ def get_parameter( parameter, file_path ):
     sys.exit(0)
 
 def write_parameter( parameter, file_path ):
-  if not os.path.exists('config'):
+    
-    os.makedirs('config')
+    if not os.path.exists('config'):
-  print("Setting up fediverse parameters...")
+        os.makedirs('config')
-  print("\n")
+    print("Setting up fediverse parameters...")
-  fediverse_db = input("fediverse db name: ")
+    print("\n")
-  fediverse_db_user = input("fediverse db user: ")
+    fediverse_db = input("fediverse db name: ")
+    fediverse_db_user = input("fediverse db user: ")
+    mastodon_db = input("Mastodon db name: ")
+    mastodon_db_user = input("Mastodon db user: ")
+  
+    with open(file_path, "w") as text_file:
 
-  with open(file_path, "w") as text_file:
+        print("fediverse_db: {}".format(fediverse_db), file=text_file)
-    print("fediverse_db: {}".format(fediverse_db), file=text_file)
+        print("fediverse_db_user: {}".format(fediverse_db_user), file=text_file)
-    print("fediverse_db_user: {}".format(fediverse_db_user), file=text_file)
+        print("mastodon_db: {}".format(mastodon_db), file=text_file)
+        print("mastodon_db_user: {}".format(mastodon_db_user), file=text_file)
 
 def create_table(db, db_user, table, sql):
 
-  conn = None
+    conn = None
-  try:
 
-    conn = psycopg2.connect(database = db, user = db_user, password = "", host = "/var/run/postgresql", port = "5432")
+    try:
-    cur = conn.cursor()
 
-    print("Creating table.. "+table)
+        conn = psycopg2.connect(database = db, user = db_user, password = "", host = "/var/run/postgresql", port = "5432")
-    # Create the table in PostgreSQL database
+        cur = conn.cursor()
-    cur.execute(sql)
 
-    conn.commit()
+        print("Creating table.. "+table)
-    print("Table "+table+" created!")
+        # Create the table in PostgreSQL database
-    print("\n")
+        cur.execute(sql)
 
-  except (Exception, psycopg2.DatabaseError) as error:
+        conn.commit()
 
-    print(error)
+        print("Table "+table+" created!")
+        print("\n")
 
-  finally:
+    except (Exception, psycopg2.DatabaseError) as error:
+
+        print(error)
+
+    finally:
+
+        if conn is not None:
+
+            conn.close()
+
+###############################################################################
+# main
+
+if __name__ == '__main__':
+
+    # Load configuration from config file
+    config_filepath = "config/db_config.txt"
+    fediverse_db = get_parameter("fediverse_db", config_filepath)
+    fediverse_db_user = get_parameter("fediverse_db_user", config_filepath)
+    mastodon_db = get_parameter("mastodon_db", config_filepath)
+    mastodon_db_user = get_parameter("mastodon_db_user", config_filepath)
+
+    ############################################################
+    # create database
+    ############################################################
+
+    conn = None
+
+    try:
+
+        conn = psycopg2.connect(dbname='postgres',
+            user=fediverse_db_user, host='',
+            password='')
+
+        conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
+
+        cur = conn.cursor()
+
+        print("Creating database " + fediverse_db + ". Please wait...")
+
+        cur.execute(sql.SQL("CREATE DATABASE {}").format(
+              sql.Identifier(fediverse_db))
+            )
+        print("Database " + fediverse_db + " created!")
+
+    except (Exception, psycopg2.DatabaseError) as error:
+
+        print(error)
+
+    finally:
+
+        if conn is not None:
+
+            conn.close()
+
+    #############################################################################################
+
+    try:
+
+        conn = None
+        conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
+
+    except (Exception, psycopg2.DatabaseError) as error:
+
+        print(error)
+        # Load configuration from config file
+        os.remove("db_config.txt")
+        print("Exiting. Run db-setup again with right parameters")
+        sys.exit(0)
 
     if conn is not None:
 
-      conn.close()
+        print("\n")
+        print("fediverse parameters saved to db-config.txt!")
+        print("\n")
 
-#############################################################################################
+    ############################################################
+    # Create needed tables
+    ############################################################
 
-# Load configuration from config file
+    print("Creating table...")
-config_filepath = "config/db_config.txt"
-fediverse_db = get_parameter("fediverse_db", config_filepath)
-fediverse_db_user = get_parameter("fediverse_db_user", config_filepath)
 
-############################################################
+    ########################################
-# create database
-############################################################
 
-conn = None
+    db = fediverse_db
+    db_user = fediverse_db_user
+    table = "world"
+    sql = "create table "+table+" (server varchar(200) PRIMARY KEY, federated_with varchar(200), updated_at timestamptz, saved_at timestamptz, checked boolean)"
+    create_table(db, db_user, table, sql)
 
-try:
+    table = "fediverse"
+    sql = "create table "+table+" (server varchar(200) PRIMARY KEY, users INT, updated_at timestamptz, software varchar(50), version varchar(100), first_checked_at timestamptz, last_checked_at timestamptz, downs int)"
+    create_table(db, db_user, table, sql)
 
-  conn = psycopg2.connect(dbname='postgres',
+    table = "totals"
-      user=fediverse_db_user, host='',
+    sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, total_servers INT, total_users INT)"
-      password='')
+    create_table(db, db_user, table, sql)
 
-  conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
+    table = "evo"
+    sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, servers INT, users INT)"
+    create_table(db, db_user, table, sql)
 
-  cur = conn.cursor()
+    table = "closed"
+    sql = "create table "+table+" (server varchar(200) PRIMARY KEY, software varchar(10), closed boolean)"
+    create_table(db, db_user, table, sql)
 
-  print("Creating database " + fediverse_db + ". Please wait...")
+    table = "botreplies"
+    sql = "create table "+table+" (status_id bigint PRIMARY KEY, query_user varchar(40), status_created_at timestamptz)"
+    create_table(db, db_user, table, sql)
 
-  cur.execute(sql.SQL("CREATE DATABASE {}").format(
+    #####################################
-          sql.Identifier(fediverse_db))
-      )
-  print("Database " + fediverse_db + " created!")
 
-except (Exception, psycopg2.DatabaseError) as error:
+    print("Done!")
-
+    print("Now you can run setup.py!")
-  print(error)
+    print("\n")
-
-finally:
-
-  if conn is not None:
-
-    conn.close()
-
-#############################################################################################
-
-try:
-
-  conn = None
-  conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
-
-except (Exception, psycopg2.DatabaseError) as error:
-
-  print(error)
-  # Load configuration from config file
-  os.remove("db_config.txt")
-  print("Exiting. Run db-setup again with right parameters")
-  sys.exit(0)
-
-if conn is not None:
-
-  print("\n")
-  print("fediverse parameters saved to db-config.txt!")
-  print("\n")
-
-############################################################
-# Create needed tables
-############################################################
-
-print("Creating table...")
-
-########################################
-
-db = fediverse_db
-db_user = fediverse_db_user
-table = "world"
-sql = "create table "+table+" (server varchar(200) PRIMARY KEY, federated_with varchar(200), updated_at timestamptz, saved_at timestamptz), checked boolean"
-create_table(db, db_user, table, sql)
-
-table = "fediverse"
-sql = "create table "+table+" (server varchar(200) PRIMARY KEY, users INT, updated_at timestamptz, software varchar(50), version varchar(100))"
-create_table(db, db_user, table, sql)
-
-table = "totals"
-sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, total_servers INT, total_users INT)"
-create_table(db, db_user, table, sql)
-
-table = "evo"
-sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, servers INT, users INT)"
-create_table(db, db_user, table, sql)
-
-#####################################
-
-print("Done!")
-print("Now you can run setup.py!")
-print("\n")

fediverse.py

@@ -14,12 +14,22 @@ import aiohttp
 import asyncio
 import socket
 import matplotlib.pyplot as plt
-
+import pdb
 plt.style.use('seaborn')
 
 start_time = time.time()
 
-apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?', '/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?', '/wp-json/nodeinfo/2.0?']
+apis = ['/nodeinfo/2.0?',
+        '/nodeinfo/2.0.json?',
+        '/main/nodeinfo/2.0?',
+        '/api/statusnet/config?',
+        '/api/nodeinfo/2.0.json?',
+        '/api/nodeinfo?',
+        '/api/v1/instance?',
+        '/wp-json/nodeinfo/2.0?',
+        '/api/v1/instance/nodeinfo/2.0?',
+        '/.well-known/x-nodeinfo2?'
+        ]
 
 client_exceptions = (
     aiohttp.ClientResponseError,
@@ -53,7 +63,7 @@ def get_alive_servers(server):
 
         cur = conn.cursor()
 
-        cur.execute("select alive, software, users_api, version from fediverse where server=(%s)", (server,))
+        cur.execute("select alive, software, users_api, version, first_checked_at, downs from fediverse where server=(%s)", (server,))
 
         row = cur.fetchone()
 
@@ -63,6 +73,8 @@ def get_alive_servers(server):
             serv_soft = row[1]
             serv_api = row[2]
             soft_version = row[3]
+            first_checked_at = row[4]
+            downs_qty = row[5]
 
         cur.close()
 
@@ -79,7 +91,9 @@ def get_alive_servers(server):
 
         try:
 
-            data = requests.get('https://' + server + serv_api, timeout=3)
+            user_agent = {'User-agent': 'Mozilla/5.0'}
+
+            data = requests.get('https://' + server + serv_api, headers = user_agent, timeout=3)
 
             if serv_soft == "mastodon":
                 if serv_api == '/nodeinfo/2.0?':
@@ -149,18 +163,46 @@ def get_alive_servers(server):
                         alive = True
 
                 except:
-                    
+
+                    users = 0
+                    soft_version = ""
+
+            if serv_soft == 'funkwhale':
+
+                try:
+                    users = data.json()['usage']['users']['total']
+                    soft_version = data.json()['software']['version']
+                    alive = True
+                except:
+                    users = 0
+                    soft_version = ""
+
+            if serv_soft == 'socialhome':
+
+                try:
+                    users = data.json()['usage']['users']['total']
+                    soft_version = data.json()['server']['version']
+                    alive = True
+                except:
                     users = 0
                     soft_version = ""
 
             if alive:
-                
+
+                if downs_qty != None:
+
+                    downs = downs_qty
+
+                else:
+
+                    downs = 0
+
                 if soft_version != "" and soft_version is not None:
                     print("Server " + str(server) + " (" + serv_soft + " " + soft_version + ") is alive!")
                 else:
                     print("Server " + str(server) + " (" + serv_soft + ") is alive!")
 
-                insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
+                insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version, first_checked_at, last_checked_at, downs) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
 
                 conn = None
 
@@ -170,9 +212,16 @@ def get_alive_servers(server):
 
                     cur = conn.cursor()
 
-                    cur.execute(insert_sql, (server, users, now, serv_soft, alive, serv_api, soft_version))
+                    cur.execute(insert_sql, (server, users, now, serv_soft, alive, serv_api, soft_version, now, now, downs))
+
+                    if first_checked_at != None:
+
+                        cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (users, now, serv_soft, alive, serv_api, soft_version, now, downs, server))
+
+                    else:
+
+                         cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), first_checked_at=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (users, now, serv_soft, alive, serv_api, soft_version, now, now, downs, server))
 
-                    cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)", (users, now, serv_soft, alive, serv_api, soft_version, server))
 
                     cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
 
@@ -258,6 +307,14 @@ def get_alive_servers(server):
 
         if alive is False:
 
+            if downs_qty != None:
+
+                downs = downs_qty + 1
+
+            else:
+
+                downs = 1
+
             conn = None
 
             try:
@@ -267,7 +324,13 @@ def get_alive_servers(server):
 
                 cur = conn.cursor()
 
-                cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s) where server=(%s)", (now, alive, server))
+                if first_checked_at != None:
+
+                    cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), first_checked_at=(%s), downs=(%s) where server=(%s)", (now, alive, now, downs, server))
+
+                else:
+
+                    cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), downs=(%s) where server=(%s)", (now, alive, downs, server))
 
                 cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
 
@@ -329,8 +392,11 @@ async def getsoft(server):
 
     url = 'https://' + server
 
+    user_agent = {'User-agent': 'Mozilla/5.0'}
+
     timeout = aiohttp.ClientTimeout(total=3)
-    async with aiohttp.ClientSession(timeout=timeout) as session:
+
+    async with aiohttp.ClientSession(headers=user_agent, timeout=timeout) as session:
         for api in apis:
             try:
                 async with session.get(url+api) as response:
@@ -871,23 +937,15 @@ if __name__ == '__main__':
             project_servers = soft_total_servers[i]
             len_pr_soft = len(project_soft)
 
-            if project_soft == 'writefreely':
+            toot_text += f":{project_soft}: {project_users:,} {project_servers:,}\n"
-
-                str_len = 11
-
-            else:
-
-                str_len = 13
-
-            toot_text += f"{':'+project_soft+':':<11}" + f"{project_users:>{str_len},}" + " " + f"{project_servers:>5,}" + "\n"
 
             i += 1
 
         print("Tooting...")
         print(toot_text)
 
-        servers_image_id = mastodon.media_post('servers.png', "image/png").id
+        servers_image_id = mastodon.media_post('servers.png', "image/png", description='servers graph').id
 
-        users_image_id = mastodon.media_post('users.png', "image/png").id
+        users_image_id = mastodon.media_post('users.png', "image/png", description='users graph').id
 
         mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, users_image_id})

fetchservers.py

@@ -10,8 +10,17 @@ import aiohttp
 import asyncio
 import socket
 
-apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?',
+apis = ['/nodeinfo/2.0?',
-        '/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?', '/wp-json/nodeinfo/2.0?']
+        '/nodeinfo/2.0.json?',
+        '/main/nodeinfo/2.0?',
+        '/api/statusnet/config?',
+        '/api/nodeinfo/2.0.json?',
+        '/api/nodeinfo?',
+        '/api/v1/instance?',
+        '/wp-json/nodeinfo/2.0?',
+        '/api/v1/instance/nodeinfo/2.0?',
+        '/.well-known/x-nodeinfo2?'
+        ]
 
 client_exceptions = (
     aiohttp.ClientResponseError,
@@ -24,15 +33,17 @@ client_exceptions = (
 
 
 def is_json(myjson):
+
     try:
         json_object = json.loads(myjson)
     except ValueError as e:
         return False
     return True
 
-
 def write_api(server, software, users, alive, api, soft_version):
+
     insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
+
     conn = None
 
     try:
@@ -46,7 +57,8 @@ def write_api(server, software, users, alive, api, soft_version):
 
         cur.execute(
             "UPDATE fediverse SET updated_at=(%s), software=(%s), users=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)",
-            (now, software, users, alive, api, soft_version, server))
+            (now, software, users, alive, api, soft_version, server)
+            )
 
         cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
 
@@ -63,9 +75,8 @@ def write_api(server, software, users, alive, api, soft_version):
         if conn is not None:
             conn.close()
 
-
 async def getsoft(server):
-    
+
     try:
 
         socket.gethostbyname(server)
@@ -79,33 +90,61 @@ async def getsoft(server):
 
     url = 'https://' + server
 
+    user_agent = {'User-agent': 'Mozilla/5.0'}
+
     timeout = aiohttp.ClientTimeout(total=3)
-    async with aiohttp.ClientSession(timeout=timeout) as session:
+    async with aiohttp.ClientSession(timeout=timeout, headers=user_agent) as session:
         for api in apis:
+
             try:
+
                 async with session.get(url + api) as response:
                     if response.status == 200:
                         try:
                             response_json = await response.json()
                         except:
                             pass
+
             except aiohttp.ClientConnectorError as err:
+
                 pass
+
             else:
+
                 if response.status == 200 and api != '/api/v1/instance?':
-                    try:
+
-                        soft = response_json['software']['name']
+                    if api != '/.well-known/x-nodeinfo2?':
-                        soft = soft.lower()
+
-                        soft_version = response_json['software']['version']
+                        try:
-                        users = response_json['usage']['users']['total']
+                            soft = response_json['software']['name']
-                        if users > 1000000:
+                            soft = soft.lower()
+                            soft_version = response_json['software']['version']
+                            users = response_json['usage']['users']['total']
+                            if users > 1000000:
+                                return
+                            alive = True
+                            write_api(server, soft, users, alive, api, soft_version)
+                            print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
                             return
-                        alive = True
+                        except:
-                        write_api(server, soft, users, alive, api, soft_version)
+                            pass
-                        print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
+                    else:
-                        return
+
-                    except:
+                        try:
-                        pass
+                            soft = response_json['server']['software']
+                            soft = soft.lower()
+                            soft_version = response_json['server']['version']
+                            users = response_json['usage']['users']['total']
+                            if users > 1000000:
+                                return
+                            alive = True
+                            if soft == 'socialhome':
+                                write_api(server, soft, users, alive, api, soft_version)
+                                print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
+                            return
+                        except:
+                            pass
+
                 if response.status == 200 and soft == '' and api == "/api/v1/instance?":
                     soft = 'mastodon'
                     users = response_json['stats']['user_count']
@@ -116,11 +155,10 @@ async def getsoft(server):
                     write_api(server, soft, users, alive, api)
                     print("Server " + server + " (" + soft + ") is alive!")
 
-
 def getserver(server, x):
 
     server = server[0].rstrip('.').lower()
-    
+
     if server.find(".") == -1:
         return
     if server.find("@") != -1:
@@ -140,8 +178,6 @@ def getserver(server, x):
 
         pass
 
-
-# Returns the parameter from the specified file
 def get_parameter(parameter, file_path):
     # Check if secrets file exists
     if not os.path.isfile(file_path):
@@ -159,14 +195,20 @@ def get_parameter(parameter, file_path):
     sys.exit(0)
 
 
-# Load configuration from config file
+def get_config():
-config_filepath = "config/config.txt"
-mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
 
-# Load database config from db_config file
+    # Load configuration from config file
-db_config_filepath = "config/db_config.txt"
+    config_filepath = "config/config.txt"
-fediverse_db = get_parameter("fediverse_db", db_config_filepath)
+    mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
-fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath)
+    return mastodon_hostname
+
+def get_db_config():
+
+    # Load database config from db_config file
+    db_config_filepath = "config/db_config.txt"
+    fediverse_db = get_parameter("fediverse_db", db_config_filepath)
+    fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath)
+    return (fediverse_db, fediverse_db_user)
 
 ###############################################################################
 # main
@@ -176,6 +218,10 @@ if __name__ == '__main__':
     now = datetime.now()
     start_time = time.time()
 
+    mastodon_hostname = get_config()
+
+    fediverse_db, fediverse_db_user = get_db_config()
+
     world_servers = []
 
     try:
@@ -207,6 +253,7 @@ if __name__ == '__main__':
     finally:
 
         if conn is not None:
+
             conn.close()
 
     ###########################################################################

getpeers.py

@@ -47,11 +47,13 @@ def write_servers(world_servers):
         i += 1
 
 def get_peers(peer):
-    
+
     try:
 
-        response = requests.get('https://' + peer + peers_api, timeout=2)       
+        user_agent = {'User-agent': 'Mozilla/5.0'}
-        
+
+        response = requests.get('https://' + peer + peers_api, headers = user_agent, timeout=3)
+
         response_json = response.json()
 
         if response.status_code == 200:
@@ -59,7 +61,7 @@ def get_peers(peer):
             try:
 
                 print("Server: " + peer + ", " + "federated with " + str(len(response_json)) + " servers")
-    
+
                 for peer_peer in response_json:
 
                     exist_count = world_peers.count(peer_peer)
@@ -111,10 +113,9 @@ if __name__ == '__main__':
     start_time = time.time()
 
     updated_at = datetime.now()
-    
+
     peers_api = '/api/v1/instance/peers?'
-    lemmy_api = '/api/v2/site?'
+
-    
     # Load configuration from config file
     config_filepath = "config/config.txt"
     mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
@@ -123,17 +124,19 @@ if __name__ == '__main__':
     db_config_filepath = "config/db_config.txt"
     fediverse_db = get_parameter("fediverse_db", db_config_filepath)
     fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath)
-    
+
     world_peers = []
-    
+
-    res = requests.get('https://' + mastodon_hostname + peers_api)
+    user_agent = {'User-agent': 'Mozilla/5.0'}
+
+    res = requests.get('https://' + mastodon_hostname + peers_api, headers = user_agent, timeout=3)
 
     hostname_peers = res.json()
 
     for peer in hostname_peers:
 
         exist_count = world_peers.count(peer)
-        
+
         if exist_count == 0:
 
             world_peers.append(peer)
@@ -148,6 +151,6 @@ if __name__ == '__main__':
         p_thread.join()
 
     write_servers(world_peers)
-    
+
     exec_time = str(round((time.time() - start_time), 2))
     print(exec_time)

requirements.txt

@@ -3,3 +3,4 @@ psycopg2-binary>=2.8.4
 aiohttp>=3.6.2
 aiodns>=2.0.0
 matplotlib>=3.3.4
+humanfriendly>=9.2

setup.py

@@ -1,6 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
 import getpass
 from mastodon import Mastodon
 from mastodon.Mastodon import MastodonMalformedEventError, MastodonNetworkError, MastodonReadTimeout, MastodonAPIError, MastodonIllegalArgumentError
@@ -9,130 +6,167 @@ import os
 import sys
 
 def create_dir():
-  if not os.path.exists('secrets'):
+
-    os.makedirs('secrets')
+    if not os.path.exists('secrets'):
+        os.makedirs('secrets')
 
 def create_file():
-  if not os.path.exists('secrets/secrets.txt'):
+
-    with open('secrets/secrets.txt', 'w'): pass
+    if not os.path.exists('secrets/secrets.txt'):
-    print(secrets_filepath + " created!")
+        with open('secrets/secrets.txt', 'w'): pass
+        print(secrets_filepath + " created!")
 
 def create_config():
-  if not os.path.exists('config'):
+
-    os.makedirs('config')
+    if not os.path.exists('config'):
-  if not os.path.exists(config_filepath):
+        os.makedirs('config')
-    print(config_filepath + " created!")
+    if not os.path.exists(config_filepath):
-    with open('config/config.txt', 'w'): pass
+        print(config_filepath + " created!")
+        with open('config/config.txt', 'w'): pass
 
 def write_params():
-  with open(secrets_filepath, 'a') as the_file:
+
-    print("Writing secrets parameter names to " + secrets_filepath)
+    with open(secrets_filepath, 'a') as the_file:
-    the_file.write('uc_client_id: \n'+'uc_client_secret: \n'+'uc_access_token: \n')
+        print("Writing secrets parameter names to " + secrets_filepath)
+        the_file.write('uc_client_id: \n'+'uc_client_secret: \n'+'uc_access_token: \n')
 
 def write_config():
-  with open(config_filepath, 'a') as the_file:
+
-    the_file.write('mastodon_hostname: \n')
+    with open(config_filepath, 'a') as the_file:
-    print("adding parameter name 'mastodon_hostname' to "+ config_filepath)
+        the_file.write('mastodon_hostname: \n')
+        print("adding parameter name 'mastodon_hostname' to "+ config_filepath)
 
 def read_client_lines(self):
-  client_path = 'app_clientcred.txt'
+
-  with open(client_path) as fp:
+    client_path = 'app_clientcred.txt'
-    line = fp.readline()
+
-    cnt = 1
+    with open(client_path) as fp:
-    while line:
+
-       if cnt == 1:
+        line = fp.readline()
-         print("Writing client id to " + secrets_filepath)
+        cnt = 1
-         modify_file(secrets_filepath, "uc_client_id: ", value=line.rstrip())
+        while line:
-       elif cnt == 2:
+           if cnt == 1:
-         print("Writing client secret to " + secrets_filepath)
+               print("Writing client id to " + secrets_filepath)
-         modify_file(secrets_filepath, "uc_client_secret: ", value=line.rstrip())
+               modify_file(secrets_filepath, "uc_client_id: ", value=line.rstrip())
-       line = fp.readline()
+           elif cnt == 2:
-       cnt += 1
+               print("Writing client secret to " + secrets_filepath)
+               modify_file(secrets_filepath, "uc_client_secret: ", value=line.rstrip())
+           line = fp.readline()
+           cnt += 1
 
 def read_token_line(self):
-  token_path = 'app_usercred.txt'
+
-  with open(token_path) as fp:
+    token_path = 'app_usercred.txt'
-    line = fp.readline()
+
-    print("Writing access token to " + secrets_filepath)
+    with open(token_path) as fp:
-    modify_file(secrets_filepath, "uc_access_token: ", value=line.rstrip())
+
+        line = fp.readline()
+        print("Writing access token to " + secrets_filepath)
+        modify_file(secrets_filepath, "uc_access_token: ", value=line.rstrip())
 
 def read_config_line():
-  with open(config_filepath) as fp:
+
-    line = fp.readline()
+    with open(config_filepath) as fp:
-    modify_file(config_filepath, "mastodon_hostname: ", value=hostname)
+
+        line = fp.readline()
+        modify_file(config_filepath, "mastodon_hostname: ", value=hostname)
+        modify_file(config_filepath, "bot_username: ", value=bot_username)
 
 def log_in():
-  error = 0
-  try:
-    global hostname
-    hostname = input("Enter Mastodon hostname: ")
-    user_name = input("User name, ex. user@" + hostname +"? ")
-    user_password = getpass.getpass("User password? ")
-    app_name = input("This app name? ")
-    Mastodon.create_app(app_name, scopes=["read","write"],
-      to_file="app_clientcred.txt", api_base_url=hostname)
-    mastodon = Mastodon(client_id = "app_clientcred.txt", api_base_url = hostname)
-    mastodon.log_in(
-       user_name,
-       user_password,
-       scopes = ["read", "write"],
-       to_file = "app_usercred.txt"
-    )
-  except MastodonIllegalArgumentError as i_error:
-    error = 1
-    if os.path.exists("secrets/secrets.txt"):
-      print("Removing secrets/secrets.txt file..")
-      os.remove("secrets/secrets.txt")
-    if os.path.exists("app_clientcred.txt"):
-      print("Removing app_clientcred.txt file..")
-      os.remove("app_clientcred.txt")
-    sys.exit(i_error)
-  except MastodonNetworkError as n_error:
-    error = 1
-    if os.path.exists("secrets/secrets.txt"):
-      print("Removing secrets/secrets.txt file..")
-      os.remove("secrets/secrets.txt")
-    if os.path.exists("app_clientcred.txt"):
-      print("Removing app_clientcred.txt file..")
-      os.remove("app_clientcred.txt")
-    sys.exit(n_error)
-  except MastodonReadTimeout as r_error:
-    error = 1
-    if os.path.exists("secrets/secrets.txt"):
-      print("Removing secrets/secrets.txt file..")
-      os.remove("secrets/secrets.txt")
-    if os.path.exists("app_clientcred.txt"):
-      print("Removing app_clientcred.txt file..")
-      os.remove("app_clientcred.txt")
-    sys.exit(r_error)
-  except MastodonAPIError as a_error:
-    error = 1
-    if os.path.exists("secrets/secrets.txt"):
-      print("Removing secrets/secrets.txt file..")
-      os.remove("secrets/secrets.txt")
-    if os.path.exists("app_clientcred.txt"):
-      print("Removing app_clientcred.txt file..")
-      os.remove("app_clientcred.txt")
-    sys.exit(a_error)
-  finally:
-    if error == 0:
 
-      create_dir()
+    error = 0
-      create_file()
+
-      write_params()
+    try:
-      client_path = 'app_clientcred.txt'
+
-      read_client_lines(client_path)
+        global hostname
-      token_path = 'app_usercred.txt'
+        hostname = input("Enter Mastodon hostname: ")
-      read_token_line(token_path)
+        user_name = input("User name, ex. user@" + hostname +"? ")
-      if os.path.exists("app_clientcred.txt"):
+        user_password = getpass.getpass("User password? ")
-        print("Removing app_clientcred.txt temp file..")
+        bot_username = input("Bot's username, ex. fediverse: ")
-        os.remove("app_clientcred.txt")
+        app_name = input("This app name? ")
-      if os.path.exists("app_usercred.txt"):
+        Mastodon.create_app(
-        print("Removing app_usercred.txt temp file..")
+            app_name,
-        os.remove("app_usercred.txt")
+            scopes=["read","write"],
-      print("Secrets setup done!\n")
+            to_file="app_clientcred.txt",
+            api_base_url=hostname
+            )
+        mastodon = Mastodon(client_id = "app_clientcred.txt", api_base_url = hostname)
+        mastodon.log_in(
+            user_name,
+            user_password,
+            scopes = ["read", "write"],
+            to_file = "app_usercred.txt"
+            )
+
+    except MastodonIllegalArgumentError as i_error:
+
+        error = 1
+
+        if os.path.exists("secrets/secrets.txt"):
+            print("Removing secrets/secrets.txt file..")
+            os.remove("secrets/secrets.txt")
+        if os.path.exists("app_clientcred.txt"):
+            print("Removing app_clientcred.txt file..")
+            os.remove("app_clientcred.txt")
+        sys.exit(i_error)
+
+    except MastodonNetworkError as n_error:
+
+        error = 1
+
+        if os.path.exists("secrets/secrets.txt"):
+            print("Removing secrets/secrets.txt file..")
+            os.remove("secrets/secrets.txt")
+        if os.path.exists("app_clientcred.txt"):
+            print("Removing app_clientcred.txt file..")
+            os.remove("app_clientcred.txt")
+        sys.exit(n_error)
+
+    except MastodonReadTimeout as r_error:
+
+        error = 1
+
+        if os.path.exists("secrets/secrets.txt"):
+            print("Removing secrets/secrets.txt file..")
+            os.remove("secrets/secrets.txt")
+        if os.path.exists("app_clientcred.txt"):
+            print("Removing app_clientcred.txt file..")
+            os.remove("app_clientcred.txt")
+        sys.exit(r_error)
+
+    except MastodonAPIError as a_error:
+
+        error = 1
+
+        if os.path.exists("secrets/secrets.txt"):
+            print("Removing secrets/secrets.txt file..")
+            os.remove("secrets/secrets.txt")
+        if os.path.exists("app_clientcred.txt"):
+            print("Removing app_clientcred.txt file..")
+            os.remove("app_clientcred.txt")
+        sys.exit(a_error)
+
+    finally:
+
+        if error == 0:
+
+            create_dir()
+            create_file()
+            write_params()
+            client_path = 'app_clientcred.txt'
+            read_client_lines(client_path)
+            token_path = 'app_usercred.txt'
+            read_token_line(token_path)
+            if os.path.exists("app_clientcred.txt"):
+                print("Removing app_clientcred.txt temp file..")
+                os.remove("app_clientcred.txt")
+            if os.path.exists("app_usercred.txt"):
+                print("Removing app_usercred.txt temp file..")
+                os.remove("app_usercred.txt")
+            print("Secrets setup done!\n")
 
 def modify_file(file_name,pattern,value=""):
+
     fh=fileinput.input(file_name,inplace=True)
     for line in fh:
         replacement=pattern + value
@@ -140,7 +174,6 @@ def modify_file(file_name,pattern,value=""):
         sys.stdout.write(line)
     fh.close()
 
-# Returns the parameter from the specified file
 def get_parameter( parameter, file_path ):
     # Check if secrets file exists
     if not os.path.isfile(file_path):
@@ -157,7 +190,6 @@ def get_parameter( parameter, file_path ):
     print(file_path + " Missing parameter %s "%parameter)
     sys.exit(0)
 
-# Returns the parameter from the specified file
 def get_hostname( parameter, config_filepath ):
     # Check if secrets file exists
     if not os.path.isfile(config_filepath):
@@ -177,12 +209,18 @@ def get_hostname( parameter, config_filepath ):
     print("setup done!")
     sys.exit(0)
 
-# Load secrets from secrets file
+###############################################################################
-secrets_filepath = "secrets/secrets.txt"
+# main
-uc_client_id     = get_parameter("uc_client_id",     secrets_filepath)
-uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
-uc_access_token  = get_parameter("uc_access_token",  secrets_filepath)
 
-# Load configuration from config file
+if __name__ == '__main__':
-config_filepath = "config/config.txt"
+
-mastodon_hostname = get_hostname("mastodon_hostname", config_filepath)
+    # Load secrets from secrets file
+    secrets_filepath = "secrets/secrets.txt"
+    uc_client_id     = get_parameter("uc_client_id",     secrets_filepath)
+    uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
+    uc_access_token  = get_parameter("uc_access_token",  secrets_filepath)
+
+    # Load configuration from config file
+    config_filepath = "config/config.txt"
+    mastodon_hostname = get_hostname("mastodon_hostname", config_filepath)
+    bot_username = get_parameter("bot_username", config_filepath)