Improved servers nodeinfo detection

fediverse.py

@@ -1,3 +1,5 @@
+from multiprocessing import set_start_method
+from multiprocessing import get_context
 import time
 import urllib3
 from datetime import datetime
@@ -7,11 +9,10 @@ import json
 import sys
 import os.path
 import requests
-import psycopg2
-from itertools import product
-import multiprocessing
 import aiohttp
 import asyncio
+import psycopg2
+from itertools import product
 import socket
 import matplotlib.pyplot as plt
 import pdb
@@ -19,18 +20,6 @@ plt.style.use('seaborn')
 
 start_time = time.time()
 
-apis = ['/nodeinfo/2.0?',
-        '/nodeinfo/2.0.json?',
-        '/main/nodeinfo/2.0?',
-        '/api/statusnet/config?',
-        '/api/nodeinfo/2.0.json?',
-        '/api/nodeinfo?',
-        '/api/v1/instance?',
-        '/wp-json/nodeinfo/2.0?',
-        '/api/v1/instance/nodeinfo/2.0?',
-        '/.well-known/x-nodeinfo2?'
-        ]
-
 client_exceptions = (
     aiohttp.ClientResponseError,
     aiohttp.ClientConnectionError,
@@ -40,121 +29,91 @@ client_exceptions = (
     socket.gaierror,
 )
 
-def is_json(myjson):
+class Server:
 
-    try:
+    name = 'Server'
-        json_object = json.loads(myjson)
-    except ValueError as e:
-        print(e)
-        return False
-    return True
 
-def get_alive_servers(server):
+    def __init_(self, server, software, users, alive, api, soft_version):
 
-    serv_api = ''
+        self.server = server
-    serv_soft = ''
+        self.software = software
-    soft_version = ''
+        self.users = users
+        self.alive = alive
+        self.api = api
+        self.version = self.soft_version
 
-    try:
+    def get_alive_servers(self, *args):
 
-        conn = None
+        fediverse_db, fediverse_db_user = db_config()
 
-        conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
+        now = datetime.now()
 
-        cur = conn.cursor()
+        if len(args) != 0:
 
-        cur.execute("select alive, software, users_api, version, first_checked_at, downs from fediverse where server=(%s)", (server,))
+            self.server = args[0]
-
-        row = cur.fetchone()
-
-        if row is not None:
-
-            was_alive = row[0]
-            serv_soft = row[1]
-            serv_api = row[2]
-            soft_version = row[3]
-            first_checked_at = row[4]
-            downs_qty = row[5]
-
-        cur.close()
-
-    except (Exception, psycopg2.DatabaseError) as error:
-
-        print(error)
-
-    finally:
-
-        if conn is not None:
-            conn.close()
-
-        alive = False
 
         try:
 
-            user_agent = {'User-agent': 'Mozilla/5.0'}
+            conn = None
 
-            data = requests.get('https://' + server + serv_api, headers = user_agent, timeout=3)
+            conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
+
+            cur = conn.cursor()
+
+            cur.execute("select alive, software, users_api, version, first_checked_at, downs from fediverse where server=(%s)", (self.server,))
+
+            row = cur.fetchone()
+
+            if row is not None:
+
+                was_alive = row[0]
+                self.software = row[1]
+                self.api = row[2]
+                self.soft_version = row[3]
+                first_checked_at = row[4]
+                downs_qty = row[5]
+
+            cur.close()
+
+        except (Exception, psycopg2.DatabaseError) as error:
+
+            print(error)
+
+        finally:
+
+            if conn is not None:
+
+                conn.close()
+
+            alive = False
+
+            try:
+
+                user_agent = {'User-agent': 'Mozilla/5.0'}
+
+                data = requests.get('https://' + self.server + self.api, headers = user_agent, timeout=3)
 
-            if serv_soft == "mastodon":
-                if serv_api == '/nodeinfo/2.0?':
-                    try:
-                        users = data.json()['usage']['users']['total']
-                        soft_version = data.json()['software']['version']
-                        alive = True
-                    except:
-                        users = 0
-                        soft_version = ""
-                if serv_api == '/nodeinfo/2.0.json?':
-                    try:
-                        users = data.json()['usage']['users']['total']
-                        soft_version = data.json()['software']['version']
-                        alive = True
-                    except:
-                        users = 0
-                        soft_version = ""
-                elif serv_api == '/api/v1/instance?':
-                    try:
-                        users = data.json()['stats']['user_count']
-                        soft_version = data.json()['version']
-                        alive = True
-                    except:
-                        users = 0
-                        soft_version = ""
-            if serv_soft == "pleroma" or serv_soft == "diaspora" or serv_soft == "peertube" or serv_soft == "pixelfed" or serv_soft == "hubzilla" or serv_soft == "writefreely" or serv_soft == "friendica":
                 try:
-                    users = data.json()['usage']['users']['total']
+
-                    soft_version = data.json()['software']['version']
+                    self.users = data.json()['usage']['users']['total']
-                    alive = True
+
-                except:
+                    if self.users == 0:
-                    users = 0
+
-                    soft_version = ""
+                        self.users = data.json()['usage']['users']['activeHalfyear']
-            if serv_soft == "gnusocialv2" or serv_soft == "gnusocial":
+
-                try:
+                    if self.software == 'socialhome':
-                    users = data.json()['usage']['users']['total']
+
-                    if users == 0:
+                        self.soft_version = data.json()['server']['version']
-                        users = data.json()['usage']['users']['activeHalfyear']
+
-                    soft_version = data.json()['software']['version']
+                    else:
-                    alive = True
+
-                except:
+                        self.soft_version = data.json()['software']['version']
-                    users = 0
+
-                    soft_version = ""
+                    if self.software == "wordpress" and "activitypub" in data.json()['protocols']:
-            if serv_soft == "plume" or serv_soft == 'red' or serv_soft == "misskey" or serv_soft == "zap" or serv_soft == "prismo" or serv_soft == "ravenvale" or serv_soft == "osada" or serv_soft == "groundpolis":
-                try:
-                    users = data.json()['usage']['users']['total']
-                    soft_version = data.json()['software']['version']
-                    alive = True
-                except:
-                    users = 0
-                    soft_version = ""
-            if serv_soft == "ganggo" or serv_soft == "squs" or serv_soft == "dolphin" or serv_soft == "lemmy" or serv_soft == "wordpress":
-                try:
-                    users = data.json()['usage']['users']['total']
-                    soft_version = data.json()['software']['version']
-                    if serv_soft == "wordpress" and "activitypub" in data.json()['protocols']:
 
                         alive = True
 
-                    elif serv_soft == "wordpress" and "activitypub" not in data.json()['protocols']:
+                    elif self.software == "wordpress" and "activitypub" not in data.json()['protocols']:
 
                         alive = False
 
@@ -164,297 +123,173 @@ def get_alive_servers(server):
 
                 except:
 
-                    users = 0
+                    self.users = 0
-                    soft_version = ""
 
-            if serv_soft == 'funkwhale':
+                    self.soft_version = ""
 
-                try:
+                else: 
-                    users = data.json()['usage']['users']['total']
-                    soft_version = data.json()['software']['version']
-                    alive = True
-                except:
-                    users = 0
-                    soft_version = ""
 
-            if serv_soft == 'socialhome':
+                    if self.api == '/api/v1/instance?':
 
-                try:
+                        try:
-                    users = data.json()['usage']['users']['total']
+                            self.users = data.json()['stats']['user_count']
-                    soft_version = data.json()['server']['version']
+                            self.soft_version = data.json()['version']
-                    alive = True
+                            alive = True
-                except:
+                        except:
-                    users = 0
+                            self.users = 0
-                    soft_version = ""
+                            self.soft_version = ""
 
-            if alive:
+                if alive:
+
+                    if downs_qty != None:
+
+                        downs = downs_qty
+
+                    else:
+
+                        downs = 0
+
+                    if self.soft_version != "" and self.soft_version is not None:
+
+                        print(f'\n** Server {self.server} ({self.software} {self.soft_version}) is alive! **')
+
+                    else:
+
+                        print(f'\n** Server {self.server} ({self.software}) is alive! **')
+
+                    insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version, first_checked_at, last_checked_at, downs) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
+
+                    conn = None
+
+                    try:
+
+                        conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
+
+                        cur = conn.cursor()
+
+                        cur.execute(insert_sql, (self.server, self.users, now, self.software, alive, self.api, self.soft_version, now, now, downs))
+
+                        if first_checked_at != None:
+
+                            cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (self.users, now, self.software, alive, self.api, self.soft_version, now, downs, self.server))
+
+                        else:
+
+                            cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), first_checked_at=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (self.users, now, self.software, alive, self.api, self.soft_version, now, now, downs, self.server))
+
+
+                        cur.execute("UPDATE world SET checked='t' where server=(%s)", (self.server,))
+
+                        conn.commit()
+
+                        cur.close()
+
+                    except (Exception, psycopg2.DatabaseError) as error:
+
+                        print(error)
+
+                    finally:
+
+                        if conn is not None:
+
+                            conn.close()
+
+            except urllib3.exceptions.ProtocolError as protoerr:
+
+                print_dead(self.server)
+                pass
+
+            except requests.exceptions.ChunkedEncodingError as chunkerr:
+
+                print_dead(self.server)
+                pass
+
+            except KeyError as e:
+
+                print_dead(self.server)
+                pass
+
+            except ValueError as verr:
+
+                print_dead(self.server)
+                pass
+
+            except requests.exceptions.SSLError as errssl:
+
+                print_dead(self.server)
+                pass
+
+            except requests.exceptions.HTTPError as errh:
+
+                print_dead(self.server)
+                pass
+
+            except requests.exceptions.ConnectionError as errc:
+
+                print_dead(self.server)
+                pass
+
+            except requests.exceptions.Timeout as errt:
+
+                print_dead(self.server)
+                pass
+
+            except requests.exceptions.RequestException as err:
+
+                print_dead(self.server)
+                pass
+
+            except socket.gaierror as gai_error:
+
+                print_dead(self.server)
+                pass
+
+            if not alive:
 
                 if downs_qty != None:
 
-                    downs = downs_qty
+                    downs = downs_qty + 1
 
                 else:
 
-                    downs = 0
+                    downs = 1
-
-                if soft_version != "" and soft_version is not None:
-                    print("Server " + str(server) + " (" + serv_soft + " " + soft_version + ") is alive!")
-                else:
-                    print("Server " + str(server) + " (" + serv_soft + ") is alive!")
-
-                insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version, first_checked_at, last_checked_at, downs) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
 
                 conn = None
 
                 try:
 
-                    conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
+                    conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="",
+                                            host="/var/run/postgresql", port="5432")
 
                     cur = conn.cursor()
 
-                    cur.execute(insert_sql, (server, users, now, serv_soft, alive, serv_api, soft_version, now, now, downs))
-
                     if first_checked_at != None:
 
-                        cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (users, now, serv_soft, alive, serv_api, soft_version, now, downs, server))
+                        cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), first_checked_at=(%s), downs=(%s) where server=(%s)", (now, alive, now, downs, self.server))
 
                     else:
 
-                         cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), first_checked_at=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (users, now, serv_soft, alive, serv_api, soft_version, now, now, downs, server))
+                        cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), downs=(%s) where server=(%s)", (now, alive, downs, self.server))
 
-
+                    cur.execute("UPDATE world SET checked='t' where server=(%s)", (self.server,))
-                    cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
 
                     conn.commit()
 
                     cur.close()
 
                 except (Exception, psycopg2.DatabaseError) as error:
+
                     print(error)
 
                 finally:
 
                     if conn is not None:
+
                         conn.close()
 
-        except urllib3.exceptions.ProtocolError as protoerr:
+def print_dead(server):
 
-            print(protoerr)
+    print(f'\nServer {server} is dead :-(')
-            print("Server " + server + " is dead :-(")
-            alive = False
-            pass
-
-        except requests.exceptions.ChunkedEncodingError as chunkerr:
-
-            print(chunkerr)
-            print("Server " + server + " is dead :-(")
-            alive = False
-            pass
-
-        except KeyError as e:
-
-            print(e)
-            print("Server " + server + " is dead :-(")
-            alive = False
-            pass
-
-        except ValueError as verr:
-
-            print(verr)
-            print("Server " + server + " is dead :-(")
-            alive = False
-            pass
-
-        except requests.exceptions.SSLError as errssl:
-
-            print(errssl)
-            print("Server " + server + " is dead :-(")
-            alive = False
-            pass
-
-        except requests.exceptions.HTTPError as errh:
-
-            print(errh)
-            print("Server " + server + " is dead :-(")
-            alive = False
-            pass
-
-        except requests.exceptions.ConnectionError as errc:
-
-            print(errc)
-            print("Server " + server + " is dead :-(")
-            alive = False
-            pass
-
-        except requests.exceptions.Timeout as errt:
-
-            print(errt)
-            print("Server " + server + " is dead :-(")
-            alive = False
-            pass
-
-        except requests.exceptions.RequestException as err:
-
-            print(err)
-            print("Server " + server + " is dead :-(")
-            alive = False
-            pass
-
-        except socket.gaierror as gai_error:
-
-            print(gai_error)
-            pass
-
-        if alive is False:
-
-            if downs_qty != None:
-
-                downs = downs_qty + 1
-
-            else:
-
-                downs = 1
-
-            conn = None
-
-            try:
-
-                conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="",
-                                        host="/var/run/postgresql", port="5432")
-
-                cur = conn.cursor()
-
-                if first_checked_at != None:
-
-                    cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), first_checked_at=(%s), downs=(%s) where server=(%s)", (now, alive, now, downs, server))
-
-                else:
-
-                    cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), downs=(%s) where server=(%s)", (now, alive, downs, server))
-
-                cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
-
-                conn.commit()
-
-                cur.close()
-
-            except (Exception, psycopg2.DatabaseError) as error:
-                print(error)
-
-            finally:
-
-                if conn is not None:
-                    conn.close()
-
-def write_api(server, software, users, alive, api, soft_version):
-
-    insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
-    conn = None
-
-    try:
-
-        conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
-
-        cur = conn.cursor()
-
-        cur.execute(insert_sql, (server, now, software, users, alive, api, soft_version))
-
-        cur.execute("UPDATE fediverse SET updated_at=(%s), software=(%s), users=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)", (now, software, users, alive, api, soft_version, server))
-
-        cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
-
-        conn.commit()
-
-        cur.close()
-
-    except (Exception, psycopg2.DatabaseError) as error:
-
-        print(error)
-
-    finally:
-
-        if conn is not None:
-
-            conn.close()
-
-async def getsoft(server):
-
-    try:
-
-        socket.gethostbyname(server)
-
-    except socket.gaierror:
-
-        pass
-        return
-
-    soft = ''
-
-    url = 'https://' + server
-
-    user_agent = {'User-agent': 'Mozilla/5.0'}
-
-    timeout = aiohttp.ClientTimeout(total=3)
-
-    async with aiohttp.ClientSession(headers=user_agent, timeout=timeout) as session:
-        for api in apis:
-            try:
-                async with session.get(url+api) as response:
-                    if response.status == 200:
-                        try:
-                            response_json = await response.json()
-                        except:
-                            pass
-            except aiohttp.ClientConnectorError as err:
-                print(err)
-                pass
-            else:
-                if response.status == 200 and api != '/api/v1/instance?':
-                    try:
-                        soft = response_json['software']['name']
-                        soft = soft.lower()
-                        soft_version = response_json['software']['version']
-                        users = response_json['usage']['users']['total']
-                        if users > 1000000:
-                            return
-                        alive = True
-                        write_api(server, soft, users, alive, api, soft_version)
-                        print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
-                        return
-                    except:
-                        pass
-                if response.status == 200 and soft == '' and api == "/api/v1/instance?":
-                    soft = 'mastodon'
-                    users = response_json['stats']['user_count']
-                    soft_version = response_json['version']
-                    if users > 1000000:
-                        return
-                    alive = True
-                    write_api(server, soft, users, alive, api)
-                    print("Server " + server + " (" + soft + ") is alive!")
-
-def getserver(server, x):
-
-    server = server[0].rstrip('.').lower()
-
-    if server.find(".") == -1:
-        return
-    if server.find("@") != -1:
-        return
-    if server.find("/") != -1:
-        return
-    if server.find(":") != -1:
-        return
-
-    try:
-
-        loop = asyncio.get_event_loop()
-        coroutines = [getsoft(server)]
-        soft = loop.run_until_complete(asyncio.gather(*coroutines, return_exceptions=True))
-
-    except:
-
-        pass
 
 def set_world_servers_check_to_false():
 
@@ -562,7 +397,6 @@ def usage():
     print('usage: python ' + sys.argv[0] + ' --multi' + ' (multiprocessing, fast)')
     print('usage: python ' + sys.argv[0] + ' --mono' + ' (one process, slow)')
 
-# Returns the parameter from the specified file
 def get_parameter(parameter, file_path):
 
     # Check if secrets file exists
@@ -594,8 +428,8 @@ if __name__ == '__main__':
     elif len(sys.argv) == 2:
 
         if sys.argv[1] == '--multi':
-    
+
-            now = datetime.now()
+            set_start_method("spawn")
 
             mastodon, mastodon_hostname = mastodon()
 
@@ -608,12 +442,16 @@ if __name__ == '__main__':
 
             alive_servers = get_last_checked_servers()
 
+            getservers = Server()
+
             ###########################################################################
             # multiprocessing!
 
-            nprocs = multiprocessing.cpu_count()
+            with get_context("spawn").Pool(processes=32) as pool:
-            with multiprocessing.Pool(processes=64) as pool:
+
-                results = pool.starmap(get_alive_servers, product(alive_servers))
+                res = pool.starmap(getservers.get_alive_servers, product(alive_servers))
+                pool.close()
+                pool.join()
 
         elif sys.argv[1] == '--mono':
 
@@ -624,17 +462,22 @@ if __name__ == '__main__':
             fediverse_db, fediverse_db_user = db_config()
 
             total_servers = 0
+
             total_users = 0
 
             set_world_servers_check_to_false()
 
             alive_servers = get_last_checked_servers()
 
+            getservers = Server()
+
             i = 0
 
             while i < len(alive_servers):
 
-                get_alive_servers(alive_servers[i])
+                getservers.server = alive_servers[i]
+
+                getservers.get_alive_servers()
 
                 i += 1
 
@@ -645,6 +488,8 @@ if __name__ == '__main__':
         ###########################################################################
         # get current total servers and users, get users from every software
 
+        now = datetime.now()
+
         gettotals_sql = "select count(server), sum(users) from fediverse where alive"
         get_soft_totals_sql = "select software, sum(users) as users, count(server) as servers from fediverse where users != 0 and alive group by software order by users desc"
         soft_total_project = []
@@ -948,4 +793,4 @@ if __name__ == '__main__':
 
         users_image_id = mastodon.media_post('users.png', "image/png", description='users graph').id
 
-        mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, users_image_id})
+        #mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, users_image_id})

fetchservers.py

@@ -1,3 +1,6 @@
+from multiprocessing import set_start_method
+from multiprocessing import get_context
+from itertools import product
 import time
 from datetime import datetime
 import os
@@ -5,22 +8,10 @@ import json
 import sys
 import os.path
 import psycopg2
-from multiprocessing import Pool, Manager
 import aiohttp
 import asyncio
 import socket
-
+import pdb
-apis = ['/nodeinfo/2.0?',
-        '/nodeinfo/2.0.json?',
-        '/main/nodeinfo/2.0?',
-        '/api/statusnet/config?',
-        '/api/nodeinfo/2.0.json?',
-        '/api/nodeinfo?',
-        '/api/v1/instance?',
-        '/wp-json/nodeinfo/2.0?',
-        '/api/v1/instance/nodeinfo/2.0?',
-        '/.well-known/x-nodeinfo2?'
-        ]
 
 client_exceptions = (
     aiohttp.ClientResponseError,
@@ -31,28 +22,24 @@ client_exceptions = (
     socket.gaierror,
 )
 
-
-def is_json(myjson):
-
-    try:
-        json_object = json.loads(myjson)
-    except ValueError as e:
-        return False
-    return True
-
 def write_api(server, software, users, alive, api, soft_version):
 
+    now = datetime.now()
+
+    fediverse_db, fediverse_db_user = get_db_config()
+
     insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
 
     conn = None
 
     try:
 
-        conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql",
+        conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
-                                port="5432")
 
         cur = conn.cursor()
 
+        print(f'Writing {server} nodeinfo data...')
+
         cur.execute(insert_sql, (server, now, software, users, alive, api, soft_version))
 
         cur.execute(
@@ -73,16 +60,20 @@ def write_api(server, software, users, alive, api, soft_version):
     finally:
 
         if conn is not None:
+
             conn.close()
 
 async def getsoft(server):
 
+    fediverse_db, fediverse_db_user = get_db_config()
+
     try:
 
         socket.gethostbyname(server)
 
-    except socket.gaierror:
+    except socket.gaierror as g_error:
 
+        print(f'Server {server} error: {g_error}')
         pass
         return
 
@@ -93,71 +84,134 @@ async def getsoft(server):
     user_agent = {'User-agent': 'Mozilla/5.0'}
 
     timeout = aiohttp.ClientTimeout(total=3)
+
     async with aiohttp.ClientSession(timeout=timeout, headers=user_agent) as session:
-        for api in apis:
 
-            try:
+        try:
+
+            async with session.get(url + '/.well-known/nodeinfo') as response:
+
+                if response.status == 200:
+
+                    try:
+
+                        response_json = await response.json()
+
+                        nodeinfo = response_json['links'][0]['href'].replace(f'https://{server}','')
+
+                    except:
+
+                        pass
+                else:
+
+                    print(f'Server {server} not responding: {response.status}')
+
+                    pass
+
+                async with session.get(url + nodeinfo) as nodeinfo_response:
+
+                    if nodeinfo_response.status == 200:
 
-                async with session.get(url + api) as response:
-                    if response.status == 200:
                         try:
-                            response_json = await response.json()
+
+                            nodeinfo_json = await nodeinfo_response.json()
+
                         except:
+
                             pass
 
-            except aiohttp.ClientConnectorError as err:
+                    else:
 
-                pass
+                        print(f"Server {server}'s nodeinfo not responding: {response.status}")
+
+                        pass
+
+        except aiohttp.ClientConnectorError as cc_err:
+
+            pass
+
+        except aiohttp.client_exceptions.ClientConnectorSSLError as ccssl_as:
+
+            pass
+
+        else:
+
+            if nodeinfo_response.status == 200 and nodeinfo != '/api/v1/instance?':
+
+                if nodeinfo != '/.well-known/x-nodeinfo2?':
+
+                    try:
+
+                        soft = nodeinfo_json['software']['name']
+                        soft = soft.lower()
+                        soft_version = nodeinfo_json['software']['version']
+                        users = nodeinfo_json['usage']['users']['total']
+                        if users > 1000000:
+                            return
+                        alive = True
+
+                        write_api(server, soft, users, alive, nodeinfo, soft_version)
+
+                        print('*********************************************************************')
+                        print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
+                        print('*********************************************************************')
+
+                        return
+
+                    except:
+
+                        pass
+
+                else:
+
+                    try:
+
+                        soft = nodeinfo_json['server']['software']
+                        soft = soft.lower()
+                        soft_version = nodeinfo_json['server']['version']
+                        users = nodeinfo_json['usage']['users']['total']
+                        if users > 1000000:
+                            return
+                        alive = True
+
+                        if soft == 'socialhome':
+
+                            write_api(server, soft, users, alive, api, soft_version)
+
+                            print('*********************************************************************')
+                            print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
+                            print('*********************************************************************')
+
+                        return
+
+                    except:
+
+                        pass
+
+            if nodeinfo_response.status == 200 and soft == '' and nodeinfo == "/api/v1/instance?":
+
+                soft = 'mastodon'
+                users = nodeinfo_json['stats']['user_count']
+                soft_version = nodeinfo_json['version']
+                if users > 1000000:
+                    return
+                alive = True
+
+                write_api(server, soft, users, alive, api)
+
+                print('*********************************************************************')
+                print("Server " + server + " (" + soft + ") is alive!")
+                print('*********************************************************************')
 
             else:
 
-                if response.status == 200 and api != '/api/v1/instance?':
+                print(f'Server {server} is dead')
 
-                    if api != '/.well-known/x-nodeinfo2?':
+def getserver(server, *args):
 
-                        try:
+    if len(args) != 0:
-                            soft = response_json['software']['name']
-                            soft = soft.lower()
-                            soft_version = response_json['software']['version']
-                            users = response_json['usage']['users']['total']
-                            if users > 1000000:
-                                return
-                            alive = True
-                            write_api(server, soft, users, alive, api, soft_version)
-                            print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
-                            return
-                        except:
-                            pass
-                    else:
 
-                        try:
+        server = server[0].rstrip('.').lower()
-                            soft = response_json['server']['software']
-                            soft = soft.lower()
-                            soft_version = response_json['server']['version']
-                            users = response_json['usage']['users']['total']
-                            if users > 1000000:
-                                return
-                            alive = True
-                            if soft == 'socialhome':
-                                write_api(server, soft, users, alive, api, soft_version)
-                                print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
-                            return
-                        except:
-                            pass
-
-                if response.status == 200 and soft == '' and api == "/api/v1/instance?":
-                    soft = 'mastodon'
-                    users = response_json['stats']['user_count']
-                    soft_version = response_json['version']
-                    if users > 1000000:
-                        return
-                    alive = True
-                    write_api(server, soft, users, alive, api)
-                    print("Server " + server + " (" + soft + ") is alive!")
-
-def getserver(server, x):
-
-    server = server[0].rstrip('.').lower()
 
     if server.find(".") == -1:
         return
@@ -178,7 +232,46 @@ def getserver(server, x):
 
         pass
 
+def get_world_servers():
+
+    world_servers = []
+
+    try:
+
+        conn = None
+
+        conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
+
+        cur = conn.cursor()
+
+        # get world servers list
+
+        cur.execute("select server from world where checked='f'")
+
+        rows = cur.fetchall()
+
+        for row in rows:
+
+            world_servers.append(row[0])
+
+        cur.close()
+
+        print("Remaining servers: " + str(len(world_servers)))
+
+    except (Exception, psycopg2.DatabaseError) as error:
+
+        print(error)
+
+    finally:
+
+        if conn is not None:
+
+            conn.close()
+
+    return world_servers
+
 def get_parameter(parameter, file_path):
+
     # Check if secrets file exists
     if not os.path.isfile(file_path):
         print("File %s not found, exiting." % file_path)
@@ -194,6 +287,10 @@ def get_parameter(parameter, file_path):
     print(file_path + "  Missing parameter %s " % parameter)
     sys.exit(0)
 
+def usage():
+
+    print('usage: python ' + sys.argv[0] + ' --multi' + ' (multiprocessing, fast)')
+    print('usage: python ' + sys.argv[0] + ' --mono' + ' (one process, slow)')
 
 def get_config():
 
@@ -214,59 +311,51 @@ def get_db_config():
 # main
 
 if __name__ == '__main__':
-    
-    now = datetime.now()
-    start_time = time.time()
 
-    mastodon_hostname = get_config()
+    # usage modes
 
-    fediverse_db, fediverse_db_user = get_db_config()
+    if len(sys.argv) == 1:
 
-    world_servers = []
+        usage()
 
-    try:
+    elif len(sys.argv) == 2:
 
-        conn = None
+        if sys.argv[1] == '--multi':
 
-        conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
+            now = datetime.now()
 
-        cur = conn.cursor()
+            start_time = time.time()
 
-        # get world servers list
+            mastodon_hostname = get_config()
 
-        cur.execute("select server from world where checked='f'")
+            fediverse_db, fediverse_db_user = get_db_config()
 
-        rows = cur.fetchall()
+            world_servers = get_world_servers()
- 
-        for row in rows:
-            
-            world_servers.append(row[0])
 
-        cur.close()
+            with get_context("spawn").Pool(processes=32) as pool:
 
-        print("Remaining servers: " + str(len(world_servers)))
+                res = pool.starmap(getserver, product(world_servers))
 
-    except (Exception, psycopg2.DatabaseError) as error:
+                pool.close()
 
-        print(error)
+                pool.join()
 
-    finally:
+            print('Done.')
 
-        if conn is not None:
+        elif sys.argv[1] == '--mono':
 
-            conn.close()
+            now = datetime.now()
 
-    ###########################################################################
+            start_time = time.time()
-    # multiprocessing!
 
-    m = Manager()
+            mastodon_hostname = get_config()
-    q = m.Queue()
-    z = zip(world_servers)
 
-    serv_number = len(world_servers)
+            fediverse_db, fediverse_db_user = get_db_config()
 
-    pool_tuple = [(x, q) for x in z]
+            world_servers = get_world_servers()
-    with Pool(processes=64) as pool:
-        pool.starmap(getserver, pool_tuple)
 
-    print('Done.')
+            for server in world_servers:
+
+                getserver(server)
+
+            print('Done.')