Refactored

app/libraries/database.py

@@ -0,0 +1,675 @@
+import os
+import sys
+import psycopg2
+from psycopg2 import sql
+from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
+import uuid
+from datetime import datetime
+import pytz
+import pdb
+
+tz = pytz.timezone('Europe/Madrid')
+
+class Database():
+
+    name = 'fediverse database library'
+
+    def __init__(self, config_file=None, blocker_db=None, blocker_db_user=None, blocker_db_user_password=None):
+
+        self.config_file = "config/db_config.txt"
+        self.blocker_db = self.__get_parameter("blocker_db", self.config_file)
+        self.blocker_db_user = self.__get_parameter("blocker_db_user", self.config_file)
+        self.blocker_db_user_password = self.__get_parameter("blocker_db_user_password", self.config_file)
+
+        db_setup = self.__check_dbsetup(self)
+
+        if not db_setup:
+
+            self.blocker_db = input("\nMau database name: ")
+            self.blocker_db_user = input("\nMau database user: ")
+            self.blocker_db_user_password = input("\nMau database user password: ")
+
+            self.__createdb(self)  
+            self.__create_config(self)
+            self.__write_config(self)
+
+    def get_nodeinfo_endpoint(self, server):
+
+        try:
+
+            conn = None
+
+            conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = self.blocker_db_user_password, host = "/var/run/postgresql", port = "6432")
+
+            cur = conn.cursor()
+
+            cur.execute("select server, api from servers where server=(%s)", (server,))
+
+            row = cur.fetchone()
+
+            if row != None:
+
+                server = row[0]
+
+                api = row[1]
+
+            else:
+
+                server = ''
+
+                api = ''
+
+            cur.close()
+
+            return (server, api)
+
+        except (Exception, psycopg2.DatabaseError) as error:
+
+            sys.exit(error)
+
+        finally:
+
+            if conn is not None:
+
+                conn.close()
+
+    def get_servers(self, software):
+
+        servers_list = []
+
+        try:
+
+            conn = None
+
+            conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = "", host = "/var/run/postgresql", port = "5432")
+
+            cur = conn.cursor()
+
+            cur.execute("select server from servers where software=(%s)", (software,))
+
+            rows = cur.fetchall()
+
+            for row in rows:
+
+                servers_list.append(row[0])
+
+            cur.close()
+
+        except (Exception, psycopg2.DatabaseError) as error:
+
+            sys.exit(error)
+
+        finally:
+
+            if conn is not None:
+
+                conn.close()
+        
+        return (servers_list)
+
+    def save_time(self, program, start, finish):
+
+        insert_sql = "INSERT INTO execution_time(program, start, finish) VALUES(%s,%s,%s) ON CONFLICT DO NOTHING"
+
+        conn = None
+
+        try:
+
+            conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = self.blocker_db_user_password, host = "/var/run/postgresql", port = "6432")
+
+            cur = conn.cursor()
+
+            cur.execute(insert_sql, (program, start, finish,))
+
+            cur.execute("UPDATE execution_time SET start=(%s), finish=(%s) where program=(%s)", (start, finish, program))
+
+            conn.commit()
+
+            cur.close()
+
+        except (Exception, psycopg2.DatabaseError) as error:
+
+            print(error)
+
+        finally:
+
+            if conn is not None:
+
+                conn.close()
+
+    def check_peer(self, server):
+
+        found = False
+
+        select_server_sql = 'select server from servers where server=(%s)'
+
+        select_dead_server_sql = 'select server from deadservers where server=(%s)'
+
+        try:
+
+            conn = None
+
+            conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432")
+
+            cur = conn.cursor()
+
+            # check server
+
+            cur.execute(select_server_sql, (server,))
+
+            row = cur.fetchone()
+
+            if row != None:
+
+                found = True
+
+            cur.execute(select_dead_server_sql, (server,))
+
+            row = cur.fetchone()
+
+            if row != None:
+
+                found = True
+
+            cur.close()
+
+        except (Exception, psycopg2.DatabaseError) as error:
+
+            print(error)
+
+        finally:
+
+            if conn is not None:
+
+                conn.close()
+
+        return found
+
+    def get_not_updated_servers(self):
+
+        not_updated_servers = []
+
+        try:
+
+            conn = None
+
+            conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432")
+
+            cur = conn.cursor()
+
+            # get saved servers list
+
+            cur.execute("select server from servers where updated_at is null")
+
+            rows = cur.fetchall()
+
+            for row in rows:
+
+                not_updated_servers.append(row[0])
+
+            cur.close()
+
+            print("Not updated servers: " + str(len(not_updated_servers)))
+
+        except (Exception, psycopg2.DatabaseError) as error:
+
+            print(error)
+
+        finally:
+
+            if conn is not None:
+
+                conn.close()
+
+        return not_updated_servers
+
+    def get_saved_servers(self):
+
+        saved_servers = []
+
+        try:
+
+            conn = None
+
+            conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432")
+
+            cur = conn.cursor()
+
+            # get saved servers list
+
+            cur.execute("select server from servers where api is not null")
+
+            rows = cur.fetchall()
+
+            for row in rows:
+
+                saved_servers.append(row[0])
+
+            cur.close()
+
+            print("Saved servers: " + str(len(saved_servers)))
+
+        except (Exception, psycopg2.DatabaseError) as error:
+
+            print(error)
+
+        finally:
+
+            if conn is not None:
+
+                conn.close()
+
+        return saved_servers
+
+    def get_dead_servers(self):
+
+        dead_servers = []
+
+        try:
+
+            conn = None
+
+            conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432")
+
+            cur = conn.cursor()
+
+            # get dead servers list
+
+            cur.execute("select server from deadservers")
+
+            rows = cur.fetchall()
+
+            for row in rows:
+
+                dead_servers.append(row[0])
+
+            cur.close()
+
+            print("Dead servers: " + str(len(dead_servers)))
+
+        except (Exception, psycopg2.DatabaseError) as error:
+
+            print(error)
+
+        finally:
+
+            if conn is not None:
+
+                conn.close()
+
+        return dead_servers
+
+    def soft_totals(self):
+
+        # get current total servers and users, get users from every software
+
+        now = datetime.now()
+
+        gettotals_sql = "select count(server), sum(users), sum(mau) from mau where alive"
+        get_soft_totals_sql = "select software, sum(users) as users, sum(mau) as mau, count(server) as servers from mau where users != 0 and mau is not null and alive group by software order by mau desc"
+
+        soft_total_project = []
+        soft_total_users = []
+        soft_total_mau = []
+        soft_total_servers = []
+
+        try:
+
+            conn = None
+
+            conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432")
+
+            cur = conn.cursor()
+
+            cur.execute(gettotals_sql)
+
+            row = cur.fetchone()
+
+            total_servers = row[0]
+
+            total_users = row[1]
+
+            total_mau = row[2]
+
+            cur.execute(get_soft_totals_sql)
+
+            rows = cur.fetchall()
+
+            for row in rows:
+
+                soft_total_project.append(row[0])
+                soft_total_users.append(row[1])
+                soft_total_mau.append(row[2])
+                soft_total_servers.append(row[3])
+
+            cur.close()
+
+        except (Exception, psycopg2.DatabaseError) as error:
+
+            print(error)
+
+        finally:
+
+            if conn is not None:
+
+                conn.close()
+
+        return (soft_total_project, soft_total_users, soft_total_mau, soft_total_servers, total_servers, total_users, total_mau)
+
+    def get_last_checked_servers(self):
+
+        # get last checked servers from fediverse DB
+
+        alive_servers = []
+
+        try:
+
+            conn = None
+
+            conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432")
+
+            cur = conn.cursor()
+
+            # get world servers list
+
+            cur.execute("select server from world where server in (select server from fediverse where users_api != '')")
+
+            rows = cur.fetchall()
+
+            for row in rows:
+
+                alive_servers.append(row[0])
+
+            cur.close()
+
+        except (Exception, psycopg2.DatabaseError) as error:
+
+            print(error)
+
+        finally:
+
+            if conn is not None:
+
+                conn.close()
+
+        return alive_servers
+
+    def write_alive_server(self, server, software, version, users, mau, alive):
+
+        now = datetime.now()
+
+        insert_sql = "INSERT INTO mau(server, software, version, users, mau, alive, updated_at) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
+
+        conn = None
+
+        try:
+
+            conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432")
+
+            cur = conn.cursor()
+
+            cur.execute(insert_sql, (server, software, version, users, mau, alive, now))
+
+            cur.execute("UPDATE mau SET software=(%s), version=(%s), users=(%s), mau=(%s), alive=(%s), updated_at=(%s) where server=(%s)", (software, version, users, mau, alive, now, server))
+
+            conn.commit()
+
+            cur.close()
+
+        except (Exception, psycopg2.DatabaseError) as error:
+
+            print(error)
+
+        finally:
+
+            if conn is not None:
+
+                conn.close()
+
+    def write_not_alive_server(self, server):
+
+        now = datetime.now()
+
+        update_sql = "UPDATE mau set alive='f', updated_at=(%s) where server=(%s)"
+
+        conn = None
+
+        try:
+
+            conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432")
+
+            cur = conn.cursor()
+
+            cur.execute(update_sql, (now, server))
+
+            conn.commit()
+
+            cur.close()
+
+        except (Exception, psycopg2.DatabaseError) as error:
+
+            print(error)
+
+        finally:
+
+            if conn is not None:
+
+                conn.close()
+
+
+    def write_server(self, server, api, software):
+
+        now = datetime.now()
+
+        insert_sql = "INSERT INTO servers(server, api, software, created_at) VALUES(%s, %s, %s, %s) ON CONFLICT DO NOTHING"
+
+        conn = None
+
+        try:
+
+            conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432")
+
+            cur = conn.cursor()
+
+            cur.execute(insert_sql, (server, api, software, now))
+
+            conn.commit()
+
+            cur.close()
+
+        except (Exception, psycopg2.DatabaseError) as error:
+
+            print(error)
+
+        finally:
+
+            if conn is not None:
+
+                conn.close()
+
+    def update_peer(self, server):
+
+        now = datetime.now()
+
+        update_sql = "UPDATE servers set updated_at=(%s) where server=(%s)"
+
+        conn = None
+
+        try:
+
+            conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432")
+
+            cur = conn.cursor()
+
+            cur.execute(update_sql, (now, server))
+
+            conn.commit()
+
+            cur.close()
+
+        except (Exception, psycopg2.DatabaseError) as error:
+
+            print(error)
+
+        finally:
+
+            if conn is not None:
+
+                conn.close()
+
+    def update_server(self, server, api):
+
+        now = datetime.now()
+
+        update_sql = "UPDATE servers set server=(%s), api=(%s), updated_at=(%s) where server=(%s)"
+
+        conn = None
+
+        try:
+
+            conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432")
+
+            cur = conn.cursor()
+
+            cur.execute(update_sql, (server, api, now, server))
+
+            conn.commit()
+
+            cur.close()
+
+        except (Exception, psycopg2.DatabaseError) as error:
+
+            print(error)
+
+        finally:
+
+            if conn is not None:
+
+                conn.close()
+
+    @staticmethod
+    def __check_dbsetup(self):
+
+        db_setup = False
+
+        try:
+
+            conn = None
+
+            conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = self.blocker_db_user_password, host = "/var/run/postgresql", port = "6432")
+
+            db_setup = True
+
+        except (Exception, psycopg2.DatabaseError) as error:
+
+            print(error)
+
+        return db_setup
+
+    @staticmethod
+    def __createdb(self):
+
+        conn = None
+
+        try:
+
+            conn = psycopg2.connect(dbname='postgres',
+                user=self.blocker_db_user, host='',
+                password=self.blocker_db_user_password)
+
+            conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
+
+            cur = conn.cursor()
+
+            print(f"Creating database {self.blocker_db}. Please wait...")
+
+            cur.execute(sql.SQL("CREATE DATABASE {}").format(
+                sql.Identifier(self.blocker_db))
+            )
+            print(f"Database {self.blocker_db} created!\n")
+
+            self.__dbtables_schemes(self)
+
+        except (Exception, psycopg2.DatabaseError) as error:
+
+            print(error)
+
+        finally:
+
+            if conn is not None:
+
+                conn.close()
+
+    @staticmethod
+    def __dbtables_schemes(self):
+
+        db = self.blocker_db
+        table = "servers"
+        sql = "create table "+table+" (server varchar(200) PRIMARY KEY, api varchar(50), software varchar(50), created_at timestamptz, updated_at timestamptz, downs int)"
+        self.__create_table(self, table, sql)
+
+    @staticmethod
+    def __create_table(self, table, sql):
+
+        conn = None
+
+        try:
+
+            conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = self.blocker_db_user_password, host = "/var/run/postgresql", port = "6432")
+
+            cur = conn.cursor()
+
+            print(f"Creating table {table}")
+
+            cur.execute(sql)
+
+            conn.commit()
+
+            print(f"Table {table} created!\n")
+
+        except (Exception, psycopg2.DatabaseError) as error:
+
+            print(error)
+
+        finally:
+
+            if conn is not None:
+
+                conn.close()
+
+    def __get_parameter(self, parameter, config_file):
+
+        if not os.path.isfile(config_file):
+            print(f"File {config_file} not found..")
+            return
+
+        with open( config_file ) as f:
+            for line in f:
+                if line.startswith( parameter ):
+                    return line.replace(parameter + ":", "").strip()
+
+        print(f"{config_file} Missing parameter {parameter}")
+
+        sys.exit(0)
+
+    @staticmethod
+    def __create_config(self):
+
+        if not os.path.exists('config'):
+
+            os.makedirs('config')
+
+        if not os.path.exists(self.config_file):
+
+            print(self.config_file + " created!")
+            with open(self.config_file, 'w'): pass
+
+    @staticmethod
+    def __write_config(self):
+
+        with open(self.config_file, 'a') as the_file:
+
+            the_file.write(f'blocker_db: {self.blocker_db}\nblocker_db_user: {self.blocker_db_user}\nblocker_db_user_password: {self.blocker_db_user_password}')
+            print(f"adding parameters to {self.config_file}\n")
+
+

app/libraries/domainblocks.py

@@ -0,0 +1,114 @@
+import os
+import sys
+from app.libraries.setup import Setup
+from mastodon import Mastodon
+from mastodon.Mastodon import MastodonNotFoundError, MastodonNetworkError, MastodonReadTimeout, MastodonAPIError, MastodonUnauthorizedError, MastodonIllegalArgumentError
+import requests
+import pdb
+
+class DomainBlocks():
+
+    name = 'Domain blocks for Mastodon'
+
+    def __init__(self, mastodon_hostname=None, domain_block_api=None, session=None):
+
+        self.domain_blocks_api = '/api/v1/admin/domain_blocks'
+        setup = Setup()
+        self.mastodon_hostname = setup.mastodon_hostname 
+        self.mastodon_app_token = { 'Authorization': 'Bearer %s'%setup.mastodon_app_token }
+
+        if session:
+            self.session = session
+        else:
+            self.session = requests.Session()
+
+    def domain_blocks_create(self, server, priv_comment, pub_comment):
+
+        data = {
+                'domain': server,
+                'severity': 'suspend',
+                'reject_media': 'true',
+                'reject_reports': 'true',
+                'private_comment': priv_comment,
+                'public_comment': pub_comment,
+                'obfuscate': 'true',
+                }
+        
+        endpoint = f'https://{self.mastodon_hostname}/{self.domain_blocks_api}'
+        
+        response = self.__api_request('POST', endpoint, data)
+
+        if response.ok:
+
+            print(f"Done, {server} is now suspended")
+
+        else:
+
+            pass
+
+    def __api_request(self, method, endpoint, data={}):
+
+        response = None
+
+        try:
+
+            kwargs = dict(data=data)
+
+            response = self.session.request(method, url = endpoint, headers = self.mastodon_app_token, **kwargs)
+
+        except Exception as e:
+
+            raise MastodonNetworkError(f"Could not complete request: {e}")
+
+        if response is None:
+
+            raise MastodonIllegalArgumentError("Illegal request.")
+
+        if not response.ok:
+
+                try:
+                    if isinstance(response, dict) and 'error' in response:
+                        error_msg = response['error']
+                    elif isinstance(response, str):
+                        error_msg = response
+                    else:
+                        error_msg = None
+                except ValueError:
+                    error_msg = None
+
+                if response.status_code == 404:
+                    ex_type = MastodonNotFoundError
+                    if not error_msg:
+                        error_msg = 'Endpoint not found.'
+                        # this is for compatibility with older versions
+                        # which raised MastodonAPIError('Endpoint not found.')
+                        # on any 404
+                elif response.status_code == 401:
+                    ex_type = MastodonUnauthorizedError
+                elif response.status_code == 422:
+                    return response
+                elif response.status_code == 500:
+                    ex_type = MastodonInternalServerError
+                elif response.status_code == 502:
+                    ex_type = MastodonBadGatewayError
+                elif response.status_code == 503:
+                    ex_type = MastodonServiceUnavailableError
+                elif response.status_code == 504:
+                    ex_type = MastodonGatewayTimeoutError
+                elif response.status_code >= 500 and \
+                    response.status_code <= 511:
+                    ex_type = MastodonServerError
+                else:
+                    ex_type = MastodonAPIError
+
+                raise ex_type(
+                    'Mastodon API returned error',
+                    response.status_code,
+                    response.reason,
+                    error_msg)
+
+        else:
+
+            return response
+
+

app/libraries/nodeinfo.py

@@ -0,0 +1,70 @@
+import time
+from datetime import datetime
+import os
+import json
+import sys
+import os.path
+import requests
+import urllib3
+import socket
+from app.libraries.database import Database
+from app.libraries.setup import Setup
+import pdb
+
+def is_json(myjson):
+
+    try:
+        json_object = json.loads(myjson)
+    except ValueError as e:
+        return False
+    return True
+
+class Nodeinfo():
+
+    name = "Query server nodeinfo data"
+
+    def __init__(self, server=None, db=None, setup=None, soft=None):
+
+        self.server = server
+        self.db = Database()
+        self.setup = Setup()
+        self.soft = ''
+
+    def getdata(self, server, api):
+
+        if server.find(".") == -1:
+            return
+        if server.find("@") != -1:
+            return
+        if server.find("/") != -1:
+            return
+        if server.find(":") != -1:
+            return
+
+        url = 'https://' + server
+
+        self.soft = ''
+
+        try:
+
+            response = requests.get(url + api, headers = self.setup.user_agent, timeout=3)
+
+            if response.status_code == 200:
+
+                try:
+
+                    self.soft = response.json()['software']['name']
+                    self.soft = self.soft.lower()
+
+                    print(f"Server {server} ({self.soft} is alive!")
+                    print('*********************************************************************')
+
+                except:
+
+                    pass
+
+        except:
+
+            pass
+
+        return (server, self.soft)

app/libraries/peers.py

@@ -0,0 +1,264 @@
+import time
+from datetime import datetime
+import os
+import json
+import sys
+import os.path
+from app.libraries.setup import Setup
+from app.libraries.database import Database
+import requests
+import urllib3
+import socket
+
+class Peers():
+
+    name = "Get peers from server"
+
+    def __init__(self, server=None, db=None, setup=None):
+
+        self.server = server
+        self.db = Database()
+        self.setup = Setup()
+
+    def getpeers(self, server):
+
+        try:
+
+            res = requests.get(f'https://{server}/{self.setup.peers_api}, headers = self.setup.user_agent, timeout=3')
+
+            peers = res.json()
+
+            if isinstance(peers, list):
+
+                self.db.update_peer(server)
+
+                return peers
+
+            else:
+
+                return None
+
+        except requests.exceptions.SSLError as errssl:
+
+            pass
+
+        except requests.exceptions.HTTPError as errh:
+
+            pass
+
+        except requests.exceptions.ConnectionError as errc:
+
+            pass
+
+        except requests.exceptions.ReadTimeout as to_err:
+
+            pass
+
+        except requests.exceptions.TooManyRedirects as tmr_err:
+
+            pass
+
+        except urllib3.exceptions.LocationParseError as lp_err:
+
+            pass
+
+        except requests.exceptions.InvalidURL as iu_err:
+
+            pass
+
+        except requests.exceptions.ChunkedEncodingError as chunk_err:
+
+            print(f'ChunkedEncodingError! {server}')
+            pass
+
+        except requests.exceptions.JSONDecodeError as jsondec_err:
+
+            print(jsondec_err)
+            pass
+
+        print('*** Not peers api!')
+        return None
+
+    def getapi(self, server):
+
+        if server.find(".") == -1:
+            return
+        if server.find("@") != -1:
+            return
+        if server.find("/") != -1:
+            return
+        if server.find(":") != -1:
+            return
+
+        is_nodeinfo = False
+
+        nodeinfo = ''
+
+        url = 'https://' + server
+
+        try:
+
+            response = requests.get(url + '/.well-known/nodeinfo', headers = self.setup.user_agent, timeout=3)
+
+            if response.status_code == 200:
+
+                try:
+
+                    response_json = response.json()
+
+                    if len(response_json['links']) == 1:
+
+                        start = [pos for pos, char in enumerate(response_json['links'][0]['href']) if char == '/'][1]
+                        end = [pos for pos, char in enumerate(response_json['links'][0]['href']) if char == '/'][2]
+
+                        server = response_json['links'][0]['href'][start+1:end]
+
+                        server_idx = response_json['links'][0]['href'].index(server)
+
+                        nodeinfo = response_json['links'][0]['href'][server_idx:].replace(server, '')
+
+
+                    elif len(response_json['links']) == 2:
+
+                        start = [pos for pos, char in enumerate(response_json['links'][1]['href']) if char == '/'][1]
+                        end = [pos for pos, char in enumerate(response_json['links'][1]['href']) if char == '/'][2]
+
+                        server = response_json['links'][1]['href'][start+1:end]
+
+                        server_idx = response_json['links'][1]['href'].index(server)
+
+                        nodeinfo = response_json['links'][1]['href'][server_idx:].replace(server, '')
+
+                except:
+
+                    pass
+
+        except requests.exceptions.ChunkedEncodingError as chunk_err:
+
+            pass
+
+        except requests.exceptions.InvalidSchema as invalid_err:
+
+            pass
+
+        except requests.exceptions.SSLError as errssl:
+
+            pass
+
+        except requests.exceptions.HTTPError as errh:
+
+            pass
+
+        except requests.exceptions.ConnectionError as errc:
+
+            pass
+
+        except requests.exceptions.ReadTimeout as to_err:
+
+            pass
+
+        except requests.exceptions.TooManyRedirects as tmr_err:
+
+            pass
+
+        except urllib3.exceptions.LocationParseError as lp_err:
+
+            pass
+
+        except requests.exceptions.InvalidURL as iu_err:
+
+            pass
+
+        except requests.exceptions.ChunkedEncodingError as chunk_err:
+
+            print(f'ChunkedEncodingError! {server}')
+            pass
+
+        return nodeinfo
+
+    def updateapi(self, server):
+
+        if server.find(".") == -1:
+            return
+        if server.find("@") != -1:
+            return
+        if server.find("/") != -1:
+            return
+        if server.find(":") != -1:
+            return
+
+        nodeinfo = ''
+
+        url = 'https://' + server
+
+        try:
+
+            response = requests.get(url + '/.well-known/nodeinfo', headers = self.setup.user_agent, timeout=3)
+
+            if response.status_code == 200:
+
+                try:
+
+                    response_json = response.json()
+
+                    if len(response_json['links']) == 1:
+
+                        start = [pos for pos, char in enumerate(response_json['links'][0]['href']) if char == '/'][1]
+                        end = [pos for pos, char in enumerate(response_json['links'][0]['href']) if char == '/'][2]
+
+                        server = response_json['links'][0]['href'][start+1:end]
+
+                        server_idx = response_json['links'][0]['href'].index(server)
+
+                        nodeinfo = response_json['links'][0]['href'][server_idx:].replace(server, '')
+
+                    elif len(response_json['links']) == 2:
+
+                        start = [pos for pos, char in enumerate(response_json['links'][1]['href']) if char == '/'][1]
+                        end = [pos for pos, char in enumerate(response_json['links'][1]['href']) if char == '/'][2]
+
+                        server = response_json['links'][1]['href'][start+1:end]
+
+                        server_idx = response_json['links'][1]['href'].index(server)
+
+                        nodeinfo = response_json['links'][1]['href'][server_idx:].replace(server, '')
+
+                except:
+
+                    pass
+
+        except requests.exceptions.SSLError as errssl:
+
+            pass
+
+        except requests.exceptions.HTTPError as errh:
+
+            pass
+
+        except requests.exceptions.ConnectionError as errc:
+
+            pass
+
+        except requests.exceptions.ReadTimeout as to_err:
+
+            pass
+
+        except requests.exceptions.TooManyRedirects as tmr_err:
+
+            pass
+
+        except urllib3.exceptions.LocationParseError as lp_err:
+
+            pass
+
+        except requests.exceptions.InvalidURL as iu_err:
+
+            pass
+
+        except requests.exceptions.ChunkedEncodingError as chunk_err:
+
+            print(f'ChunkedEncodingError! {server}')
+            pass
+
+        return server, nodeinfo
+

app/libraries/setup.py

@@ -0,0 +1,157 @@
+import os
+import sys
+from datetime import datetime
+import pytz
+from mastodon import Mastodon
+from mastodon.Mastodon import MastodonMalformedEventError, MastodonNetworkError, MastodonReadTimeout, MastodonAPIError, MastodonIllegalArgumentError
+import pdb
+
+class Setup():
+
+    name = 'fediverse setup'
+
+    def __init__(self, config_file=None, mastodon_hostname=None, peers_api=None, user_agent=None, secrets_filepath=None, mastodon_app_token=None):
+
+        self.config_file = "config/config.txt"
+        self.mastodon_hostname = self.__get_parameter("mastodon_hostname", self.config_file)
+        self.peers_api = '/api/v1/instance/peers?'
+        self.user_agent = {'User-agent': "fediverse's stats (fediverse@mastodont.cat)"}
+
+        self.secrets_filepath = 'secrets/secrets.txt'
+
+        is_setup = self.__check_mastodon_setup(self)
+
+        if is_setup:
+
+            self.mastodon_app_token = self.__get_mastodon_parameter("mastodon_app_token", self.secrets_filepath)
+
+        else:
+
+            self.mastodon_app_token = self.mastodon_setup(self)
+
+    @staticmethod
+    def __check_mastodon_setup(self):
+
+        is_setup = False
+
+        if not os.path.isfile(self.secrets_filepath):
+            print(f"File {self.secrets_filepath} not found, running setup.")
+        else:
+            is_setup = True
+
+        return is_setup
+
+    @staticmethod
+    def mastodon_setup(self):
+
+        if not os.path.exists('secrets'):
+            os.makedirs('secrets')
+
+        self.mastodon_user = input("Mastodon user login? ")
+        self.mastodon_password = input("Mastodon user password? ")
+        self.app_name = 'fediverse'
+
+        self.mastodon_app_token = self.mastodon_log_in()
+
+        if not os.path.exists(self.secrets_filepath):
+            with open(self.secrets_filepath, 'w'): pass
+            print(f"{self.secrets_filepath} created!")
+
+        with open(self.secrets_filepath, 'a') as the_file:
+            print("Writing Mastodon parameters to " + self.secrets_filepath)
+            the_file.write(f'mastodon_app_token: {self.mastodon_app_token}')
+
+        return self.mastodon_app_token
+
+    def mastodon_log_in(self):
+
+        token = ''
+
+        try:
+
+            response = Mastodon.create_app(
+                self.app_name,
+                scopes=["read","write", "admin:read", "admin:write"],
+                to_file=None,
+                api_base_url=self.mastodon_hostname
+                )
+            client_id = response[0]
+            client_secret = response[1]
+
+            mastodon = Mastodon(client_id = client_id, client_secret = client_secret, api_base_url = self.mastodon_hostname)
+        
+            token = mastodon.log_in(
+                self.mastodon_user,
+                self.mastodon_password,
+                scopes = ["read", "write", "admin:read", "admin:write"],
+                to_file = None
+            )
+
+            print('Log in succesful!')
+
+        except MastodonIllegalArgumentError as i_error:
+
+            sys.stdout.write(f'\n{str(i_error)}\n')
+
+        except MastodonNetworkError as n_error:
+
+            sys.stdout.write(f'\n{str(n_error)}\n')
+
+        except MastodonReadTimeout as r_error:
+
+            sys.stdout.write(f'\n{str(r_error)}\n')
+
+        except MastodonAPIError as a_error:
+
+            sys.stdout.write(f'\n{str(a_error)}\n')
+
+        finally:
+
+            return token
+
+    def __get_parameter(self, parameter, config_file):
+
+        if not os.path.isfile(config_file):
+            print(f"File {config_file} not found..")
+
+            self.mastodon_hostname = input("\nMastodon hostname: ")
+
+            self.__create_config(self)
+            self.__write_config(self)
+
+        with open( self.config_file ) as f:
+            for line in f:
+                if line.startswith( parameter ):
+                    return line.replace(parameter + ":", "").strip()
+
+    def __get_mastodon_parameter(self, parameter, secrets_filepath):
+
+        if not os.path.isfile(secrets_filepath):
+            print(f"File {secrets_filepath} not found..")
+
+            self.sign_in()
+
+        with open( self.secrets_filepath ) as f:
+            for line in f:
+                if line.startswith( parameter ):
+                    return line.replace(parameter + ":", "").strip()
+
+    @staticmethod
+    def __create_config(self):
+
+        if not os.path.exists('config'):
+
+            os.makedirs('config')
+
+        if not os.path.exists(self.config_file):
+
+            print(self.config_file + " created!")
+            with open(self.config_file, 'w'): pass
+
+    @staticmethod
+    def __write_config(self):
+
+        with open(self.config_file, 'a') as the_file:
+
+            the_file.write(f'mastodon_hostname: {self.mastodon_hostname}')
+            print(f"adding parameters to {self.config_file}\n")

blocksoft.py

@@ -1,487 +1,25 @@
 import os
 import sys
+from app.libraries.setup import Setup
+from app.libraries.database import Database
+from app.libraries.domainblocks import DomainBlocks
 from mastodon import Mastodon
-from mastodon.Mastodon import MastodonNotFoundError, MastodonNetworkError, MastodonReadTimeout, MastodonAPIError, MastodonUnauthorizedError, MastodonIllegalArgumentError
-import psycopg2
-from psycopg2 import sql
-from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
-import getpass
-import fileinput,re
-import requests
 
-class DomainBlocks():
-
-    name = 'Domain blocks for Mastodon'
-
-    def __init__(self, mastodon_hostname=None, domain_block_api=None, session=None):
-
-        self.domain_blocks_api = '/api/v1/admin/domain_blocks'
-        self.config_file = "config/config.txt"
-        self.secrets_file = "secrets/secrets.txt"
-
-        is_setup = self.__check_setup(self.secrets_file)
-
-        if is_setup:
-
-            self.__uc_client_id     = self.__get_parameter("uc_client_id",     self.secrets_file)
-            self.__uc_client_secret = self.__get_parameter("uc_client_secret", self.secrets_file)
-            self.__uc_access_token  = self.__get_parameter("uc_access_token",  self.secrets_file)
-
-            self.blocker_db, self.blocker_db_user = self.db_config()
-
-            self.mastodon, self.mastodon_hostname, self.headers = self.log_in() 
-
-        else:
-
-            while(True):
-
-                logged_in, self.mastodon, self.mastodon_hostname = self.setup()
-                
-                if not logged_in:
-
-                    print("\nLog in failed! Try again.\n")
-
-                else:
-
-                    break
-
-        db_setup = self.__check_dbsetup(self)
-
-        if not db_setup:
-
-            self.__createdb(self)  
-        
-        if session:
-            self.session = session
-        else:
-            self.session = requests.Session()
-
-    def get_servers(self, software):
-
-        servers_list = []
-
-        try:
-
-            conn = None
-
-            conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = "", host = "/var/run/postgresql", port = "5432")
-
-            cur = conn.cursor()
-
-            cur.execute("select server from blocker where software=(%s)", (software,))
-
-            rows = cur.fetchall()
-
-            for row in rows:
-
-                servers_list.append(row[0])
-
-            cur.close()
-
-        except (Exception, psycopg2.DatabaseError) as error:
-
-            sys.exit(error)
-
-        finally:
-
-            if conn is not None:
-
-                conn.close()
-        
-        return (servers_list)
-
-    def log_in(self):
-
-        uc_client_id     = self.__get_parameter("uc_client_id",     self.secrets_file)
-        uc_client_secret = self.__get_parameter("uc_client_secret", self.secrets_file)
-        uc_access_token  = self.__get_parameter("uc_access_token",  self.secrets_file)
-
-        self.mastodon_hostname = self.__get_parameter("mastodon_hostname", self.config_file)
-
-        mastodon = Mastodon(
-            client_id = uc_client_id,
-            client_secret = uc_client_secret,
-            access_token = uc_access_token,
-            api_base_url = 'https://' + self.mastodon_hostname,
-        )
-
-        headers={ 'Authorization': 'Bearer %s'%uc_access_token }
-
-        return (mastodon, self.mastodon_hostname, headers)
-    
-    def domain_blocks_create(self, server, priv_comment, pub_comment):
-
-        data = {
-                'domain': server,
-                'severity': 'suspend',
-                'reject_media': 'true',
-                'reject_reports': 'true',
-                'private_comment': priv_comment,
-                'public_comment': pub_comment,
-                'obfuscate': 'true',
-                }
-        
-        endpoint = f'https://{self.mastodon_hostname}/{self.domain_blocks_api}'
-        
-        response = self.__api_request('POST', endpoint, data)
-
-        if response.ok:
-
-            print(f"Done, {server} is now suspended")
-
-        else:
-
-            pass
-
-    @staticmethod
-    def __check_setup(file_path):
-
-        is_setup = False
-
-        if not os.path.isfile(file_path):
-            print(f"File {file_path} not found, running setup.")
-            return
-        else:
-            is_setup = True
-            return is_setup
-
-    def setup(self):
-
-        logged_in = False
-
-        try:
-
-            self.mastodon_hostname = input("Enter Mastodon hostname (or 'q' to exit): ")
-
-            if self.mastodon_hostname == 'q':
-
-                sys.exit("Bye")
-
-            user_name = input("Bot user name, ex. john? ")
-            user_password = getpass.getpass("Bot password? ")
-            app_name = input("Bot App name? ")
-            self.blocker_db = input("Blocker's database name: ")
-            self.blocker_db_user = input("blocker's database user: ")
-            
-            client_id, client_secret = Mastodon.create_app(
-                    app_name,
-                    scopes = ["read", "write", "admin:read", "admin:write"],
-                    to_file="app_clientcred.txt",
-                    api_base_url=self.mastodon_hostname
-                    )
-
-            mastodon = Mastodon(client_id = "app_clientcred.txt", api_base_url = self.mastodon_hostname)
-
-            mastodon.log_in(
-                user_name,
-                user_password,
-                scopes = ["read", "write", "admin:read", "admin:write"],
-                to_file = "app_usercred.txt"
-            )
-
-            if os.path.isfile("app_usercred.txt"):
-
-                print(f"Log in succesful!")
-                logged_in = True
-
-            if not os.path.exists('secrets'):
-                os.makedirs('secrets')
-
-            if not os.path.exists(self.secrets_file):
-                with open(self.secrets_file, 'w'): pass
-                print(f"{self.secrets_file} created!")
-
-            with open(self.secrets_file, 'a') as the_file:
-                print(f"Writing secrets parameter names to {self.secrets_file}")
-                the_file.write('uc_client_id: \n'+'uc_client_secret: \n'+'uc_access_token: \n')
-
-            client_path = 'app_clientcred.txt'
-
-            with open(client_path) as fp:
-
-                line = fp.readline()
-                cnt = 1
-
-                while line:
-
-                    if cnt == 1:
-
-                        print(f"Writing client id to {self.secrets_file}")
-                        self.__modify_file(self, self.secrets_file, "uc_client_id: ", value=line.rstrip())
-
-                    elif cnt == 2:
-
-                        print(f"Writing client secret to {self.secrets_file}")
-                        self.__modify_file(self, self.secrets_file, "uc_client_secret: ", value=line.rstrip())
-
-                    line = fp.readline()
-                    cnt += 1
-
-            token_path = 'app_usercred.txt'
-
-            with open(token_path) as fp:
-
-                line = fp.readline()
-                print(f"Writing access token to {self.secrets_file}")
-                self.__modify_file(self, self.secrets_file, "uc_access_token: ", value=line.rstrip())
-
-            if os.path.exists("app_clientcred.txt"):
-
-                print("Removing app_clientcred.txt temp file..")
-                os.remove("app_clientcred.txt")
-
-            if os.path.exists("app_usercred.txt"):
-
-                print("Removing app_usercred.txt temp file..")
-                os.remove("app_usercred.txt")
-
-            self.__create_config(self)
-            self.__write_config(self)
-            
-            print("Secrets setup done!\n")
-            
-            with open(self.config_file, 'a') as the_file:
-                print(f"Writing Mastodon hostname parameter to {self.config_file}")
-                the_file.write(f'blocker_db: {self.blocker_db}\n')
-                the_file.write(f'blocker_db_user: {self.blocker_db_user}\n')
-
-            print("Blocker setup done!\n")
-            
-        except MastodonIllegalArgumentError as i_error:
-
-            sys.stdout.write(f'\n{str(i_error)}\n')
-
-        except MastodonNetworkError as n_error:
-
-            sys.stdout.write(f'\n{str(n_error)}\n')
-
-        except MastodonReadTimeout as r_error:
-
-            sys.stdout.write(f'\n{str(r_error)}\n')
-
-        except MastodonAPIError as a_error:
-
-            sys.stdout.write(f'\n{str(a_error)}\n')
-
-        return (logged_in, mastodon, self.mastodon_hostname)
-
-    def __api_request(self, method, endpoint, data={}):
-
-        response = None
-
-        try:
-
-            kwargs = dict(data=data)
-
-            response = self.session.request(method, url = endpoint, headers = self.headers, **kwargs)
-
-        except Exception as e:
-
-            raise MastodonNetworkError(f"Could not complete request: {e}")
-
-        if response is None:
-
-            raise MastodonIllegalArgumentError("Illegal request.")
-
-        if not response.ok:
-
-                try:
-                    if isinstance(response, dict) and 'error' in response:
-                        error_msg = response['error']
-                    elif isinstance(response, str):
-                        error_msg = response
-                    else:
-                        error_msg = None
-                except ValueError:
-                    error_msg = None
-
-                if response.status_code == 404:
-                    ex_type = MastodonNotFoundError
-                    if not error_msg:
-                        error_msg = 'Endpoint not found.'
-                        # this is for compatibility with older versions
-                        # which raised MastodonAPIError('Endpoint not found.')
-                        # on any 404
-                elif response.status_code == 401:
-                    ex_type = MastodonUnauthorizedError
-                elif response.status_code == 422:
-                    return response
-                elif response.status_code == 500:
-                    ex_type = MastodonInternalServerError
-                elif response.status_code == 502:
-                    ex_type = MastodonBadGatewayError
-                elif response.status_code == 503:
-                    ex_type = MastodonServiceUnavailableError
-                elif response.status_code == 504:
-                    ex_type = MastodonGatewayTimeoutError
-                elif response.status_code >= 500 and \
-                    response.status_code <= 511:
-                    ex_type = MastodonServerError
-                else:
-                    ex_type = MastodonAPIError
-
-                raise ex_type(
-                    'Mastodon API returned error',
-                    response.status_code,
-                    response.reason,
-                    error_msg)
-
-        else:
-
-            return response
-
-    def db_config(self):
-
-        self.blocker_db = self.__get_parameter("blocker_db", self.config_file)
-        self.blocker_db_user = self.__get_parameter("blocker_db_user", self.config_file)
-
-        return (self.blocker_db, self.blocker_db_user)
-
-    @staticmethod
-    def __check_dbsetup(self):
-
-        dbsetup = False
-
-        try:
-
-            conn = None
-
-            conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = "", host = "/var/run/postgresql", port = "5432")
-
-            dbsetup = True
-
-        except (Exception, psycopg2.DatabaseError) as error:
-
-            print(error)
-
-        return dbsetup
-
-    @staticmethod
-    def __createdb(self):
-
-        conn = None
-
-        try:
-
-            conn = psycopg2.connect(dbname='postgres',
-                user=self.blocker_db_user, host='',
-                password='')
-
-            conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
-
-            cur = conn.cursor()
-
-            print(f"Creating database {self.blocker_db}. Please wait...")
-
-            cur.execute(sql.SQL("CREATE DATABASE {}").format(
-                sql.Identifier(self.blocker_db))
-            )
-            print(f"Database {self.blocker_db} created!\n")
-
-            self.__dbtables_schemes(self)
-
-        except (Exception, psycopg2.DatabaseError) as error:
-
-            print(error)
-
-        finally:
-
-            if conn is not None:
-
-                conn.close()
-
-    @staticmethod
-    def __dbtables_schemes(self):
-
-        table = "blocker"
-        sql = "create table "+table+" (server varchar(200), users int, updated_at timestamptz, software varchar(50), alive boolean, users_api varchar(50), version  varchar(100), first_checked_at timestamptz, last_checked_at timestamptz, downs int)"
-        self.__create_table(self, table, sql)
-
-        table = "execution_time"
-        sql = "create table "+table+" (program varchar(30), start timestamptz, finish timestamptz)"
-        self.__create_table(self, table, sql)
-
-    @staticmethod
-    def __create_table(self, table, sql):
-
-        conn = None
-
-        try:
-
-            conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = "", host = "/var/run/postgresql", port = "5432")
-            cur = conn.cursor()
-
-            print(f"Creating table {table}")
-            cur.execute(sql)
-
-            conn.commit()
-            print(f"Table {table} created!\n")
-
-        except (Exception, psycopg2.DatabaseError) as error:
-
-            print(error)
-
-        finally:
-
-            if conn is not None:
-
-                conn.close()
-
-    def __get_parameter(self, parameter, config_file):
-
-        if not os.path.isfile(config_file):
-            print(f"File {config_file} not found, exiting.")
-            sys.exit(0)
-
-        with open( config_file ) as f:
-            for line in f:
-                if line.startswith( parameter ):
-                    return line.replace(parameter + ":", "").strip()
-
-        print(f"{config_file} Missing parameter {parameter}")
-
-        sys.exit(0)
-
-    @staticmethod
-    def __modify_file(self, file_name, pattern,value=""):
-
-        fh=fileinput.input(file_name,inplace=True)
-
-        for line in fh:
-
-            replacement=pattern + value
-            line=re.sub(pattern,replacement,line)
-            sys.stdout.write(line)
-
-        fh.close()
-
-    @staticmethod
-    def __create_config(self):
-
-        if not os.path.exists('config'):
-
-            os.makedirs('config')
-
-        if not os.path.exists(self.config_file):
-
-            print(self.config_file + " created!")
-            with open('config/config.txt', 'w'): pass
-
-    @staticmethod
-    def __write_config(self):
-
-        with open(self.config_file, 'a') as the_file:
-
-            the_file.write(f'mastodon_hostname: {self.mastodon_hostname}\n')
-            print(f"adding parameter 'mastodon_hostname' to {self.config_file}\n")
-
-###############################################################################
 # main
 
 if __name__ == '__main__':
 
+    setup = Setup()
+
+    db = Database()
+
     blocker = DomainBlocks()
 
+    mastodon = Mastodon(
+        access_token = setup.mastodon_app_token,
+        api_base_url= setup.mastodon_hostname
+        )
+
     soft_list = 'software.txt'
 
     soft_file = open(soft_list, 'r')
@@ -500,11 +38,11 @@ if __name__ == '__main__':
 
         print(f'checking software {software}...')
 
-        servers_list = blocker.get_servers(software)
+        servers_list = db.get_servers(software)
 
         for server in servers_list:
 
-            blocker.domain_blocks_create(server, private_comment, public_comment )
+            blocker.domain_blocks_create(server, private_comment, public_comment)
 
 
 

peers.py

@@ -1,535 +1,42 @@
-import os
-import sys
-import time
-from datetime import datetime
-import requests
-import urllib3
-import json
-import psycopg2
-from psycopg2 import sql
-from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
-#import ray
-import pdb
+from app.libraries.setup import Setup
+from app.libraries.database import Database
+from app.libraries.peers import Peers
+from app.libraries.nodeinfo import Nodeinfo
+import ray
 
-#ray.init(num_cpus = 32) # Specify this system CPUs.
+ray.init(num_cpus = 25) # Specify this system CPUs.
 
-class Peers:
+@ray.remote
+def peerapi(peer):
 
-    name = 'Mastodon server peers'
+    api = peers.getapi(peer) # get peer nodeinfo's URL
 
-    def __init__(self, mastodon_hostname=None, peers_api=None):
-
-        self.peers_api = '/api/v1/instance/peers'
-        self.config_path = "config/config.txt"
-        
-        self.apis = ['/api/v1/instance?',
-            '/nodeinfo/2.0?',
-            '/nodeinfo/2.0.json?',
-            '/main/nodeinfo/2.0?',
-            '/api/statusnet/config?',
-            '/api/nodeinfo/2.0.json?',
-            '/api/nodeinfo?',
-            '/wp-json/nodeinfo/2.0?',
-            '/api/v1/instance/nodeinfo/2.0?',
-            '/.well-known/x-nodeinfo2?'
-            ]
-
-        is_setup = self.__check_setup(self)
-
-        if is_setup:
-
-            self.mastodon_hostname = self.__get_parameter("mastodon_hostname", self.config_path)
-            self.blocker_db = self.__get_parameter("blocker_db", self.config_path)
-            self.blocker_db_user = self.__get_parameter("blocker_db_user", self.config_path)
-        
-        else:
-
-            self.mastodon_hostname, self.blocker_db, self.blocker_db_user = self.__setup(self)
-
-        db_setup = self.__check_dbsetup(self)
-
-        if not db_setup:
-
-            self.__createdb(self)        
-    
-    def get_peers(self):
-
-        user_agent = {'User-agent': "fediverse's stats (fediverse@soc.catala.digital)"}
-
-        res = requests.get(f'https://{self.mastodon_hostname}{self.peers_api}', headers = user_agent, timeout=3)
-
-        peers = res.json()
-
-        return peers
-
-    def getsoft(self, server):
-
-        if server.find(".") == -1:
-            return
-        if server.find("@") != -1:
-            return
-        if server.find("/") != -1:
-            return
-        if server.find(":") != -1:
-            return
-
-        soft = ''
-
-        is_nodeinfo = False
-
-        url = 'https://' + server
-
-        user_agent = {'User-agent': "fediverse's stats (fediverse@soc.catala.digital)"}
+    if api != '':
 
         try:
 
-            response = requests.get(url + '/.well-known/nodeinfo', headers = user_agent, timeout=3)
+            server, software = nd.getdata(peer, api)
 
-            if response.status_code == 200:
+            db.write_server(peer, api, software)
 
-                try:
+            print(f'server: {peer}, api: {api}, software: {software}')
 
-                    response_json = response.json()
-
-                    nodeinfo = response_json['links'][0]['href'].replace(f'https://{server}','')
-
-                    try:
-
-                        nodeinfo_data = requests.get(url + nodeinfo, headers = user_agent, timeout=3)
-
-                        if nodeinfo_data.status_code == 200:
-
-                            nodeinfo_json = nodeinfo_data.json()
-
-                            is_nodeinfo = True
-
-                        else:
-
-                            print(f"Server {server}'s nodeinfo not responding: error code {nodeinfo_data.status_code}")
-
-                    except:
-
-                        pass
-
-                except:
-
-                    print(f'Server {server} not responding: error code {response.status_code}')
-                    print('*********************************************************************')
-
-                    pass
-            else:
-
-                for api in self.apis:
-
-                    try:
-
-                        response = requests.get(url + api, headers = user_agent, timeout=3)
-
-                        if is_json(response.text):
-
-                            nodeinfo_json = response.json()
-
-                            if 'software' in nodeinfo_json:
-
-                                nodeinfo = api
-
-                                is_nodeinfo = True
-
-                                break
-
-                            elif 'title' in nodeinfo_json:
-
-                                if nodeinfo_json['title'] == 'Zap':
-
-                                    nodeinfo = api
-
-                                    is_nodeinfo = True
-
-                                    soft = 'zap'
-
-                                    break
-
-                            elif 'version' in nodeinfo_json:
-
-                                nodeinfo = api
-
-                                is_nodeinfo = True
-
-                                break
-
-                    except:
-
-                        pass
-
-        except requests.exceptions.SSLError as errssl:
+        except:
 
             pass
 
-        except requests.exceptions.HTTPError as errh:
-
-            pass
-
-        except requests.exceptions.ConnectionError as errc:
-
-            pass
-
-        except requests.exceptions.ReadTimeout as to_err:
-
-            pass
-
-        except requests.exceptions.TooManyRedirects as tmr_err:
-
-            pass
-
-        except urllib3.exceptions.LocationParseError as lp_err:
-
-            pass
-
-        except requests.exceptions.InvalidURL as iu_err:
-
-            pass
-
-        except requests.exceptions.ChunkedEncodingError as chunk_err:
-
-            print(f'ChunkedEncodingError! {server}')
-            pass
-
-        except ray.exceptions.RaySystemError as ray_sys_error:
-
-            print(ray_sys_error) 
-            pass
-
-        else:
-
-            if is_nodeinfo:
-
-                if nodeinfo != '/api/v1/instance?':
-
-                    if nodeinfo != '/.well-known/x-nodeinfo2?':
-
-                        try:
-
-                            soft = nodeinfo_json['software']['name']
-                            soft = soft.lower()
-                            soft_version = nodeinfo_json['software']['version']
-                            users = nodeinfo_json['usage']['users']['total']
-                            alive = True
-
-                            self.write_api(server, soft, users, alive, nodeinfo, soft_version)
-
-                            print(f"Server {server} ({soft} {soft_version}) is alive!")
-                            print('*********************************************************************')
-
-                            return
-
-                        except:
-
-                            pass
-
-                    else:
-
-                        try:
-
-                            soft = nodeinfo_json['server']['software']
-                            soft = soft.lower()
-                            soft_version = nodeinfo_json['server']['version']
-                            users = nodeinfo_json['usage']['users']['total']
-                            alive = True
-
-                            if soft == 'socialhome':
-
-                                self.write_api(server, soft, users, alive, nodeinfo, soft_version)
-
-                                print('*********************************************************************')
-                                print(f"Server {serve}r ({soft} {soft_version}) is alive!")
-                                print('*********************************************************************')
-
-                            return
-
-                        except:
-
-                            pass
-
-                if soft == '' and nodeinfo == "/api/v1/instance?":
-
-                    soft = 'mastodon'
-
-                    try:
-
-                        users = nodeinfo_json['stats']['user_count']
-
-                        if users > 1000000:
-
-                            return
-
-                    except:
-
-                        users = 0
-
-                    try:
-
-                        soft_version = nodeinfo_json['version']
-
-                    except:
-
-                        soft_version = 'unknown'
-
-                    alive = True
-
-                    self.write_api(self, server, soft, users, alive, nodeinfo, soft_version)
-
-                    print('*********************************************************************')
-                    print(f"Server {server} ({soft}) is alive!")
-
-                elif soft == 'zap' and nodeinfo == "/api/v1/instance?":
-
-                    soft = 'zap'
-                    users = nodeinfo_json['stats']['user_count']
-                    soft_version = nodeinfo_json['version']
-                    alive = True
-
-                    print(server, soft, users, alive, api)
-
-                    print('*********************************************************************')
-                    print(f"Server {server} ({soft}) is alive!")
-
-            else:
-
-                print(f'Server {server} is dead')
-                print('*********************************************************************')    
-
-    def write_api(self, server, software, users, alive, api, soft_version):
-
-        insert_sql = "INSERT INTO blocker(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
-
-        conn = None
-
-        try:
-
-            conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password="", host="/var/run/postgresql", port="5432")
-
-            cur = conn.cursor()
-
-            print(f'Writing {server} nodeinfo data...')
-
-            cur.execute(insert_sql, (server, now, software, users, alive, api, soft_version))
-
-            cur.execute(
-                "UPDATE blocker SET updated_at=(%s), software=(%s), users=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)",
-                (now, software, users, alive, api, soft_version, server)
-                )
-
-            conn.commit()
-
-            cur.close()
-
-        except (Exception, psycopg2.DatabaseError) as error:
-
-            print(error)
-
-        finally:
-
-            if conn is not None:
-
-                conn.close()
-
-    def save_time(self, program, start, finish):
-
-        insert_sql = "INSERT INTO execution_time(program, start, finish) VALUES(%s,%s,%s) ON CONFLICT DO NOTHING"
-
-        conn = None
-
-        try:
-
-            conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = "", host = "/var/run/postgresql", port = "5432")
-
-            cur = conn.cursor()
-
-            cur.execute(insert_sql, (program, start, finish,))
-
-            cur.execute("UPDATE execution_time SET start=(%s), finish=(%s) where program=(%s)", (start, finish, program))
-
-            conn.commit()
-
-            cur.close()
-
-        except (Exception, psycopg2.DatabaseError) as error:
-
-            print(error)
-
-        finally:
-
-            if conn is not None:
-
-                conn.close()
-
-    @staticmethod
-    def __check_setup(self):
-
-        is_setup = False
-
-        if not os.path.isfile(self.config_path):
-            print(f"File {self.config_path} not found, running setup.")
-        else:
-            is_setup = True
-
-        return is_setup
-    
-    @staticmethod
-    def __setup(self):
-
-        if not os.path.exists('config'):
-            os.makedirs('config')
-
-        self.mastodon_hostname = input("Mastodon hostname: ")
-        self.blocker_db = input("Blocker database name: ")
-        self.blocker_db_user = input("Blocker database user: ")
-
-        if not os.path.exists(self.config_path):
-            with open(self.config_path, 'w'): pass
-            print(f"{self.config_path} created!")
-
-        with open(self.config_path, 'a') as the_file:
-            print("Writing Blocker parameters to " + self.config_path)
-            the_file.write(f'mastodon_hostname: {self.mastodon_hostname}\n')
-            the_file.write(f'blocker_db: {self.blocker_db}\n')
-            the_file.write(f'blocker_db_user: {self.blocker_db_user}\n')
-
-        return (self.mastodon_hostname, self.blocker_db, self.blocker_db_user)
-
-    @staticmethod
-    def __check_dbsetup(self):
-
-        dbsetup = False
-
-        try:
-
-            conn = None
-
-            conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = "", host = "/var/run/postgresql", port = "5432")
-
-            dbsetup = True
-
-        except (Exception, psycopg2.DatabaseError) as error:
-
-            print(error)
-
-        return dbsetup
-
-    @staticmethod
-    def __createdb(self):
-
-        conn = None
-
-        try:
-
-            conn = psycopg2.connect(dbname='postgres',
-                user=self.blocker_db_user, host='',
-                password='')
-
-            conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
-
-            cur = conn.cursor()
-
-            print(f"Creating database {self.blocker_db}. Please wait...")
-
-            cur.execute(sql.SQL("CREATE DATABASE {}").format(
-                sql.Identifier(self.blocker_db))
-            )
-            print(f"Database {self.blocker_db} created!\n")
-
-            self.__dbtables_schemes(self)
-
-        except (Exception, psycopg2.DatabaseError) as error:
-
-            print(error)
-
-        finally:
-
-            if conn is not None:
-
-                conn.close()
-
-    @staticmethod
-    def __dbtables_schemes(self):
-
-        table = "blocker"
-        sql = f"""create table {table} (server varchar(200) PRIMARY KEY, users INT, updated_at timestamptz, software varchar(50), alive boolean, users_api varchar(50), 
-            version varchar(100), first_checked_at timestamptz, last_checked_at timestamptz, downs int)"""
-        self.__create_table(self, table, sql)
-
-        table = "execution_time"
-        sql = "create table "+table+" (program varchar(30) PRIMARY KEY, start timestamptz, finish timestamptz)"
-        self.__create_table(self, table, sql)
-        
-    @staticmethod
-    def __create_table(self, table, sql):
-
-        conn = None
-
-        try:
-
-            conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = "", host = "/var/run/postgresql", port = "5432")
-            cur = conn.cursor()
-
-            print(f"Creating table {table}")
-            cur.execute(sql)
-
-            conn.commit()
-            print(f"Table {table} created!\n")
-
-        except (Exception, psycopg2.DatabaseError) as error:
-
-            print(error)
-
-        finally:
-
-            if conn is not None:
-
-                conn.close()
-
-    @staticmethod
-    def __get_parameter(parameter, file_path ):
-
-        with open( file_path ) as f:
-            for line in f:
-                if line.startswith( parameter ):
-                    return line.replace(parameter + ":", "").strip()
-
-        print(f'{file_path} Missing parameter {parameter}')
-        sys.exit(0)
-
-###############################################################################
-# main
-
 if __name__ == '__main__':
 
-    obj = Peers()
-    
-    peers = obj.get_peers()
+    db = Database()
 
-    print(f"{obj.mastodon_hostname}'s peers: {len(peers)}") 
+    setup = Setup()
 
-    now = datetime.now()
+    peers = Peers()
 
-    start = datetime.now()
+    nd = Nodeinfo()
 
-    program = obj.name
-
-    finish = start
-
-    obj.save_time(program, start, finish)
-
-    for peer in peers:
-
-        obj.getsoft(peer)
-
-    #results = ray.get([obj.getsoft.remote(server) for server in peers])
-
-    finish = datetime.now()
-
-    print(f"duration = {finish - start}.\nprocessed servers: {len(peers)}")
-
-    obj.save_time(program, start, finish)
+    peers_list = peers.getpeers(setup.mastodon_hostname) #get peers from Mastodon host
 
+    if peers_list != None:
 
+        ray.get([peerapi.remote(peer) for peer in peers_list])

requirements.txt

@@ -1,3 +1,4 @@
 requests
 psycopg2-binary
 Mastodonplus.py
+ray