diff --git a/app/libraries/database.py b/app/libraries/database.py new file mode 100644 index 0000000..68739ec --- /dev/null +++ b/app/libraries/database.py @@ -0,0 +1,675 @@ +import os +import sys +import psycopg2 +from psycopg2 import sql +from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT +import uuid +from datetime import datetime +import pytz +import pdb + +tz = pytz.timezone('Europe/Madrid') + +class Database(): + + name = 'fediverse database library' + + def __init__(self, config_file=None, blocker_db=None, blocker_db_user=None, blocker_db_user_password=None): + + self.config_file = "config/db_config.txt" + self.blocker_db = self.__get_parameter("blocker_db", self.config_file) + self.blocker_db_user = self.__get_parameter("blocker_db_user", self.config_file) + self.blocker_db_user_password = self.__get_parameter("blocker_db_user_password", self.config_file) + + db_setup = self.__check_dbsetup(self) + + if not db_setup: + + self.blocker_db = input("\nMau database name: ") + self.blocker_db_user = input("\nMau database user: ") + self.blocker_db_user_password = input("\nMau database user password: ") + + self.__createdb(self) + self.__create_config(self) + self.__write_config(self) + + def get_nodeinfo_endpoint(self, server): + + try: + + conn = None + + conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = self.blocker_db_user_password, host = "/var/run/postgresql", port = "6432") + + cur = conn.cursor() + + cur.execute("select server, api from servers where server=(%s)", (server,)) + + row = cur.fetchone() + + if row != None: + + server = row[0] + + api = row[1] + + else: + + server = '' + + api = '' + + cur.close() + + return (server, api) + + except (Exception, psycopg2.DatabaseError) as error: + + sys.exit(error) + + finally: + + if conn is not None: + + conn.close() + + def get_servers(self, software): + + servers_list = [] + + try: + + conn = None + + conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = "", host = "/var/run/postgresql", port = "5432") + + cur = conn.cursor() + + cur.execute("select server from servers where software=(%s)", (software,)) + + rows = cur.fetchall() + + for row in rows: + + servers_list.append(row[0]) + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + sys.exit(error) + + finally: + + if conn is not None: + + conn.close() + + return (servers_list) + + def save_time(self, program, start, finish): + + insert_sql = "INSERT INTO execution_time(program, start, finish) VALUES(%s,%s,%s) ON CONFLICT DO NOTHING" + + conn = None + + try: + + conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = self.blocker_db_user_password, host = "/var/run/postgresql", port = "6432") + + cur = conn.cursor() + + cur.execute(insert_sql, (program, start, finish,)) + + cur.execute("UPDATE execution_time SET start=(%s), finish=(%s) where program=(%s)", (start, finish, program)) + + conn.commit() + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + def check_peer(self, server): + + found = False + + select_server_sql = 'select server from servers where server=(%s)' + + select_dead_server_sql = 'select server from deadservers where server=(%s)' + + try: + + conn = None + + conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432") + + cur = conn.cursor() + + # check server + + cur.execute(select_server_sql, (server,)) + + row = cur.fetchone() + + if row != None: + + found = True + + cur.execute(select_dead_server_sql, (server,)) + + row = cur.fetchone() + + if row != None: + + found = True + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + return found + + def get_not_updated_servers(self): + + not_updated_servers = [] + + try: + + conn = None + + conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432") + + cur = conn.cursor() + + # get saved servers list + + cur.execute("select server from servers where updated_at is null") + + rows = cur.fetchall() + + for row in rows: + + not_updated_servers.append(row[0]) + + cur.close() + + print("Not updated servers: " + str(len(not_updated_servers))) + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + return not_updated_servers + + def get_saved_servers(self): + + saved_servers = [] + + try: + + conn = None + + conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432") + + cur = conn.cursor() + + # get saved servers list + + cur.execute("select server from servers where api is not null") + + rows = cur.fetchall() + + for row in rows: + + saved_servers.append(row[0]) + + cur.close() + + print("Saved servers: " + str(len(saved_servers))) + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + return saved_servers + + def get_dead_servers(self): + + dead_servers = [] + + try: + + conn = None + + conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432") + + cur = conn.cursor() + + # get dead servers list + + cur.execute("select server from deadservers") + + rows = cur.fetchall() + + for row in rows: + + dead_servers.append(row[0]) + + cur.close() + + print("Dead servers: " + str(len(dead_servers))) + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + return dead_servers + + def soft_totals(self): + + # get current total servers and users, get users from every software + + now = datetime.now() + + gettotals_sql = "select count(server), sum(users), sum(mau) from mau where alive" + get_soft_totals_sql = "select software, sum(users) as users, sum(mau) as mau, count(server) as servers from mau where users != 0 and mau is not null and alive group by software order by mau desc" + + soft_total_project = [] + soft_total_users = [] + soft_total_mau = [] + soft_total_servers = [] + + try: + + conn = None + + conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432") + + cur = conn.cursor() + + cur.execute(gettotals_sql) + + row = cur.fetchone() + + total_servers = row[0] + + total_users = row[1] + + total_mau = row[2] + + cur.execute(get_soft_totals_sql) + + rows = cur.fetchall() + + for row in rows: + + soft_total_project.append(row[0]) + soft_total_users.append(row[1]) + soft_total_mau.append(row[2]) + soft_total_servers.append(row[3]) + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + return (soft_total_project, soft_total_users, soft_total_mau, soft_total_servers, total_servers, total_users, total_mau) + + def get_last_checked_servers(self): + + # get last checked servers from fediverse DB + + alive_servers = [] + + try: + + conn = None + + conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432") + + cur = conn.cursor() + + # get world servers list + + cur.execute("select server from world where server in (select server from fediverse where users_api != '')") + + rows = cur.fetchall() + + for row in rows: + + alive_servers.append(row[0]) + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + return alive_servers + + def write_alive_server(self, server, software, version, users, mau, alive): + + now = datetime.now() + + insert_sql = "INSERT INTO mau(server, software, version, users, mau, alive, updated_at) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING" + + conn = None + + try: + + conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432") + + cur = conn.cursor() + + cur.execute(insert_sql, (server, software, version, users, mau, alive, now)) + + cur.execute("UPDATE mau SET software=(%s), version=(%s), users=(%s), mau=(%s), alive=(%s), updated_at=(%s) where server=(%s)", (software, version, users, mau, alive, now, server)) + + conn.commit() + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + def write_not_alive_server(self, server): + + now = datetime.now() + + update_sql = "UPDATE mau set alive='f', updated_at=(%s) where server=(%s)" + + conn = None + + try: + + conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432") + + cur = conn.cursor() + + cur.execute(update_sql, (now, server)) + + conn.commit() + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + + def write_server(self, server, api, software): + + now = datetime.now() + + insert_sql = "INSERT INTO servers(server, api, software, created_at) VALUES(%s, %s, %s, %s) ON CONFLICT DO NOTHING" + + conn = None + + try: + + conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432") + + cur = conn.cursor() + + cur.execute(insert_sql, (server, api, software, now)) + + conn.commit() + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + def update_peer(self, server): + + now = datetime.now() + + update_sql = "UPDATE servers set updated_at=(%s) where server=(%s)" + + conn = None + + try: + + conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432") + + cur = conn.cursor() + + cur.execute(update_sql, (now, server)) + + conn.commit() + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + def update_server(self, server, api): + + now = datetime.now() + + update_sql = "UPDATE servers set server=(%s), api=(%s), updated_at=(%s) where server=(%s)" + + conn = None + + try: + + conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432") + + cur = conn.cursor() + + cur.execute(update_sql, (server, api, now, server)) + + conn.commit() + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + @staticmethod + def __check_dbsetup(self): + + db_setup = False + + try: + + conn = None + + conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = self.blocker_db_user_password, host = "/var/run/postgresql", port = "6432") + + db_setup = True + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + return db_setup + + @staticmethod + def __createdb(self): + + conn = None + + try: + + conn = psycopg2.connect(dbname='postgres', + user=self.blocker_db_user, host='', + password=self.blocker_db_user_password) + + conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) + + cur = conn.cursor() + + print(f"Creating database {self.blocker_db}. Please wait...") + + cur.execute(sql.SQL("CREATE DATABASE {}").format( + sql.Identifier(self.blocker_db)) + ) + print(f"Database {self.blocker_db} created!\n") + + self.__dbtables_schemes(self) + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + @staticmethod + def __dbtables_schemes(self): + + db = self.blocker_db + table = "servers" + sql = "create table "+table+" (server varchar(200) PRIMARY KEY, api varchar(50), software varchar(50), created_at timestamptz, updated_at timestamptz, downs int)" + self.__create_table(self, table, sql) + + @staticmethod + def __create_table(self, table, sql): + + conn = None + + try: + + conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = self.blocker_db_user_password, host = "/var/run/postgresql", port = "6432") + + cur = conn.cursor() + + print(f"Creating table {table}") + + cur.execute(sql) + + conn.commit() + + print(f"Table {table} created!\n") + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + def __get_parameter(self, parameter, config_file): + + if not os.path.isfile(config_file): + print(f"File {config_file} not found..") + return + + with open( config_file ) as f: + for line in f: + if line.startswith( parameter ): + return line.replace(parameter + ":", "").strip() + + print(f"{config_file} Missing parameter {parameter}") + + sys.exit(0) + + @staticmethod + def __create_config(self): + + if not os.path.exists('config'): + + os.makedirs('config') + + if not os.path.exists(self.config_file): + + print(self.config_file + " created!") + with open(self.config_file, 'w'): pass + + @staticmethod + def __write_config(self): + + with open(self.config_file, 'a') as the_file: + + the_file.write(f'blocker_db: {self.blocker_db}\nblocker_db_user: {self.blocker_db_user}\nblocker_db_user_password: {self.blocker_db_user_password}') + print(f"adding parameters to {self.config_file}\n") + + diff --git a/app/libraries/domainblocks.py b/app/libraries/domainblocks.py new file mode 100644 index 0000000..40b2325 --- /dev/null +++ b/app/libraries/domainblocks.py @@ -0,0 +1,114 @@ +import os +import sys +from app.libraries.setup import Setup +from mastodon import Mastodon +from mastodon.Mastodon import MastodonNotFoundError, MastodonNetworkError, MastodonReadTimeout, MastodonAPIError, MastodonUnauthorizedError, MastodonIllegalArgumentError +import requests +import pdb + +class DomainBlocks(): + + name = 'Domain blocks for Mastodon' + + def __init__(self, mastodon_hostname=None, domain_block_api=None, session=None): + + self.domain_blocks_api = '/api/v1/admin/domain_blocks' + setup = Setup() + self.mastodon_hostname = setup.mastodon_hostname + self.mastodon_app_token = { 'Authorization': 'Bearer %s'%setup.mastodon_app_token } + + if session: + self.session = session + else: + self.session = requests.Session() + + def domain_blocks_create(self, server, priv_comment, pub_comment): + + data = { + 'domain': server, + 'severity': 'suspend', + 'reject_media': 'true', + 'reject_reports': 'true', + 'private_comment': priv_comment, + 'public_comment': pub_comment, + 'obfuscate': 'true', + } + + endpoint = f'https://{self.mastodon_hostname}/{self.domain_blocks_api}' + + response = self.__api_request('POST', endpoint, data) + + if response.ok: + + print(f"Done, {server} is now suspended") + + else: + + pass + + def __api_request(self, method, endpoint, data={}): + + response = None + + try: + + kwargs = dict(data=data) + + response = self.session.request(method, url = endpoint, headers = self.mastodon_app_token, **kwargs) + + except Exception as e: + + raise MastodonNetworkError(f"Could not complete request: {e}") + + if response is None: + + raise MastodonIllegalArgumentError("Illegal request.") + + if not response.ok: + + try: + if isinstance(response, dict) and 'error' in response: + error_msg = response['error'] + elif isinstance(response, str): + error_msg = response + else: + error_msg = None + except ValueError: + error_msg = None + + if response.status_code == 404: + ex_type = MastodonNotFoundError + if not error_msg: + error_msg = 'Endpoint not found.' + # this is for compatibility with older versions + # which raised MastodonAPIError('Endpoint not found.') + # on any 404 + elif response.status_code == 401: + ex_type = MastodonUnauthorizedError + elif response.status_code == 422: + return response + elif response.status_code == 500: + ex_type = MastodonInternalServerError + elif response.status_code == 502: + ex_type = MastodonBadGatewayError + elif response.status_code == 503: + ex_type = MastodonServiceUnavailableError + elif response.status_code == 504: + ex_type = MastodonGatewayTimeoutError + elif response.status_code >= 500 and \ + response.status_code <= 511: + ex_type = MastodonServerError + else: + ex_type = MastodonAPIError + + raise ex_type( + 'Mastodon API returned error', + response.status_code, + response.reason, + error_msg) + + else: + + return response + + diff --git a/app/libraries/nodeinfo.py b/app/libraries/nodeinfo.py new file mode 100644 index 0000000..fb4c9a5 --- /dev/null +++ b/app/libraries/nodeinfo.py @@ -0,0 +1,70 @@ +import time +from datetime import datetime +import os +import json +import sys +import os.path +import requests +import urllib3 +import socket +from app.libraries.database import Database +from app.libraries.setup import Setup +import pdb + +def is_json(myjson): + + try: + json_object = json.loads(myjson) + except ValueError as e: + return False + return True + +class Nodeinfo(): + + name = "Query server nodeinfo data" + + def __init__(self, server=None, db=None, setup=None, soft=None): + + self.server = server + self.db = Database() + self.setup = Setup() + self.soft = '' + + def getdata(self, server, api): + + if server.find(".") == -1: + return + if server.find("@") != -1: + return + if server.find("/") != -1: + return + if server.find(":") != -1: + return + + url = 'https://' + server + + self.soft = '' + + try: + + response = requests.get(url + api, headers = self.setup.user_agent, timeout=3) + + if response.status_code == 200: + + try: + + self.soft = response.json()['software']['name'] + self.soft = self.soft.lower() + + print(f"Server {server} ({self.soft} is alive!") + print('*********************************************************************') + + except: + + pass + + except: + + pass + + return (server, self.soft) diff --git a/app/libraries/peers.py b/app/libraries/peers.py new file mode 100644 index 0000000..ae9a52f --- /dev/null +++ b/app/libraries/peers.py @@ -0,0 +1,264 @@ +import time +from datetime import datetime +import os +import json +import sys +import os.path +from app.libraries.setup import Setup +from app.libraries.database import Database +import requests +import urllib3 +import socket + +class Peers(): + + name = "Get peers from server" + + def __init__(self, server=None, db=None, setup=None): + + self.server = server + self.db = Database() + self.setup = Setup() + + def getpeers(self, server): + + try: + + res = requests.get(f'https://{server}/{self.setup.peers_api}, headers = self.setup.user_agent, timeout=3') + + peers = res.json() + + if isinstance(peers, list): + + self.db.update_peer(server) + + return peers + + else: + + return None + + except requests.exceptions.SSLError as errssl: + + pass + + except requests.exceptions.HTTPError as errh: + + pass + + except requests.exceptions.ConnectionError as errc: + + pass + + except requests.exceptions.ReadTimeout as to_err: + + pass + + except requests.exceptions.TooManyRedirects as tmr_err: + + pass + + except urllib3.exceptions.LocationParseError as lp_err: + + pass + + except requests.exceptions.InvalidURL as iu_err: + + pass + + except requests.exceptions.ChunkedEncodingError as chunk_err: + + print(f'ChunkedEncodingError! {server}') + pass + + except requests.exceptions.JSONDecodeError as jsondec_err: + + print(jsondec_err) + pass + + print('*** Not peers api!') + return None + + def getapi(self, server): + + if server.find(".") == -1: + return + if server.find("@") != -1: + return + if server.find("/") != -1: + return + if server.find(":") != -1: + return + + is_nodeinfo = False + + nodeinfo = '' + + url = 'https://' + server + + try: + + response = requests.get(url + '/.well-known/nodeinfo', headers = self.setup.user_agent, timeout=3) + + if response.status_code == 200: + + try: + + response_json = response.json() + + if len(response_json['links']) == 1: + + start = [pos for pos, char in enumerate(response_json['links'][0]['href']) if char == '/'][1] + end = [pos for pos, char in enumerate(response_json['links'][0]['href']) if char == '/'][2] + + server = response_json['links'][0]['href'][start+1:end] + + server_idx = response_json['links'][0]['href'].index(server) + + nodeinfo = response_json['links'][0]['href'][server_idx:].replace(server, '') + + + elif len(response_json['links']) == 2: + + start = [pos for pos, char in enumerate(response_json['links'][1]['href']) if char == '/'][1] + end = [pos for pos, char in enumerate(response_json['links'][1]['href']) if char == '/'][2] + + server = response_json['links'][1]['href'][start+1:end] + + server_idx = response_json['links'][1]['href'].index(server) + + nodeinfo = response_json['links'][1]['href'][server_idx:].replace(server, '') + + except: + + pass + + except requests.exceptions.ChunkedEncodingError as chunk_err: + + pass + + except requests.exceptions.InvalidSchema as invalid_err: + + pass + + except requests.exceptions.SSLError as errssl: + + pass + + except requests.exceptions.HTTPError as errh: + + pass + + except requests.exceptions.ConnectionError as errc: + + pass + + except requests.exceptions.ReadTimeout as to_err: + + pass + + except requests.exceptions.TooManyRedirects as tmr_err: + + pass + + except urllib3.exceptions.LocationParseError as lp_err: + + pass + + except requests.exceptions.InvalidURL as iu_err: + + pass + + except requests.exceptions.ChunkedEncodingError as chunk_err: + + print(f'ChunkedEncodingError! {server}') + pass + + return nodeinfo + + def updateapi(self, server): + + if server.find(".") == -1: + return + if server.find("@") != -1: + return + if server.find("/") != -1: + return + if server.find(":") != -1: + return + + nodeinfo = '' + + url = 'https://' + server + + try: + + response = requests.get(url + '/.well-known/nodeinfo', headers = self.setup.user_agent, timeout=3) + + if response.status_code == 200: + + try: + + response_json = response.json() + + if len(response_json['links']) == 1: + + start = [pos for pos, char in enumerate(response_json['links'][0]['href']) if char == '/'][1] + end = [pos for pos, char in enumerate(response_json['links'][0]['href']) if char == '/'][2] + + server = response_json['links'][0]['href'][start+1:end] + + server_idx = response_json['links'][0]['href'].index(server) + + nodeinfo = response_json['links'][0]['href'][server_idx:].replace(server, '') + + elif len(response_json['links']) == 2: + + start = [pos for pos, char in enumerate(response_json['links'][1]['href']) if char == '/'][1] + end = [pos for pos, char in enumerate(response_json['links'][1]['href']) if char == '/'][2] + + server = response_json['links'][1]['href'][start+1:end] + + server_idx = response_json['links'][1]['href'].index(server) + + nodeinfo = response_json['links'][1]['href'][server_idx:].replace(server, '') + + except: + + pass + + except requests.exceptions.SSLError as errssl: + + pass + + except requests.exceptions.HTTPError as errh: + + pass + + except requests.exceptions.ConnectionError as errc: + + pass + + except requests.exceptions.ReadTimeout as to_err: + + pass + + except requests.exceptions.TooManyRedirects as tmr_err: + + pass + + except urllib3.exceptions.LocationParseError as lp_err: + + pass + + except requests.exceptions.InvalidURL as iu_err: + + pass + + except requests.exceptions.ChunkedEncodingError as chunk_err: + + print(f'ChunkedEncodingError! {server}') + pass + + return server, nodeinfo + diff --git a/app/libraries/setup.py b/app/libraries/setup.py new file mode 100644 index 0000000..6dff671 --- /dev/null +++ b/app/libraries/setup.py @@ -0,0 +1,157 @@ +import os +import sys +from datetime import datetime +import pytz +from mastodon import Mastodon +from mastodon.Mastodon import MastodonMalformedEventError, MastodonNetworkError, MastodonReadTimeout, MastodonAPIError, MastodonIllegalArgumentError +import pdb + +class Setup(): + + name = 'fediverse setup' + + def __init__(self, config_file=None, mastodon_hostname=None, peers_api=None, user_agent=None, secrets_filepath=None, mastodon_app_token=None): + + self.config_file = "config/config.txt" + self.mastodon_hostname = self.__get_parameter("mastodon_hostname", self.config_file) + self.peers_api = '/api/v1/instance/peers?' + self.user_agent = {'User-agent': "fediverse's stats (fediverse@mastodont.cat)"} + + self.secrets_filepath = 'secrets/secrets.txt' + + is_setup = self.__check_mastodon_setup(self) + + if is_setup: + + self.mastodon_app_token = self.__get_mastodon_parameter("mastodon_app_token", self.secrets_filepath) + + else: + + self.mastodon_app_token = self.mastodon_setup(self) + + @staticmethod + def __check_mastodon_setup(self): + + is_setup = False + + if not os.path.isfile(self.secrets_filepath): + print(f"File {self.secrets_filepath} not found, running setup.") + else: + is_setup = True + + return is_setup + + @staticmethod + def mastodon_setup(self): + + if not os.path.exists('secrets'): + os.makedirs('secrets') + + self.mastodon_user = input("Mastodon user login? ") + self.mastodon_password = input("Mastodon user password? ") + self.app_name = 'fediverse' + + self.mastodon_app_token = self.mastodon_log_in() + + if not os.path.exists(self.secrets_filepath): + with open(self.secrets_filepath, 'w'): pass + print(f"{self.secrets_filepath} created!") + + with open(self.secrets_filepath, 'a') as the_file: + print("Writing Mastodon parameters to " + self.secrets_filepath) + the_file.write(f'mastodon_app_token: {self.mastodon_app_token}') + + return self.mastodon_app_token + + def mastodon_log_in(self): + + token = '' + + try: + + response = Mastodon.create_app( + self.app_name, + scopes=["read","write", "admin:read", "admin:write"], + to_file=None, + api_base_url=self.mastodon_hostname + ) + client_id = response[0] + client_secret = response[1] + + mastodon = Mastodon(client_id = client_id, client_secret = client_secret, api_base_url = self.mastodon_hostname) + + token = mastodon.log_in( + self.mastodon_user, + self.mastodon_password, + scopes = ["read", "write", "admin:read", "admin:write"], + to_file = None + ) + + print('Log in succesful!') + + except MastodonIllegalArgumentError as i_error: + + sys.stdout.write(f'\n{str(i_error)}\n') + + except MastodonNetworkError as n_error: + + sys.stdout.write(f'\n{str(n_error)}\n') + + except MastodonReadTimeout as r_error: + + sys.stdout.write(f'\n{str(r_error)}\n') + + except MastodonAPIError as a_error: + + sys.stdout.write(f'\n{str(a_error)}\n') + + finally: + + return token + + def __get_parameter(self, parameter, config_file): + + if not os.path.isfile(config_file): + print(f"File {config_file} not found..") + + self.mastodon_hostname = input("\nMastodon hostname: ") + + self.__create_config(self) + self.__write_config(self) + + with open( self.config_file ) as f: + for line in f: + if line.startswith( parameter ): + return line.replace(parameter + ":", "").strip() + + def __get_mastodon_parameter(self, parameter, secrets_filepath): + + if not os.path.isfile(secrets_filepath): + print(f"File {secrets_filepath} not found..") + + self.sign_in() + + with open( self.secrets_filepath ) as f: + for line in f: + if line.startswith( parameter ): + return line.replace(parameter + ":", "").strip() + + @staticmethod + def __create_config(self): + + if not os.path.exists('config'): + + os.makedirs('config') + + if not os.path.exists(self.config_file): + + print(self.config_file + " created!") + with open(self.config_file, 'w'): pass + + @staticmethod + def __write_config(self): + + with open(self.config_file, 'a') as the_file: + + the_file.write(f'mastodon_hostname: {self.mastodon_hostname}') + print(f"adding parameters to {self.config_file}\n") diff --git a/blocksoft.py b/blocksoft.py index 4608c31..2947fa6 100644 --- a/blocksoft.py +++ b/blocksoft.py @@ -1,487 +1,25 @@ import os import sys +from app.libraries.setup import Setup +from app.libraries.database import Database +from app.libraries.domainblocks import DomainBlocks from mastodon import Mastodon -from mastodon.Mastodon import MastodonNotFoundError, MastodonNetworkError, MastodonReadTimeout, MastodonAPIError, MastodonUnauthorizedError, MastodonIllegalArgumentError -import psycopg2 -from psycopg2 import sql -from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT -import getpass -import fileinput,re -import requests -class DomainBlocks(): - - name = 'Domain blocks for Mastodon' - - def __init__(self, mastodon_hostname=None, domain_block_api=None, session=None): - - self.domain_blocks_api = '/api/v1/admin/domain_blocks' - self.config_file = "config/config.txt" - self.secrets_file = "secrets/secrets.txt" - - is_setup = self.__check_setup(self.secrets_file) - - if is_setup: - - self.__uc_client_id = self.__get_parameter("uc_client_id", self.secrets_file) - self.__uc_client_secret = self.__get_parameter("uc_client_secret", self.secrets_file) - self.__uc_access_token = self.__get_parameter("uc_access_token", self.secrets_file) - - self.blocker_db, self.blocker_db_user = self.db_config() - - self.mastodon, self.mastodon_hostname, self.headers = self.log_in() - - else: - - while(True): - - logged_in, self.mastodon, self.mastodon_hostname = self.setup() - - if not logged_in: - - print("\nLog in failed! Try again.\n") - - else: - - break - - db_setup = self.__check_dbsetup(self) - - if not db_setup: - - self.__createdb(self) - - if session: - self.session = session - else: - self.session = requests.Session() - - def get_servers(self, software): - - servers_list = [] - - try: - - conn = None - - conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = "", host = "/var/run/postgresql", port = "5432") - - cur = conn.cursor() - - cur.execute("select server from blocker where software=(%s)", (software,)) - - rows = cur.fetchall() - - for row in rows: - - servers_list.append(row[0]) - - cur.close() - - except (Exception, psycopg2.DatabaseError) as error: - - sys.exit(error) - - finally: - - if conn is not None: - - conn.close() - - return (servers_list) - - def log_in(self): - - uc_client_id = self.__get_parameter("uc_client_id", self.secrets_file) - uc_client_secret = self.__get_parameter("uc_client_secret", self.secrets_file) - uc_access_token = self.__get_parameter("uc_access_token", self.secrets_file) - - self.mastodon_hostname = self.__get_parameter("mastodon_hostname", self.config_file) - - mastodon = Mastodon( - client_id = uc_client_id, - client_secret = uc_client_secret, - access_token = uc_access_token, - api_base_url = 'https://' + self.mastodon_hostname, - ) - - headers={ 'Authorization': 'Bearer %s'%uc_access_token } - - return (mastodon, self.mastodon_hostname, headers) - - def domain_blocks_create(self, server, priv_comment, pub_comment): - - data = { - 'domain': server, - 'severity': 'suspend', - 'reject_media': 'true', - 'reject_reports': 'true', - 'private_comment': priv_comment, - 'public_comment': pub_comment, - 'obfuscate': 'true', - } - - endpoint = f'https://{self.mastodon_hostname}/{self.domain_blocks_api}' - - response = self.__api_request('POST', endpoint, data) - - if response.ok: - - print(f"Done, {server} is now suspended") - - else: - - pass - - @staticmethod - def __check_setup(file_path): - - is_setup = False - - if not os.path.isfile(file_path): - print(f"File {file_path} not found, running setup.") - return - else: - is_setup = True - return is_setup - - def setup(self): - - logged_in = False - - try: - - self.mastodon_hostname = input("Enter Mastodon hostname (or 'q' to exit): ") - - if self.mastodon_hostname == 'q': - - sys.exit("Bye") - - user_name = input("Bot user name, ex. john? ") - user_password = getpass.getpass("Bot password? ") - app_name = input("Bot App name? ") - self.blocker_db = input("Blocker's database name: ") - self.blocker_db_user = input("blocker's database user: ") - - client_id, client_secret = Mastodon.create_app( - app_name, - scopes = ["read", "write", "admin:read", "admin:write"], - to_file="app_clientcred.txt", - api_base_url=self.mastodon_hostname - ) - - mastodon = Mastodon(client_id = "app_clientcred.txt", api_base_url = self.mastodon_hostname) - - mastodon.log_in( - user_name, - user_password, - scopes = ["read", "write", "admin:read", "admin:write"], - to_file = "app_usercred.txt" - ) - - if os.path.isfile("app_usercred.txt"): - - print(f"Log in succesful!") - logged_in = True - - if not os.path.exists('secrets'): - os.makedirs('secrets') - - if not os.path.exists(self.secrets_file): - with open(self.secrets_file, 'w'): pass - print(f"{self.secrets_file} created!") - - with open(self.secrets_file, 'a') as the_file: - print(f"Writing secrets parameter names to {self.secrets_file}") - the_file.write('uc_client_id: \n'+'uc_client_secret: \n'+'uc_access_token: \n') - - client_path = 'app_clientcred.txt' - - with open(client_path) as fp: - - line = fp.readline() - cnt = 1 - - while line: - - if cnt == 1: - - print(f"Writing client id to {self.secrets_file}") - self.__modify_file(self, self.secrets_file, "uc_client_id: ", value=line.rstrip()) - - elif cnt == 2: - - print(f"Writing client secret to {self.secrets_file}") - self.__modify_file(self, self.secrets_file, "uc_client_secret: ", value=line.rstrip()) - - line = fp.readline() - cnt += 1 - - token_path = 'app_usercred.txt' - - with open(token_path) as fp: - - line = fp.readline() - print(f"Writing access token to {self.secrets_file}") - self.__modify_file(self, self.secrets_file, "uc_access_token: ", value=line.rstrip()) - - if os.path.exists("app_clientcred.txt"): - - print("Removing app_clientcred.txt temp file..") - os.remove("app_clientcred.txt") - - if os.path.exists("app_usercred.txt"): - - print("Removing app_usercred.txt temp file..") - os.remove("app_usercred.txt") - - self.__create_config(self) - self.__write_config(self) - - print("Secrets setup done!\n") - - with open(self.config_file, 'a') as the_file: - print(f"Writing Mastodon hostname parameter to {self.config_file}") - the_file.write(f'blocker_db: {self.blocker_db}\n') - the_file.write(f'blocker_db_user: {self.blocker_db_user}\n') - - print("Blocker setup done!\n") - - except MastodonIllegalArgumentError as i_error: - - sys.stdout.write(f'\n{str(i_error)}\n') - - except MastodonNetworkError as n_error: - - sys.stdout.write(f'\n{str(n_error)}\n') - - except MastodonReadTimeout as r_error: - - sys.stdout.write(f'\n{str(r_error)}\n') - - except MastodonAPIError as a_error: - - sys.stdout.write(f'\n{str(a_error)}\n') - - return (logged_in, mastodon, self.mastodon_hostname) - - def __api_request(self, method, endpoint, data={}): - - response = None - - try: - - kwargs = dict(data=data) - - response = self.session.request(method, url = endpoint, headers = self.headers, **kwargs) - - except Exception as e: - - raise MastodonNetworkError(f"Could not complete request: {e}") - - if response is None: - - raise MastodonIllegalArgumentError("Illegal request.") - - if not response.ok: - - try: - if isinstance(response, dict) and 'error' in response: - error_msg = response['error'] - elif isinstance(response, str): - error_msg = response - else: - error_msg = None - except ValueError: - error_msg = None - - if response.status_code == 404: - ex_type = MastodonNotFoundError - if not error_msg: - error_msg = 'Endpoint not found.' - # this is for compatibility with older versions - # which raised MastodonAPIError('Endpoint not found.') - # on any 404 - elif response.status_code == 401: - ex_type = MastodonUnauthorizedError - elif response.status_code == 422: - return response - elif response.status_code == 500: - ex_type = MastodonInternalServerError - elif response.status_code == 502: - ex_type = MastodonBadGatewayError - elif response.status_code == 503: - ex_type = MastodonServiceUnavailableError - elif response.status_code == 504: - ex_type = MastodonGatewayTimeoutError - elif response.status_code >= 500 and \ - response.status_code <= 511: - ex_type = MastodonServerError - else: - ex_type = MastodonAPIError - - raise ex_type( - 'Mastodon API returned error', - response.status_code, - response.reason, - error_msg) - - else: - - return response - - def db_config(self): - - self.blocker_db = self.__get_parameter("blocker_db", self.config_file) - self.blocker_db_user = self.__get_parameter("blocker_db_user", self.config_file) - - return (self.blocker_db, self.blocker_db_user) - - @staticmethod - def __check_dbsetup(self): - - dbsetup = False - - try: - - conn = None - - conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = "", host = "/var/run/postgresql", port = "5432") - - dbsetup = True - - except (Exception, psycopg2.DatabaseError) as error: - - print(error) - - return dbsetup - - @staticmethod - def __createdb(self): - - conn = None - - try: - - conn = psycopg2.connect(dbname='postgres', - user=self.blocker_db_user, host='', - password='') - - conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) - - cur = conn.cursor() - - print(f"Creating database {self.blocker_db}. Please wait...") - - cur.execute(sql.SQL("CREATE DATABASE {}").format( - sql.Identifier(self.blocker_db)) - ) - print(f"Database {self.blocker_db} created!\n") - - self.__dbtables_schemes(self) - - except (Exception, psycopg2.DatabaseError) as error: - - print(error) - - finally: - - if conn is not None: - - conn.close() - - @staticmethod - def __dbtables_schemes(self): - - table = "blocker" - sql = "create table "+table+" (server varchar(200), users int, updated_at timestamptz, software varchar(50), alive boolean, users_api varchar(50), version varchar(100), first_checked_at timestamptz, last_checked_at timestamptz, downs int)" - self.__create_table(self, table, sql) - - table = "execution_time" - sql = "create table "+table+" (program varchar(30), start timestamptz, finish timestamptz)" - self.__create_table(self, table, sql) - - @staticmethod - def __create_table(self, table, sql): - - conn = None - - try: - - conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = "", host = "/var/run/postgresql", port = "5432") - cur = conn.cursor() - - print(f"Creating table {table}") - cur.execute(sql) - - conn.commit() - print(f"Table {table} created!\n") - - except (Exception, psycopg2.DatabaseError) as error: - - print(error) - - finally: - - if conn is not None: - - conn.close() - - def __get_parameter(self, parameter, config_file): - - if not os.path.isfile(config_file): - print(f"File {config_file} not found, exiting.") - sys.exit(0) - - with open( config_file ) as f: - for line in f: - if line.startswith( parameter ): - return line.replace(parameter + ":", "").strip() - - print(f"{config_file} Missing parameter {parameter}") - - sys.exit(0) - - @staticmethod - def __modify_file(self, file_name, pattern,value=""): - - fh=fileinput.input(file_name,inplace=True) - - for line in fh: - - replacement=pattern + value - line=re.sub(pattern,replacement,line) - sys.stdout.write(line) - - fh.close() - - @staticmethod - def __create_config(self): - - if not os.path.exists('config'): - - os.makedirs('config') - - if not os.path.exists(self.config_file): - - print(self.config_file + " created!") - with open('config/config.txt', 'w'): pass - - @staticmethod - def __write_config(self): - - with open(self.config_file, 'a') as the_file: - - the_file.write(f'mastodon_hostname: {self.mastodon_hostname}\n') - print(f"adding parameter 'mastodon_hostname' to {self.config_file}\n") - -############################################################################### # main if __name__ == '__main__': + setup = Setup() + + db = Database() + blocker = DomainBlocks() + mastodon = Mastodon( + access_token = setup.mastodon_app_token, + api_base_url= setup.mastodon_hostname + ) + soft_list = 'software.txt' soft_file = open(soft_list, 'r') @@ -500,11 +38,11 @@ if __name__ == '__main__': print(f'checking software {software}...') - servers_list = blocker.get_servers(software) + servers_list = db.get_servers(software) for server in servers_list: - blocker.domain_blocks_create(server, private_comment, public_comment ) + blocker.domain_blocks_create(server, private_comment, public_comment) diff --git a/peers.py b/peers.py index 45bd4c5..e5aafc4 100644 --- a/peers.py +++ b/peers.py @@ -1,535 +1,42 @@ -import os -import sys -import time -from datetime import datetime -import requests -import urllib3 -import json -import psycopg2 -from psycopg2 import sql -from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT -#import ray -import pdb +from app.libraries.setup import Setup +from app.libraries.database import Database +from app.libraries.peers import Peers +from app.libraries.nodeinfo import Nodeinfo +import ray -#ray.init(num_cpus = 32) # Specify this system CPUs. +ray.init(num_cpus = 25) # Specify this system CPUs. -class Peers: +@ray.remote +def peerapi(peer): - name = 'Mastodon server peers' + api = peers.getapi(peer) # get peer nodeinfo's URL - def __init__(self, mastodon_hostname=None, peers_api=None): - - self.peers_api = '/api/v1/instance/peers' - self.config_path = "config/config.txt" - - self.apis = ['/api/v1/instance?', - '/nodeinfo/2.0?', - '/nodeinfo/2.0.json?', - '/main/nodeinfo/2.0?', - '/api/statusnet/config?', - '/api/nodeinfo/2.0.json?', - '/api/nodeinfo?', - '/wp-json/nodeinfo/2.0?', - '/api/v1/instance/nodeinfo/2.0?', - '/.well-known/x-nodeinfo2?' - ] - - is_setup = self.__check_setup(self) - - if is_setup: - - self.mastodon_hostname = self.__get_parameter("mastodon_hostname", self.config_path) - self.blocker_db = self.__get_parameter("blocker_db", self.config_path) - self.blocker_db_user = self.__get_parameter("blocker_db_user", self.config_path) - - else: - - self.mastodon_hostname, self.blocker_db, self.blocker_db_user = self.__setup(self) - - db_setup = self.__check_dbsetup(self) - - if not db_setup: - - self.__createdb(self) - - def get_peers(self): - - user_agent = {'User-agent': "fediverse's stats (fediverse@soc.catala.digital)"} - - res = requests.get(f'https://{self.mastodon_hostname}{self.peers_api}', headers = user_agent, timeout=3) - - peers = res.json() - - return peers - - def getsoft(self, server): - - if server.find(".") == -1: - return - if server.find("@") != -1: - return - if server.find("/") != -1: - return - if server.find(":") != -1: - return - - soft = '' - - is_nodeinfo = False - - url = 'https://' + server - - user_agent = {'User-agent': "fediverse's stats (fediverse@soc.catala.digital)"} + if api != '': try: - response = requests.get(url + '/.well-known/nodeinfo', headers = user_agent, timeout=3) + server, software = nd.getdata(peer, api) - if response.status_code == 200: + db.write_server(peer, api, software) - try: + print(f'server: {peer}, api: {api}, software: {software}') - response_json = response.json() - - nodeinfo = response_json['links'][0]['href'].replace(f'https://{server}','') - - try: - - nodeinfo_data = requests.get(url + nodeinfo, headers = user_agent, timeout=3) - - if nodeinfo_data.status_code == 200: - - nodeinfo_json = nodeinfo_data.json() - - is_nodeinfo = True - - else: - - print(f"Server {server}'s nodeinfo not responding: error code {nodeinfo_data.status_code}") - - except: - - pass - - except: - - print(f'Server {server} not responding: error code {response.status_code}') - print('*********************************************************************') - - pass - else: - - for api in self.apis: - - try: - - response = requests.get(url + api, headers = user_agent, timeout=3) - - if is_json(response.text): - - nodeinfo_json = response.json() - - if 'software' in nodeinfo_json: - - nodeinfo = api - - is_nodeinfo = True - - break - - elif 'title' in nodeinfo_json: - - if nodeinfo_json['title'] == 'Zap': - - nodeinfo = api - - is_nodeinfo = True - - soft = 'zap' - - break - - elif 'version' in nodeinfo_json: - - nodeinfo = api - - is_nodeinfo = True - - break - - except: - - pass - - except requests.exceptions.SSLError as errssl: + except: pass - except requests.exceptions.HTTPError as errh: - - pass - - except requests.exceptions.ConnectionError as errc: - - pass - - except requests.exceptions.ReadTimeout as to_err: - - pass - - except requests.exceptions.TooManyRedirects as tmr_err: - - pass - - except urllib3.exceptions.LocationParseError as lp_err: - - pass - - except requests.exceptions.InvalidURL as iu_err: - - pass - - except requests.exceptions.ChunkedEncodingError as chunk_err: - - print(f'ChunkedEncodingError! {server}') - pass - - except ray.exceptions.RaySystemError as ray_sys_error: - - print(ray_sys_error) - pass - - else: - - if is_nodeinfo: - - if nodeinfo != '/api/v1/instance?': - - if nodeinfo != '/.well-known/x-nodeinfo2?': - - try: - - soft = nodeinfo_json['software']['name'] - soft = soft.lower() - soft_version = nodeinfo_json['software']['version'] - users = nodeinfo_json['usage']['users']['total'] - alive = True - - self.write_api(server, soft, users, alive, nodeinfo, soft_version) - - print(f"Server {server} ({soft} {soft_version}) is alive!") - print('*********************************************************************') - - return - - except: - - pass - - else: - - try: - - soft = nodeinfo_json['server']['software'] - soft = soft.lower() - soft_version = nodeinfo_json['server']['version'] - users = nodeinfo_json['usage']['users']['total'] - alive = True - - if soft == 'socialhome': - - self.write_api(server, soft, users, alive, nodeinfo, soft_version) - - print('*********************************************************************') - print(f"Server {serve}r ({soft} {soft_version}) is alive!") - print('*********************************************************************') - - return - - except: - - pass - - if soft == '' and nodeinfo == "/api/v1/instance?": - - soft = 'mastodon' - - try: - - users = nodeinfo_json['stats']['user_count'] - - if users > 1000000: - - return - - except: - - users = 0 - - try: - - soft_version = nodeinfo_json['version'] - - except: - - soft_version = 'unknown' - - alive = True - - self.write_api(self, server, soft, users, alive, nodeinfo, soft_version) - - print('*********************************************************************') - print(f"Server {server} ({soft}) is alive!") - - elif soft == 'zap' and nodeinfo == "/api/v1/instance?": - - soft = 'zap' - users = nodeinfo_json['stats']['user_count'] - soft_version = nodeinfo_json['version'] - alive = True - - print(server, soft, users, alive, api) - - print('*********************************************************************') - print(f"Server {server} ({soft}) is alive!") - - else: - - print(f'Server {server} is dead') - print('*********************************************************************') - - def write_api(self, server, software, users, alive, api, soft_version): - - insert_sql = "INSERT INTO blocker(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING" - - conn = None - - try: - - conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password="", host="/var/run/postgresql", port="5432") - - cur = conn.cursor() - - print(f'Writing {server} nodeinfo data...') - - cur.execute(insert_sql, (server, now, software, users, alive, api, soft_version)) - - cur.execute( - "UPDATE blocker SET updated_at=(%s), software=(%s), users=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)", - (now, software, users, alive, api, soft_version, server) - ) - - conn.commit() - - cur.close() - - except (Exception, psycopg2.DatabaseError) as error: - - print(error) - - finally: - - if conn is not None: - - conn.close() - - def save_time(self, program, start, finish): - - insert_sql = "INSERT INTO execution_time(program, start, finish) VALUES(%s,%s,%s) ON CONFLICT DO NOTHING" - - conn = None - - try: - - conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = "", host = "/var/run/postgresql", port = "5432") - - cur = conn.cursor() - - cur.execute(insert_sql, (program, start, finish,)) - - cur.execute("UPDATE execution_time SET start=(%s), finish=(%s) where program=(%s)", (start, finish, program)) - - conn.commit() - - cur.close() - - except (Exception, psycopg2.DatabaseError) as error: - - print(error) - - finally: - - if conn is not None: - - conn.close() - - @staticmethod - def __check_setup(self): - - is_setup = False - - if not os.path.isfile(self.config_path): - print(f"File {self.config_path} not found, running setup.") - else: - is_setup = True - - return is_setup - - @staticmethod - def __setup(self): - - if not os.path.exists('config'): - os.makedirs('config') - - self.mastodon_hostname = input("Mastodon hostname: ") - self.blocker_db = input("Blocker database name: ") - self.blocker_db_user = input("Blocker database user: ") - - if not os.path.exists(self.config_path): - with open(self.config_path, 'w'): pass - print(f"{self.config_path} created!") - - with open(self.config_path, 'a') as the_file: - print("Writing Blocker parameters to " + self.config_path) - the_file.write(f'mastodon_hostname: {self.mastodon_hostname}\n') - the_file.write(f'blocker_db: {self.blocker_db}\n') - the_file.write(f'blocker_db_user: {self.blocker_db_user}\n') - - return (self.mastodon_hostname, self.blocker_db, self.blocker_db_user) - - @staticmethod - def __check_dbsetup(self): - - dbsetup = False - - try: - - conn = None - - conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = "", host = "/var/run/postgresql", port = "5432") - - dbsetup = True - - except (Exception, psycopg2.DatabaseError) as error: - - print(error) - - return dbsetup - - @staticmethod - def __createdb(self): - - conn = None - - try: - - conn = psycopg2.connect(dbname='postgres', - user=self.blocker_db_user, host='', - password='') - - conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) - - cur = conn.cursor() - - print(f"Creating database {self.blocker_db}. Please wait...") - - cur.execute(sql.SQL("CREATE DATABASE {}").format( - sql.Identifier(self.blocker_db)) - ) - print(f"Database {self.blocker_db} created!\n") - - self.__dbtables_schemes(self) - - except (Exception, psycopg2.DatabaseError) as error: - - print(error) - - finally: - - if conn is not None: - - conn.close() - - @staticmethod - def __dbtables_schemes(self): - - table = "blocker" - sql = f"""create table {table} (server varchar(200) PRIMARY KEY, users INT, updated_at timestamptz, software varchar(50), alive boolean, users_api varchar(50), - version varchar(100), first_checked_at timestamptz, last_checked_at timestamptz, downs int)""" - self.__create_table(self, table, sql) - - table = "execution_time" - sql = "create table "+table+" (program varchar(30) PRIMARY KEY, start timestamptz, finish timestamptz)" - self.__create_table(self, table, sql) - - @staticmethod - def __create_table(self, table, sql): - - conn = None - - try: - - conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = "", host = "/var/run/postgresql", port = "5432") - cur = conn.cursor() - - print(f"Creating table {table}") - cur.execute(sql) - - conn.commit() - print(f"Table {table} created!\n") - - except (Exception, psycopg2.DatabaseError) as error: - - print(error) - - finally: - - if conn is not None: - - conn.close() - - @staticmethod - def __get_parameter(parameter, file_path ): - - with open( file_path ) as f: - for line in f: - if line.startswith( parameter ): - return line.replace(parameter + ":", "").strip() - - print(f'{file_path} Missing parameter {parameter}') - sys.exit(0) - -############################################################################### -# main - if __name__ == '__main__': - obj = Peers() - - peers = obj.get_peers() + db = Database() - print(f"{obj.mastodon_hostname}'s peers: {len(peers)}") + setup = Setup() - now = datetime.now() + peers = Peers() - start = datetime.now() + nd = Nodeinfo() - program = obj.name - - finish = start - - obj.save_time(program, start, finish) - - for peer in peers: - - obj.getsoft(peer) - - #results = ray.get([obj.getsoft.remote(server) for server in peers]) - - finish = datetime.now() - - print(f"duration = {finish - start}.\nprocessed servers: {len(peers)}") - - obj.save_time(program, start, finish) + peers_list = peers.getpeers(setup.mastodon_hostname) #get peers from Mastodon host + if peers_list != None: + ray.get([peerapi.remote(peer) for peer in peers_list]) diff --git a/requirements.txt b/requirements.txt index 5201f8e..3328928 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ requests psycopg2-binary Mastodonplus.py +ray