import os import sys import psycopg2 from psycopg2 import sql from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT import uuid from datetime import datetime import pytz import pdb tz = pytz.timezone('Europe/Madrid') class Database(): name = 'fediverse database library' def __init__(self, config_file=None, blocker_db=None, blocker_db_user=None, blocker_db_user_password=None): self.config_file = "config/db_config.txt" self.blocker_db = self.__get_parameter("blocker_db", self.config_file) self.blocker_db_user = self.__get_parameter("blocker_db_user", self.config_file) self.blocker_db_user_password = self.__get_parameter("blocker_db_user_password", self.config_file) db_setup = self.__check_dbsetup(self) if not db_setup: self.blocker_db = input("\nMau database name: ") self.blocker_db_user = input("\nMau database user: ") self.blocker_db_user_password = input("\nMau database user password: ") self.__createdb(self) self.__create_config(self) self.__write_config(self) def get_nodeinfo_endpoint(self, server): try: conn = None conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = self.blocker_db_user_password, host = "/var/run/postgresql", port = "6432") cur = conn.cursor() cur.execute("select server, api from servers where server=(%s)", (server,)) row = cur.fetchone() if row != None: server = row[0] api = row[1] else: server = '' api = '' cur.close() return (server, api) except (Exception, psycopg2.DatabaseError) as error: sys.exit(error) finally: if conn is not None: conn.close() def get_servers(self, software): servers_list = [] try: conn = None conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = "", host = "/var/run/postgresql", port = "5432") cur = conn.cursor() cur.execute("select server from servers where software=(%s)", (software,)) rows = cur.fetchall() for row in rows: servers_list.append(row[0]) cur.close() except (Exception, psycopg2.DatabaseError) as error: sys.exit(error) finally: if conn is not None: conn.close() return (servers_list) def save_time(self, program, start, finish): insert_sql = "INSERT INTO execution_time(program, start, finish) VALUES(%s,%s,%s) ON CONFLICT DO NOTHING" conn = None try: conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = self.blocker_db_user_password, host = "/var/run/postgresql", port = "6432") cur = conn.cursor() cur.execute(insert_sql, (program, start, finish,)) cur.execute("UPDATE execution_time SET start=(%s), finish=(%s) where program=(%s)", (start, finish, program)) conn.commit() cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() def check_peer(self, server): found = False select_server_sql = 'select server from servers where server=(%s)' select_dead_server_sql = 'select server from deadservers where server=(%s)' try: conn = None conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432") cur = conn.cursor() # check server cur.execute(select_server_sql, (server,)) row = cur.fetchone() if row != None: found = True cur.execute(select_dead_server_sql, (server,)) row = cur.fetchone() if row != None: found = True cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() return found def get_not_updated_servers(self): not_updated_servers = [] try: conn = None conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432") cur = conn.cursor() # get saved servers list cur.execute("select server from servers where updated_at is null") rows = cur.fetchall() for row in rows: not_updated_servers.append(row[0]) cur.close() print("Not updated servers: " + str(len(not_updated_servers))) except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() return not_updated_servers def get_saved_servers(self): saved_servers = [] try: conn = None conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432") cur = conn.cursor() # get saved servers list cur.execute("select server from servers where api is not null") rows = cur.fetchall() for row in rows: saved_servers.append(row[0]) cur.close() print("Saved servers: " + str(len(saved_servers))) except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() return saved_servers def get_dead_servers(self): dead_servers = [] try: conn = None conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432") cur = conn.cursor() # get dead servers list cur.execute("select server from deadservers") rows = cur.fetchall() for row in rows: dead_servers.append(row[0]) cur.close() print("Dead servers: " + str(len(dead_servers))) except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() return dead_servers def soft_totals(self): # get current total servers and users, get users from every software now = datetime.now() gettotals_sql = "select count(server), sum(users), sum(mau) from mau where alive" get_soft_totals_sql = "select software, sum(users) as users, sum(mau) as mau, count(server) as servers from mau where users != 0 and mau is not null and alive group by software order by mau desc" soft_total_project = [] soft_total_users = [] soft_total_mau = [] soft_total_servers = [] try: conn = None conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432") cur = conn.cursor() cur.execute(gettotals_sql) row = cur.fetchone() total_servers = row[0] total_users = row[1] total_mau = row[2] cur.execute(get_soft_totals_sql) rows = cur.fetchall() for row in rows: soft_total_project.append(row[0]) soft_total_users.append(row[1]) soft_total_mau.append(row[2]) soft_total_servers.append(row[3]) cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() return (soft_total_project, soft_total_users, soft_total_mau, soft_total_servers, total_servers, total_users, total_mau) def get_last_checked_servers(self): # get last checked servers from fediverse DB alive_servers = [] try: conn = None conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432") cur = conn.cursor() # get world servers list cur.execute("select server from world where server in (select server from fediverse where users_api != '')") rows = cur.fetchall() for row in rows: alive_servers.append(row[0]) cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() return alive_servers def write_alive_server(self, server, software, version, users, mau, alive): now = datetime.now() insert_sql = "INSERT INTO mau(server, software, version, users, mau, alive, updated_at) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING" conn = None try: conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432") cur = conn.cursor() cur.execute(insert_sql, (server, software, version, users, mau, alive, now)) cur.execute("UPDATE mau SET software=(%s), version=(%s), users=(%s), mau=(%s), alive=(%s), updated_at=(%s) where server=(%s)", (software, version, users, mau, alive, now, server)) conn.commit() cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() def write_not_alive_server(self, server): now = datetime.now() update_sql = "UPDATE mau set alive='f', updated_at=(%s) where server=(%s)" conn = None try: conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432") cur = conn.cursor() cur.execute(update_sql, (now, server)) conn.commit() cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() def write_server(self, server, api, software): now = datetime.now() insert_sql = "INSERT INTO servers(server, api, software, created_at) VALUES(%s, %s, %s, %s) ON CONFLICT DO NOTHING" conn = None try: conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432") cur = conn.cursor() cur.execute(insert_sql, (server, api, software, now)) conn.commit() cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() def update_peer(self, server): now = datetime.now() update_sql = "UPDATE servers set updated_at=(%s) where server=(%s)" conn = None try: conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432") cur = conn.cursor() cur.execute(update_sql, (now, server)) conn.commit() cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() def update_server(self, server, api): now = datetime.now() update_sql = "UPDATE servers set server=(%s), api=(%s), updated_at=(%s) where server=(%s)" conn = None try: conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432") cur = conn.cursor() cur.execute(update_sql, (server, api, now, server)) conn.commit() cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() @staticmethod def __check_dbsetup(self): db_setup = False try: conn = None conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = self.blocker_db_user_password, host = "/var/run/postgresql", port = "6432") db_setup = True except (Exception, psycopg2.DatabaseError) as error: print(error) return db_setup @staticmethod def __createdb(self): conn = None try: conn = psycopg2.connect(dbname='postgres', user=self.blocker_db_user, host='', password=self.blocker_db_user_password) conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) cur = conn.cursor() print(f"Creating database {self.blocker_db}. Please wait...") cur.execute(sql.SQL("CREATE DATABASE {}").format( sql.Identifier(self.blocker_db)) ) print(f"Database {self.blocker_db} created!\n") self.__dbtables_schemes(self) except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() @staticmethod def __dbtables_schemes(self): db = self.blocker_db table = "servers" sql = "create table "+table+" (server varchar(200) PRIMARY KEY, api varchar(50), software varchar(50), created_at timestamptz, updated_at timestamptz, downs int)" self.__create_table(self, table, sql) @staticmethod def __create_table(self, table, sql): conn = None try: conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = self.blocker_db_user_password, host = "/var/run/postgresql", port = "6432") cur = conn.cursor() print(f"Creating table {table}") cur.execute(sql) conn.commit() print(f"Table {table} created!\n") except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() def __get_parameter(self, parameter, config_file): if not os.path.isfile(config_file): print(f"File {config_file} not found..") return with open( config_file ) as f: for line in f: if line.startswith( parameter ): return line.replace(parameter + ":", "").strip() print(f"{config_file} Missing parameter {parameter}") sys.exit(0) @staticmethod def __create_config(self): if not os.path.exists('config'): os.makedirs('config') if not os.path.exists(self.config_file): print(self.config_file + " created!") with open(self.config_file, 'w'): pass @staticmethod def __write_config(self): with open(self.config_file, 'a') as the_file: the_file.write(f'blocker_db: {self.blocker_db}\nblocker_db_user: {self.blocker_db_user}\nblocker_db_user_password: {self.blocker_db_user_password}') print(f"adding parameters to {self.config_file}\n")