import os import sys import psycopg2 from psycopg2 import sql from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT import uuid from datetime import datetime import pytz tz = pytz.timezone('Europe/Madrid') class Database(): name = 'fediverse database library' def __init__(self, config_file=None, fediverse_db=None, fediverse_db_user=None, fediverse_db_user_password=None, fediverse_db_host='/var/run/postgresql', fediverse_db_port='6432'): self.config_file = config_file or "config/db_config.txt" self.fediverse_db = fediverse_db or self.__get_parameter("fediverse_db", self.config_file) self.fediverse_db_user = fediverse_db_user or self.__get_parameter("fediverse_db_user", self.config_file) self.fediverse_db_user_password = fediverse_db_user_password or self.__get_parameter("fediverse_db_user_password", self.config_file) self.fediverse_db_host = fediverse_db_host self.fediverse_db_port = fediverse_db_port db_setup = self.__check_dbsetup(self) if not db_setup: self.fediverse_db = input("\nFediverse database name: ") self.fediverse_db_user = input("\nFediverse database user: ") self.fediverse_db_user_password = input("\nFediverse database user password: ") self.__createdb(self) self.__create_config(self) self.__write_config(self) def write_server(self, server, federated_with, now): now = datetime.now() insert_sql = "INSERT INTO world(server, federated_with, updated_at, saved_at, checked) VALUES(%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING" conn = None try: conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = self.fediverse_db_host, port = self.fediverse_db_port) cur = conn.cursor() cur.execute(insert_sql, (server, federated_with, now, now, 'f')) print(f'writing {server} to world database') conn.commit() cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() def save_time(self, program, start, finish): insert_sql = "INSERT INTO execution_time(program, start, finish) VALUES(%s,%s,%s) ON CONFLICT DO NOTHING" conn = None try: conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = self.fediverse_db_host, port = self.fediverse_db_port) cur = conn.cursor() cur.execute(insert_sql, (program, start, finish,)) cur.execute("UPDATE execution_time SET start=(%s), finish=(%s) where program=(%s)", (start, finish, program)) conn.commit() cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() def write_api(self, server, software, users, mau, alive, api, soft_version): now = datetime.now() insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, mau, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING" conn = None try: conn = psycopg2.connect(database=self.fediverse_db, user=self.fediverse_db_user, password = self.fediverse_db_user_password, host="/var/run/postgresql", port="6432") cur = conn.cursor() print(f'Writing {server} nodeinfo data...') cur.execute(insert_sql, (server, now, software, users, mau, alive, api, soft_version)) cur.execute( "UPDATE fediverse SET updated_at=(%s), software=(%s), users=(%s), mau=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)", (now, software, users, mau, alive, api, soft_version, server) ) cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,)) conn.commit() cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() def get_world_servers(self): world_servers = [] try: conn = None conn = psycopg2.connect(database=self.fediverse_db, user=self.fediverse_db_user, password = self.fediverse_db_user_password, host="/var/run/postgresql", port="6432") cur = conn.cursor() # get world servers list cur.execute("select server from world where checked='f'") rows = cur.fetchall() for row in rows: world_servers.append(row[0]) cur.close() print("Remaining servers: " + str(len(world_servers))) except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() return world_servers def get_last_checked_servers(self): # get last checked servers from fediverse DB alive_servers = [] try: conn = None conn = psycopg2.connect(database=self.fediverse_db, user=self.fediverse_db_user, password = self.fediverse_db_user_password, host="/var/run/postgresql", port="6432") cur = conn.cursor() # get world servers list cur.execute("select server from world where server in (select server from fediverse where users_api != '')") rows = cur.fetchall() for row in rows: alive_servers.append(row[0]) cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() return alive_servers def set_world_servers_check_to_false(self): # set all world servers's checked column to False try: conn = None conn = psycopg2.connect(database=self.fediverse_db, user=self.fediverse_db_user, password = self.fediverse_db_user_password, host="/var/run/postgresql", port="6432") cur = conn.cursor() cur.execute("UPDATE world SET checked='f'") conn.commit() cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() def delete_dead_servers(self): conn = None try: conn = psycopg2.connect(database=self.fediverse_db, user=self.fediverse_db_user, password = self.fediverse_db_user_password, host="/var/run/postgresql", port="6432") cur = conn.cursor() cur.execute("select server from fediverse where downs > '14' and not alive and now() - first_checked_at > interval '7 days'") rows = cur.fetchall() for row in rows: print(f'Deleting server {row[0]}...') cur.execute("delete from fediverse where server=(%s)", (row[0],)) conn.commit() cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() def write_alive_server(self, server, software, soft_version, alive, api, users, downs, first_checked_at, mau): now = datetime.now() insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version, first_checked_at, last_checked_at, downs, mau) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING" conn = None try: conn = psycopg2.connect(database=self.fediverse_db, user=self.fediverse_db_user, password = self.fediverse_db_user_password, host="/var/run/postgresql", port="6432") cur = conn.cursor() cur.execute(insert_sql, (server, users, now, software, alive, api, soft_version, now, now, downs, mau)) if first_checked_at != None: cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), last_checked_at=(%s), downs=(%s), mau=(%s) where server=(%s)", (users, now, software, alive, api, soft_version, now, downs, mau, server)) else: cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), first_checked_at=(%s), last_checked_at=(%s), downs=(%s), mau=(%s) where server=(%s)", (users, now, software, alive, api, soft_version, now, now, downs, mau, server)) cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,)) conn.commit() cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() def write_not_alive_server(self, server, software, soft_version, alive, api, users, downs, first_checked_at, mau): now = datetime.now() conn = None try: conn = psycopg2.connect(database=self.fediverse_db, user=self.fediverse_db_user, password = self.fediverse_db_user_password, host="/var/run/postgresql", port="6432") cur = conn.cursor() if first_checked_at != None: cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), downs=(%s), mau=(%s) where server=(%s)", (now, alive, downs, mau, server)) else: cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), first_checked_at=(%s), downs=(%s), mau=(%s) where server=(%s)", (now, alive, now, downs, mau, server)) cur.execute("UPDATE world SET checked='f' where server=(%s)", (server,)) conn.commit() cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() def write_blocked_software(self, server, software, soft_version, alive, api, users, downs, first_checked_at): now = datetime.now() insert_sql = "INSERT INTO blocked(server, users, updated_at, software, alive, users_api, version, first_checked_at, last_checked_at, downs) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING" conn = None try: conn = psycopg2.connect(database=self.fediverse_db, user=self.fediverse_db_user, password = self.fediverse_db_user_password, host="/var/run/postgresql", port="6432") cur = conn.cursor() cur.execute(insert_sql, (server, users, now, software, alive, api, soft_version, now, now, downs)) if first_checked_at != None: cur.execute("UPDATE blocked SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (users, now, software, alive, api, soft_version, now, downs, server)) else: cur.execute("UPDATE blocked SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), first_checked_at=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (users, now, software, alive, api, soft_version, now, now, downs, server)) cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,)) conn.commit() cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() def soft_totals(self): # get current total servers and users, get users from every software now = datetime.now() gettotals_sql = "select count(server), sum(users), sum(mau) from fediverse where alive" get_soft_totals_sql = "select software, sum(users) as users, sum(mau) as mau, count(server) as servers from fediverse where users != 0 and mau is not null and alive group by software order by mau desc" soft_total_project = [] soft_total_users = [] soft_total_mau = [] soft_total_servers = [] try: conn = None conn = psycopg2.connect(database=self.fediverse_db, user=self.fediverse_db_user, password = self.fediverse_db_user_password, host="/var/run/postgresql", port="6432") cur = conn.cursor() cur.execute(gettotals_sql) row = cur.fetchone() total_servers = row[0] total_users = row[1] total_mau = row[2] cur.execute(get_soft_totals_sql) rows = cur.fetchall() for row in rows: soft_total_project.append(row[0]) soft_total_users.append(row[1]) soft_total_mau.append(row[2]) soft_total_servers.append(row[3]) cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() return (soft_total_project, soft_total_users, soft_total_mau, soft_total_servers, total_servers, total_users, total_mau) def last_values(self, total_servers, total_users, total_mau): # get last check values and write current total ones now = datetime.now() evo_servers = 0 evo_users = 0 evo_mau = 0 select_sql = "select total_servers, total_users, total_mau from totals order by datetime desc limit 1" insert_sql = "INSERT INTO totals(datetime, total_servers, total_users, total_mau) VALUES(%s,%s,%s,%s)" try: conn = None conn = psycopg2.connect(database=self.fediverse_db, user=self.fediverse_db_user, password = self.fediverse_db_user_password, host="/var/run/postgresql", port="6432") cur = conn.cursor() cur.execute(select_sql) row = cur.fetchone() if row is not None: servers_before = row[0] users_before = row[1] mau_before = row[2] else: servers_before = 0 users_before = 0 mau_before = 0 cur.execute(insert_sql, (now, total_servers, total_users, total_mau)) conn.commit() cur.close() evo_servers = total_servers - servers_before evo_users = total_users - users_before evo_mau = total_mau - mau_before except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() return (evo_servers, evo_users, evo_mau) def write_evo(self, evo_servers, evo_users, evo_mau): # write evo values now = datetime.now() insert_sql = "INSERT INTO evo(datetime, servers, users, mau) VALUES(%s,%s,%s,%s)" conn = None try: conn = psycopg2.connect(database=self.fediverse_db, user=self.fediverse_db_user, password = self.fediverse_db_user_password, host="/var/run/postgresql", port="6432") cur = conn.cursor() cur.execute(insert_sql, (now, evo_servers, evo_users, evo_mau)) conn.commit() cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() def last_world_datetime(self): # get world's last update datetime conn = None try: conn = psycopg2.connect(database=self.fediverse_db, user=self.fediverse_db_user, password = self.fediverse_db_user_password, host="/var/run/postgresql", port="6432") cur = conn.cursor() cur.execute("select updated_at from world order by updated_at desc limit 1") row = cur.fetchone() last_update = row[0] last_update = last_update.strftime('%m/%d/%Y, %H:%M:%S') cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() return last_update def max(self): # get max servers and mau conn = None try: conn = psycopg2.connect(database=self.fediverse_db, user=self.fediverse_db_user, password = self.fediverse_db_user_password, host="/var/run/postgresql", port="6432") cur = conn.cursor() cur.execute("select MAX(total_servers) from totals") row = cur.fetchone() if row is not None: max_servers = row[0] else: max_servers = 0 cur.execute("select MAX(total_mau) from totals") row = cur.fetchone() if row is not None: max_mau = row[0] else: max_mau = 0 cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() return (max_servers, max_mau) def get_plots(self): # get plots servers_plots = [] mau_plots = [] global_week = [] global_servers = [] global_users = [] global_mau = [] conn = None try: conn = psycopg2.connect(database=self.fediverse_db, user=self.fediverse_db_user, password = self.fediverse_db_user_password, host="/var/run/postgresql", port="6432") cur = conn.cursor() cur.execute("select total_servers, total_mau from totals order by datetime desc limit 14") rows = cur.fetchall() for row in rows: servers_plots.append(row[0]) mau_plots.append(row[1]) cur.execute("select distinct on (date_trunc('week', datetime)) datetime::TIMESTAMP::DATE, total_servers, total_users, total_mau from totals group by date_trunc('month', datetime), datetime") rows = cur.fetchall() for row in rows: global_week.append(row[0]) global_servers.append(row[1]) global_users.append(row[2]) global_mau.append(row[3]) cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() return (servers_plots, mau_plots, global_week, global_servers, global_users, global_mau) def get_server_data(self, server): was_alive = None software = None api = None soft_version = None first_checked_at = None downs_qty = None mau = 0 try: conn = None conn = psycopg2.connect(database=self.fediverse_db, user=self.fediverse_db_user, password = self.fediverse_db_user_password, host="/var/run/postgresql", port="6432") cur = conn.cursor() cur.execute("select alive, software, users_api, version, first_checked_at, downs, mau from fediverse where server=(%s)", (server,)) row = cur.fetchone() if row is not None: was_alive = row[0] software = row[1] api = row[2] soft_version = row[3] first_checked_at = row[4] downs_qty = row[5] mau = row[6] cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() return (was_alive, software, api, soft_version, first_checked_at, downs_qty, mau) def get_soft_data(self, search_soft): try: conn = None conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = self.fediverse_db_host, port = self.fediverse_db_port) cur = conn.cursor() cur.execute("select count(server), sum(users), sum(mau) from fediverse where software=(%s) and alive", (search_soft,)) row = cur.fetchone() if row != None: servers = row[0] users = row[1] mau = row[2] else: servers = 0 users = 0 mau = 0 cur.close() return (servers, users, mau) except (Exception, psycopg2.DatabaseError) as error: sys.exit(error) finally: if conn is not None: conn.close() def fediquery_server_data(self, search_server): try: conn = None conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = self.fediverse_db_host, port = self.fediverse_db_port) cur = conn.cursor() cur.execute("select server, software, version, users, mau, alive from fediverse where server=(%s)", (search_server,)) row = cur.fetchone() if row != None: server = row[0] software = row[1] version = row[2] users = row[3] if row[4] != None: mau = row[4] else: mau = 0 alive = row[5] else: server = '' software = '' version = '' users = 0 mau = 0 alive = False cur.close() return (server, software, version, users, mau, alive) except (Exception, psycopg2.DatabaseError) as error: sys.exit(error) finally: if conn is not None: conn.close() def get_uptime(self): try: conn = None conn = psycopg2.connect(database=self.fediverse_db, user=self.fediverse_db_user, password=self.fediverse_db_user_password, host="/var/run/postgresql", port="6432") cur = conn.cursor() cur.execute("select count(server) from fediverse where alive") row = cur.fetchone() alive_servers = row[0] cur.execute("select MAX(date_trunc('day', last_checked_at) - date_trunc('days', first_checked_at)) from fediverse;") row = cur.fetchone() max_uptime = row[0] cur.execute("select count(server) from fediverse where date_trunc('days', last_checked_at)-date_trunc('days',first_checked_at)=(%s)", (max_uptime,)) row = cur.fetchone() if row is not None: best_servers = row[0] cur.execute("select software, count(*) as servers from fediverse where date_trunc('days',last_checked_at)-date_trunc('days', first_checked_at)=(%s) group by software order by servers desc", (max_uptime,)) rows = cur.fetchall() software_lst = [] servers_lst = [] for row in rows: software_lst.append(row[0]) servers_lst.append(row[1]) cur.close() return (alive_servers, max_uptime, best_servers, software_lst, servers_lst) except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() def get_percentage(self, uptime_servers, software): try: conn = None conn = psycopg2.connect(database=self.fediverse_db, user=self.fediverse_db_user, password=self.fediverse_db_user_password, host="/var/run/postgresql", port="6432") cur = conn.cursor() cur.execute("select count(server) from fediverse where alive and software=(%s)", (software,)) row = cur.fetchone() if row is not None: soft_total_servers = row[0] cur.close() soft_uptime_percent = round((uptime_servers * 100) / soft_total_servers, 1) return (soft_uptime_percent) except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() def check_connection(self): try: conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = self.fediverse_db_host, port = self.fediverse_db_port) finally: conn.close() @staticmethod def __check_dbsetup(self): db_setup = False try: conn = None conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = self.fediverse_db_host, port = self.fediverse_db_port) db_setup = True except (Exception, psycopg2.DatabaseError) as error: print(error) return db_setup @staticmethod def __createdb(self): conn = None try: conn = psycopg2.connect(dbname='postgres', user=self.fediverse_db_user, host='', password=self.fediverse_db_user_password) conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) cur = conn.cursor() print(f"Creating database {self.fediverse_db}. Please wait...") cur.execute(sql.SQL("CREATE DATABASE {}").format( sql.Identifier(self.fediverse_db)) ) print(f"Database {self.fediverse_db} created!\n") self.__dbtables_schemes(self) except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() @staticmethod def __dbtables_schemes(self): db = self.fediverse_db table = "world" sql = "create table "+table+" (server varchar(200) PRIMARY KEY, federated_with varchar(200), updated_at timestamptz, saved_at timestamptz, checked boolean)" self.__create_table(self, table, sql) table = "fediverse" sql = "create table "+table+" (server varchar(200) PRIMARY KEY, users INT, updated_at timestamptz, software varchar(50), alive boolean, users_api varchar(50), version varchar(100), first_checked_at timestamptz, last_checked_at timestamptz, downs int, mau int)" self.__create_table(self, table, sql) table = "blocked" sql = "create table "+table+" (server varchar(200) PRIMARY KEY, users INT, updated_at timestamptz, software varchar(50), alive boolean, users_api varchar(50), version varchar(100), first_checked_at timestamptz, last_checked_at timestamptz, downs int)" self.__create_table(self, table, sql) table = "totals" sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, total_servers INT, total_users INT)" self.__create_table(self, table, sql) table = "evo" sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, servers INT, users INT)" self.__create_table(self, table, sql) table = "execution_time" sql = "create table "+table+" (program varchar(20) PRIMARY KEY, start timestamptz, finish timestamptz)" self.__create_table(self, table, sql) @staticmethod def __create_table(self, table, sql): conn = None try: conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = self.fediverse_db_host, port = self.fediverse_db_port) cur = conn.cursor() print(f"Creating table {table}") cur.execute(sql) conn.commit() print(f"Table {table} created!\n") except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() def __get_parameter(self, parameter, config_file): if not os.path.isfile(config_file): print(f"File {config_file} not found..") return with open( config_file ) as f: for line in f: if line.startswith( parameter ): return line.replace(parameter + ":", "").strip() print(f"{config_file} Missing parameter {parameter}") sys.exit(0) @staticmethod def __create_config(self): if not os.path.exists('config'): os.makedirs('config') if not os.path.exists(self.config_file): print(self.config_file + " created!") with open(self.config_file, 'w'): pass @staticmethod def __write_config(self): with open(self.config_file, 'a') as the_file: the_file.write(f'fediverse_db: {self.fediverse_db}\nfediverse_db_user: {self.fediverse_db_user}\nfediverse_db_user_password: {self.fediverse_db_user_password}') print(f"adding parameters to {self.config_file}\n")