From f12c26cd8bb0b70fb5bcdc2a18f9d76fcfc32ed8 Mon Sep 17 00:00:00 2001 From: spla Date: Thu, 5 Jan 2023 01:08:04 +0100 Subject: [PATCH] Added Database management Class --- database.py | 989 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 989 insertions(+) create mode 100644 database.py diff --git a/database.py b/database.py new file mode 100644 index 0000000..e39c21c --- /dev/null +++ b/database.py @@ -0,0 +1,989 @@ +import os +import sys +import psycopg2 +from psycopg2 import sql +from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT +import uuid +from datetime import datetime +import pytz +import pdb + +tz = pytz.timezone('Europe/Madrid') + +class Database(): + + name = 'fediverse database library' + + def __init__(self, config_file=None, fediverse_db=None, fediverse_db_user=None, fediverse_db_user_password=None): + + self.config_file = "config/db_config.txt" + self.fediverse_db = self.__get_parameter("fediverse_db", self.config_file) + self.fediverse_db_user = self.__get_parameter("fediverse_db_user", self.config_file) + self.fediverse_db_user_password = self.__get_parameter("fediverse_db_user_password", self.config_file) + + db_setup = self.__check_dbsetup(self) + + if not db_setup: + + self.fediverse_db = input("\nFediverse database name: ") + self.fediverse_db_user = input("\nFediverse database user: ") + self.fediverse_db_user_password = input("\nFediverse database user password: ") + + self.__createdb(self) + self.__create_config(self) + self.__write_config(self) + + def write_server(self, server, federated_with, now): + + now = datetime.now() + + insert_sql = "INSERT INTO world(server, federated_with, updated_at, saved_at, checked) VALUES(%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING" + + conn = None + + try: + + conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = "/var/run/postgresql", port = "6432") + + cur = conn.cursor() + + cur.execute(insert_sql, (server, federated_with, now, now, 'f')) + + print(f'writing {server} to world database') + + conn.commit() + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + def save_time(self, program, start, finish): + + insert_sql = "INSERT INTO execution_time(program, start, finish) VALUES(%s,%s,%s) ON CONFLICT DO NOTHING" + + conn = None + + try: + + conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = "/var/run/postgresql", port = "6432") + + cur = conn.cursor() + + cur.execute(insert_sql, (program, start, finish,)) + + cur.execute("UPDATE execution_time SET start=(%s), finish=(%s) where program=(%s)", (start, finish, program)) + + conn.commit() + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + def write_api(self, server, software, users, alive, api, soft_version): + + now = datetime.now() + + insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING" + + conn = None + + try: + + conn = psycopg2.connect(database=self.fediverse_db, user=self.fediverse_db_user, password = self.fediverse_db_user_password, host="/var/run/postgresql", port="6432") + + cur = conn.cursor() + + print(f'Writing {server} nodeinfo data...') + + cur.execute(insert_sql, (server, now, software, users, alive, api, soft_version)) + + cur.execute( + "UPDATE fediverse SET updated_at=(%s), software=(%s), users=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)", + (now, software, users, alive, api, soft_version, server) + ) + + cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,)) + + conn.commit() + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + def get_world_servers(self): + + world_servers = [] + + try: + + conn = None + + conn = psycopg2.connect(database=self.fediverse_db, user=self.fediverse_db_user, password = self.fediverse_db_user_password, host="/var/run/postgresql", port="6432") + + cur = conn.cursor() + + # get world servers list + + cur.execute("select server from world where checked='f'") + + rows = cur.fetchall() + + for row in rows: + + world_servers.append(row[0]) + + cur.close() + + print("Remaining servers: " + str(len(world_servers))) + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + return world_servers + + def get_last_checked_servers(self): + + # get last checked servers from fediverse DB + + alive_servers = [] + + try: + + conn = None + + conn = psycopg2.connect(database=self.fediverse_db, user=self.fediverse_db_user, password = self.fediverse_db_user_password, host="/var/run/postgresql", port="6432") + + cur = conn.cursor() + + # get world servers list + + cur.execute("select server from world where server in (select server from fediverse where users_api != '')") + + rows = cur.fetchall() + + for row in rows: + + alive_servers.append(row[0]) + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + return alive_servers + + def set_world_servers_check_to_false(self): + + # set all world servers's checked column to False + + try: + + conn = None + + conn = psycopg2.connect(database=self.fediverse_db, user=self.fediverse_db_user, password = self.fediverse_db_user_password, host="/var/run/postgresql", port="6432") + + cur = conn.cursor() + + cur.execute("UPDATE world SET checked='f'") + + conn.commit() + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + def delete_dead_servers(self): + + conn = None + + try: + + conn = psycopg2.connect(database=self.fediverse_db, user=self.fediverse_db_user, password = self.fediverse_db_user_password, host="/var/run/postgresql", port="6432") + + cur = conn.cursor() + + cur.execute("select server from fediverse where downs > '14' and not alive and now() - first_checked_at > interval '7 days'") + + rows = cur.fetchall() + + for row in rows: + + print(f'Deleting server {row[0]}...') + + cur.execute("delete from fediverse where server=(%s)", (row[0],)) + + conn.commit() + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + def write_alive_server(self, server, software, soft_version, alive, api, users, downs, first_checked_at, mau): + + now = datetime.now() + + insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version, first_checked_at, last_checked_at, downs, mau) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING" + + conn = None + + try: + + conn = psycopg2.connect(database=self.fediverse_db, user=self.fediverse_db_user, password = self.fediverse_db_user_password, host="/var/run/postgresql", port="6432") + + cur = conn.cursor() + + cur.execute(insert_sql, (server, users, now, software, alive, api, soft_version, now, now, downs, mau)) + + if first_checked_at != None: + + cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), last_checked_at=(%s), downs=(%s), mau=(%s) where server=(%s)", (users, now, software, alive, api, soft_version, now, downs, mau, server)) + + else: + + cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), first_checked_at=(%s), last_checked_at=(%s), downs=(%s), mau=(%s) where server=(%s)", (users, now, software, alive, api, soft_version, now, now, downs, mau, server)) + + cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,)) + + conn.commit() + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + def write_not_alive_server(self, server, software, soft_version, alive, api, users, downs, first_checked_at): + + now = datetime.now() + + conn = None + + try: + + conn = psycopg2.connect(database=self.fediverse_db, user=self.fediverse_db_user, password = self.fediverse_db_user_password, host="/var/run/postgresql", port="6432") + + cur = conn.cursor() + + if first_checked_at != None: + + cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), downs=(%s) where server=(%s)", (now, alive, downs, server)) + + else: + + cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), first_checked_at=(%s), downs=(%s) where server=(%s)", (now, alive, now, downs, server)) + + cur.execute("UPDATE world SET checked='f' where server=(%s)", (server,)) + + conn.commit() + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + def write_blocked_software(self, server, software, soft_version, alive, api, users, downs, first_checked_at): + + now = datetime.now() + + insert_sql = "INSERT INTO blocked(server, users, updated_at, software, alive, users_api, version, first_checked_at, last_checked_at, downs) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING" + + conn = None + + try: + + conn = psycopg2.connect(database=self.fediverse_db, user=self.fediverse_db_user, password = self.fediverse_db_user_password, host="/var/run/postgresql", port="6432") + + cur = conn.cursor() + + cur.execute(insert_sql, (server, users, now, software, alive, api, soft_version, now, now, downs)) + + if first_checked_at != None: + + cur.execute("UPDATE blocked SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (users, now, software, alive, api, soft_version, now, downs, server)) + + else: + + cur.execute("UPDATE blocked SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), first_checked_at=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (users, now, software, alive, api, soft_version, now, now, downs, server)) + + cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,)) + + conn.commit() + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + def soft_totals(self): + + # get current total servers and users, get users from every software + + now = datetime.now() + + gettotals_sql = "select count(server), sum(users), sum(mau) from fediverse where alive" + get_soft_totals_sql = "select software, sum(users) as users, sum(mau) as mau, count(server) as servers from fediverse where users != 0 and mau is not null and alive group by software order by mau desc" + + soft_total_project = [] + soft_total_users = [] + soft_total_mau = [] + soft_total_servers = [] + + try: + + conn = None + + conn = psycopg2.connect(database=self.fediverse_db, user=self.fediverse_db_user, password = self.fediverse_db_user_password, host="/var/run/postgresql", port="6432") + + cur = conn.cursor() + + cur.execute(gettotals_sql) + + row = cur.fetchone() + + total_servers = row[0] + + total_users = row[1] + + total_mau = row[2] + + cur.execute(get_soft_totals_sql) + + rows = cur.fetchall() + + for row in rows: + + soft_total_project.append(row[0]) + soft_total_users.append(row[1]) + soft_total_mau.append(row[2]) + soft_total_servers.append(row[3]) + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + return (soft_total_project, soft_total_users, soft_total_mau, soft_total_servers, total_servers, total_users, total_mau) + + def last_values(self, total_servers, total_users, total_mau): + + # get last check values and write current total ones + + now = datetime.now() + + evo_servers = 0 + + evo_users = 0 + + evo_mau = 0 + + select_sql = "select total_servers, total_users, total_mau from totals order by datetime desc limit 1" + + insert_sql = "INSERT INTO totals(datetime, total_servers, total_users, total_mau) VALUES(%s,%s,%s,%s)" + + try: + + conn = None + + conn = psycopg2.connect(database=self.fediverse_db, user=self.fediverse_db_user, password = self.fediverse_db_user_password, host="/var/run/postgresql", port="6432") + + cur = conn.cursor() + + cur.execute(select_sql) + + row = cur.fetchone() + + if row is not None: + + servers_before = row[0] + + users_before = row[1] + + mau_before = row[2] + + else: + + servers_before = 0 + + users_before = 0 + + mau_before = 0 + + cur.execute(insert_sql, (now, total_servers, total_users, total_mau)) + + conn.commit() + + cur.close() + + evo_servers = total_servers - servers_before + + evo_users = total_users - users_before + + evo_mau = total_mau - mau_before + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + return (evo_servers, evo_users, evo_mau) + + def write_evo(self, evo_servers, evo_users, evo_mau): + + # write evo values + + now = datetime.now() + + insert_sql = "INSERT INTO evo(datetime, servers, users, mau) VALUES(%s,%s,%s,%s)" + + conn = None + + try: + + conn = psycopg2.connect(database=self.fediverse_db, user=self.fediverse_db_user, password = self.fediverse_db_user_password, host="/var/run/postgresql", port="6432") + + cur = conn.cursor() + + cur.execute(insert_sql, (now, evo_servers, evo_users, evo_mau)) + + conn.commit() + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + def last_world_datetime(self): + + # get world's last update datetime + + conn = None + + try: + + conn = psycopg2.connect(database=self.fediverse_db, user=self.fediverse_db_user, password = self.fediverse_db_user_password, host="/var/run/postgresql", port="6432") + + cur = conn.cursor() + + cur.execute("select updated_at from world order by updated_at desc limit 1") + + row = cur.fetchone() + + last_update = row[0] + + last_update = last_update.strftime('%m/%d/%Y, %H:%M:%S') + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + return last_update + + def max(self): + + # get max servers and mau + + conn = None + + try: + + conn = psycopg2.connect(database=self.fediverse_db, user=self.fediverse_db_user, password = self.fediverse_db_user_password, host="/var/run/postgresql", port="6432") + + cur = conn.cursor() + + cur.execute("select MAX(total_servers) from totals") + + row = cur.fetchone() + + if row is not None: + + max_servers = row[0] + + else: + + max_servers = 0 + + cur.execute("select MAX(total_mau) from totals") + + row = cur.fetchone() + + if row is not None: + + max_mau = row[0] + + else: + + max_mau = 0 + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + return (max_servers, max_mau) + + def get_plots(self): + + # get plots + + servers_plots = [] + + mau_plots = [] + + global_week = [] + + global_servers = [] + + global_users = [] + + global_mau = [] + + conn = None + + try: + + conn = psycopg2.connect(database=self.fediverse_db, user=self.fediverse_db_user, password = self.fediverse_db_user_password, host="/var/run/postgresql", port="6432") + + cur = conn.cursor() + + cur.execute("select total_servers, total_mau from totals order by datetime desc limit 14") + + rows = cur.fetchall() + + for row in rows: + + servers_plots.append(row[0]) + + mau_plots.append(row[1]) + + cur.execute("select distinct on (date_trunc('week', datetime)) datetime::TIMESTAMP::DATE, total_servers, total_users, total_mau from totals group by date_trunc('month', datetime), datetime") + + rows = cur.fetchall() + + for row in rows: + + global_week.append(row[0]) + + global_servers.append(row[1]) + + global_users.append(row[2]) + + global_mau.append(row[3]) + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + return (servers_plots, mau_plots, global_week, global_servers, global_users, global_mau) + + def get_server_data(self, server): + + was_alive = None + software = None + api = None + soft_version = None + first_checked_at = None + downs_qty = None + + try: + + conn = None + + conn = psycopg2.connect(database=self.fediverse_db, user=self.fediverse_db_user, password = self.fediverse_db_user_password, host="/var/run/postgresql", port="6432") + + cur = conn.cursor() + + cur.execute("select alive, software, users_api, version, first_checked_at, downs from fediverse where server=(%s)", (server,)) + + row = cur.fetchone() + + if row is not None: + + was_alive = row[0] + software = row[1] + api = row[2] + soft_version = row[3] + first_checked_at = row[4] + downs_qty = row[5] + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + return (was_alive, software, api, soft_version, first_checked_at, downs_qty) + + def get_soft_data(self, search_soft): + + try: + + conn = None + + conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = "/var/run/postgresql", port = "6432") + + cur = conn.cursor() + + cur.execute("select count(server), sum(users), sum(mau) from fediverse where software=(%s) and alive", (search_soft,)) + + row = cur.fetchone() + + if row != None: + + servers = row[0] + + users = row[1] + + mau = row[2] + + else: + + servers = 0 + + users = 0 + + mau = 0 + + cur.close() + + return (servers, users, mau) + + except (Exception, psycopg2.DatabaseError) as error: + + sys.exit(error) + + finally: + + if conn is not None: + + conn.close() + + def fediquery_server_data(search_server): + + try: + + conn = None + + conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "Wer42B2Ir0b0t1c5", host = "/var/run/postgresql", port = "6432") + + cur = conn.cursor() + + cur.execute("select server, software, version, users, mau, alive from fediverse where server=(%s)", (search_server,)) + + row = cur.fetchone() + + if row != None: + + server = row[0] + + software = row[1] + + version = row[2] + + users = row[3] + + if row[4] != None: + + mau = row[4] + + else: + + mau = 0 + + alive = row[5] + + else: + + server = '' + + software = '' + + version = '' + + users = 0 + + mau = 0 + + alive = False + + cur.close() + + return (server, software, version, users, mau, alive) + + except (Exception, psycopg2.DatabaseError) as error: + + sys.exit(error) + + finally: + + if conn is not None: + + conn.close() + + @staticmethod + def __check_dbsetup(self): + + db_setup = False + + try: + + conn = None + + conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = "/var/run/postgresql", port = "6432") + + db_setup = True + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + return db_setup + + @staticmethod + def __createdb(self): + + conn = None + + try: + + conn = psycopg2.connect(dbname='postgres', + user=self.fediverse_db_user, host='', + password=self.fediverse_db_user_password) + + conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) + + cur = conn.cursor() + + print(f"Creating database {self.fediverse_db}. Please wait...") + + cur.execute(sql.SQL("CREATE DATABASE {}").format( + sql.Identifier(self.fediverse_db)) + ) + print(f"Database {self.fediverse_db} created!\n") + + self.__dbtables_schemes(self) + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + @staticmethod + def __dbtables_schemes(self): + + db = self.fediverse_db + table = "world" + sql = "create table "+table+" (server varchar(200) PRIMARY KEY, federated_with varchar(200), updated_at timestamptz, saved_at timestamptz, checked boolean)" + self.__create_table(self, table, sql) + + table = "fediverse" + sql = "create table "+table+" (server varchar(200) PRIMARY KEY, users INT, updated_at timestamptz, software varchar(50), alive boolean, users_api varchar(50), version varchar(100), first_checked_at timestamptz, last_checked_at timestamptz, downs int, mau int)" + self.__create_table(self, table, sql) + + table = "blocked" + sql = "create table "+table+" (server varchar(200) PRIMARY KEY, users INT, updated_at timestamptz, software varchar(50), alive boolean, users_api varchar(50), version varchar(100), first_checked_at timestamptz, last_checked_at timestamptz, downs int)" + self.__create_table(self, table, sql) + + table = "totals" + sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, total_servers INT, total_users INT)" + self.__create_table(self, table, sql) + + table = "evo" + sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, servers INT, users INT)" + self.__create_table(self, table, sql) + + table = "execution_time" + sql = "create table "+table+" (program varchar(20) PRIMARY KEY, start timestamptz, finish timestamptz)" + self.__create_table(self, table, sql) + + @staticmethod + def __create_table(self, table, sql): + + conn = None + + try: + + conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = "/var/run/postgresql", port = "6432") + + cur = conn.cursor() + + print(f"Creating table {table}") + + cur.execute(sql) + + conn.commit() + + print(f"Table {table} created!\n") + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + def __get_parameter(self, parameter, config_file): + + if not os.path.isfile(config_file): + print(f"File {config_file} not found..") + return + + with open( config_file ) as f: + for line in f: + if line.startswith( parameter ): + return line.replace(parameter + ":", "").strip() + + print(f"{config_file} Missing parameter {parameter}") + + sys.exit(0) + + @staticmethod + def __create_config(self): + + if not os.path.exists('config'): + + os.makedirs('config') + + if not os.path.exists(self.config_file): + + print(self.config_file + " created!") + with open(self.config_file, 'w'): pass + + @staticmethod + def __write_config(self): + + with open(self.config_file, 'a') as the_file: + + the_file.write(f'fediverse_db: {self.fediverse_db}\nfediverse_db_user: {self.fediverse_db_user}\nfediverse_db_user_password: {self.fediverse_db_user_password}') + print(f"adding parameters to {self.config_file}\n") + +