676 líneas
16 KiB
Python
676 líneas
16 KiB
Python
import os
|
|
import sys
|
|
import psycopg2
|
|
from psycopg2 import sql
|
|
from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
|
|
import uuid
|
|
from datetime import datetime
|
|
import pytz
|
|
import pdb
|
|
|
|
tz = pytz.timezone('Europe/Madrid')
|
|
|
|
class Database():
|
|
|
|
name = 'fediverse database library'
|
|
|
|
def __init__(self, config_file=None, blocker_db=None, blocker_db_user=None, blocker_db_user_password=None):
|
|
|
|
self.config_file = "config/db_config.txt"
|
|
self.blocker_db = self.__get_parameter("blocker_db", self.config_file)
|
|
self.blocker_db_user = self.__get_parameter("blocker_db_user", self.config_file)
|
|
self.blocker_db_user_password = self.__get_parameter("blocker_db_user_password", self.config_file)
|
|
|
|
db_setup = self.__check_dbsetup(self)
|
|
|
|
if not db_setup:
|
|
|
|
self.blocker_db = input("\nMau database name: ")
|
|
self.blocker_db_user = input("\nMau database user: ")
|
|
self.blocker_db_user_password = input("\nMau database user password: ")
|
|
|
|
self.__createdb(self)
|
|
self.__create_config(self)
|
|
self.__write_config(self)
|
|
|
|
def get_nodeinfo_endpoint(self, server):
|
|
|
|
try:
|
|
|
|
conn = None
|
|
|
|
conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = self.blocker_db_user_password, host = "/var/run/postgresql", port = "6432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
cur.execute("select server, api from servers where server=(%s)", (server,))
|
|
|
|
row = cur.fetchone()
|
|
|
|
if row != None:
|
|
|
|
server = row[0]
|
|
|
|
api = row[1]
|
|
|
|
else:
|
|
|
|
server = ''
|
|
|
|
api = ''
|
|
|
|
cur.close()
|
|
|
|
return (server, api)
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
sys.exit(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
def get_servers(self, software):
|
|
|
|
servers_list = []
|
|
|
|
try:
|
|
|
|
conn = None
|
|
|
|
conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
cur.execute("select server from servers where software=(%s)", (software,))
|
|
|
|
rows = cur.fetchall()
|
|
|
|
for row in rows:
|
|
|
|
servers_list.append(row[0])
|
|
|
|
cur.close()
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
sys.exit(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
return (servers_list)
|
|
|
|
def save_time(self, program, start, finish):
|
|
|
|
insert_sql = "INSERT INTO execution_time(program, start, finish) VALUES(%s,%s,%s) ON CONFLICT DO NOTHING"
|
|
|
|
conn = None
|
|
|
|
try:
|
|
|
|
conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = self.blocker_db_user_password, host = "/var/run/postgresql", port = "6432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
cur.execute(insert_sql, (program, start, finish,))
|
|
|
|
cur.execute("UPDATE execution_time SET start=(%s), finish=(%s) where program=(%s)", (start, finish, program))
|
|
|
|
conn.commit()
|
|
|
|
cur.close()
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
def check_peer(self, server):
|
|
|
|
found = False
|
|
|
|
select_server_sql = 'select server from servers where server=(%s)'
|
|
|
|
select_dead_server_sql = 'select server from deadservers where server=(%s)'
|
|
|
|
try:
|
|
|
|
conn = None
|
|
|
|
conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
# check server
|
|
|
|
cur.execute(select_server_sql, (server,))
|
|
|
|
row = cur.fetchone()
|
|
|
|
if row != None:
|
|
|
|
found = True
|
|
|
|
cur.execute(select_dead_server_sql, (server,))
|
|
|
|
row = cur.fetchone()
|
|
|
|
if row != None:
|
|
|
|
found = True
|
|
|
|
cur.close()
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
return found
|
|
|
|
def get_not_updated_servers(self):
|
|
|
|
not_updated_servers = []
|
|
|
|
try:
|
|
|
|
conn = None
|
|
|
|
conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
# get saved servers list
|
|
|
|
cur.execute("select server from servers where updated_at is null")
|
|
|
|
rows = cur.fetchall()
|
|
|
|
for row in rows:
|
|
|
|
not_updated_servers.append(row[0])
|
|
|
|
cur.close()
|
|
|
|
print("Not updated servers: " + str(len(not_updated_servers)))
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
return not_updated_servers
|
|
|
|
def get_saved_servers(self):
|
|
|
|
saved_servers = []
|
|
|
|
try:
|
|
|
|
conn = None
|
|
|
|
conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
# get saved servers list
|
|
|
|
cur.execute("select server from servers where api is not null")
|
|
|
|
rows = cur.fetchall()
|
|
|
|
for row in rows:
|
|
|
|
saved_servers.append(row[0])
|
|
|
|
cur.close()
|
|
|
|
print("Saved servers: " + str(len(saved_servers)))
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
return saved_servers
|
|
|
|
def get_dead_servers(self):
|
|
|
|
dead_servers = []
|
|
|
|
try:
|
|
|
|
conn = None
|
|
|
|
conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
# get dead servers list
|
|
|
|
cur.execute("select server from deadservers")
|
|
|
|
rows = cur.fetchall()
|
|
|
|
for row in rows:
|
|
|
|
dead_servers.append(row[0])
|
|
|
|
cur.close()
|
|
|
|
print("Dead servers: " + str(len(dead_servers)))
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
return dead_servers
|
|
|
|
def soft_totals(self):
|
|
|
|
# get current total servers and users, get users from every software
|
|
|
|
now = datetime.now()
|
|
|
|
gettotals_sql = "select count(server), sum(users), sum(mau) from mau where alive"
|
|
get_soft_totals_sql = "select software, sum(users) as users, sum(mau) as mau, count(server) as servers from mau where users != 0 and mau is not null and alive group by software order by mau desc"
|
|
|
|
soft_total_project = []
|
|
soft_total_users = []
|
|
soft_total_mau = []
|
|
soft_total_servers = []
|
|
|
|
try:
|
|
|
|
conn = None
|
|
|
|
conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
cur.execute(gettotals_sql)
|
|
|
|
row = cur.fetchone()
|
|
|
|
total_servers = row[0]
|
|
|
|
total_users = row[1]
|
|
|
|
total_mau = row[2]
|
|
|
|
cur.execute(get_soft_totals_sql)
|
|
|
|
rows = cur.fetchall()
|
|
|
|
for row in rows:
|
|
|
|
soft_total_project.append(row[0])
|
|
soft_total_users.append(row[1])
|
|
soft_total_mau.append(row[2])
|
|
soft_total_servers.append(row[3])
|
|
|
|
cur.close()
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
return (soft_total_project, soft_total_users, soft_total_mau, soft_total_servers, total_servers, total_users, total_mau)
|
|
|
|
def get_last_checked_servers(self):
|
|
|
|
# get last checked servers from fediverse DB
|
|
|
|
alive_servers = []
|
|
|
|
try:
|
|
|
|
conn = None
|
|
|
|
conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
# get world servers list
|
|
|
|
cur.execute("select server from world where server in (select server from fediverse where users_api != '')")
|
|
|
|
rows = cur.fetchall()
|
|
|
|
for row in rows:
|
|
|
|
alive_servers.append(row[0])
|
|
|
|
cur.close()
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
return alive_servers
|
|
|
|
def write_alive_server(self, server, software, version, users, mau, alive):
|
|
|
|
now = datetime.now()
|
|
|
|
insert_sql = "INSERT INTO mau(server, software, version, users, mau, alive, updated_at) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
|
|
|
|
conn = None
|
|
|
|
try:
|
|
|
|
conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
cur.execute(insert_sql, (server, software, version, users, mau, alive, now))
|
|
|
|
cur.execute("UPDATE mau SET software=(%s), version=(%s), users=(%s), mau=(%s), alive=(%s), updated_at=(%s) where server=(%s)", (software, version, users, mau, alive, now, server))
|
|
|
|
conn.commit()
|
|
|
|
cur.close()
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
def write_not_alive_server(self, server):
|
|
|
|
now = datetime.now()
|
|
|
|
update_sql = "UPDATE mau set alive='f', updated_at=(%s) where server=(%s)"
|
|
|
|
conn = None
|
|
|
|
try:
|
|
|
|
conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
cur.execute(update_sql, (now, server))
|
|
|
|
conn.commit()
|
|
|
|
cur.close()
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
|
|
def write_server(self, server, api, software):
|
|
|
|
now = datetime.now()
|
|
|
|
insert_sql = "INSERT INTO servers(server, api, software, created_at) VALUES(%s, %s, %s, %s) ON CONFLICT DO NOTHING"
|
|
|
|
conn = None
|
|
|
|
try:
|
|
|
|
conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
cur.execute(insert_sql, (server, api, software, now))
|
|
|
|
conn.commit()
|
|
|
|
cur.close()
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
def update_peer(self, server):
|
|
|
|
now = datetime.now()
|
|
|
|
update_sql = "UPDATE servers set updated_at=(%s) where server=(%s)"
|
|
|
|
conn = None
|
|
|
|
try:
|
|
|
|
conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
cur.execute(update_sql, (now, server))
|
|
|
|
conn.commit()
|
|
|
|
cur.close()
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
def update_server(self, server, api):
|
|
|
|
now = datetime.now()
|
|
|
|
update_sql = "UPDATE servers set server=(%s), api=(%s), updated_at=(%s) where server=(%s)"
|
|
|
|
conn = None
|
|
|
|
try:
|
|
|
|
conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
cur.execute(update_sql, (server, api, now, server))
|
|
|
|
conn.commit()
|
|
|
|
cur.close()
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
@staticmethod
|
|
def __check_dbsetup(self):
|
|
|
|
db_setup = False
|
|
|
|
try:
|
|
|
|
conn = None
|
|
|
|
conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = self.blocker_db_user_password, host = "/var/run/postgresql", port = "6432")
|
|
|
|
db_setup = True
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
return db_setup
|
|
|
|
@staticmethod
|
|
def __createdb(self):
|
|
|
|
conn = None
|
|
|
|
try:
|
|
|
|
conn = psycopg2.connect(dbname='postgres',
|
|
user=self.blocker_db_user, host='',
|
|
password=self.blocker_db_user_password)
|
|
|
|
conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
|
|
|
|
cur = conn.cursor()
|
|
|
|
print(f"Creating database {self.blocker_db}. Please wait...")
|
|
|
|
cur.execute(sql.SQL("CREATE DATABASE {}").format(
|
|
sql.Identifier(self.blocker_db))
|
|
)
|
|
print(f"Database {self.blocker_db} created!\n")
|
|
|
|
self.__dbtables_schemes(self)
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
@staticmethod
|
|
def __dbtables_schemes(self):
|
|
|
|
db = self.blocker_db
|
|
table = "servers"
|
|
sql = "create table "+table+" (server varchar(200) PRIMARY KEY, api varchar(50), software varchar(50), created_at timestamptz, updated_at timestamptz, downs int)"
|
|
self.__create_table(self, table, sql)
|
|
|
|
@staticmethod
|
|
def __create_table(self, table, sql):
|
|
|
|
conn = None
|
|
|
|
try:
|
|
|
|
conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = self.blocker_db_user_password, host = "/var/run/postgresql", port = "6432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
print(f"Creating table {table}")
|
|
|
|
cur.execute(sql)
|
|
|
|
conn.commit()
|
|
|
|
print(f"Table {table} created!\n")
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
def __get_parameter(self, parameter, config_file):
|
|
|
|
if not os.path.isfile(config_file):
|
|
print(f"File {config_file} not found..")
|
|
return
|
|
|
|
with open( config_file ) as f:
|
|
for line in f:
|
|
if line.startswith( parameter ):
|
|
return line.replace(parameter + ":", "").strip()
|
|
|
|
print(f"{config_file} Missing parameter {parameter}")
|
|
|
|
sys.exit(0)
|
|
|
|
@staticmethod
|
|
def __create_config(self):
|
|
|
|
if not os.path.exists('config'):
|
|
|
|
os.makedirs('config')
|
|
|
|
if not os.path.exists(self.config_file):
|
|
|
|
print(self.config_file + " created!")
|
|
with open(self.config_file, 'w'): pass
|
|
|
|
@staticmethod
|
|
def __write_config(self):
|
|
|
|
with open(self.config_file, 'a') as the_file:
|
|
|
|
the_file.write(f'blocker_db: {self.blocker_db}\nblocker_db_user: {self.blocker_db_user}\nblocker_db_user_password: {self.blocker_db_user_password}')
|
|
print(f"adding parameters to {self.config_file}\n")
|
|
|
|
|