diff --git a/README.md b/README.md index ab6bb41..299bb5e 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Within Python Virtual Environment: 3. Run `python fediverse.py` to query world alive servers API. It gets data from server's nodeinfo. -4. Use your favourite scheduling method to set `python fediverse.py` to run twice daily, `python fetchservers.py` one time daily and `fediquery.py` to run every minute. +4. Use your favourite scheduling method to set `python fediverse.py` to run twice daily, `python fetchservers.py` one time daily, `python fediquery.py` to run every minute and `python uptime.py' every minute to publish best fediverse uptime. 18.2.2021 - New feature! Added [Lemmy project](https://join.lemmy.ml) 12.5.2021 - New feature! Added Wordpress support. The code can now detect Wordpress instances with ActivityPub enabled plugin. diff --git a/database.py b/database.py index 42342f4..6ae0375 100644 --- a/database.py +++ b/database.py @@ -841,6 +841,98 @@ class Database(): conn.close() + def get_uptime(self): + + try: + + conn = None + + conn = psycopg2.connect(database=self.fediverse_db, user=self.fediverse_db_user, password=self.fediverse_db_user_password, host="/var/run/postgresql", port="6432") + + cur = conn.cursor() + + cur.execute("select count(server) from fediverse where alive") + + row = cur.fetchone() + + alive_servers = row[0] + + cur.execute("select MAX(date_trunc('day', last_checked_at) - date_trunc('days', first_checked_at)) from fediverse;") + + row = cur.fetchone() + + max_uptime = row[0] + + cur.execute("select count(server) from fediverse where date_trunc('days', last_checked_at)-date_trunc('days',first_checked_at)=(%s)", (max_uptime,)) + + row = cur.fetchone() + + if row is not None: + + best_servers = row[0] + + cur.execute("select software, count(*) as servers from fediverse where date_trunc('days',last_checked_at)-date_trunc('days', first_checked_at)=(%s) group by software order by servers desc", (max_uptime,)) + + rows = cur.fetchall() + + software_lst = [] + + servers_lst = [] + + for row in rows: + + software_lst.append(row[0]) + + servers_lst.append(row[1]) + + cur.close() + + return (alive_servers, max_uptime, best_servers, software_lst, servers_lst) + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + def get_percentage(self, uptime_servers, software): + + try: + + conn = None + + conn = psycopg2.connect(database=self.fediverse_db, user=self.fediverse_db_user, password=self.fediverse_db_user_password, host="/var/run/postgresql", port="6432") + + cur = conn.cursor() + + cur.execute("select count(server) from fediverse where alive and software=(%s)", (software,)) + + row = cur.fetchone() + + if row is not None: + + soft_total_servers = row[0] + + cur.close() + + soft_uptime_percent = round((uptime_servers * 100) / soft_total_servers, 1) + + return (soft_uptime_percent) + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + @staticmethod def __check_dbsetup(self): diff --git a/requirements.txt b/requirements.txt index b19d060..1c7b826 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ ray Mastodon.py matplotlib pandas +humanfriendly diff --git a/uptime.py b/uptime.py index f0a0908..6c41e81 100644 --- a/uptime.py +++ b/uptime.py @@ -2,162 +2,24 @@ import time import os from datetime import datetime, timedelta import humanfriendly +from setup import Setup +from database import Database from mastodon import Mastodon -import psycopg2 - -def get_uptime(): - - try: - - conn = None - - conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432") - - cur = conn.cursor() - - cur.execute("select count(server) from fediverse where alive") - - row = cur.fetchone() - - alive_servers = row[0] - - cur.execute("select MAX(last_checked_at - first_checked_at) from fediverse") - - row = cur.fetchone() - - max_uptime = row[0] - - cur.execute("select count(server) from fediverse where last_checked_at-first_checked_at=(%s)", (max_uptime,)) - - row = cur.fetchone() - - if row is not None: - - best_servers = row[0] - - cur.execute("select software, count(*) as servers from fediverse where last_checked_at-first_checked_at=(%s) group by software order by servers desc", (max_uptime,)) - - rows = cur.fetchall() - - software_lst = [] - - servers_lst = [] - - for row in rows: - - software_lst.append(row[0]) - - servers_lst.append(row[1]) - - cur.close() - - return (alive_servers, max_uptime, best_servers, software_lst, servers_lst) - - except (Exception, psycopg2.DatabaseError) as error: - - print(error) - - finally: - - if conn is not None: - - conn.close() - -def get_percentage(uptime_servers, software): - - try: - - conn = None - - conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432") - - cur = conn.cursor() - - cur.execute("select count(server) from fediverse where alive and software=(%s)", (software,)) - - row = cur.fetchone() - - if row is not None: - - soft_total_servers = row[0] - - cur.close() - - soft_uptime_percent = round((uptime_servers * 100) / soft_total_servers, 1) - - return (soft_uptime_percent) - - except (Exception, psycopg2.DatabaseError) as error: - - print(error) - - finally: - - if conn is not None: - - conn.close() - -def mastodon(): - - # Load secrets from secrets file - secrets_filepath = "secrets/uptime_secrets.txt" - uc_client_id = get_parameter("uc_client_id", secrets_filepath) - uc_client_secret = get_parameter("uc_client_secret", secrets_filepath) - uc_access_token = get_parameter("uc_access_token", secrets_filepath) - - # Load configuration from config file - config_filepath = "config/config.txt" - mastodon_hostname = get_parameter("mastodon_hostname", config_filepath) - - # Initialise Mastodon API - mastodon = Mastodon( - client_id=uc_client_id, - client_secret=uc_client_secret, - access_token=uc_access_token, - api_base_url='https://' + mastodon_hostname, - ) - - # Initialise access headers - headers = {'Authorization': 'Bearer %s'%uc_access_token} - - return (mastodon, mastodon_hostname) - -def db_config(): - - # Load db configuration from config file - config_filepath = "config/db_config.txt" - fediverse_db = get_parameter("fediverse_db", config_filepath) - fediverse_db_user = get_parameter("fediverse_db_user", config_filepath) - - return (fediverse_db, fediverse_db_user) - -# Returns the parameter from the specified file -def get_parameter(parameter, file_path): - - # Check if secrets file exists - if not os.path.isfile(file_path): - print("File %s not found, exiting."%file_path) - sys.exit(0) - - # Find parameter in file - with open(file_path) as f: - for line in f: - if line.startswith(parameter): - return line.replace(parameter + ":", "").strip() - - # Cannot find parameter, exit - print(file_path + " Missing parameter %s "%parameter) - sys.exit(0) # main if __name__ == '__main__': - mastodon, mastodon_hostname = mastodon() + setup = Setup() - fediverse_db, fediverse_db_user = db_config() - - alive_servers, max_uptime, best_servers, software_lst, servers_lst = get_uptime() + mastodon = Mastodon( + access_token = setup.mastodon_app_token, + api_base_url= setup.mastodon_hostname + ) + + db = Database() + + alive_servers, max_uptime, best_servers, software_lst, servers_lst = db.get_uptime() toot_text = '\nAlive servers: ' + str(alive_servers) toot_text += '\n\n' @@ -169,7 +31,7 @@ if __name__ == '__main__': i = 0 while i < len(software_lst): - soft_percent = get_percentage(servers_lst[i], software_lst[i]) + soft_percent = db.get_percentage(servers_lst[i], software_lst[i]) toot_text += ':' + str(software_lst[i]) + ': ' + str(servers_lst[i]) + ' (' + str(soft_percent) + '%)\n' if len(toot_text) > 470: diff --git a/uptime_setup.py b/uptime_setup.py deleted file mode 100644 index bfbe972..0000000 --- a/uptime_setup.py +++ /dev/null @@ -1,188 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import getpass -from mastodon import Mastodon -from mastodon.Mastodon import MastodonMalformedEventError, MastodonNetworkError, MastodonReadTimeout, MastodonAPIError, MastodonIllegalArgumentError -import fileinput,re -import os -import sys - -def create_dir(): - if not os.path.exists('secrets'): - os.makedirs('secrets') - -def create_file(): - if not os.path.exists('secrets/uptime_secrets.txt'): - with open('secrets/uptime_secrets.txt', 'w'): pass - print(secrets_filepath + " created!") - -def create_config(): - if not os.path.exists('config'): - os.makedirs('config') - if not os.path.exists(config_filepath): - print(config_filepath + " created!") - with open('config/config.txt', 'w'): pass - -def write_params(): - with open(secrets_filepath, 'a') as the_file: - print("Writing secrets parameter names to " + secrets_filepath) - the_file.write('uc_client_id: \n'+'uc_client_secret: \n'+'uc_access_token: \n') - -def write_config(): - with open(config_filepath, 'a') as the_file: - the_file.write('mastodon_hostname: \n') - print("adding parameter name 'mastodon_hostname' to "+ config_filepath) - -def read_client_lines(self): - client_path = 'app_clientcred.txt' - with open(client_path) as fp: - line = fp.readline() - cnt = 1 - while line: - if cnt == 1: - print("Writing client id to " + secrets_filepath) - modify_file(secrets_filepath, "uc_client_id: ", value=line.rstrip()) - elif cnt == 2: - print("Writing client secret to " + secrets_filepath) - modify_file(secrets_filepath, "uc_client_secret: ", value=line.rstrip()) - line = fp.readline() - cnt += 1 - -def read_token_line(self): - token_path = 'app_usercred.txt' - with open(token_path) as fp: - line = fp.readline() - print("Writing access token to " + secrets_filepath) - modify_file(secrets_filepath, "uc_access_token: ", value=line.rstrip()) - -def read_config_line(): - with open(config_filepath) as fp: - line = fp.readline() - modify_file(config_filepath, "mastodon_hostname: ", value=hostname) - -def log_in(): - error = 0 - try: - global hostname - hostname = input("Enter Mastodon hostname: ") - user_name = input("User name, ex. user@" + hostname +"? ") - user_password = getpass.getpass("User password? ") - app_name = input("This app name? ") - Mastodon.create_app(app_name, scopes=["read","write"], - to_file="app_clientcred.txt", api_base_url=hostname) - mastodon = Mastodon(client_id = "app_clientcred.txt", api_base_url = hostname) - mastodon.log_in( - user_name, - user_password, - scopes = ["read", "write"], - to_file = "app_usercred.txt" - ) - except MastodonIllegalArgumentError as i_error: - error = 1 - if os.path.exists("secrets/uptime_secrets.txt"): - print("Removing secrets/uptime_secrets.txt file..") - os.remove("secrets/uptime_secrets.txt") - if os.path.exists("app_clientcred.txt"): - print("Removing app_clientcred.txt file..") - os.remove("app_clientcred.txt") - sys.exit(i_error) - except MastodonNetworkError as n_error: - error = 1 - if os.path.exists("secrets/uptime_secrets.txt"): - print("Removing secrets/uptime_secrets.txt file..") - os.remove("secrets/uptime_secrets.txt") - if os.path.exists("app_clientcred.txt"): - print("Removing app_clientcred.txt file..") - os.remove("app_clientcred.txt") - sys.exit(n_error) - except MastodonReadTimeout as r_error: - error = 1 - if os.path.exists("secrets/uptime_secrets.txt"): - print("Removing secrets/uptime_secrets.txt file..") - os.remove("secrets/uptime_secrets.txt") - if os.path.exists("app_clientcred.txt"): - print("Removing app_clientcred.txt file..") - os.remove("app_clientcred.txt") - sys.exit(r_error) - except MastodonAPIError as a_error: - error = 1 - if os.path.exists("secrets/uptime_secrets.txt"): - print("Removing secrets/uptime_secrets.txt file..") - os.remove("secrets/uptime_secrets.txt") - if os.path.exists("app_clientcred.txt"): - print("Removing app_clientcred.txt file..") - os.remove("app_clientcred.txt") - sys.exit(a_error) - finally: - if error == 0: - - create_dir() - create_file() - write_params() - client_path = 'app_clientcred.txt' - read_client_lines(client_path) - token_path = 'app_usercred.txt' - read_token_line(token_path) - if os.path.exists("app_clientcred.txt"): - print("Removing app_clientcred.txt temp file..") - os.remove("app_clientcred.txt") - if os.path.exists("app_usercred.txt"): - print("Removing app_usercred.txt temp file..") - os.remove("app_usercred.txt") - print("uptime secrets setup done!\n") - -def modify_file(file_name,pattern,value=""): - fh=fileinput.input(file_name,inplace=True) - for line in fh: - replacement=pattern + value - line=re.sub(pattern,replacement,line) - sys.stdout.write(line) - fh.close() - -# Returns the parameter from the specified file -def get_parameter( parameter, file_path ): - # Check if secrets file exists - if not os.path.isfile(file_path): - print("File %s not found, creating it."%file_path) - log_in() - - # Find parameter in file - with open( file_path ) as f: - for line in f: - if line.startswith( parameter ): - return line.replace(parameter + ":", "").strip() - - # Cannot find parameter, exit - print(file_path + " Missing parameter %s "%parameter) - sys.exit(0) - -# Returns the parameter from the specified file -def get_hostname( parameter, config_filepath ): - # Check if secrets file exists - if not os.path.isfile(config_filepath): - print("File %s not found, creating it."%config_filepath) - create_config() - - # Find parameter in file - with open( config_filepath ) as f: - for line in f: - if line.startswith( parameter ): - return line.replace(parameter + ":", "").strip() - - # Cannot find parameter, exit - print(config_filepath + " Missing parameter %s "%parameter) - write_config() - read_config_line() - print("setup done!") - sys.exit(0) - -# Load secrets from secrets file -secrets_filepath = "secrets/uptime_secrets.txt" -uc_client_id = get_parameter("uc_client_id", secrets_filepath) -uc_client_secret = get_parameter("uc_client_secret", secrets_filepath) -uc_access_token = get_parameter("uc_access_token", secrets_filepath) - -# Load configuration from config file -#config_filepath = "config/config.txt" -#mastodon_hostname = get_hostname("mastodon_hostname", config_filepath)