From a806bbe85980b27b0fd7351c31d99ead9322146a Mon Sep 17 00:00:00 2001 From: quimnuss Date: Sun, 8 Jan 2023 18:57:35 +0100 Subject: [PATCH] reestructure project following usual python structure --- federation/__init__.py | 3 + database.py => federation/database.py | 0 setup.py => federation/setup.py | 3 +- fediquery.py => scripts/fediquery.py | 0 fediverse.py => scripts/fediverse.py | 91 ++++++++++++++-------- fetchservers.py => scripts/fetchservers.py | 0 query.py => scripts/query.py | 0 uptime.py => scripts/uptime.py | 0 tests/test_database.py | 0 9 files changed, 63 insertions(+), 34 deletions(-) create mode 100644 federation/__init__.py rename database.py => federation/database.py (100%) rename setup.py => federation/setup.py (99%) rename fediquery.py => scripts/fediquery.py (100%) rename fediverse.py => scripts/fediverse.py (73%) rename fetchservers.py => scripts/fetchservers.py (100%) rename query.py => scripts/query.py (100%) rename uptime.py => scripts/uptime.py (100%) create mode 100644 tests/test_database.py diff --git a/federation/__init__.py b/federation/__init__.py new file mode 100644 index 0000000..78c2567 --- /dev/null +++ b/federation/__init__.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python3 + +from .database import Database diff --git a/database.py b/federation/database.py similarity index 100% rename from database.py rename to federation/database.py diff --git a/setup.py b/federation/setup.py similarity index 99% rename from setup.py rename to federation/setup.py index 11d2ee5..f42e1d0 100644 --- a/setup.py +++ b/federation/setup.py @@ -4,7 +4,6 @@ from datetime import datetime import pytz from mastodon import Mastodon from mastodon.Mastodon import MastodonMalformedEventError, MastodonNetworkError, MastodonReadTimeout, MastodonAPIError, MastodonIllegalArgumentError -import pdb class Setup(): @@ -79,7 +78,7 @@ class Setup(): client_secret = response[1] mastodon = Mastodon(client_id = client_id, client_secret = client_secret, api_base_url = self.mastodon_hostname) - + token = mastodon.log_in( self.mastodon_user, self.mastodon_password, diff --git a/fediquery.py b/scripts/fediquery.py similarity index 100% rename from fediquery.py rename to scripts/fediquery.py diff --git a/fediverse.py b/scripts/fediverse.py similarity index 73% rename from fediverse.py rename to scripts/fediverse.py index e0cd167..cdef1d2 100644 --- a/fediverse.py +++ b/scripts/fediverse.py @@ -6,7 +6,7 @@ import urllib3 import requests import socket from setup import Setup -from database import Database +from federation import Database from mastodon import Mastodon from matplotlib import pyplot as plt import matplotlib.dates as mdates @@ -31,7 +31,8 @@ plt.rc('figure', titlesize=BIGGER_SIZE) # fontsize of the figure title mdates.set_epoch('2000-01-01T00:00:00') y_formatter = ScalarFormatter(useOffset=False) -ray.init(num_cpus = 25) # Specify this system CPUs. +ray.init(num_cpus=25) # Specify this system CPUs. + class Server: @@ -54,7 +55,8 @@ class Server: try: - data = requests.get('https://' + self + api, headers = setup.user_agent, timeout=3) + data = requests.get('https://' + self + api, + headers=setup.user_agent, timeout=3) nodeinfo_json = data.json() @@ -65,7 +67,8 @@ class Server: users = 1 - mau = nodeinfo_json.get('usage').get('users').get('activeMonth') or '0' + mau = nodeinfo_json.get('usage').get( + 'users').get('activeMonth') or '0' if software == 'socialhome': @@ -91,7 +94,7 @@ class Server: soft_version = "" - else: + else: if api == '/api/v1/instance': @@ -115,7 +118,8 @@ class Server: if soft_version != "" and soft_version is not None: - print(f'\n** Server {self} ({software} {soft_version}) is alive! **') + print( + f'\n** Server {self} ({software} {soft_version}) is alive! **') else: @@ -123,11 +127,13 @@ class Server: if software != 'birdsitelive': - db.write_alive_server(self, software, soft_version, alive, api, users, downs, first_checked_at, mau) + db.write_alive_server( + self, software, soft_version, alive, api, users, downs, first_checked_at, mau) else: - db.write_blocked_software(self, software, soft_version, alive, api, users, downs, first_checked_at) + db.write_blocked_software( + self, software, soft_version, alive, api, users, downs, first_checked_at) except urllib3.exceptions.ProtocolError as protoerr: @@ -205,10 +211,12 @@ class Server: return (self, software, soft_version, alive, api, users, downs, first_checked_at, mau) + def print_dead(server): print(f'\nServer {server} is dead :-(') + if __name__ == '__main__': db = Database() @@ -226,9 +234,9 @@ if __name__ == '__main__': now = start mastodon = Mastodon( - access_token = setup.mastodon_app_token, - api_base_url= setup.mastodon_hostname - ) + access_token=setup.mastodon_app_token, + api_base_url=setup.mastodon_hostname + ) total_servers = 0 @@ -242,9 +250,11 @@ if __name__ == '__main__': ray_start = time.time() - results = ray.get([getservers.get_alive_servers.remote(server) for server in alive_servers]) + results = ray.get([getservers.get_alive_servers.remote(server) + for server in alive_servers]) - print(f"duration = {time.time() - ray_start}.\nprocessed servers: {len(results)}") + print( + f"duration = {time.time() - ray_start}.\nprocessed servers: {len(results)}") # get current total servers and users, get users from every software @@ -252,7 +262,8 @@ if __name__ == '__main__': # get last check values and write current total ones - evo_servers, evo_users, evo_mau = db.last_values(total_servers, total_users, total_mau) + evo_servers, evo_users, evo_mau = db.last_values( + total_servers, total_users, total_mau) # write evo values @@ -273,11 +284,14 @@ if __name__ == '__main__': ############################################################################### # generate graphs - plt.plot([-6, -5, -4, -3, -2, -1, 0], [servers_plots[6], servers_plots[5], servers_plots[4], servers_plots[3], servers_plots[2], servers_plots[1], servers_plots[0]], marker='o', color='mediumseagreen') + plt.plot([-6, -5, -4, -3, -2, -1, 0], [servers_plots[6], servers_plots[5], servers_plots[4], servers_plots[3], + servers_plots[2], servers_plots[1], servers_plots[0]], marker='o', color='mediumseagreen') - plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_servers, max_servers, max_servers, max_servers, max_servers, max_servers, max_servers], color='red') + plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_servers, max_servers, max_servers, + max_servers, max_servers, max_servers, max_servers], color='red') - plt.title('fediverse: total alive servers (max: ' + str(f"{max_servers:,}" + ')'), loc='right', color='blue') + plt.title('fediverse: total alive servers (max: ' + + str(f"{max_servers:,}" + ')'), loc='right', color='blue') plt.xlabel('Last seven days') @@ -291,13 +305,17 @@ if __name__ == '__main__': plt.close() - plt.plot([-6, -5, -4, -3, -2, -1, 0], [mau_plots[6], mau_plots[5], mau_plots[4], mau_plots[3], mau_plots[2], mau_plots[1], mau_plots[0]], marker='o', color='royalblue') + plt.plot([-6, -5, -4, -3, -2, -1, 0], [mau_plots[6], mau_plots[5], mau_plots[4], + mau_plots[3], mau_plots[2], mau_plots[1], mau_plots[0]], marker='o', color='royalblue') - plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_mau, max_mau, max_mau, max_mau, max_mau, max_mau, max_mau], color='red') + plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_mau, max_mau, max_mau, + max_mau, max_mau, max_mau, max_mau], color='red') - plt.title('fediverse: total MAU (max: ' + str(f"{max_mau:,}" + ')'), loc='right', color='royalblue') + plt.title('fediverse: total MAU (max: ' + + str(f"{max_mau:,}" + ')'), loc='right', color='royalblue') - plt.legend(('mau', 'max'), shadow=True, loc=(0.01, 0.80), handlelength=1.5, fontsize=10) + plt.legend(('mau', 'max'), shadow=True, loc=( + 0.01, 0.80), handlelength=1.5, fontsize=10) plt.xlabel('Last seven days') @@ -310,9 +328,9 @@ if __name__ == '__main__': plt.close() df = pd.DataFrame({'date': np.array(global_week), - #'servers': np.array(global_servers), - 'users': np.array(global_users), - 'mau': np.array(global_mau)}) + #'servers': np.array(global_servers), + 'users': np.array(global_users), + 'mau': np.array(global_mau)}) df['date'] = pd.to_datetime(df['date']) @@ -349,25 +367,30 @@ if __name__ == '__main__': if evo_servers >= 0: - toot_text += "alive servers: " + str(f"{total_servers:,}") + " (+"+ str(f"{evo_servers:,}") + ") \n" + toot_text += "alive servers: " + \ + str(f"{total_servers:,}") + \ + " (+" + str(f"{evo_servers:,}") + ") \n" toot_text += "max: " + str(f"{max_servers:,}") + "\n" elif evo_servers < 0: - toot_text += "alive servers: " + str(f"{total_servers:,}") + " ("+ str(f"{evo_servers:,}") + ") \n" + toot_text += "alive servers: " + \ + str(f"{total_servers:,}") + " (" + str(f"{evo_servers:,}") + ") \n" toot_text += "max: " + str(f"{max_servers:,}") + "\n" if evo_mau >= 0: - toot_text += "total MAU: " + str(f"{total_mau:,}") + " (+"+ str(f"{evo_mau:,}") + ") \n" + toot_text += "total MAU: " + \ + str(f"{total_mau:,}") + " (+" + str(f"{evo_mau:,}") + ") \n" toot_text += "max: " + str(f"{max_mau:,}") + "\n" elif evo_mau < 0: - toot_text += "total MAU: " + str(f"{total_mau:,}") + " ("+ str(f"{evo_mau:,}") + ") \n" + toot_text += "total MAU: " + \ + str(f"{total_mau:,}") + " (" + str(f"{evo_mau:,}") + ") \n" toot_text += "max: " + str(f"{max_mau:,}") + "\n" @@ -397,13 +420,17 @@ if __name__ == '__main__': print(toot_text) - servers_image_id = mastodon.media_post('servers.png', "image/png", description='servers graph').id + servers_image_id = mastodon.media_post( + 'servers.png', "image/png", description='servers graph').id - mau_image_id = mastodon.media_post('mau.png', "image/png", description='MAU graph').id + mau_image_id = mastodon.media_post( + 'mau.png', "image/png", description='MAU graph').id - global_image_id = mastodon.media_post('global.png', "image/png", description='global graph').id + global_image_id = mastodon.media_post( + 'global.png', "image/png", description='global graph').id - mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, mau_image_id, global_image_id}) + mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={ + servers_image_id, mau_image_id, global_image_id}) db.delete_dead_servers() diff --git a/fetchservers.py b/scripts/fetchservers.py similarity index 100% rename from fetchservers.py rename to scripts/fetchservers.py diff --git a/query.py b/scripts/query.py similarity index 100% rename from query.py rename to scripts/query.py diff --git a/uptime.py b/scripts/uptime.py similarity index 100% rename from uptime.py rename to scripts/uptime.py diff --git a/tests/test_database.py b/tests/test_database.py new file mode 100644 index 0000000..e69de29