fediverse/fediverse.py

399 líneas
10 KiB
Python
Original Vista normal Històric

2022-03-11 19:53:21 +01:00
import sys
import os
2020-05-17 21:40:19 +02:00
import time
2020-05-17 21:28:03 +02:00
from datetime import datetime
2022-03-11 19:53:21 +01:00
import urllib3
2020-05-17 21:28:03 +02:00
import requests
import socket
2023-01-05 00:01:36 +01:00
from setup import Setup
from database import Database
2022-03-11 19:53:21 +01:00
from mastodon import Mastodon
2023-01-05 00:01:36 +01:00
from matplotlib import pyplot as plt
import matplotlib.dates as mdates
from matplotlib.ticker import ScalarFormatter
import numpy as np
import pandas as pd
2022-03-11 19:53:21 +01:00
import ray
2022-03-21 19:24:10 +01:00
import pdb
2023-01-05 00:01:36 +01:00
SMALL_SIZE = 6
MEDIUM_SIZE = 10
BIGGER_SIZE = 12
plt.rc('font', size=MEDIUM_SIZE) # controls default text sizes
plt.rc('axes', titlesize=MEDIUM_SIZE) # fontsize of the axes title
plt.rc('axes', labelsize=MEDIUM_SIZE) # fontsize of the x and y labels
plt.rc('xtick', labelsize=SMALL_SIZE) # fontsize of the tick labels
plt.rc('ytick', labelsize=MEDIUM_SIZE) # fontsize of the tick labels
plt.rc('legend', fontsize=MEDIUM_SIZE) # legend fontsize
plt.rc('figure', titlesize=BIGGER_SIZE) # fontsize of the figure title
mdates.set_epoch('2000-01-01T00:00:00')
y_formatter = ScalarFormatter(useOffset=False)
ray.init(num_cpus = 25) # Specify this system CPUs.
2022-03-02 22:26:05 +01:00
class Server:
2021-05-14 13:14:31 +02:00
2022-03-02 22:26:05 +01:00
name = 'Server'
2020-05-17 21:28:03 +02:00
2022-03-21 19:24:10 +01:00
def __init_(self, server=None):
2021-05-14 13:14:31 +02:00
2022-03-02 22:26:05 +01:00
self.server = server
2020-05-17 21:28:03 +02:00
2022-03-11 19:53:21 +01:00
@ray.remote
2022-03-21 19:24:10 +01:00
def get_alive_servers(self):
2021-05-14 13:14:31 +02:00
2022-03-11 19:53:21 +01:00
users = 0
2020-05-17 21:28:03 +02:00
2022-03-11 19:53:21 +01:00
downs = 0
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
was_alive, software, api, soft_version, first_checked_at, downs_qty = db.get_server_data(self)
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
alive = False
2023-01-05 00:01:36 +01:00
try:
2023-01-05 00:01:36 +01:00
data = requests.get('https://' + self + api, headers = setup.user_agent, timeout=3)
2023-01-05 00:01:36 +01:00
nodeinfo_json = data.json()
2022-03-02 22:26:05 +01:00
try:
2023-01-05 00:01:36 +01:00
users = nodeinfo_json.get('usage').get('users').get('total') or '0'
if users > 1000000:
users = 1
2023-01-05 00:01:36 +01:00
mau = nodeinfo_json.get('usage').get('users').get('activeMonth') or '0'
2023-01-05 00:01:36 +01:00
if software == 'socialhome':
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
soft_version = nodeinfo_json['server']['version']
2023-01-05 00:01:36 +01:00
else:
2023-01-05 00:01:36 +01:00
soft_version = nodeinfo_json['software']['version']
2023-01-05 00:01:36 +01:00
if software == "wordpress" and "activitypub" in nodeinfo_json['protocols']:
2023-01-05 00:01:36 +01:00
alive = True
2023-01-05 00:01:36 +01:00
elif software == "wordpress" and "activitypub" not in nodeinfo_json['protocols']:
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
alive = False
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
else:
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
alive = True
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
except:
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
soft_version = ""
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
else:
2023-01-05 00:01:36 +01:00
if api == '/api/v1/instance?':
2023-01-05 00:01:36 +01:00
try:
users = nodeinfo_json.get('stats').get('user_count') or '0'
if users > 1000000:
users = 1
2023-01-05 00:01:36 +01:00
soft_version = nodeinfo_json['version']
2022-03-02 22:26:05 +01:00
alive = True
2023-01-05 00:01:36 +01:00
mau = 0
except:
soft_version = ""
2023-01-05 00:01:36 +01:00
if alive:
2022-03-02 22:26:05 +01:00
if downs_qty != None:
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
downs = downs_qty
2023-01-05 00:01:36 +01:00
if soft_version != "" and soft_version is not None:
2023-01-05 00:01:36 +01:00
print(f'\n** Server {self} ({software} {soft_version}) is alive! **')
2023-01-05 00:01:36 +01:00
else:
2023-01-05 00:01:36 +01:00
print(f'\n** Server {self} ({software}) is alive! **')
2023-01-05 00:01:36 +01:00
if software != 'birdsitelive':
2023-01-05 00:01:36 +01:00
db.write_alive_server(self, software, soft_version, alive, api, users, downs, first_checked_at, mau)
2022-03-11 19:53:21 +01:00
2023-01-05 00:01:36 +01:00
else:
2022-03-11 19:53:21 +01:00
2023-01-05 00:01:36 +01:00
db.write_blocked_software(self, software, soft_version, alive, api, users, downs, first_checked_at)
2022-03-11 19:53:21 +01:00
2023-01-05 00:01:36 +01:00
except urllib3.exceptions.ProtocolError as protoerr:
2022-03-11 19:53:21 +01:00
2023-01-05 00:01:36 +01:00
print_dead(self)
pass
2023-01-05 00:01:36 +01:00
except requests.exceptions.ChunkedEncodingError as chunkerr:
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
print_dead(self)
pass
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
except KeyError as e:
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
print_dead(self)
pass
2023-01-05 00:01:36 +01:00
except ValueError as verr:
2023-01-05 00:01:36 +01:00
print_dead(self)
pass
2023-01-05 00:01:36 +01:00
except requests.exceptions.SSLError as errssl:
2023-01-05 00:01:36 +01:00
print_dead(self)
pass
2023-01-05 00:01:36 +01:00
except requests.exceptions.HTTPError as errh:
2023-01-05 00:01:36 +01:00
print_dead(self)
pass
2023-01-05 00:01:36 +01:00
except requests.exceptions.ConnectionError as errc:
2023-01-05 00:01:36 +01:00
print_dead(self)
pass
2022-03-11 19:53:21 +01:00
2023-01-05 00:01:36 +01:00
except requests.exceptions.Timeout as errt:
2023-01-05 00:01:36 +01:00
print_dead(self)
pass
2023-01-05 00:01:36 +01:00
except requests.exceptions.RequestException as err:
2023-01-05 00:01:36 +01:00
print_dead(self)
pass
2023-01-05 00:01:36 +01:00
except socket.gaierror as gai_error:
2023-01-05 00:01:36 +01:00
print_dead(self)
pass
2023-01-05 00:01:36 +01:00
if not alive:
2023-01-05 00:01:36 +01:00
mau = 0
2023-01-05 00:01:36 +01:00
if downs_qty != None:
2023-01-05 00:01:36 +01:00
downs = downs_qty + 1
2023-01-05 00:01:36 +01:00
else:
2023-01-05 00:01:36 +01:00
downs = 1
2023-01-05 00:01:36 +01:00
db.write_not_alive_server(self, software, soft_version, alive, api, users, downs, first_checked_at)
2023-01-05 00:01:36 +01:00
return (self, software, soft_version, alive, api, users, downs, first_checked_at, mau)
2022-03-11 19:53:21 +01:00
def print_dead(server):
print(f'\nServer {server} is dead :-(')
if __name__ == '__main__':
2023-01-05 00:01:36 +01:00
db = Database()
2023-01-05 00:01:36 +01:00
setup = Setup()
2023-01-05 00:01:36 +01:00
start = datetime.now()
2022-03-11 19:53:21 +01:00
2023-01-05 00:01:36 +01:00
program = 'fediverse'
2022-03-11 19:53:21 +01:00
2023-01-05 00:01:36 +01:00
finish = start
2023-01-05 00:01:36 +01:00
db.save_time(program, start, finish)
2022-03-11 19:53:21 +01:00
2023-01-05 00:01:36 +01:00
now = start
2022-03-11 19:53:21 +01:00
2023-01-05 00:01:36 +01:00
mastodon = Mastodon(
access_token = setup.mastodon_app_token,
api_base_url= setup.mastodon_hostname
)
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
total_servers = 0
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
total_users = 0
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
alive_servers = db.get_last_checked_servers()
2023-01-05 00:01:36 +01:00
getservers = Server()
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
getservers.now = now
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
ray_start = time.time()
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
results = ray.get([getservers.get_alive_servers.remote(server) for server in alive_servers])
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
print(f"duration = {time.time() - ray_start}.\nprocessed servers: {len(results)}")
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
# get current total servers and users, get users from every software
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
soft_total_project, soft_total_users, soft_total_mau, soft_total_servers, total_servers, total_users, total_mau = db.soft_totals()
2023-01-05 00:01:36 +01:00
# get last check values and write current total ones
2023-01-05 00:01:36 +01:00
evo_servers, evo_users, evo_mau = db.last_values(total_servers, total_users, total_mau)
2023-01-05 00:01:36 +01:00
# write evo values
2023-01-05 00:01:36 +01:00
db.write_evo(evo_servers, evo_users, evo_mau)
2023-01-05 00:01:36 +01:00
# get world's last update datetime
2023-01-05 00:01:36 +01:00
last_update = db.last_world_datetime()
2023-01-05 00:01:36 +01:00
# get max servers and mau
2023-01-05 00:01:36 +01:00
max_servers, max_mau = db.max()
2023-01-05 00:01:36 +01:00
# get plots
2023-01-05 00:01:36 +01:00
servers_plots, mau_plots, global_week, global_servers, global_users, global_mau = db.get_plots()
2023-01-05 00:01:36 +01:00
###############################################################################
# generate graphs
2023-01-05 00:01:36 +01:00
plt.plot([-6, -5, -4, -3, -2, -1, 0], [servers_plots[6], servers_plots[5], servers_plots[4], servers_plots[3], servers_plots[2], servers_plots[1], servers_plots[0]], marker='o', color='mediumseagreen')
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_servers, max_servers, max_servers, max_servers, max_servers, max_servers, max_servers], color='red')
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
plt.title('fediverse: total alive servers (max: ' + str(f"{max_servers:,}" + ')'), loc='right', color='blue')
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
plt.xlabel('Last seven days')
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
plt.ylabel('fediverse alive servers')
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
plt.legend(('servers', 'max'), shadow=True, loc=(0.01, 1.00), handlelength=1.5, fontsize=10)
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
plt.savefig('servers.png')
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
plt.close()
2023-01-05 00:01:36 +01:00
plt.plot([-6, -5, -4, -3, -2, -1, 0], [mau_plots[6], mau_plots[5], mau_plots[4], mau_plots[3], mau_plots[2], mau_plots[1], mau_plots[0]], marker='o', color='royalblue')
2023-01-05 00:01:36 +01:00
plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_mau, max_mau, max_mau, max_mau, max_mau, max_mau, max_mau], color='red')
2023-01-05 00:01:36 +01:00
plt.title('fediverse: total MAU (max: ' + str(f"{max_mau:,}" + ')'), loc='right', color='royalblue')
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
plt.legend(('mau', 'max'), shadow=True, loc=(0.01, 0.80), handlelength=1.5, fontsize=10)
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
plt.xlabel('Last seven days')
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
plt.ylabel('MAU')
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
plt.savefig('mau.png')
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
plt.close()
2023-01-05 00:01:36 +01:00
df = pd.DataFrame({'date': np.array(global_week),
#'servers': np.array(global_servers),
'users': np.array(global_users),
'mau': np.array(global_mau)})
2023-01-05 00:01:36 +01:00
df['date'] = pd.to_datetime(df['date'])
2023-01-05 00:01:36 +01:00
fig, ax = plt.subplots()
2023-01-05 00:01:36 +01:00
ax.plot(df.date, df.users, label='Registered', color='orange')
2023-01-05 00:01:36 +01:00
ax.plot(df.date, df.mau, label='MAU', color='blue')
2023-01-05 00:01:36 +01:00
plt.tick_params(rotation=45)
2023-01-05 00:01:36 +01:00
ax.set_title("fediverse's registered and Monthly Active Users")
2023-01-05 00:01:36 +01:00
ax.set_xlabel('weeks')
2023-01-05 00:01:36 +01:00
ax.set_ylabel('users')
2023-01-05 00:01:36 +01:00
ax.grid(visible=True)
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
ax.legend(title='Users')
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
ax.yaxis.set_major_formatter(y_formatter)
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
plt.savefig('global.png')
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
plt.close()
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
###############################################################################
# P O S T !
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
toot_text = "#fediverse alive servers stats" + " \n"
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
toot_text += "\n"
2023-01-05 00:01:36 +01:00
if evo_servers >= 0:
2023-01-05 00:01:36 +01:00
toot_text += "alive servers: " + str(f"{total_servers:,}") + " (+"+ str(f"{evo_servers:,}") + ") \n"
2023-01-05 00:01:36 +01:00
toot_text += "max: " + str(f"{max_servers:,}") + "\n"
2023-01-05 00:01:36 +01:00
elif evo_servers < 0:
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
toot_text += "alive servers: " + str(f"{total_servers:,}") + " ("+ str(f"{evo_servers:,}") + ") \n"
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
toot_text += "max: " + str(f"{max_servers:,}") + "\n"
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
if evo_mau >= 0:
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
toot_text += "total MAU: " + str(f"{total_mau:,}") + " (+"+ str(f"{evo_mau:,}") + ") \n"
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
toot_text += "max: " + str(f"{max_mau:,}") + "\n"
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
elif evo_mau < 0:
2023-01-05 00:01:36 +01:00
toot_text += "total MAU: " + str(f"{total_mau:,}") + " ("+ str(f"{evo_mau:,}") + ") \n"
2023-01-05 00:01:36 +01:00
toot_text += "max: " + str(f"{max_mau:,}") + "\n"
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
toot_text += "\ntop ten (MAU / servers):\n\n"
2020-05-17 21:28:03 +02:00
2023-01-05 00:01:36 +01:00
i = 0
2023-01-05 00:01:36 +01:00
while i < 10:
2023-01-05 00:01:36 +01:00
project_soft = soft_total_project[i]
2023-01-05 00:01:36 +01:00
project_mau = soft_total_mau[i]
2023-01-05 00:01:36 +01:00
project_servers = soft_total_servers[i]
2023-01-05 00:01:36 +01:00
len_pr_soft = len(project_soft)
2023-01-05 00:01:36 +01:00
if project_soft == 'ativity-relay':
2023-01-05 00:01:36 +01:00
project_soft = 'activityrelay'
2023-01-05 00:01:36 +01:00
toot_text += f":{project_soft}: {project_mau:,} / {project_servers:,}\n"
2023-01-05 00:01:36 +01:00
i += 1
2023-01-05 00:01:36 +01:00
print("Tooting...")
2023-01-05 00:01:36 +01:00
print(toot_text)
2023-01-05 00:01:36 +01:00
servers_image_id = mastodon.media_post('servers.png', "image/png", description='servers graph').id
2023-01-05 00:01:36 +01:00
mau_image_id = mastodon.media_post('mau.png', "image/png", description='MAU graph').id
2022-03-11 19:53:21 +01:00
2023-01-05 00:01:36 +01:00
global_image_id = mastodon.media_post('global.png', "image/png", description='global graph').id
2022-03-11 19:53:21 +01:00
2023-01-05 00:01:36 +01:00
mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, mau_image_id, global_image_id})
2023-01-05 00:01:36 +01:00
db.delete_dead_servers()
2023-01-05 00:01:36 +01:00
finish = datetime.now()
2022-03-11 19:53:21 +01:00
2023-01-05 00:01:36 +01:00
db.save_time(program, start, finish)