New features! Wordpress support and server&users graphs generation
This commit is contained in:
spla
pare
d8c58f5237
commit
3f8a747923
S'han modificat 2 arxius amb 416 adicions i 267 eliminacions
|
|
@ -65,6 +65,7 @@ Within Python Virtual Environment:
|
|||
| prismo | nodeinfo/2.0.json | ['software']['name'] |
|
||||
| ravenvale | nodeinfo/2.0.json | ['software']['name'] |
|
||||
| squs | nodeinfo/2.0 | ['software']['name'] |
|
||||
| wordpress | wp-json/nodeinfo/2.0 | ['software']['name'] |
|
||||
| writefreely | api/nodeinfo | ['software']['name'] |
|
||||
| zap | nodeinfo/2.0.json | ['software']['name'] |
|
||||
|
||||
|
|
@ -73,3 +74,5 @@ Within Python Virtual Environment:
|
|||
Note: install all needed packages with 'pip install package' or use 'pip install -r requirements.txt' to install them.
|
||||
|
||||
18.2.21 - New feature! Added [Lemmy project](https://join.lemmy.ml)
|
||||
12.5.21 - New feature! Added Wordpress support. The code can now detect Wordpress instances with ActivityPub enabled plugin.
|
||||
12.5.21 - New feature! New shinny creation of servers and users graphs.
|
||||
|
|
|
|||
404
fediverse.py
404
fediverse.py
|
|
@ -1,8 +1,4 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import time
|
||||
start_time = time.time()
|
||||
import urllib3
|
||||
from urllib3 import exceptions
|
||||
from datetime import datetime
|
||||
|
|
@ -32,10 +28,17 @@ from asyncio import TimeoutError
|
|||
import socket
|
||||
from socket import gaierror, gethostbyname
|
||||
|
||||
import matplotlib
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
plt.style.use('seaborn')
|
||||
|
||||
start_time = time.time()
|
||||
|
||||
from decimal import *
|
||||
getcontext().prec = 2
|
||||
|
||||
apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?', '/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?']
|
||||
apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?', '/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?','/wp-json/nodeinfo/2.0?']
|
||||
|
||||
client_exceptions = (
|
||||
aiohttp.ClientResponseError,
|
||||
|
|
@ -46,12 +49,6 @@ client_exceptions = (
|
|||
socket.gaierror,
|
||||
)
|
||||
|
||||
now = datetime.now()
|
||||
|
||||
###############################################################################
|
||||
# INITIALISATION
|
||||
###############################################################################
|
||||
|
||||
def is_json(myjson):
|
||||
try:
|
||||
json_object = json.loads(myjson)
|
||||
|
|
@ -59,9 +56,11 @@ def is_json(myjson):
|
|||
return False
|
||||
return True
|
||||
|
||||
def alive_server(server, x):
|
||||
def get_alive_servers(server):
|
||||
|
||||
server = server[0].rstrip('.').lower()
|
||||
serv_api = ''
|
||||
serv_soft = ''
|
||||
soft_version = ''
|
||||
|
||||
conn = None
|
||||
|
||||
|
|
@ -150,21 +149,36 @@ def alive_server(server, x):
|
|||
except:
|
||||
users = 0
|
||||
soft_version = ""
|
||||
if serv_soft == "ganggo" or serv_soft == "squs" or serv_soft == "dolphin":
|
||||
if serv_soft == "ganggo" or serv_soft == "squs" or serv_soft == "dolphin" or serv_soft == "lemmy" or serv_soft == "wordpress":
|
||||
try:
|
||||
users = data.json()['usage']['users']['total']
|
||||
soft_version = data.json()['software']['version']
|
||||
if serv_soft == "wordpress" and "activitypub" in data.json()['protocols']:
|
||||
|
||||
alive = True
|
||||
|
||||
elif serv_soft == "wordpress" and "activitypub" not in data.json()['protocols']:
|
||||
|
||||
alive = False
|
||||
|
||||
else:
|
||||
|
||||
alive = True
|
||||
|
||||
except:
|
||||
|
||||
users = 0
|
||||
soft_version = ""
|
||||
|
||||
if alive:
|
||||
|
||||
if soft_version != "" and soft_version != None:
|
||||
print("Server " + str(server) + " (" + serv_soft + " " + soft_version + ") is alive!")
|
||||
else:
|
||||
print("Server " + str(server) + " (" + serv_soft + ") is alive!")
|
||||
|
||||
insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
|
||||
|
||||
conn = None
|
||||
|
||||
try:
|
||||
|
|
@ -245,6 +259,11 @@ def alive_server(server, x):
|
|||
alive = False
|
||||
pass
|
||||
|
||||
except socket.gaierror as gai_error:
|
||||
|
||||
print(gai_error)
|
||||
pass
|
||||
|
||||
if alive == False:
|
||||
|
||||
conn = None
|
||||
|
|
@ -311,6 +330,7 @@ async def getsoft(server):
|
|||
|
||||
except socket.gaierror:
|
||||
|
||||
pass
|
||||
return
|
||||
|
||||
soft = ''
|
||||
|
|
@ -377,56 +397,7 @@ def getserver(server, x):
|
|||
|
||||
pass
|
||||
|
||||
# Returns the parameter from the specified file
|
||||
def get_parameter( parameter, file_path ):
|
||||
# Check if secrets file exists
|
||||
if not os.path.isfile(file_path):
|
||||
print("File %s not found, exiting."%file_path)
|
||||
sys.exit(0)
|
||||
|
||||
# Find parameter in file
|
||||
with open( file_path ) as f:
|
||||
for line in f:
|
||||
if line.startswith( parameter ):
|
||||
return line.replace(parameter + ":", "").strip()
|
||||
|
||||
# Cannot find parameter, exit
|
||||
print(file_path + " Missing parameter %s "%parameter)
|
||||
sys.exit(0)
|
||||
|
||||
# Load secrets from secrets file
|
||||
secrets_filepath = "secrets/secrets.txt"
|
||||
uc_client_id = get_parameter("uc_client_id", secrets_filepath)
|
||||
uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
|
||||
uc_access_token = get_parameter("uc_access_token", secrets_filepath)
|
||||
|
||||
# Load configuration from config file
|
||||
config_filepath = "config/config.txt"
|
||||
mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
|
||||
|
||||
# Load database config from db_config file
|
||||
db_config_filepath = "config/db_config.txt"
|
||||
fediverse_db = get_parameter("fediverse_db", db_config_filepath)
|
||||
fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath)
|
||||
|
||||
# Initialise Mastodon API
|
||||
mastodon = Mastodon(
|
||||
client_id = uc_client_id,
|
||||
client_secret = uc_client_secret,
|
||||
access_token = uc_access_token,
|
||||
api_base_url = 'https://' + mastodon_hostname,
|
||||
)
|
||||
|
||||
# Initialise access headers
|
||||
headers={ 'Authorization': 'Bearer %s'%uc_access_token }
|
||||
|
||||
###############################################################################
|
||||
# main
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
total_servers = 0
|
||||
total_users = 0
|
||||
def set_world_servers_check_to_false():
|
||||
|
||||
############################################################################
|
||||
# set all world servers's checked column to False
|
||||
|
|
@ -455,9 +426,10 @@ if __name__ == '__main__':
|
|||
|
||||
conn.close()
|
||||
|
||||
############################################################################
|
||||
# get last check servers from fediverse DB
|
||||
def get_last_checked_servers():
|
||||
|
||||
############################################################################
|
||||
# get last checked servers from fediverse DB
|
||||
|
||||
alive_servers = []
|
||||
|
||||
|
|
@ -490,93 +462,125 @@ if __name__ == '__main__':
|
|||
|
||||
conn.close()
|
||||
|
||||
###########################################################################
|
||||
# multiprocessing!
|
||||
return alive_servers
|
||||
|
||||
m = Manager()
|
||||
q = m.Queue()
|
||||
z = zip(alive_servers)
|
||||
def mastodon():
|
||||
|
||||
serv_number = len(alive_servers)
|
||||
# Load secrets from secrets file
|
||||
secrets_filepath = "secrets/secrets.txt"
|
||||
uc_client_id = get_parameter("uc_client_id", secrets_filepath)
|
||||
uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
|
||||
uc_access_token = get_parameter("uc_access_token", secrets_filepath)
|
||||
|
||||
pool_tuple = [(x, q) for x in z]
|
||||
with Pool(processes=64) as pool:
|
||||
pool.starmap(alive_server, pool_tuple)
|
||||
# Load configuration from config file
|
||||
config_filepath = "config/config.txt"
|
||||
mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
|
||||
|
||||
###########################################################################
|
||||
# Initialise Mastodon API
|
||||
mastodon = Mastodon(
|
||||
client_id = uc_client_id,
|
||||
client_secret = uc_client_secret,
|
||||
access_token = uc_access_token,
|
||||
api_base_url = 'https://' + mastodon_hostname,
|
||||
)
|
||||
|
||||
print("Getting the remaining servers from world")
|
||||
# Initialise access headers
|
||||
headers={ 'Authorization': 'Bearer %s'%uc_access_token }
|
||||
|
||||
world_servers = []
|
||||
return (mastodon, mastodon_hostname)
|
||||
|
||||
try:
|
||||
def db_config():
|
||||
|
||||
conn = None
|
||||
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
# Load db configuration from config file
|
||||
config_filepath = "config/db_config.txt"
|
||||
fediverse_db = get_parameter("fediverse_db", config_filepath)
|
||||
fediverse_db_user = get_parameter("fediverse_db_user", config_filepath)
|
||||
|
||||
cur = conn.cursor()
|
||||
return (fediverse_db, fediverse_db_user)
|
||||
|
||||
### get world servers list
|
||||
def usage():
|
||||
|
||||
cur.execute("select server from world where checked='f'")
|
||||
print('usage: python ' + sys.argv[0] + ' --multi' + ' (multiprocessing, fast)')
|
||||
print('usage: python ' + sys.argv[0] + ' --mono' + ' (one process, slow)')
|
||||
|
||||
for row in cur:
|
||||
# Returns the parameter from the specified file
|
||||
def get_parameter( parameter, file_path ):
|
||||
# Check if secrets file exists
|
||||
if not os.path.isfile(file_path):
|
||||
print("File %s not found, exiting."%file_path)
|
||||
sys.exit(0)
|
||||
|
||||
world_servers.append(row[0])
|
||||
# Find parameter in file
|
||||
with open( file_path ) as f:
|
||||
for line in f:
|
||||
if line.startswith( parameter ):
|
||||
return line.replace(parameter + ":", "").strip()
|
||||
|
||||
cur.close()
|
||||
# Cannot find parameter, exit
|
||||
print(file_path + " Missing parameter %s "%parameter)
|
||||
sys.exit(0)
|
||||
|
||||
print("Remaining servers: " + str(len(world_servers)))
|
||||
###############################################################################
|
||||
# main
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
if __name__ == '__main__':
|
||||
|
||||
print(error)
|
||||
# usage modes
|
||||
|
||||
finally:
|
||||
if len(sys.argv) == 1:
|
||||
|
||||
if conn is not None:
|
||||
usage()
|
||||
|
||||
conn.close()
|
||||
elif len(sys.argv) == 2:
|
||||
|
||||
if sys.argv[1] == '--multi':
|
||||
|
||||
now = datetime.now()
|
||||
|
||||
mastodon, mastodon_hostname = mastodon()
|
||||
|
||||
fediverse_db, fediverse_db_user = db_config()
|
||||
|
||||
total_servers = 0
|
||||
total_users = 0
|
||||
|
||||
set_world_servers_check_to_false()
|
||||
|
||||
alive_servers = get_last_checked_servers()
|
||||
|
||||
###########################################################################
|
||||
# multiprocessing!
|
||||
|
||||
m = Manager()
|
||||
q = m.Queue()
|
||||
z = zip(world_servers)
|
||||
nprocs = multiprocessing.cpu_count()
|
||||
with multiprocessing.Pool(processes=64) as pool:
|
||||
results = pool.starmap(get_alive_servers, product(alive_servers))
|
||||
|
||||
serv_number = len(world_servers)
|
||||
elif sys.argv[1] == '--mono':
|
||||
|
||||
pool_tuple = [(x, q) for x in z]
|
||||
with Pool(processes=64) as pool:
|
||||
pool.starmap(getserver, pool_tuple)
|
||||
now = datetime.now()
|
||||
|
||||
###########################################################################
|
||||
# delete not alive servers from fediverse table
|
||||
mastodon, mastodon_hostname = mastodon()
|
||||
|
||||
try:
|
||||
fediverse_db, fediverse_db_user = db_config()
|
||||
|
||||
conn = None
|
||||
total_servers = 0
|
||||
total_users = 0
|
||||
|
||||
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
set_world_servers_check_to_false()
|
||||
|
||||
cur = conn.cursor()
|
||||
alive_servers = get_last_checked_servers()
|
||||
|
||||
cur.execute("delete from fediverse where not alive")
|
||||
i = 0
|
||||
|
||||
conn.commit()
|
||||
while i < len(alive_servers):
|
||||
|
||||
cur.close()
|
||||
get_alive_servers(alive_servers[i])
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
i += 1
|
||||
|
||||
print(error)
|
||||
else:
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
usage()
|
||||
|
||||
###########################################################################
|
||||
# get current total servers and users, get users from every software
|
||||
|
|
@ -724,30 +728,172 @@ if __name__ == '__main__':
|
|||
|
||||
conn.close()
|
||||
|
||||
##############################################################################
|
||||
# get max servers and users
|
||||
|
||||
conn = None
|
||||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute("select MAX(total_servers) from totals")
|
||||
|
||||
row = cur.fetchone()
|
||||
|
||||
if row != None:
|
||||
|
||||
max_servers = row[0]
|
||||
|
||||
else:
|
||||
|
||||
max_servers = 0
|
||||
|
||||
cur.execute("select MAX(total_users) from totals")
|
||||
|
||||
row = cur.fetchone()
|
||||
|
||||
if row != None:
|
||||
|
||||
max_users = row[0]
|
||||
|
||||
else:
|
||||
|
||||
max_users = 0
|
||||
|
||||
cur.close()
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print(error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
###############################################################################
|
||||
# get plots
|
||||
|
||||
servers_plots = []
|
||||
|
||||
users_plots = []
|
||||
|
||||
conn = None
|
||||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute("select total_servers, total_users from totals order by datetime desc limit 14")
|
||||
|
||||
rows = cur.fetchall()
|
||||
|
||||
for row in rows:
|
||||
|
||||
servers_plots.append(row[0])
|
||||
|
||||
users_plots.append(row[1])
|
||||
|
||||
cur.close()
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print(error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
###############################################################################
|
||||
# generate graphs
|
||||
|
||||
plt.plot([-6, -5, -4, -3, -2, -1, 0], [servers_plots[6], servers_plots[5],servers_plots[4],servers_plots[3],servers_plots[2],servers_plots[1],servers_plots[0]], marker='o', color='mediumseagreen')
|
||||
|
||||
plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_servers, max_servers, max_servers, max_servers, max_servers, max_servers, max_servers], color='red')
|
||||
|
||||
plt.title('fediverse: total alive servers (max: ' + str(f"{max_servers:,}" + ')') , loc='right', color='blue')
|
||||
|
||||
plt.xlabel('Last seven days')
|
||||
|
||||
plt.ylabel('fediverse alive servers')
|
||||
|
||||
plt.legend(('servers', 'max'), shadow=True, loc=(0.01, 1.00), handlelength=1.5, fontsize=10)
|
||||
|
||||
plt.savefig('servers.png')
|
||||
|
||||
plt.close()
|
||||
|
||||
plt.plot([-6, -5, -4, -3, -2, -1, 0], [users_plots[6], users_plots[5], users_plots[4], users_plots[3], users_plots[2], users_plots[1], users_plots[0]], marker='o', color='royalblue')
|
||||
|
||||
plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_users, max_users, max_users, max_users, max_users, max_users, max_users], color='red')
|
||||
|
||||
plt.title('fediverse: total registered users (max: ' + str(f"{max_users:,}" + ')') , loc='right', color='royalblue')
|
||||
|
||||
plt.legend(('users', 'max'), shadow=True, loc=(0.01, 0.80), handlelength=1.5, fontsize=10)
|
||||
|
||||
plt.xlabel('Last seven days')
|
||||
|
||||
plt.ylabel('Registered users')
|
||||
|
||||
plt.savefig('users.png')
|
||||
|
||||
plt.close()
|
||||
|
||||
###############################################################################
|
||||
# T O O T !
|
||||
|
||||
toot_text = "#fediverse alive servers stats" + " \n"
|
||||
toot_text += "\n"
|
||||
if evo_servers >= 0:
|
||||
toot_text += "alive servers: " + str(total_servers) + " (+"+ str(evo_servers) + ") \n"
|
||||
toot_text += "alive servers: " + str(f"{total_servers:,}") + " (+"+ str(f"{evo_servers:,}") + ") \n"
|
||||
toot_text += "max: " + str(f"{max_servers:,}") + "\n"
|
||||
elif evo_servers < 0:
|
||||
toot_text += "alive servers: " + str(total_servers) + " ("+ str(evo_servers) + ") \n"
|
||||
toot_text += "alive servers: " + str(f"{total_servers:,}") + " ("+ str(f"{evo_servers:,}") + ") \n"
|
||||
toot_text += "max: " + str(f"{max_servers:,}") + "\n"
|
||||
if evo_users >= 0:
|
||||
toot_text += "total users: " + str(total_users) + " (+"+ str(evo_users) + ") \n"
|
||||
toot_text += "total users: " + str(f"{total_users:,}") + " (+"+ str(f"{evo_users:,}") + ") \n"
|
||||
toot_text += "max: " + str(f"{max_users:,}") + "\n"
|
||||
elif evo_users < 0:
|
||||
toot_text += "total users: " + str(total_users) + " ("+ str(evo_users) + ") \n"
|
||||
toot_text += "total users: " + str(f"{total_users:,}") + " ("+ str(f"{evo_users:,}") + ") \n"
|
||||
toot_text += "max: " + str(f"{max_users:,}") + "\n"
|
||||
|
||||
toot_text += "\n"
|
||||
toot_text += "top five (soft: users servers):" + " \n"
|
||||
toot_text += "top ten (soft users servers):" + " \n"
|
||||
toot_text += "\n"
|
||||
toot_text += '{:<8}: {:>7} {:>4}'.format(str(soft_total_project[0]), str(soft_total_users[0]), str(soft_total_servers[0])) + " \n"
|
||||
toot_text += '{:<8}: {:>7} {:>4}'.format(str(soft_total_project[1]), str(soft_total_users[1]), str(soft_total_servers[1])) + " \n"
|
||||
toot_text += '{:<8}: {:>7} {:>4}'.format(str(soft_total_project[2]), str(soft_total_users[2]), str(soft_total_servers[2])) + " \n"
|
||||
toot_text += '{:<8}: {:>7} {:>4}'.format(str(soft_total_project[3]), str(soft_total_users[3]), str(soft_total_servers[3])) + " \n"
|
||||
toot_text += '{:<8}: {:>7} {:>4}'.format(str(soft_total_project[4]), str(soft_total_users[4]), str(soft_total_servers[4])) + " \n"
|
||||
toot_text += "\n"
|
||||
toot_text += "updated at " + str(last_update) + " \n"
|
||||
|
||||
i = 0
|
||||
while i < 10:
|
||||
|
||||
project_soft = soft_total_project[i]
|
||||
project_users = soft_total_users[i]
|
||||
project_servers = soft_total_servers[i]
|
||||
len_pr_soft = len(project_soft)
|
||||
|
||||
if project_soft == 'writefreely':
|
||||
|
||||
str_len = 11
|
||||
|
||||
else:
|
||||
|
||||
str_len = 13
|
||||
|
||||
toot_text += f"{':'+project_soft+':':<11}" + f"{project_users:>{str_len},}" + " " + f"{project_servers:>5,}" + "\n"
|
||||
|
||||
i += 1
|
||||
|
||||
print("Tooting...")
|
||||
print(toot_text)
|
||||
|
||||
mastodon.status_post(toot_text, in_reply_to_id=None, )
|
||||
servers_image_id = mastodon.media_post('servers.png', "image/png").id
|
||||
|
||||
users_image_id = mastodon.media_post('users.png', "image/png").id
|
||||
|
||||
mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, users_image_id})
|
||||
|
|
|
|||
Loading…
Referencia en una nova incidència