New features! Wordpress support and server&users graphs generation

This commit is contained in:
spla 2021-05-12 16:17:33 +02:00
pare d8c58f5237
commit 3f8a747923
S'han modificat 2 arxius amb 416 adicions i 267 eliminacions

Veure arxiu

@ -65,6 +65,7 @@ Within Python Virtual Environment:
| prismo | nodeinfo/2.0.json | ['software']['name'] |
| ravenvale | nodeinfo/2.0.json | ['software']['name'] |
| squs | nodeinfo/2.0 | ['software']['name'] |
| wordpress | wp-json/nodeinfo/2.0 | ['software']['name'] |
| writefreely | api/nodeinfo | ['software']['name'] |
| zap | nodeinfo/2.0.json | ['software']['name'] |
@ -72,4 +73,6 @@ Within Python Virtual Environment:
Note: install all needed packages with 'pip install package' or use 'pip install -r requirements.txt' to install them.
18.2.21 - New feature! Added [Lemmy project](https://join.lemmy.ml)
18.2.21 - New feature! Added [Lemmy project](https://join.lemmy.ml)
12.5.21 - New feature! Added Wordpress support. The code can now detect Wordpress instances with ActivityPub enabled plugin.
12.5.21 - New feature! New shinny creation of servers and users graphs.

Veure arxiu

@ -1,8 +1,4 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import time
start_time = time.time()
import urllib3
from urllib3 import exceptions
from datetime import datetime
@ -32,10 +28,17 @@ from asyncio import TimeoutError
import socket
from socket import gaierror, gethostbyname
import matplotlib
import matplotlib.pyplot as plt
plt.style.use('seaborn')
start_time = time.time()
from decimal import *
getcontext().prec = 2
apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?', '/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?']
apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?', '/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?','/wp-json/nodeinfo/2.0?']
client_exceptions = (
aiohttp.ClientResponseError,
@ -46,12 +49,6 @@ client_exceptions = (
socket.gaierror,
)
now = datetime.now()
###############################################################################
# INITIALISATION
###############################################################################
def is_json(myjson):
try:
json_object = json.loads(myjson)
@ -59,9 +56,11 @@ def is_json(myjson):
return False
return True
def alive_server(server, x):
def get_alive_servers(server):
server = server[0].rstrip('.').lower()
serv_api = ''
serv_soft = ''
soft_version = ''
conn = None
@ -150,46 +149,61 @@ def alive_server(server, x):
except:
users = 0
soft_version = ""
if serv_soft == "ganggo" or serv_soft == "squs" or serv_soft == "dolphin":
if serv_soft == "ganggo" or serv_soft == "squs" or serv_soft == "dolphin" or serv_soft == "lemmy" or serv_soft == "wordpress":
try:
users = data.json()['usage']['users']['total']
soft_version = data.json()['software']['version']
alive = True
if serv_soft == "wordpress" and "activitypub" in data.json()['protocols']:
alive = True
elif serv_soft == "wordpress" and "activitypub" not in data.json()['protocols']:
alive = False
else:
alive = True
except:
users = 0
soft_version = ""
if soft_version != "" and soft_version != None:
print("Server " + str(server) + " (" + serv_soft + " " + soft_version + ") is alive!")
else:
print("Server " + str(server) + " (" + serv_soft + ") is alive!")
if alive:
insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
conn = None
if soft_version != "" and soft_version != None:
print("Server " + str(server) + " (" + serv_soft + " " + soft_version + ") is alive!")
else:
print("Server " + str(server) + " (" + serv_soft + ") is alive!")
try:
insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
conn = None
cur = conn.cursor()
try:
cur.execute(insert_sql, (server, users, now, serv_soft, alive, serv_api, soft_version))
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)", (users, now, serv_soft, alive, serv_api, soft_version, server))
cur = conn.cursor()
cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
cur.execute(insert_sql, (server, users, now, serv_soft, alive, serv_api, soft_version))
conn.commit()
cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)", (users, now, serv_soft, alive, serv_api, soft_version, server))
cur.close()
cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
except (Exception, psycopg2.DatabaseError) as error:
print(error)
conn.commit()
finally:
cur.close()
if conn is not None:
conn.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
except urllib3.exceptions.ProtocolError as protoerr:
@ -245,6 +259,11 @@ def alive_server(server, x):
alive = False
pass
except socket.gaierror as gai_error:
print(gai_error)
pass
if alive == False:
conn = None
@ -311,6 +330,7 @@ async def getsoft(server):
except socket.gaierror:
pass
return
soft = ''
@ -377,56 +397,7 @@ def getserver(server, x):
pass
# Returns the parameter from the specified file
def get_parameter( parameter, file_path ):
# Check if secrets file exists
if not os.path.isfile(file_path):
print("File %s not found, exiting."%file_path)
sys.exit(0)
# Find parameter in file
with open( file_path ) as f:
for line in f:
if line.startswith( parameter ):
return line.replace(parameter + ":", "").strip()
# Cannot find parameter, exit
print(file_path + " Missing parameter %s "%parameter)
sys.exit(0)
# Load secrets from secrets file
secrets_filepath = "secrets/secrets.txt"
uc_client_id = get_parameter("uc_client_id", secrets_filepath)
uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
uc_access_token = get_parameter("uc_access_token", secrets_filepath)
# Load configuration from config file
config_filepath = "config/config.txt"
mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
# Load database config from db_config file
db_config_filepath = "config/db_config.txt"
fediverse_db = get_parameter("fediverse_db", db_config_filepath)
fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath)
# Initialise Mastodon API
mastodon = Mastodon(
client_id = uc_client_id,
client_secret = uc_client_secret,
access_token = uc_access_token,
api_base_url = 'https://' + mastodon_hostname,
)
# Initialise access headers
headers={ 'Authorization': 'Bearer %s'%uc_access_token }
###############################################################################
# main
if __name__ == '__main__':
total_servers = 0
total_users = 0
def set_world_servers_check_to_false():
############################################################################
# set all world servers's checked column to False
@ -455,9 +426,10 @@ if __name__ == '__main__':
conn.close()
############################################################################
# get last check servers from fediverse DB
def get_last_checked_servers():
############################################################################
# get last checked servers from fediverse DB
alive_servers = []
@ -490,264 +462,438 @@ if __name__ == '__main__':
conn.close()
###########################################################################
# multiprocessing!
return alive_servers
m = Manager()
q = m.Queue()
z = zip(alive_servers)
def mastodon():
serv_number = len(alive_servers)
# Load secrets from secrets file
secrets_filepath = "secrets/secrets.txt"
uc_client_id = get_parameter("uc_client_id", secrets_filepath)
uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
uc_access_token = get_parameter("uc_access_token", secrets_filepath)
pool_tuple = [(x, q) for x in z]
with Pool(processes=64) as pool:
pool.starmap(alive_server, pool_tuple)
# Load configuration from config file
config_filepath = "config/config.txt"
mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
###########################################################################
# Initialise Mastodon API
mastodon = Mastodon(
client_id = uc_client_id,
client_secret = uc_client_secret,
access_token = uc_access_token,
api_base_url = 'https://' + mastodon_hostname,
)
print("Getting the remaining servers from world")
# Initialise access headers
headers={ 'Authorization': 'Bearer %s'%uc_access_token }
world_servers = []
return (mastodon, mastodon_hostname)
try:
def db_config():
conn = None
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
# Load db configuration from config file
config_filepath = "config/db_config.txt"
fediverse_db = get_parameter("fediverse_db", config_filepath)
fediverse_db_user = get_parameter("fediverse_db_user", config_filepath)
cur = conn.cursor()
return (fediverse_db, fediverse_db_user)
### get world servers list
def usage():
cur.execute("select server from world where checked='f'")
print('usage: python ' + sys.argv[0] + ' --multi' + ' (multiprocessing, fast)')
print('usage: python ' + sys.argv[0] + ' --mono' + ' (one process, slow)')
for row in cur:
# Returns the parameter from the specified file
def get_parameter( parameter, file_path ):
# Check if secrets file exists
if not os.path.isfile(file_path):
print("File %s not found, exiting."%file_path)
sys.exit(0)
world_servers.append(row[0])
# Find parameter in file
with open( file_path ) as f:
for line in f:
if line.startswith( parameter ):
return line.replace(parameter + ":", "").strip()
cur.close()
# Cannot find parameter, exit
print(file_path + " Missing parameter %s "%parameter)
sys.exit(0)
print("Remaining servers: " + str(len(world_servers)))
###############################################################################
# main
except (Exception, psycopg2.DatabaseError) as error:
if __name__ == '__main__':
print(error)
# usage modes
finally:
if len(sys.argv) == 1:
if conn is not None:
usage()
conn.close()
elif len(sys.argv) == 2:
###########################################################################
# multiprocessing!
if sys.argv[1] == '--multi':
m = Manager()
q = m.Queue()
z = zip(world_servers)
now = datetime.now()
serv_number = len(world_servers)
mastodon, mastodon_hostname = mastodon()
pool_tuple = [(x, q) for x in z]
with Pool(processes=64) as pool:
pool.starmap(getserver, pool_tuple)
fediverse_db, fediverse_db_user = db_config()
###########################################################################
# delete not alive servers from fediverse table
total_servers = 0
total_users = 0
try:
set_world_servers_check_to_false()
conn = None
alive_servers = get_last_checked_servers()
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
###########################################################################
# multiprocessing!
cur = conn.cursor()
nprocs = multiprocessing.cpu_count()
with multiprocessing.Pool(processes=64) as pool:
results = pool.starmap(get_alive_servers, product(alive_servers))
cur.execute("delete from fediverse where not alive")
elif sys.argv[1] == '--mono':
conn.commit()
now = datetime.now()
cur.close()
mastodon, mastodon_hostname = mastodon()
except (Exception, psycopg2.DatabaseError) as error:
fediverse_db, fediverse_db_user = db_config()
print(error)
total_servers = 0
total_users = 0
finally:
set_world_servers_check_to_false()
if conn is not None:
alive_servers = get_last_checked_servers()
conn.close()
i = 0
###########################################################################
# get current total servers and users, get users from every software
while i < len(alive_servers):
gettotals_sql = "select count(server), sum(users) from fediverse where alive"
get_soft_totals_sql = "select software, sum(users) as users, count(server) as servers from fediverse where users != 0 and alive group by software order by users desc"
soft_total_project = []
soft_total_users = []
soft_total_servers = []
get_alive_servers(alive_servers[i])
try:
conn = None
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute(gettotals_sql)
row = cur.fetchone()
total_servers = row[0]
total_users = row[1]
cur.execute(get_soft_totals_sql)
rows = cur.fetchall()
for row in rows:
soft_total_project.append(row[0])
soft_total_users.append(row[1])
soft_total_servers.append(row[2])
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
###########################################################################
# get last check values and write current total ones
select_sql = "select total_servers, total_users from totals order by datetime desc limit 1"
insert_sql = "INSERT INTO totals(datetime, total_servers, total_users) VALUES(%s,%s,%s)"
try:
conn = None
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute(select_sql)
row = cur.fetchone()
if row != None:
servers_before = row[0]
users_before = row[1]
i += 1
else:
servers_before = 0
users_before = 0
usage()
cur.execute(insert_sql, (now, total_servers, total_users))
###########################################################################
# get current total servers and users, get users from every software
conn.commit()
gettotals_sql = "select count(server), sum(users) from fediverse where alive"
get_soft_totals_sql = "select software, sum(users) as users, count(server) as servers from fediverse where users != 0 and alive group by software order by users desc"
soft_total_project = []
soft_total_users = []
soft_total_servers = []
cur.close()
try:
evo_servers = total_servers - servers_before
evo_users = total_users - users_before
conn = None
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
except (Exception, psycopg2.DatabaseError) as error:
cur = conn.cursor()
print(error)
cur.execute(gettotals_sql)
finally:
row = cur.fetchone()
if conn is not None:
total_servers = row[0]
total_users = row[1]
conn.close()
cur.execute(get_soft_totals_sql)
rows = cur.fetchall()
for row in rows:
soft_total_project.append(row[0])
soft_total_users.append(row[1])
soft_total_servers.append(row[2])
cur.close()
################################################################################
# write evo values
except (Exception, psycopg2.DatabaseError) as error:
insert_sql = "INSERT INTO evo(datetime, servers, users) VALUES(%s,%s,%s)"
print(error)
conn = None
finally:
try:
if conn is not None:
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
conn.close()
cur = conn.cursor()
###########################################################################
# get last check values and write current total ones
cur.execute(insert_sql, (now, evo_servers, evo_users))
select_sql = "select total_servers, total_users from totals order by datetime desc limit 1"
insert_sql = "INSERT INTO totals(datetime, total_servers, total_users) VALUES(%s,%s,%s)"
conn.commit()
try:
cur.close()
conn = None
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
except (Exception, psycopg2.DatabaseError) as error:
cur = conn.cursor()
print(error)
cur.execute(select_sql)
finally:
row = cur.fetchone()
if conn is not None:
if row != None:
conn.close()
servers_before = row[0]
users_before = row[1]
##############################################################################
# get world's last update datetime
else:
conn = None
servers_before = 0
users_before = 0
try:
cur.execute(insert_sql, (now, total_servers, total_users))
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
conn.commit()
cur = conn.cursor()
cur.close()
cur.execute("select updated_at from world order by updated_at desc limit 1")
evo_servers = total_servers - servers_before
evo_users = total_users - users_before
row = cur.fetchone()
except (Exception, psycopg2.DatabaseError) as error:
last_update = row[0]
last_update = last_update.strftime('%m/%d/%Y, %H:%M:%S')
print(error)
cur.close()
finally:
except (Exception, psycopg2.DatabaseError) as error:
if conn is not None:
print(error)
conn.close()
finally:
################################################################################
# write evo values
if conn is not None:
insert_sql = "INSERT INTO evo(datetime, servers, users) VALUES(%s,%s,%s)"
conn.close()
conn = None
###############################################################################
# T O O T !
try:
toot_text = "#fediverse alive servers stats" + " \n"
toot_text += "\n"
if evo_servers >= 0:
toot_text += "alive servers: " + str(total_servers) + " (+"+ str(evo_servers) + ") \n"
elif evo_servers < 0:
toot_text += "alive servers: " + str(total_servers) + " ("+ str(evo_servers) + ") \n"
if evo_users >= 0:
toot_text += "total users: " + str(total_users) + " (+"+ str(evo_users) + ") \n"
elif evo_users < 0:
toot_text += "total users: " + str(total_users) + " ("+ str(evo_users) + ") \n"
toot_text += "\n"
toot_text += "top five (soft: users servers):" + " \n"
toot_text += "\n"
toot_text += '{:<8}: {:>7} {:>4}'.format(str(soft_total_project[0]), str(soft_total_users[0]), str(soft_total_servers[0])) + " \n"
toot_text += '{:<8}: {:>7} {:>4}'.format(str(soft_total_project[1]), str(soft_total_users[1]), str(soft_total_servers[1])) + " \n"
toot_text += '{:<8}: {:>7} {:>4}'.format(str(soft_total_project[2]), str(soft_total_users[2]), str(soft_total_servers[2])) + " \n"
toot_text += '{:<8}: {:>7} {:>4}'.format(str(soft_total_project[3]), str(soft_total_users[3]), str(soft_total_servers[3])) + " \n"
toot_text += '{:<8}: {:>7} {:>4}'.format(str(soft_total_project[4]), str(soft_total_users[4]), str(soft_total_servers[4])) + " \n"
toot_text += "\n"
toot_text += "updated at " + str(last_update) + " \n"
print("Tooting...")
print(toot_text)
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
mastodon.status_post(toot_text, in_reply_to_id=None, )
cur = conn.cursor()
cur.execute(insert_sql, (now, evo_servers, evo_users))
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
##############################################################################
# get world's last update datetime
conn = None
try:
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute("select updated_at from world order by updated_at desc limit 1")
row = cur.fetchone()
last_update = row[0]
last_update = last_update.strftime('%m/%d/%Y, %H:%M:%S')
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
##############################################################################
# get max servers and users
conn = None
try:
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute("select MAX(total_servers) from totals")
row = cur.fetchone()
if row != None:
max_servers = row[0]
else:
max_servers = 0
cur.execute("select MAX(total_users) from totals")
row = cur.fetchone()
if row != None:
max_users = row[0]
else:
max_users = 0
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
###############################################################################
# get plots
servers_plots = []
users_plots = []
conn = None
try:
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute("select total_servers, total_users from totals order by datetime desc limit 14")
rows = cur.fetchall()
for row in rows:
servers_plots.append(row[0])
users_plots.append(row[1])
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
###############################################################################
# generate graphs
plt.plot([-6, -5, -4, -3, -2, -1, 0], [servers_plots[6], servers_plots[5],servers_plots[4],servers_plots[3],servers_plots[2],servers_plots[1],servers_plots[0]], marker='o', color='mediumseagreen')
plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_servers, max_servers, max_servers, max_servers, max_servers, max_servers, max_servers], color='red')
plt.title('fediverse: total alive servers (max: ' + str(f"{max_servers:,}" + ')') , loc='right', color='blue')
plt.xlabel('Last seven days')
plt.ylabel('fediverse alive servers')
plt.legend(('servers', 'max'), shadow=True, loc=(0.01, 1.00), handlelength=1.5, fontsize=10)
plt.savefig('servers.png')
plt.close()
plt.plot([-6, -5, -4, -3, -2, -1, 0], [users_plots[6], users_plots[5], users_plots[4], users_plots[3], users_plots[2], users_plots[1], users_plots[0]], marker='o', color='royalblue')
plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_users, max_users, max_users, max_users, max_users, max_users, max_users], color='red')
plt.title('fediverse: total registered users (max: ' + str(f"{max_users:,}" + ')') , loc='right', color='royalblue')
plt.legend(('users', 'max'), shadow=True, loc=(0.01, 0.80), handlelength=1.5, fontsize=10)
plt.xlabel('Last seven days')
plt.ylabel('Registered users')
plt.savefig('users.png')
plt.close()
###############################################################################
# T O O T !
toot_text = "#fediverse alive servers stats" + " \n"
toot_text += "\n"
if evo_servers >= 0:
toot_text += "alive servers: " + str(f"{total_servers:,}") + " (+"+ str(f"{evo_servers:,}") + ") \n"
toot_text += "max: " + str(f"{max_servers:,}") + "\n"
elif evo_servers < 0:
toot_text += "alive servers: " + str(f"{total_servers:,}") + " ("+ str(f"{evo_servers:,}") + ") \n"
toot_text += "max: " + str(f"{max_servers:,}") + "\n"
if evo_users >= 0:
toot_text += "total users: " + str(f"{total_users:,}") + " (+"+ str(f"{evo_users:,}") + ") \n"
toot_text += "max: " + str(f"{max_users:,}") + "\n"
elif evo_users < 0:
toot_text += "total users: " + str(f"{total_users:,}") + " ("+ str(f"{evo_users:,}") + ") \n"
toot_text += "max: " + str(f"{max_users:,}") + "\n"
toot_text += "\n"
toot_text += "top ten (soft users servers):" + " \n"
toot_text += "\n"
i = 0
while i < 10:
project_soft = soft_total_project[i]
project_users = soft_total_users[i]
project_servers = soft_total_servers[i]
len_pr_soft = len(project_soft)
if project_soft == 'writefreely':
str_len = 11
else:
str_len = 13
toot_text += f"{':'+project_soft+':':<11}" + f"{project_users:>{str_len},}" + " " + f"{project_servers:>5,}" + "\n"
i += 1
print("Tooting...")
print(toot_text)
servers_image_id = mastodon.media_post('servers.png', "image/png").id
users_image_id = mastodon.media_post('users.png', "image/png").id
mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, users_image_id})