This code gets all peers from running Mastodon, Pleroma or Lemmy host server and then all peers from host server's peers. Goal is to collect maximum number of alive fediverse's servers by querying their API and then post servers and registered users.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

893 lines
26 KiB

import time
import urllib3
from datetime import datetime
from mastodon import Mastodon
import os
import json
import sys
import os.path
import requests
import psycopg2
from itertools import product
import multiprocessing
import aiohttp
import asyncio
import socket
import matplotlib.pyplot as plt
plt.style.use('seaborn')
start_time = time.time()
apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?', '/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?', '/wp-json/nodeinfo/2.0?']
client_exceptions = (
aiohttp.ClientResponseError,
aiohttp.ClientConnectionError,
aiohttp.ClientConnectorError,
aiohttp.ClientError,
asyncio.TimeoutError,
socket.gaierror,
)
def is_json(myjson):
try:
json_object = json.loads(myjson)
except ValueError as e:
print(e)
return False
return True
def get_alive_servers(server):
serv_api = ''
serv_soft = ''
soft_version = ''
try:
conn = None
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor()
cur.execute("select alive, software, users_api, version from fediverse where server=(%s)", (server,))
row = cur.fetchone()
if row is not None:
was_alive = row[0]
serv_soft = row[1]
serv_api = row[2]
soft_version = row[3]
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
alive = False
try:
data = requests.get('https://' + server + serv_api, timeout=3)
if serv_soft == "mastodon":
if serv_api == '/nodeinfo/2.0?':
try:
users = data.json()['usage']['users']['total']
soft_version = data.json()['software']['version']
alive = True
except:
users = 0
soft_version = ""
if serv_api == '/nodeinfo/2.0.json?':
try:
users = data.json()['usage']['users']['total']
soft_version = data.json()['software']['version']
alive = True
except:
users = 0
soft_version = ""
elif serv_api == '/api/v1/instance?':
try:
users = data.json()['stats']['user_count']
soft_version = data.json()['version']
alive = True
except:
users = 0
soft_version = ""
if serv_soft == "pleroma" or serv_soft == "diaspora" or serv_soft == "peertube" or serv_soft == "pixelfed" or serv_soft == "hubzilla" or serv_soft == "writefreely" or serv_soft == "friendica":
try:
users = data.json()['usage']['users']['total']
soft_version = data.json()['software']['version']
alive = True
except:
users = 0
soft_version = ""
if serv_soft == "gnusocialv2" or serv_soft == "gnusocial":
try:
users = data.json()['usage']['users']['total']
if users == 0:
users = data.json()['usage']['users']['activeHalfyear']
soft_version = data.json()['software']['version']
alive = True
except:
users = 0
soft_version = ""
if serv_soft == "plume" or serv_soft == 'red' or serv_soft == "misskey" or serv_soft == "zap" or serv_soft == "prismo" or serv_soft == "ravenvale" or serv_soft == "osada" or serv_soft == "groundpolis":
try:
users = data.json()['usage']['users']['total']
soft_version = data.json()['software']['version']
alive = True
except:
users = 0
soft_version = ""
if serv_soft == "ganggo" or serv_soft == "squs" or serv_soft == "dolphin" or serv_soft == "lemmy" or serv_soft == "wordpress":
try:
users = data.json()['usage']['users']['total']
soft_version = data.json()['software']['version']
if serv_soft == "wordpress" and "activitypub" in data.json()['protocols']:
alive = True
elif serv_soft == "wordpress" and "activitypub" not in data.json()['protocols']:
alive = False
else:
alive = True
except:
users = 0
soft_version = ""
if alive:
if soft_version != "" and soft_version is not None:
print("Server " + str(server) + " (" + serv_soft + " " + soft_version + ") is alive!")
else:
print("Server " + str(server) + " (" + serv_soft + ") is alive!")
insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
conn = None
try:
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor()
cur.execute(insert_sql, (server, users, now, serv_soft, alive, serv_api, soft_version))
cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)", (users, now, serv_soft, alive, serv_api, soft_version, server))
cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
except urllib3.exceptions.ProtocolError as protoerr:
print(protoerr)
print("Server " + server + " is dead :-(")
alive = False
pass
except requests.exceptions.ChunkedEncodingError as chunkerr:
print(chunkerr)
print("Server " + server + " is dead :-(")
alive = False
pass
except KeyError as e:
print(e)
print("Server " + server + " is dead :-(")
alive = False
pass
except ValueError as verr:
print(verr)
print("Server " + server + " is dead :-(")
alive = False
pass
except requests.exceptions.SSLError as errssl:
print(errssl)
print("Server " + server + " is dead :-(")
alive = False
pass
except requests.exceptions.HTTPError as errh:
print(errh)
print("Server " + server + " is dead :-(")
alive = False
pass
except requests.exceptions.ConnectionError as errc:
print(errc)
print("Server " + server + " is dead :-(")
alive = False
pass
except requests.exceptions.Timeout as errt:
print(errt)
print("Server " + server + " is dead :-(")
alive = False
pass
except requests.exceptions.RequestException as err:
print(err)
print("Server " + server + " is dead :-(")
alive = False
pass
except socket.gaierror as gai_error:
print(gai_error)
pass
if alive is False:
conn = None
try:
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="",
host="/var/run/postgresql", port="5432")
cur = conn.cursor()
cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s) where server=(%s)", (now, alive, server))
cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
def write_api(server, software, users, alive, api, soft_version):
insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
conn = None
try:
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor()
cur.execute(insert_sql, (server, now, software, users, alive, api, soft_version))
cur.execute("UPDATE fediverse SET updated_at=(%s), software=(%s), users=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)", (now, software, users, alive, api, soft_version, server))
cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
async def getsoft(server):
try:
socket.gethostbyname(server)
except socket.gaierror:
pass
return
soft = ''
url = 'https://' + server
timeout = aiohttp.ClientTimeout(total=3)
async with aiohttp.ClientSession(timeout=timeout) as session:
for api in apis:
try:
async with session.get(url+api) as response:
if response.status == 200:
try:
response_json = await response.json()
except:
pass
except aiohttp.ClientConnectorError as err:
print(err)
pass
else:
if response.status == 200 and api != '/api/v1/instance?':
try:
soft = response_json['software']['name']
soft = soft.lower()
soft_version = response_json['software']['version']
users = response_json['usage']['users']['total']
if users > 1000000:
return
alive = True
write_api(server, soft, users, alive, api, soft_version)
print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
return
except:
pass
if response.status == 200 and soft == '' and api == "/api/v1/instance?":
soft = 'mastodon'
users = response_json['stats']['user_count']
soft_version = response_json['version']
if users > 1000000:
return
alive = True
write_api(server, soft, users, alive, api)
print("Server " + server + " (" + soft + ") is alive!")
def getserver(server, x):
server = server[0].rstrip('.').lower()
if server.find(".") == -1:
return
if server.find("@") != -1:
return
if server.find("/") != -1:
return
if server.find(":") != -1:
return
try:
loop = asyncio.get_event_loop()
coroutines = [getsoft(server)]
soft = loop.run_until_complete(asyncio.gather(*coroutines, return_exceptions=True))
except:
pass
def set_world_servers_check_to_false():
############################################################################
# set all world servers's checked column to False
try:
conn = None
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor()
cur.execute("UPDATE world SET checked='f'")
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
def get_last_checked_servers():
############################################################################
# get last checked servers from fediverse DB
alive_servers = []
try:
conn = None
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor()
# get world servers list
cur.execute("select server from world where server in (select server from fediverse where users_api != '')")
alive_servers = []
for row in cur:
alive_servers.append(row[0])
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
return alive_servers
def mastodon():
# Load secrets from secrets file
secrets_filepath = "secrets/secrets.txt"
uc_client_id = get_parameter("uc_client_id", secrets_filepath)
uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
uc_access_token = get_parameter("uc_access_token", secrets_filepath)
# Load configuration from config file
config_filepath = "config/config.txt"
mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
# Initialise Mastodon API
mastodon = Mastodon(
client_id=uc_client_id,
client_secret=uc_client_secret,
access_token=uc_access_token,
api_base_url='https://' + mastodon_hostname,
)
# Initialise access headers
headers = {'Authorization': 'Bearer %s'%uc_access_token}
return (mastodon, mastodon_hostname)
def db_config():
# Load db configuration from config file
config_filepath = "config/db_config.txt"
fediverse_db = get_parameter("fediverse_db", config_filepath)
fediverse_db_user = get_parameter("fediverse_db_user", config_filepath)
return (fediverse_db, fediverse_db_user)
def usage():
print('usage: python ' + sys.argv[0] + ' --multi' + ' (multiprocessing, fast)')
print('usage: python ' + sys.argv[0] + ' --mono' + ' (one process, slow)')
# Returns the parameter from the specified file
def get_parameter(parameter, file_path):
# Check if secrets file exists
if not os.path.isfile(file_path):
print("File %s not found, exiting."%file_path)
sys.exit(0)
# Find parameter in file
with open(file_path) as f:
for line in f:
if line.startswith(parameter):
return line.replace(parameter + ":", "").strip()
# Cannot find parameter, exit
print(file_path + " Missing parameter %s "%parameter)
sys.exit(0)
###############################################################################
# main
if __name__ == '__main__':
# usage modes
if len(sys.argv) == 1:
usage()
elif len(sys.argv) == 2:
if sys.argv[1] == '--multi':
now = datetime.now()
mastodon, mastodon_hostname = mastodon()
fediverse_db, fediverse_db_user = db_config()
total_servers = 0
total_users = 0
set_world_servers_check_to_false()
alive_servers = get_last_checked_servers()
###########################################################################
# multiprocessing!
nprocs = multiprocessing.cpu_count()
with multiprocessing.Pool(processes=64) as pool:
results = pool.starmap(get_alive_servers, product(alive_servers))
elif sys.argv[1] == '--mono':
now = datetime.now()
mastodon, mastodon_hostname = mastodon()
fediverse_db, fediverse_db_user = db_config()
total_servers = 0
total_users = 0
set_world_servers_check_to_false()
alive_servers = get_last_checked_servers()
i = 0
while i < len(alive_servers):
get_alive_servers(alive_servers[i])
i += 1
else:
usage()
###########################################################################
# get current total servers and users, get users from every software
gettotals_sql = "select count(server), sum(users) from fediverse where alive"
get_soft_totals_sql = "select software, sum(users) as users, count(server) as servers from fediverse where users != 0 and alive group by software order by users desc"
soft_total_project = []
soft_total_users = []
soft_total_servers = []
try:
conn = None
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor()
cur.execute(gettotals_sql)
row = cur.fetchone()
total_servers = row[0]
total_users = row[1]
cur.execute(get_soft_totals_sql)
rows = cur.fetchall()
for row in rows:
soft_total_project.append(row[0])
soft_total_users.append(row[1])
soft_total_servers.append(row[2])
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
###########################################################################
# get last check values and write current total ones
select_sql = "select total_servers, total_users from totals order by datetime desc limit 1"
insert_sql = "INSERT INTO totals(datetime, total_servers, total_users) VALUES(%s,%s,%s)"
try:
conn = None
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor()
cur.execute(select_sql)
row = cur.fetchone()
if row is not None:
servers_before = row[0]
users_before = row[1]
else:
servers_before = 0
users_before = 0
cur.execute(insert_sql, (now, total_servers, total_users))
conn.commit()
cur.close()
evo_servers = total_servers - servers_before
evo_users = total_users - users_before
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
################################################################################
# write evo values
insert_sql = "INSERT INTO evo(datetime, servers, users) VALUES(%s,%s,%s)"
conn = None
try:
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor()
cur.execute(insert_sql, (now, evo_servers, evo_users))
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
##############################################################################
# get world's last update datetime
conn = None
try:
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor()
cur.execute("select updated_at from world order by updated_at desc limit 1")
row = cur.fetchone()
last_update = row[0]
last_update = last_update.strftime('%m/%d/%Y, %H:%M:%S')
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
##############################################################################
# get max servers and users
conn = None
try:
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor()
cur.execute("select MAX(total_servers) from totals")
row = cur.fetchone()
if row is not None:
max_servers = row[0]
else:
max_servers = 0
cur.execute("select MAX(total_users) from totals")
row = cur.fetchone()
if row is not None:
max_users = row[0]
else:
max_users = 0
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
###############################################################################
# get plots
servers_plots = []
users_plots = []
conn = None
try:
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor()
cur.execute("select total_servers, total_users from totals order by datetime desc limit 14")
rows = cur.fetchall()
for row in rows:
servers_plots.append(row[0])
users_plots.append(row[1])
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
###############################################################################
# generate graphs
plt.plot([-6, -5, -4, -3, -2, -1, 0], [servers_plots[6], servers_plots[5], servers_plots[4], servers_plots[3], servers_plots[2], servers_plots[1], servers_plots[0]], marker='o', color='mediumseagreen')
plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_servers, max_servers, max_servers, max_servers, max_servers, max_servers, max_servers], color='red')
plt.title('fediverse: total alive servers (max: ' + str(f"{max_servers:,}" + ')'), loc='right', color='blue')
plt.xlabel('Last seven days')
plt.ylabel('fediverse alive servers')
plt.legend(('servers', 'max'), shadow=True, loc=(0.01, 1.00), handlelength=1.5, fontsize=10)
plt.savefig('servers.png')
plt.close()
plt.plot([-6, -5, -4, -3, -2, -1, 0], [users_plots[6], users_plots[5], users_plots[4], users_plots[3], users_plots[2], users_plots[1], users_plots[0]], marker='o', color='royalblue')
plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_users, max_users, max_users, max_users, max_users, max_users, max_users], color='red')
plt.title('fediverse: total registered users (max: ' + str(f"{max_users:,}" + ')'), loc='right', color='royalblue')
plt.legend(('users', 'max'), shadow=True, loc=(0.01, 0.80), handlelength=1.5, fontsize=10)
plt.xlabel('Last seven days')
plt.ylabel('Registered users')
plt.savefig('users.png')
plt.close()
###############################################################################
# T O O T !
toot_text = "#fediverse alive servers stats" + " \n"
toot_text += "\n"
if evo_servers >= 0:
toot_text += "alive servers: " + str(f"{total_servers:,}") + " (+"+ str(f"{evo_servers:,}") + ") \n"
toot_text += "max: " + str(f"{max_servers:,}") + "\n"
elif evo_servers < 0:
toot_text += "alive servers: " + str(f"{total_servers:,}") + " ("+ str(f"{evo_servers:,}") + ") \n"
toot_text += "max: " + str(f"{max_servers:,}") + "\n"
if evo_users >= 0:
toot_text += "total users: " + str(f"{total_users:,}") + " (+"+ str(f"{evo_users:,}") + ") \n"
toot_text += "max: " + str(f"{max_users:,}") + "\n"
elif evo_users < 0:
toot_text += "total users: " + str(f"{total_users:,}") + " ("+ str(f"{evo_users:,}") + ") \n"
toot_text += "max: " + str(f"{max_users:,}") + "\n"
toot_text += "\n"
toot_text += "top ten (soft users servers):" + " \n"
toot_text += "\n"
i = 0
while i < 10:
project_soft = soft_total_project[i]
project_users = soft_total_users[i]
project_servers = soft_total_servers[i]
len_pr_soft = len(project_soft)
if project_soft == 'writefreely':
str_len = 11
else:
str_len = 13
toot_text += f"{':'+project_soft+':':<11}" + f"{project_users:>{str_len},}" + " " + f"{project_servers:>5,}" + "\n"
i += 1
print("Tooting...")
print(toot_text)
servers_image_id = mastodon.media_post('servers.png', "image/png").id
users_image_id = mastodon.media_post('users.png', "image/png").id
mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, users_image_id})