forkeado de spla/fediverse
New features! Wordpress support and server&users graphs generation
This commit is contained in:
pare
d8c58f5237
commit
3f8a747923
S'han modificat 2 arxius amb 416 adicions i 267 eliminacions
|
@ -65,6 +65,7 @@ Within Python Virtual Environment:
|
||||||
| prismo | nodeinfo/2.0.json | ['software']['name'] |
|
| prismo | nodeinfo/2.0.json | ['software']['name'] |
|
||||||
| ravenvale | nodeinfo/2.0.json | ['software']['name'] |
|
| ravenvale | nodeinfo/2.0.json | ['software']['name'] |
|
||||||
| squs | nodeinfo/2.0 | ['software']['name'] |
|
| squs | nodeinfo/2.0 | ['software']['name'] |
|
||||||
|
| wordpress | wp-json/nodeinfo/2.0 | ['software']['name'] |
|
||||||
| writefreely | api/nodeinfo | ['software']['name'] |
|
| writefreely | api/nodeinfo | ['software']['name'] |
|
||||||
| zap | nodeinfo/2.0.json | ['software']['name'] |
|
| zap | nodeinfo/2.0.json | ['software']['name'] |
|
||||||
|
|
||||||
|
@ -72,4 +73,6 @@ Within Python Virtual Environment:
|
||||||
|
|
||||||
Note: install all needed packages with 'pip install package' or use 'pip install -r requirements.txt' to install them.
|
Note: install all needed packages with 'pip install package' or use 'pip install -r requirements.txt' to install them.
|
||||||
|
|
||||||
18.2.21 - New feature! Added [Lemmy project](https://join.lemmy.ml)
|
18.2.21 - New feature! Added [Lemmy project](https://join.lemmy.ml)
|
||||||
|
12.5.21 - New feature! Added Wordpress support. The code can now detect Wordpress instances with ActivityPub enabled plugin.
|
||||||
|
12.5.21 - New feature! New shinny creation of servers and users graphs.
|
||||||
|
|
674
fediverse.py
674
fediverse.py
|
@ -1,8 +1,4 @@
|
||||||
#!/usr/bin/env python
|
|
||||||
# -*- coding: utf-8 -*-
|
|
||||||
|
|
||||||
import time
|
import time
|
||||||
start_time = time.time()
|
|
||||||
import urllib3
|
import urllib3
|
||||||
from urllib3 import exceptions
|
from urllib3 import exceptions
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
@ -32,10 +28,17 @@ from asyncio import TimeoutError
|
||||||
import socket
|
import socket
|
||||||
from socket import gaierror, gethostbyname
|
from socket import gaierror, gethostbyname
|
||||||
|
|
||||||
|
import matplotlib
|
||||||
|
import matplotlib.pyplot as plt
|
||||||
|
|
||||||
|
plt.style.use('seaborn')
|
||||||
|
|
||||||
|
start_time = time.time()
|
||||||
|
|
||||||
from decimal import *
|
from decimal import *
|
||||||
getcontext().prec = 2
|
getcontext().prec = 2
|
||||||
|
|
||||||
apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?', '/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?']
|
apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?', '/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?','/wp-json/nodeinfo/2.0?']
|
||||||
|
|
||||||
client_exceptions = (
|
client_exceptions = (
|
||||||
aiohttp.ClientResponseError,
|
aiohttp.ClientResponseError,
|
||||||
|
@ -46,12 +49,6 @@ client_exceptions = (
|
||||||
socket.gaierror,
|
socket.gaierror,
|
||||||
)
|
)
|
||||||
|
|
||||||
now = datetime.now()
|
|
||||||
|
|
||||||
###############################################################################
|
|
||||||
# INITIALISATION
|
|
||||||
###############################################################################
|
|
||||||
|
|
||||||
def is_json(myjson):
|
def is_json(myjson):
|
||||||
try:
|
try:
|
||||||
json_object = json.loads(myjson)
|
json_object = json.loads(myjson)
|
||||||
|
@ -59,9 +56,11 @@ def is_json(myjson):
|
||||||
return False
|
return False
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def alive_server(server, x):
|
def get_alive_servers(server):
|
||||||
|
|
||||||
server = server[0].rstrip('.').lower()
|
serv_api = ''
|
||||||
|
serv_soft = ''
|
||||||
|
soft_version = ''
|
||||||
|
|
||||||
conn = None
|
conn = None
|
||||||
|
|
||||||
|
@ -150,46 +149,61 @@ def alive_server(server, x):
|
||||||
except:
|
except:
|
||||||
users = 0
|
users = 0
|
||||||
soft_version = ""
|
soft_version = ""
|
||||||
if serv_soft == "ganggo" or serv_soft == "squs" or serv_soft == "dolphin":
|
if serv_soft == "ganggo" or serv_soft == "squs" or serv_soft == "dolphin" or serv_soft == "lemmy" or serv_soft == "wordpress":
|
||||||
try:
|
try:
|
||||||
users = data.json()['usage']['users']['total']
|
users = data.json()['usage']['users']['total']
|
||||||
soft_version = data.json()['software']['version']
|
soft_version = data.json()['software']['version']
|
||||||
alive = True
|
if serv_soft == "wordpress" and "activitypub" in data.json()['protocols']:
|
||||||
|
|
||||||
|
alive = True
|
||||||
|
|
||||||
|
elif serv_soft == "wordpress" and "activitypub" not in data.json()['protocols']:
|
||||||
|
|
||||||
|
alive = False
|
||||||
|
|
||||||
|
else:
|
||||||
|
|
||||||
|
alive = True
|
||||||
|
|
||||||
except:
|
except:
|
||||||
|
|
||||||
users = 0
|
users = 0
|
||||||
soft_version = ""
|
soft_version = ""
|
||||||
|
|
||||||
if soft_version != "" and soft_version != None:
|
if alive:
|
||||||
print("Server " + str(server) + " (" + serv_soft + " " + soft_version + ") is alive!")
|
|
||||||
else:
|
|
||||||
print("Server " + str(server) + " (" + serv_soft + ") is alive!")
|
|
||||||
|
|
||||||
insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
|
if soft_version != "" and soft_version != None:
|
||||||
conn = None
|
print("Server " + str(server) + " (" + serv_soft + " " + soft_version + ") is alive!")
|
||||||
|
else:
|
||||||
|
print("Server " + str(server) + " (" + serv_soft + ") is alive!")
|
||||||
|
|
||||||
try:
|
insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
|
||||||
|
|
||||||
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
conn = None
|
||||||
|
|
||||||
cur = conn.cursor()
|
try:
|
||||||
|
|
||||||
cur.execute(insert_sql, (server, users, now, serv_soft, alive, serv_api, soft_version))
|
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||||
|
|
||||||
cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)", (users, now, serv_soft, alive, serv_api, soft_version, server))
|
cur = conn.cursor()
|
||||||
|
|
||||||
cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
|
cur.execute(insert_sql, (server, users, now, serv_soft, alive, serv_api, soft_version))
|
||||||
|
|
||||||
conn.commit()
|
cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)", (users, now, serv_soft, alive, serv_api, soft_version, server))
|
||||||
|
|
||||||
cur.close()
|
cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
|
||||||
|
|
||||||
except (Exception, psycopg2.DatabaseError) as error:
|
conn.commit()
|
||||||
print(error)
|
|
||||||
|
|
||||||
finally:
|
cur.close()
|
||||||
|
|
||||||
if conn is not None:
|
except (Exception, psycopg2.DatabaseError) as error:
|
||||||
conn.close()
|
print(error)
|
||||||
|
|
||||||
|
finally:
|
||||||
|
|
||||||
|
if conn is not None:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
except urllib3.exceptions.ProtocolError as protoerr:
|
except urllib3.exceptions.ProtocolError as protoerr:
|
||||||
|
|
||||||
|
@ -245,6 +259,11 @@ def alive_server(server, x):
|
||||||
alive = False
|
alive = False
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
except socket.gaierror as gai_error:
|
||||||
|
|
||||||
|
print(gai_error)
|
||||||
|
pass
|
||||||
|
|
||||||
if alive == False:
|
if alive == False:
|
||||||
|
|
||||||
conn = None
|
conn = None
|
||||||
|
@ -311,6 +330,7 @@ async def getsoft(server):
|
||||||
|
|
||||||
except socket.gaierror:
|
except socket.gaierror:
|
||||||
|
|
||||||
|
pass
|
||||||
return
|
return
|
||||||
|
|
||||||
soft = ''
|
soft = ''
|
||||||
|
@ -377,56 +397,7 @@ def getserver(server, x):
|
||||||
|
|
||||||
pass
|
pass
|
||||||
|
|
||||||
# Returns the parameter from the specified file
|
def set_world_servers_check_to_false():
|
||||||
def get_parameter( parameter, file_path ):
|
|
||||||
# Check if secrets file exists
|
|
||||||
if not os.path.isfile(file_path):
|
|
||||||
print("File %s not found, exiting."%file_path)
|
|
||||||
sys.exit(0)
|
|
||||||
|
|
||||||
# Find parameter in file
|
|
||||||
with open( file_path ) as f:
|
|
||||||
for line in f:
|
|
||||||
if line.startswith( parameter ):
|
|
||||||
return line.replace(parameter + ":", "").strip()
|
|
||||||
|
|
||||||
# Cannot find parameter, exit
|
|
||||||
print(file_path + " Missing parameter %s "%parameter)
|
|
||||||
sys.exit(0)
|
|
||||||
|
|
||||||
# Load secrets from secrets file
|
|
||||||
secrets_filepath = "secrets/secrets.txt"
|
|
||||||
uc_client_id = get_parameter("uc_client_id", secrets_filepath)
|
|
||||||
uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
|
|
||||||
uc_access_token = get_parameter("uc_access_token", secrets_filepath)
|
|
||||||
|
|
||||||
# Load configuration from config file
|
|
||||||
config_filepath = "config/config.txt"
|
|
||||||
mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
|
|
||||||
|
|
||||||
# Load database config from db_config file
|
|
||||||
db_config_filepath = "config/db_config.txt"
|
|
||||||
fediverse_db = get_parameter("fediverse_db", db_config_filepath)
|
|
||||||
fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath)
|
|
||||||
|
|
||||||
# Initialise Mastodon API
|
|
||||||
mastodon = Mastodon(
|
|
||||||
client_id = uc_client_id,
|
|
||||||
client_secret = uc_client_secret,
|
|
||||||
access_token = uc_access_token,
|
|
||||||
api_base_url = 'https://' + mastodon_hostname,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Initialise access headers
|
|
||||||
headers={ 'Authorization': 'Bearer %s'%uc_access_token }
|
|
||||||
|
|
||||||
###############################################################################
|
|
||||||
# main
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
|
|
||||||
total_servers = 0
|
|
||||||
total_users = 0
|
|
||||||
|
|
||||||
############################################################################
|
############################################################################
|
||||||
# set all world servers's checked column to False
|
# set all world servers's checked column to False
|
||||||
|
@ -455,9 +426,10 @@ if __name__ == '__main__':
|
||||||
|
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
############################################################################
|
def get_last_checked_servers():
|
||||||
# get last check servers from fediverse DB
|
|
||||||
|
|
||||||
|
############################################################################
|
||||||
|
# get last checked servers from fediverse DB
|
||||||
|
|
||||||
alive_servers = []
|
alive_servers = []
|
||||||
|
|
||||||
|
@ -490,264 +462,438 @@ if __name__ == '__main__':
|
||||||
|
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
###########################################################################
|
return alive_servers
|
||||||
# multiprocessing!
|
|
||||||
|
|
||||||
m = Manager()
|
def mastodon():
|
||||||
q = m.Queue()
|
|
||||||
z = zip(alive_servers)
|
|
||||||
|
|
||||||
serv_number = len(alive_servers)
|
# Load secrets from secrets file
|
||||||
|
secrets_filepath = "secrets/secrets.txt"
|
||||||
|
uc_client_id = get_parameter("uc_client_id", secrets_filepath)
|
||||||
|
uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
|
||||||
|
uc_access_token = get_parameter("uc_access_token", secrets_filepath)
|
||||||
|
|
||||||
pool_tuple = [(x, q) for x in z]
|
# Load configuration from config file
|
||||||
with Pool(processes=64) as pool:
|
config_filepath = "config/config.txt"
|
||||||
pool.starmap(alive_server, pool_tuple)
|
mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
|
||||||
|
|
||||||
###########################################################################
|
# Initialise Mastodon API
|
||||||
|
mastodon = Mastodon(
|
||||||
|
client_id = uc_client_id,
|
||||||
|
client_secret = uc_client_secret,
|
||||||
|
access_token = uc_access_token,
|
||||||
|
api_base_url = 'https://' + mastodon_hostname,
|
||||||
|
)
|
||||||
|
|
||||||
print("Getting the remaining servers from world")
|
# Initialise access headers
|
||||||
|
headers={ 'Authorization': 'Bearer %s'%uc_access_token }
|
||||||
|
|
||||||
world_servers = []
|
return (mastodon, mastodon_hostname)
|
||||||
|
|
||||||
try:
|
def db_config():
|
||||||
|
|
||||||
conn = None
|
# Load db configuration from config file
|
||||||
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
config_filepath = "config/db_config.txt"
|
||||||
|
fediverse_db = get_parameter("fediverse_db", config_filepath)
|
||||||
|
fediverse_db_user = get_parameter("fediverse_db_user", config_filepath)
|
||||||
|
|
||||||
cur = conn.cursor()
|
return (fediverse_db, fediverse_db_user)
|
||||||
|
|
||||||
### get world servers list
|
def usage():
|
||||||
|
|
||||||
cur.execute("select server from world where checked='f'")
|
print('usage: python ' + sys.argv[0] + ' --multi' + ' (multiprocessing, fast)')
|
||||||
|
print('usage: python ' + sys.argv[0] + ' --mono' + ' (one process, slow)')
|
||||||
|
|
||||||
for row in cur:
|
# Returns the parameter from the specified file
|
||||||
|
def get_parameter( parameter, file_path ):
|
||||||
|
# Check if secrets file exists
|
||||||
|
if not os.path.isfile(file_path):
|
||||||
|
print("File %s not found, exiting."%file_path)
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
world_servers.append(row[0])
|
# Find parameter in file
|
||||||
|
with open( file_path ) as f:
|
||||||
|
for line in f:
|
||||||
|
if line.startswith( parameter ):
|
||||||
|
return line.replace(parameter + ":", "").strip()
|
||||||
|
|
||||||
cur.close()
|
# Cannot find parameter, exit
|
||||||
|
print(file_path + " Missing parameter %s "%parameter)
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
print("Remaining servers: " + str(len(world_servers)))
|
###############################################################################
|
||||||
|
# main
|
||||||
|
|
||||||
except (Exception, psycopg2.DatabaseError) as error:
|
if __name__ == '__main__':
|
||||||
|
|
||||||
print(error)
|
# usage modes
|
||||||
|
|
||||||
finally:
|
if len(sys.argv) == 1:
|
||||||
|
|
||||||
if conn is not None:
|
usage()
|
||||||
|
|
||||||
conn.close()
|
elif len(sys.argv) == 2:
|
||||||
|
|
||||||
###########################################################################
|
if sys.argv[1] == '--multi':
|
||||||
# multiprocessing!
|
|
||||||
|
|
||||||
m = Manager()
|
now = datetime.now()
|
||||||
q = m.Queue()
|
|
||||||
z = zip(world_servers)
|
|
||||||
|
|
||||||
serv_number = len(world_servers)
|
mastodon, mastodon_hostname = mastodon()
|
||||||
|
|
||||||
pool_tuple = [(x, q) for x in z]
|
fediverse_db, fediverse_db_user = db_config()
|
||||||
with Pool(processes=64) as pool:
|
|
||||||
pool.starmap(getserver, pool_tuple)
|
|
||||||
|
|
||||||
###########################################################################
|
total_servers = 0
|
||||||
# delete not alive servers from fediverse table
|
total_users = 0
|
||||||
|
|
||||||
try:
|
set_world_servers_check_to_false()
|
||||||
|
|
||||||
conn = None
|
alive_servers = get_last_checked_servers()
|
||||||
|
|
||||||
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
###########################################################################
|
||||||
|
# multiprocessing!
|
||||||
|
|
||||||
cur = conn.cursor()
|
nprocs = multiprocessing.cpu_count()
|
||||||
|
with multiprocessing.Pool(processes=64) as pool:
|
||||||
|
results = pool.starmap(get_alive_servers, product(alive_servers))
|
||||||
|
|
||||||
cur.execute("delete from fediverse where not alive")
|
elif sys.argv[1] == '--mono':
|
||||||
|
|
||||||
conn.commit()
|
now = datetime.now()
|
||||||
|
|
||||||
cur.close()
|
mastodon, mastodon_hostname = mastodon()
|
||||||
|
|
||||||
except (Exception, psycopg2.DatabaseError) as error:
|
fediverse_db, fediverse_db_user = db_config()
|
||||||
|
|
||||||
print(error)
|
total_servers = 0
|
||||||
|
total_users = 0
|
||||||
|
|
||||||
finally:
|
set_world_servers_check_to_false()
|
||||||
|
|
||||||
if conn is not None:
|
alive_servers = get_last_checked_servers()
|
||||||
|
|
||||||
conn.close()
|
i = 0
|
||||||
|
|
||||||
###########################################################################
|
while i < len(alive_servers):
|
||||||
# get current total servers and users, get users from every software
|
|
||||||
|
|
||||||
gettotals_sql = "select count(server), sum(users) from fediverse where alive"
|
get_alive_servers(alive_servers[i])
|
||||||
get_soft_totals_sql = "select software, sum(users) as users, count(server) as servers from fediverse where users != 0 and alive group by software order by users desc"
|
|
||||||
soft_total_project = []
|
|
||||||
soft_total_users = []
|
|
||||||
soft_total_servers = []
|
|
||||||
|
|
||||||
try:
|
i += 1
|
||||||
|
|
||||||
conn = None
|
|
||||||
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
|
||||||
|
|
||||||
cur = conn.cursor()
|
|
||||||
|
|
||||||
cur.execute(gettotals_sql)
|
|
||||||
|
|
||||||
row = cur.fetchone()
|
|
||||||
|
|
||||||
total_servers = row[0]
|
|
||||||
total_users = row[1]
|
|
||||||
|
|
||||||
cur.execute(get_soft_totals_sql)
|
|
||||||
rows = cur.fetchall()
|
|
||||||
for row in rows:
|
|
||||||
soft_total_project.append(row[0])
|
|
||||||
soft_total_users.append(row[1])
|
|
||||||
soft_total_servers.append(row[2])
|
|
||||||
cur.close()
|
|
||||||
|
|
||||||
except (Exception, psycopg2.DatabaseError) as error:
|
|
||||||
|
|
||||||
print(error)
|
|
||||||
|
|
||||||
finally:
|
|
||||||
|
|
||||||
if conn is not None:
|
|
||||||
|
|
||||||
conn.close()
|
|
||||||
|
|
||||||
###########################################################################
|
|
||||||
# get last check values and write current total ones
|
|
||||||
|
|
||||||
select_sql = "select total_servers, total_users from totals order by datetime desc limit 1"
|
|
||||||
insert_sql = "INSERT INTO totals(datetime, total_servers, total_users) VALUES(%s,%s,%s)"
|
|
||||||
|
|
||||||
try:
|
|
||||||
|
|
||||||
conn = None
|
|
||||||
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
|
||||||
|
|
||||||
cur = conn.cursor()
|
|
||||||
|
|
||||||
cur.execute(select_sql)
|
|
||||||
|
|
||||||
row = cur.fetchone()
|
|
||||||
|
|
||||||
if row != None:
|
|
||||||
|
|
||||||
servers_before = row[0]
|
|
||||||
users_before = row[1]
|
|
||||||
|
|
||||||
else:
|
else:
|
||||||
|
|
||||||
servers_before = 0
|
usage()
|
||||||
users_before = 0
|
|
||||||
|
|
||||||
cur.execute(insert_sql, (now, total_servers, total_users))
|
###########################################################################
|
||||||
|
# get current total servers and users, get users from every software
|
||||||
|
|
||||||
conn.commit()
|
gettotals_sql = "select count(server), sum(users) from fediverse where alive"
|
||||||
|
get_soft_totals_sql = "select software, sum(users) as users, count(server) as servers from fediverse where users != 0 and alive group by software order by users desc"
|
||||||
|
soft_total_project = []
|
||||||
|
soft_total_users = []
|
||||||
|
soft_total_servers = []
|
||||||
|
|
||||||
cur.close()
|
try:
|
||||||
|
|
||||||
evo_servers = total_servers - servers_before
|
conn = None
|
||||||
evo_users = total_users - users_before
|
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||||
|
|
||||||
except (Exception, psycopg2.DatabaseError) as error:
|
cur = conn.cursor()
|
||||||
|
|
||||||
print(error)
|
cur.execute(gettotals_sql)
|
||||||
|
|
||||||
finally:
|
row = cur.fetchone()
|
||||||
|
|
||||||
if conn is not None:
|
total_servers = row[0]
|
||||||
|
total_users = row[1]
|
||||||
|
|
||||||
conn.close()
|
cur.execute(get_soft_totals_sql)
|
||||||
|
rows = cur.fetchall()
|
||||||
|
for row in rows:
|
||||||
|
soft_total_project.append(row[0])
|
||||||
|
soft_total_users.append(row[1])
|
||||||
|
soft_total_servers.append(row[2])
|
||||||
|
cur.close()
|
||||||
|
|
||||||
################################################################################
|
except (Exception, psycopg2.DatabaseError) as error:
|
||||||
# write evo values
|
|
||||||
|
|
||||||
insert_sql = "INSERT INTO evo(datetime, servers, users) VALUES(%s,%s,%s)"
|
print(error)
|
||||||
|
|
||||||
conn = None
|
finally:
|
||||||
|
|
||||||
try:
|
if conn is not None:
|
||||||
|
|
||||||
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
conn.close()
|
||||||
|
|
||||||
cur = conn.cursor()
|
###########################################################################
|
||||||
|
# get last check values and write current total ones
|
||||||
|
|
||||||
cur.execute(insert_sql, (now, evo_servers, evo_users))
|
select_sql = "select total_servers, total_users from totals order by datetime desc limit 1"
|
||||||
|
insert_sql = "INSERT INTO totals(datetime, total_servers, total_users) VALUES(%s,%s,%s)"
|
||||||
|
|
||||||
conn.commit()
|
try:
|
||||||
|
|
||||||
cur.close()
|
conn = None
|
||||||
|
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||||
|
|
||||||
except (Exception, psycopg2.DatabaseError) as error:
|
cur = conn.cursor()
|
||||||
|
|
||||||
print(error)
|
cur.execute(select_sql)
|
||||||
|
|
||||||
finally:
|
row = cur.fetchone()
|
||||||
|
|
||||||
if conn is not None:
|
if row != None:
|
||||||
|
|
||||||
conn.close()
|
servers_before = row[0]
|
||||||
|
users_before = row[1]
|
||||||
|
|
||||||
##############################################################################
|
else:
|
||||||
# get world's last update datetime
|
|
||||||
|
|
||||||
conn = None
|
servers_before = 0
|
||||||
|
users_before = 0
|
||||||
|
|
||||||
try:
|
cur.execute(insert_sql, (now, total_servers, total_users))
|
||||||
|
|
||||||
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
conn.commit()
|
||||||
|
|
||||||
cur = conn.cursor()
|
cur.close()
|
||||||
|
|
||||||
cur.execute("select updated_at from world order by updated_at desc limit 1")
|
evo_servers = total_servers - servers_before
|
||||||
|
evo_users = total_users - users_before
|
||||||
|
|
||||||
row = cur.fetchone()
|
except (Exception, psycopg2.DatabaseError) as error:
|
||||||
|
|
||||||
last_update = row[0]
|
print(error)
|
||||||
last_update = last_update.strftime('%m/%d/%Y, %H:%M:%S')
|
|
||||||
|
|
||||||
cur.close()
|
finally:
|
||||||
|
|
||||||
except (Exception, psycopg2.DatabaseError) as error:
|
if conn is not None:
|
||||||
|
|
||||||
print(error)
|
conn.close()
|
||||||
|
|
||||||
finally:
|
################################################################################
|
||||||
|
# write evo values
|
||||||
|
|
||||||
if conn is not None:
|
insert_sql = "INSERT INTO evo(datetime, servers, users) VALUES(%s,%s,%s)"
|
||||||
|
|
||||||
conn.close()
|
conn = None
|
||||||
|
|
||||||
###############################################################################
|
try:
|
||||||
# T O O T !
|
|
||||||
|
|
||||||
toot_text = "#fediverse alive servers stats" + " \n"
|
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||||
toot_text += "\n"
|
|
||||||
if evo_servers >= 0:
|
|
||||||
toot_text += "alive servers: " + str(total_servers) + " (+"+ str(evo_servers) + ") \n"
|
|
||||||
elif evo_servers < 0:
|
|
||||||
toot_text += "alive servers: " + str(total_servers) + " ("+ str(evo_servers) + ") \n"
|
|
||||||
if evo_users >= 0:
|
|
||||||
toot_text += "total users: " + str(total_users) + " (+"+ str(evo_users) + ") \n"
|
|
||||||
elif evo_users < 0:
|
|
||||||
toot_text += "total users: " + str(total_users) + " ("+ str(evo_users) + ") \n"
|
|
||||||
toot_text += "\n"
|
|
||||||
toot_text += "top five (soft: users servers):" + " \n"
|
|
||||||
toot_text += "\n"
|
|
||||||
toot_text += '{:<8}: {:>7} {:>4}'.format(str(soft_total_project[0]), str(soft_total_users[0]), str(soft_total_servers[0])) + " \n"
|
|
||||||
toot_text += '{:<8}: {:>7} {:>4}'.format(str(soft_total_project[1]), str(soft_total_users[1]), str(soft_total_servers[1])) + " \n"
|
|
||||||
toot_text += '{:<8}: {:>7} {:>4}'.format(str(soft_total_project[2]), str(soft_total_users[2]), str(soft_total_servers[2])) + " \n"
|
|
||||||
toot_text += '{:<8}: {:>7} {:>4}'.format(str(soft_total_project[3]), str(soft_total_users[3]), str(soft_total_servers[3])) + " \n"
|
|
||||||
toot_text += '{:<8}: {:>7} {:>4}'.format(str(soft_total_project[4]), str(soft_total_users[4]), str(soft_total_servers[4])) + " \n"
|
|
||||||
toot_text += "\n"
|
|
||||||
toot_text += "updated at " + str(last_update) + " \n"
|
|
||||||
print("Tooting...")
|
|
||||||
print(toot_text)
|
|
||||||
|
|
||||||
mastodon.status_post(toot_text, in_reply_to_id=None, )
|
cur = conn.cursor()
|
||||||
|
|
||||||
|
cur.execute(insert_sql, (now, evo_servers, evo_users))
|
||||||
|
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
cur.close()
|
||||||
|
|
||||||
|
except (Exception, psycopg2.DatabaseError) as error:
|
||||||
|
|
||||||
|
print(error)
|
||||||
|
|
||||||
|
finally:
|
||||||
|
|
||||||
|
if conn is not None:
|
||||||
|
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
##############################################################################
|
||||||
|
# get world's last update datetime
|
||||||
|
|
||||||
|
conn = None
|
||||||
|
|
||||||
|
try:
|
||||||
|
|
||||||
|
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||||
|
|
||||||
|
cur = conn.cursor()
|
||||||
|
|
||||||
|
cur.execute("select updated_at from world order by updated_at desc limit 1")
|
||||||
|
|
||||||
|
row = cur.fetchone()
|
||||||
|
|
||||||
|
last_update = row[0]
|
||||||
|
last_update = last_update.strftime('%m/%d/%Y, %H:%M:%S')
|
||||||
|
|
||||||
|
cur.close()
|
||||||
|
|
||||||
|
except (Exception, psycopg2.DatabaseError) as error:
|
||||||
|
|
||||||
|
print(error)
|
||||||
|
|
||||||
|
finally:
|
||||||
|
|
||||||
|
if conn is not None:
|
||||||
|
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
##############################################################################
|
||||||
|
# get max servers and users
|
||||||
|
|
||||||
|
conn = None
|
||||||
|
|
||||||
|
try:
|
||||||
|
|
||||||
|
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||||
|
|
||||||
|
cur = conn.cursor()
|
||||||
|
|
||||||
|
cur.execute("select MAX(total_servers) from totals")
|
||||||
|
|
||||||
|
row = cur.fetchone()
|
||||||
|
|
||||||
|
if row != None:
|
||||||
|
|
||||||
|
max_servers = row[0]
|
||||||
|
|
||||||
|
else:
|
||||||
|
|
||||||
|
max_servers = 0
|
||||||
|
|
||||||
|
cur.execute("select MAX(total_users) from totals")
|
||||||
|
|
||||||
|
row = cur.fetchone()
|
||||||
|
|
||||||
|
if row != None:
|
||||||
|
|
||||||
|
max_users = row[0]
|
||||||
|
|
||||||
|
else:
|
||||||
|
|
||||||
|
max_users = 0
|
||||||
|
|
||||||
|
cur.close()
|
||||||
|
|
||||||
|
except (Exception, psycopg2.DatabaseError) as error:
|
||||||
|
|
||||||
|
print(error)
|
||||||
|
|
||||||
|
finally:
|
||||||
|
|
||||||
|
if conn is not None:
|
||||||
|
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
###############################################################################
|
||||||
|
# get plots
|
||||||
|
|
||||||
|
servers_plots = []
|
||||||
|
|
||||||
|
users_plots = []
|
||||||
|
|
||||||
|
conn = None
|
||||||
|
|
||||||
|
try:
|
||||||
|
|
||||||
|
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||||
|
|
||||||
|
cur = conn.cursor()
|
||||||
|
|
||||||
|
cur.execute("select total_servers, total_users from totals order by datetime desc limit 14")
|
||||||
|
|
||||||
|
rows = cur.fetchall()
|
||||||
|
|
||||||
|
for row in rows:
|
||||||
|
|
||||||
|
servers_plots.append(row[0])
|
||||||
|
|
||||||
|
users_plots.append(row[1])
|
||||||
|
|
||||||
|
cur.close()
|
||||||
|
|
||||||
|
except (Exception, psycopg2.DatabaseError) as error:
|
||||||
|
|
||||||
|
print(error)
|
||||||
|
|
||||||
|
finally:
|
||||||
|
|
||||||
|
if conn is not None:
|
||||||
|
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
###############################################################################
|
||||||
|
# generate graphs
|
||||||
|
|
||||||
|
plt.plot([-6, -5, -4, -3, -2, -1, 0], [servers_plots[6], servers_plots[5],servers_plots[4],servers_plots[3],servers_plots[2],servers_plots[1],servers_plots[0]], marker='o', color='mediumseagreen')
|
||||||
|
|
||||||
|
plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_servers, max_servers, max_servers, max_servers, max_servers, max_servers, max_servers], color='red')
|
||||||
|
|
||||||
|
plt.title('fediverse: total alive servers (max: ' + str(f"{max_servers:,}" + ')') , loc='right', color='blue')
|
||||||
|
|
||||||
|
plt.xlabel('Last seven days')
|
||||||
|
|
||||||
|
plt.ylabel('fediverse alive servers')
|
||||||
|
|
||||||
|
plt.legend(('servers', 'max'), shadow=True, loc=(0.01, 1.00), handlelength=1.5, fontsize=10)
|
||||||
|
|
||||||
|
plt.savefig('servers.png')
|
||||||
|
|
||||||
|
plt.close()
|
||||||
|
|
||||||
|
plt.plot([-6, -5, -4, -3, -2, -1, 0], [users_plots[6], users_plots[5], users_plots[4], users_plots[3], users_plots[2], users_plots[1], users_plots[0]], marker='o', color='royalblue')
|
||||||
|
|
||||||
|
plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_users, max_users, max_users, max_users, max_users, max_users, max_users], color='red')
|
||||||
|
|
||||||
|
plt.title('fediverse: total registered users (max: ' + str(f"{max_users:,}" + ')') , loc='right', color='royalblue')
|
||||||
|
|
||||||
|
plt.legend(('users', 'max'), shadow=True, loc=(0.01, 0.80), handlelength=1.5, fontsize=10)
|
||||||
|
|
||||||
|
plt.xlabel('Last seven days')
|
||||||
|
|
||||||
|
plt.ylabel('Registered users')
|
||||||
|
|
||||||
|
plt.savefig('users.png')
|
||||||
|
|
||||||
|
plt.close()
|
||||||
|
|
||||||
|
###############################################################################
|
||||||
|
# T O O T !
|
||||||
|
|
||||||
|
toot_text = "#fediverse alive servers stats" + " \n"
|
||||||
|
toot_text += "\n"
|
||||||
|
if evo_servers >= 0:
|
||||||
|
toot_text += "alive servers: " + str(f"{total_servers:,}") + " (+"+ str(f"{evo_servers:,}") + ") \n"
|
||||||
|
toot_text += "max: " + str(f"{max_servers:,}") + "\n"
|
||||||
|
elif evo_servers < 0:
|
||||||
|
toot_text += "alive servers: " + str(f"{total_servers:,}") + " ("+ str(f"{evo_servers:,}") + ") \n"
|
||||||
|
toot_text += "max: " + str(f"{max_servers:,}") + "\n"
|
||||||
|
if evo_users >= 0:
|
||||||
|
toot_text += "total users: " + str(f"{total_users:,}") + " (+"+ str(f"{evo_users:,}") + ") \n"
|
||||||
|
toot_text += "max: " + str(f"{max_users:,}") + "\n"
|
||||||
|
elif evo_users < 0:
|
||||||
|
toot_text += "total users: " + str(f"{total_users:,}") + " ("+ str(f"{evo_users:,}") + ") \n"
|
||||||
|
toot_text += "max: " + str(f"{max_users:,}") + "\n"
|
||||||
|
|
||||||
|
toot_text += "\n"
|
||||||
|
toot_text += "top ten (soft users servers):" + " \n"
|
||||||
|
toot_text += "\n"
|
||||||
|
|
||||||
|
i = 0
|
||||||
|
while i < 10:
|
||||||
|
|
||||||
|
project_soft = soft_total_project[i]
|
||||||
|
project_users = soft_total_users[i]
|
||||||
|
project_servers = soft_total_servers[i]
|
||||||
|
len_pr_soft = len(project_soft)
|
||||||
|
|
||||||
|
if project_soft == 'writefreely':
|
||||||
|
|
||||||
|
str_len = 11
|
||||||
|
|
||||||
|
else:
|
||||||
|
|
||||||
|
str_len = 13
|
||||||
|
|
||||||
|
toot_text += f"{':'+project_soft+':':<11}" + f"{project_users:>{str_len},}" + " " + f"{project_servers:>5,}" + "\n"
|
||||||
|
|
||||||
|
i += 1
|
||||||
|
|
||||||
|
print("Tooting...")
|
||||||
|
print(toot_text)
|
||||||
|
|
||||||
|
servers_image_id = mastodon.media_post('servers.png', "image/png").id
|
||||||
|
|
||||||
|
users_image_id = mastodon.media_post('users.png', "image/png").id
|
||||||
|
|
||||||
|
mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, users_image_id})
|
||||||
|
|
Loading…
Referencia en una nova incidència