From d82ad780d250c2bf838cdf126a55ad180efa911a Mon Sep 17 00:00:00 2001 From: spla Date: Sun, 4 Sep 2022 09:54:27 +0200 Subject: [PATCH] Removed registered users info. Added Monthly Active Users (MAU) --- db-setup.py | 6 +- fediverse.py | 168 ++++++++++++++++++++++++++++++++++++++++-------- fetchservers.py | 2 + 3 files changed, 148 insertions(+), 28 deletions(-) diff --git a/db-setup.py b/db-setup.py index 65bfdca..c2f9d80 100644 --- a/db-setup.py +++ b/db-setup.py @@ -151,7 +151,11 @@ if __name__ == '__main__': create_table(db, db_user, table, sql) table = "fediverse" - sql = "create table "+table+" (server varchar(200) PRIMARY KEY, users INT, updated_at timestamptz, software varchar(50), version varchar(100), first_checked_at timestamptz, last_checked_at timestamptz, downs int)" + sql = "create table "+table+" (server varchar(200) PRIMARY KEY, users INT, updated_at timestamptz, software varchar(50), alive boolean, users_api varchar(50), version varchar(100), first_checked_at timestamptz, last_checked_at timestamptz, downs int)" + create_table(db, db_user, table, sql) + + table = "blocked" + sql = "create table "+table+" (server varchar(200) PRIMARY KEY, users INT, updated_at timestamptz, software varchar(50), alive boolean, users_api varchar(50), version varchar(100), first_checked_at timestamptz, last_checked_at timestamptz, downs int)" create_table(db, db_user, table, sql) table = "totals" diff --git a/fediverse.py b/fediverse.py index 77e5951..0f9a2d0 100644 --- a/fediverse.py +++ b/fediverse.py @@ -77,6 +77,14 @@ class Server: users = data.json()['usage']['users']['activeHalfyear'] + if 'activeMonth' in data.json()['usage']['users'].keys(): + + mau = data.json()['usage']['users']['activeMonth'] + + else: + + mau = 0 + if software == 'socialhome': soft_version = data.json()['server']['version'] @@ -109,6 +117,7 @@ class Server: users = data.json()['stats']['user_count'] soft_version = data.json()['version'] alive = True + mau = 0 except: soft_version = "" @@ -126,7 +135,13 @@ class Server: print(f'\n** Server {self} ({software}) is alive! **') - write_alive_server(self, software, soft_version, alive, api, users, downs, first_checked_at) + if software != 'birdsitelive': + + write_alive_server(self, software, soft_version, alive, api, users, downs, first_checked_at, mau) + + else: + + write_blocked_software(self, software, soft_version, alive, api, users, downs, first_checked_at) except urllib3.exceptions.ProtocolError as protoerr: @@ -180,6 +195,8 @@ class Server: if not alive: + mau = 0 + if downs_qty != None: downs = downs_qty + 1 @@ -190,11 +207,49 @@ class Server: write_not_alive_server(self, software, soft_version, alive, api, users, downs, first_checked_at) - return (self, software, soft_version, alive, api, users, downs, first_checked_at) + return (self, software, soft_version, alive, api, users, downs, first_checked_at, mau) -def write_alive_server(server, software, soft_version, alive, api, users, downs, first_checked_at): +def write_alive_server(server, software, soft_version, alive, api, users, downs, first_checked_at, mau): - insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version, first_checked_at, last_checked_at, downs) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING" + insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version, first_checked_at, last_checked_at, downs, mau) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING" + + conn = None + + try: + + conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432") + + cur = conn.cursor() + + cur.execute(insert_sql, (server, users, now, software, alive, api, soft_version, now, now, downs, mau)) + + if first_checked_at != None: + + cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), last_checked_at=(%s), downs=(%s), mau=(%s) where server=(%s)", (users, now, software, alive, api, soft_version, now, downs, mau, server)) + + else: + + cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), first_checked_at=(%s), last_checked_at=(%s), downs=(%s), mau=(%s) where server=(%s)", (users, now, software, alive, api, soft_version, now, now, downs, mau, server)) + + cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,)) + + conn.commit() + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + +def write_blocked_software(server, software, soft_version, alive, api, users, downs, first_checked_at): + + insert_sql = "INSERT INTO blocked(server, users, updated_at, software, alive, users_api, version, first_checked_at, last_checked_at, downs) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING" conn = None @@ -208,11 +263,11 @@ def write_alive_server(server, software, soft_version, alive, api, users, downs, if first_checked_at != None: - cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (users, now, software, alive, api, soft_version, now, downs, server)) + cur.execute("UPDATE blocked SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (users, now, software, alive, api, soft_version, now, downs, server)) else: - cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), first_checked_at=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (users, now, software, alive, api, soft_version, now, now, downs, server)) + cur.execute("UPDATE blocked SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), first_checked_at=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (users, now, software, alive, api, soft_version, now, now, downs, server)) cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,)) @@ -505,10 +560,12 @@ if __name__ == '__main__': now = datetime.now() - gettotals_sql = "select count(server), sum(users) from fediverse where alive" - get_soft_totals_sql = "select software, sum(users) as users, count(server) as servers from fediverse where users != 0 and alive group by software order by users desc" + gettotals_sql = "select count(server), sum(users), sum(mau) from fediverse where alive" + get_soft_totals_sql = "select software, sum(users) as users, sum(mau) as mau, count(server) as servers from fediverse where users != 0 and mau is not null and alive group by software order by mau desc" + soft_total_project = [] soft_total_users = [] + soft_total_mau = [] soft_total_servers = [] try: @@ -527,6 +584,8 @@ if __name__ == '__main__': total_users = row[1] + total_mau = row[2] + cur.execute(get_soft_totals_sql) rows = cur.fetchall() @@ -535,7 +594,8 @@ if __name__ == '__main__': soft_total_project.append(row[0]) soft_total_users.append(row[1]) - soft_total_servers.append(row[2]) + soft_total_mau.append(row[2]) + soft_total_servers.append(row[3]) cur.close() @@ -552,9 +612,9 @@ if __name__ == '__main__': ########################################################################### # get last check values and write current total ones - select_sql = "select total_servers, total_users from totals order by datetime desc limit 1" + select_sql = "select total_servers, total_users, total_mau from totals order by datetime desc limit 1" - insert_sql = "INSERT INTO totals(datetime, total_servers, total_users) VALUES(%s,%s,%s)" + insert_sql = "INSERT INTO totals(datetime, total_servers, total_users, total_mau) VALUES(%s,%s,%s,%s)" try: @@ -574,13 +634,17 @@ if __name__ == '__main__': users_before = row[1] + mau_before = row[2] + else: servers_before = 0 users_before = 0 - cur.execute(insert_sql, (now, total_servers, total_users)) + mau_before = 0 + + cur.execute(insert_sql, (now, total_servers, total_users, total_mau)) conn.commit() @@ -590,6 +654,8 @@ if __name__ == '__main__': evo_users = total_users - users_before + evo_mau = total_mau - mau_before + except (Exception, psycopg2.DatabaseError) as error: print(error) @@ -603,7 +669,7 @@ if __name__ == '__main__': ################################################################################ # write evo values - insert_sql = "INSERT INTO evo(datetime, servers, users) VALUES(%s,%s,%s)" + insert_sql = "INSERT INTO evo(datetime, servers, users, mau) VALUES(%s,%s,%s,%s)" conn = None @@ -613,7 +679,7 @@ if __name__ == '__main__': cur = conn.cursor() - cur.execute(insert_sql, (now, evo_servers, evo_users)) + cur.execute(insert_sql, (now, evo_servers, evo_users, evo_mau)) conn.commit() @@ -661,7 +727,7 @@ if __name__ == '__main__': conn.close() ############################################################################## - # get max servers and users + # get max servers, users and mau conn = None @@ -695,6 +761,18 @@ if __name__ == '__main__': max_users = 0 + cur.execute("select MAX(total_mau) from totals") + + row = cur.fetchone() + + if row is not None: + + max_mau = row[0] + + else: + + max_mau = 0 + cur.close() except (Exception, psycopg2.DatabaseError) as error: @@ -714,6 +792,8 @@ if __name__ == '__main__': users_plots = [] + mau_plots = [] + conn = None try: @@ -722,7 +802,7 @@ if __name__ == '__main__': cur = conn.cursor() - cur.execute("select total_servers, total_users from totals order by datetime desc limit 14") + cur.execute("select total_servers, total_users, total_mau from totals order by datetime desc limit 14") rows = cur.fetchall() @@ -732,6 +812,8 @@ if __name__ == '__main__': users_plots.append(row[1]) + mau_plots.append(row[2]) + cur.close() except (Exception, psycopg2.DatabaseError) as error: @@ -779,6 +861,22 @@ if __name__ == '__main__': plt.close() + plt.plot([-6, -5, -4, -3, -2, -1, 0], [mau_plots[6], mau_plots[5], mau_plots[4], mau_plots[3], mau_plots[2], mau_plots[1], mau_plots[0]], marker='o', color='royalblue') + + plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_mau, max_mau, max_mau, max_mau, max_mau, max_mau, max_mau], color='red') + + plt.title('fediverse: total MAU (max: ' + str(f"{max_mau:,}" + ')'), loc='right', color='royalblue') + + plt.legend(('mau', 'max'), shadow=True, loc=(0.01, 0.80), handlelength=1.5, fontsize=10) + + plt.xlabel('Last seven days') + + plt.ylabel('MAU') + + plt.savefig('mau.png') + + plt.close() + ############################################################################### # T O O T ! @@ -798,19 +896,31 @@ if __name__ == '__main__': toot_text += "max: " + str(f"{max_servers:,}") + "\n" - if evo_users >= 0: + #if evo_users >= 0: - toot_text += "total users: " + str(f"{total_users:,}") + " (+"+ str(f"{evo_users:,}") + ") \n" + # toot_text += "total users: " + str(f"{total_users:,}") + " (+"+ str(f"{evo_users:,}") + ") \n" - toot_text += "max: " + str(f"{max_users:,}") + "\n" + # toot_text += "max: " + str(f"{max_users:,}") + "\n" - elif evo_users < 0: + #elif evo_users < 0: - toot_text += "total users: " + str(f"{total_users:,}") + " ("+ str(f"{evo_users:,}") + ") \n" + # toot_text += "total users: " + str(f"{total_users:,}") + " ("+ str(f"{evo_users:,}") + ") \n" - toot_text += "max: " + str(f"{max_users:,}") + "\n" + # toot_text += "max: " + str(f"{max_users:,}") + "\n" - toot_text += "\ntop ten (soft users servers):\n\n" + if evo_mau >= 0: + + toot_text += "total MAU: " + str(f"{total_mau:,}") + " (+"+ str(f"{evo_mau:,}") + ") \n" + + toot_text += "max: " + str(f"{max_mau:,}") + "\n" + + elif evo_mau < 0: + + toot_text += "total MAU: " + str(f"{total_mau:,}") + " ("+ str(f"{evo_mau:,}") + ") \n" + + toot_text += "max: " + str(f"{max_mau:,}") + "\n" + + toot_text += "\ntop ten (soft MAU servers):\n\n" i = 0 @@ -818,13 +928,15 @@ if __name__ == '__main__': project_soft = soft_total_project[i] - project_users = soft_total_users[i] + #project_users = soft_total_users[i] + + project_mau = soft_total_mau[i] project_servers = soft_total_servers[i] len_pr_soft = len(project_soft) - toot_text += f":{project_soft}: {project_users:,} {project_servers:,}\n" + toot_text += f":{project_soft}: {project_mau:,} {project_servers:,}\n" i += 1 @@ -834,9 +946,11 @@ if __name__ == '__main__': servers_image_id = mastodon.media_post('servers.png', "image/png", description='servers graph').id - users_image_id = mastodon.media_post('users.png', "image/png", description='users graph').id + #users_image_id = mastodon.media_post('users.png', "image/png", description='users graph').id - mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, users_image_id}) + mau_image_id = mastodon.media_post('mau.png', "image/png", description='MAU graph').id + + mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, mau_image_id}) delete_dead_servers() diff --git a/fetchservers.py b/fetchservers.py index d49bd47..a486b5a 100644 --- a/fetchservers.py +++ b/fetchservers.py @@ -20,8 +20,10 @@ from ray.exceptions import ( ) apis = ['/api/v1/instance?', + '/api/v1/nodeinfo?', '/nodeinfo/2.0?', '/nodeinfo/2.0.json?', + '/nodeinfo/2.1.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?', '/api/nodeinfo/2.0.json?',