From 043ab501ff14a7b8209fe45d967a4c0b5da7c31c Mon Sep 17 00:00:00 2001 From: spla Date: Tue, 19 May 2020 23:33:55 +0200 Subject: [PATCH] disabled tqdm, added get server's software --- fediverse.py | 472 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 287 insertions(+), 185 deletions(-) diff --git a/fediverse.py b/fediverse.py index 9bf76e4..f205fe4 100644 --- a/fediverse.py +++ b/fediverse.py @@ -23,9 +23,7 @@ import psycopg2 from itertools import product from multiprocessing import Pool, Lock, Process, Queue, current_process, Manager -import queue import multiprocessing -import tqdm from decimal import * getcontext().prec = 2 @@ -41,24 +39,24 @@ def is_json(myjson): return False return True -def getserver(server): +def getserver(server, x): - #server = server[0].rstrip('.').lower() + server = server[0].rstrip('.').lower() if server.find(".") == -1: - #print(server + "is Not a real domain!") + print(server + " is Not a real domain!") return if server.find("@") != -1: - #print(server + "is Not a real domain!") + print(server + " is Not a real domain!") return if server.find("/") != -1: - #print("Removing substring after first / from " + server) - server = server.split("/", 1)[0] + print(server + " is Not a real domain!") + return if server.find(":") != -1: - #print("Removing substring after first : from " + server) - server = server.split(":", 1)[0] + print(server + " is Not a real domain!") + return - global mastodont + global mastodon global pleroma global gnusocial global zap @@ -76,44 +74,46 @@ def getserver(server): global writefreely global ravenvale global diaspora + global dolphin - global pl_users + pl_users = 0 global total_pl_users - global mast_users + mast_users = 0 global total_mast_users - global gs_users + gs_users = 0 global total_gs_users - global usuaris_zap + zap_users = 0 global total_zap_users - global usuaris_plume + plume_users = 0 global total_plume_users - global usuaris_hubzilla + hubzilla_users = 0 global total_hubzilla_users - global usuaris_misskey + misskey_users = 0 global total_misskey_users - global usuaris_prismo + prismo_users = 0 global total_prismo_users - global usuaris_osada + osada_users = 0 global total_osada_users - global usuaris_gpolis + gpolis_users = 0 global total_gpolis_users - global usuaris_ggg + ggg_users = 0 global total_ggg_users - global usuaris_squs + squs_users = 0 global total_squs_users - global peertube_users + peertube_users = 0 global total_peertube_users - global diaspora_users + diaspora_users = 0 global total_diaspora_users - global usuaris_friendica + friendica_users = 0 global total_friendica_users - global usuaris_pixelfed + pixelfed_users = 0 global total_pixelfed_users - global usuaris_writefreely + writefreely_users = 0 global total_writefreely_users - global ravenvale_users + ravenvale_users = 0 global total_ravenvale_users - global usuaris_total + dolphin_users = 0 + global total_dolphin_users check_diaspora = False check_peertube = False @@ -132,8 +132,9 @@ def getserver(server): check_writefreely = False check_raven = False check_gnusocial2 = False + check_dolphin = False - usuaris = 0 + users = 0 try: @@ -154,33 +155,39 @@ def getserver(server): if nodeinfo.status_code == 200 and check_pleroma == True: - #print("Servidor Pleroma: ") + print("Servidor Pleroma: ") + soft = "pleroma" pl_users = nodeinfo.json()['usage']['users']['total'] total_pl_users = total_pl_users + pl_users - usuaris = pl_users - #print(server, pl_users) + users = pl_users + print(server, pl_users) else: if type(res.json()) != int: if res.json().get('stats') != None: - #print("Servidor Mastodon: ") + print("Servidor Mastodon: ") + soft = "mastodon" mast_users = res.json()['stats']['user_count'] else: mast_users = 0 + soft = "mastodon" else: mast_users = 0 - if mast_users != None: + if mast_users != None and mast_users < 999999: total_mast_users = total_mast_users + mast_users - usuaris = mast_users - #print(server, mast_users) + users = mast_users + print(server, mast_users) - #print("\n") + print("\n") - insert_sql = "INSERT INTO fediverse(server, users, updated_at) VALUES(%s,%s,%s) ON CONFLICT DO NOTHING" + if users > 1000000: + return + + insert_sql = "INSERT INTO fediverse(server, users, updated_at, software) VALUES(%s,%s,%s,%s) ON CONFLICT DO NOTHING" conn = None try: @@ -189,9 +196,9 @@ def getserver(server): cur = conn.cursor() - cur.execute(insert_sql, (server, usuaris, now)) + cur.execute(insert_sql, (server, users, now, soft)) - cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s) where server=(%s)", (usuaris, now, server)) + cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s) where server=(%s)", (users, now, soft, server)) conn.commit() @@ -209,7 +216,7 @@ def getserver(server): else: return - + except urllib3.exceptions.ProtocolError as protoerr: pass @@ -292,9 +299,11 @@ def getserver(server): check_squs = True elif soft == "friendica" or soft == "Friendica": check_friendica = True + elif soft == "dolphin": + check_dolphin = True if is_json(wf_nodeinfo.text) == True and soft == "": - if wf_nodeinfo.json().get('error') != None and wf_nodeinfo.json().get('status') == None: + #if wf_nodeinfo.json().get('error') != None and wf_nodeinfo.json().get('status') == None: soft = wf_nodeinfo.json()['software']['name'] if soft == "writefreely": check_writefreely = True @@ -322,185 +331,218 @@ def getserver(server): pass - if check_diaspora == False and check_peertube == False and check_zap == False and check_plume == False and check_hubzilla == False and check_misskey == False and check_prismo == False and check_osada == False and check_groundpolis == False and check_ganggo == False and check_squs == False and check_gnusocial == False and check_friendica == False and check_pixelfed == False and check_writefreely == False and check_raven == False and check_gnusocial2 == False: + if check_diaspora == False and check_peertube == False and check_zap == False and check_plume == False and check_hubzilla == False and check_misskey == False and check_prismo == False and check_osada == False and check_groundpolis == False and check_ganggo == False and check_squs == False and check_gnusocial == False and check_friendica == False and check_pixelfed == False and check_writefreely == False and check_raven == False and check_gnusocial2 == False and check_dolphin == False: api_reply = 0 pass - elif check_diaspora == True or check_peertube == True or check_zap == True or check_plume == True or check_hubzilla == True or check_misskey == True or check_prismo == True or check_osada == True or check_groundpolis == True or check_ganggo == True or check_squs == True or check_gnusocial == True or check_friendica == True or check_pixelfed == True or check_writefreely == True or check_raven == True or check_gnusocial2 == True: + elif check_diaspora == True or check_peertube == True or check_zap == True or check_plume == True or check_hubzilla == True or check_misskey == True or check_prismo == True or check_osada == True or check_groundpolis == True or check_ganggo == True or check_squs == True or check_gnusocial == True or check_friendica == True or check_pixelfed == True or check_writefreely == True or check_raven == True or check_gnusocial2 == True or check_dolphin == True: api_reply = 1 if nodeinfo.ok == True and check_peertube == True: - #print("Servidor Peertube") + print("Servidor Peertube") + soft = "peertube" peertube_users = nodeinfo.json()['usage']['users']['total'] total_peertube_users = total_peertube_users + peertube_users - usuaris = peertube_users - #print(server, peertube_users) - #print("\n") + users = peertube_users + print(server, peertube_users) + print("\n") if nodeinfo.ok == True and check_diaspora == True: - #print("Servidor Diaspora") + print("Servidor Diaspora") + soft = "diaspora" diaspora_users = nodeinfo.json()['usage']['users']['total'] total_diaspora_users = total_diaspora_users + diaspora_users - usuaris = diaspora_users - #print(server, diaspora_users) - #print("\n") + users = diaspora_users + print(server, diaspora_users) + print("\n") elif nodeinfo.ok == True and check_raven == True: - #print("Servidor Ravenvale") + print("Servidor Ravenvale") + soft = "ravenvale" ravenvale_users = nodeinfo.json()['usage']['users']['total'] total_ravenvale_users = total_ravenvale_users + ravenvale_users - usuaris = ravenvale_users - #print(server, ravenvale_users) - #print("\n") + users = ravenvale_users + print(server, ravenvale_users) + print("\n") elif others_nodeinfo.ok == True and check_zap == True: - #print("Servidor Zap") - usuaris_zap = others_nodeinfo.json()['usage']['users']['total'] - total_zap_users = total_zap_users + usuaris_zap - usuaris = usuaris_zap - #print(server, usuaris_zap) - #print("\n") + print("Servidor Zap") + soft = "zap" + zap_users = others_nodeinfo.json()['usage']['users']['total'] + total_zap_users = total_zap_users + zap_users + users = zap_users + print(server, zap_users) + print("\n") elif others_nodeinfo.ok == True and check_plume == True: - #print("Servidor Plume") - usuaris_plume = others_nodeinfo.json()['usage']['users']['total'] - total_plume_users = total_plume_users + usuaris_plume - usuaris = usuaris_plume - #print(server, usuaris_plume) - #print("\n") + print("Servidor Plume") + soft = "plume" + plume_users = others_nodeinfo.json()['usage']['users']['total'] + total_plume_users = total_plume_users + plume_users + users = plume_users + print(server, plume_users) + print("\n") elif others_nodeinfo.ok == True and check_hubzilla == True: - #print("Servidor Hubzilla") - usuaris_hubzilla = others_nodeinfo.json()['usage']['users']['total'] - total_hubzilla_users = total_hubzilla_users + usuaris_hubzilla - usuaris = usuaris_hubzilla - #print(server, usuaris_hubzilla) - #print("\n") + print("Servidor Hubzilla") + soft = "hubzilla" + hubzilla_users = others_nodeinfo.json()['usage']['users']['total'] + total_hubzilla_users = total_hubzilla_users + hubzilla_users + users = hubzilla_users + print(server, hubzilla_users) + print("\n") elif others_nodeinfo.ok == True and check_misskey == True: - #print("Servidor Misskey") - usuaris_misskey = 0 - total_misskey_users = total_misskey_users + usuaris_misskey - usuaris = usuaris_misskey - #print(server, usuaris_misskey) - #print("\n") + print("Servidor Misskey") + soft = "misskey" + misskey_users = others_nodeinfo.json()['usage']['users']['total'] + if misskey_users == 0: + misskey_users = others_nodeinfo.json()['usage']['users']['activeHalfyear'] + total_misskey_users = total_misskey_users + misskey_users + users = misskey_users + print(server, misskey_users) + print("\n") elif others_nodeinfo.ok == True and check_prismo == True: - #print("Servidor Prismo") - usuaris_prismo = others_nodeinfo.json()['usage']['users']['total'] - total_prismo_users = total_prismo_users + usuaris_prismo - usuaris = usuaris_prismo - #print(server, usuaris_prismo) - #print("\n") + print("Servidor Prismo") + soft = "prismo" + prismo_users = others_nodeinfo.json()['usage']['users']['total'] + total_prismo_users = total_prismo_users + prismo_users + users = prismo_users + print(server, prismo_users) + print("\n") elif others_nodeinfo.ok == True and check_osada == True: - #print("Servidor Osada") - usuaris_osada = others_nodeinfo.json()['usage']['users']['total'] - total_osada_users = total_osada_users + usuaris_osada - usuaris = usuaris_osada - #print(server, usuaris_osada) - #print("\n") + print("Servidor Osada") + soft = "osada" + osada_users = others_nodeinfo.json()['usage']['users']['total'] + total_osada_users = total_osada_users + osada_users + users = osada_users + print(server, osada_users) + print("\n") elif others_nodeinfo.ok == True and check_groundpolis == True: - #print("Servidor Groundpolis") - usuaris_gpolis = others_nodeinfo.json()['usage']['users']['total'] - total_gpolis_users = total_gpolis_users + usuaris_gpolis - usuaris = usuaris_gpolis - #print(server, usuaris_gpolis) - #print("\n") + print("Servidor Groundpolis") + soft = "groundpolis" + gpolis_users = others_nodeinfo.json()['usage']['users']['total'] + total_gpolis_users = total_gpolis_users + gpolis_users + users = gpolis_users + print(server, gpolis_users) + print("\n") elif others_nodeinfo.ok == True and check_ganggo == True: - #print("Servidor Ganggo") - usuaris_ggg = others_nodeinfo.json()['usage']['users']['total'] - total_ggg_users = total_ggg_users + usuaris_ggg - usuaris = usuaris_ggg - #print(server, usuaris_ggg) - #print("\n") + print("Servidor Ganggo") + soft = "ganggo" + ggg_users = others_nodeinfo.json()['usage']['users']['total'] + total_ggg_users = total_ggg_users + ggg_users + users = ggg_users + print(server, ggg_users) + print("\n") elif others_nodeinfo.ok == True and check_squs == True: - #print("Servidor Squs") - usuaris_squs = others_nodeinfo.json()['usage']['users']['total'] - total_squs_users = total_squs_users + usuaris_squs - usuaris = usuaris_squs - #print(server, usuaris_squs) - #print("\n") + print("Servidor Squs") + soft = "squs" + squs_users = others_nodeinfo.json()['usage']['users']['total'] + total_squs_users = total_squs_users + squs_users + users = squs_users + print(server, squs_users) + print("\n") elif others_nodeinfo.ok == True and check_friendica == True: - #print("Servidor Friendica") + print("Servidor Friendica") + soft = "friendica" if others_nodeinfo.json()['usage'] != []: - usuaris_friendica = others_nodeinfo.json()['usage']['users']['total'] + friendica_users = others_nodeinfo.json()['usage']['users']['total'] else: - usuaris_friendica = 0 - total_friendica_users = total_friendica_users + usuaris_friendica - usuaris = usuaris_friendica - #print(server, usuaris_friendica) - #print("\n") + friendica_users = 0 + total_friendica_users = total_friendica_users + friendica_users + users = friendica_users + print(server, friendica_users) + print("\n") check_friendica = False + elif others_nodeinfo.ok == True and check_dolphin == True: + + print("Servidor Dolphin") + soft = "dolphin" + if others_nodeinfo.json()['usage'] != []: + dolphin_users = others_nodeinfo.json()['usage']['users']['total'] + else: + dolphin_users = 0 + total_dolphin_users = total_dolphin_users + dolphin_users + users = dolphin_users + print(server, dolphin_users) + print("\n") + check_dolphin = False + elif gs_nodeinfo.ok == True and check_gnusocial == True: - #print("Servidor GNU Social") + print("Servidor GNU Social") + soft = "gnusocial" gs_users = gs_nodeinfo.json()['usage']['users']['total'] if gs_users == 0: - gs_users = gs_nodeinfo.json()['usage']['users']['activeHalfyear'] + gs_users = gs_nodeinfo.json()['usage']['users']['activeHalfyear'] total_gs_users = total_gs_users + gs_users - usuaris = gs_users - #print(server, gs_users) - #print("\n") + users = gs_users + print(server, gs_users) + print("\n") elif statusnet.ok == True and check_friendica == True: #statusnet.json()['site']['friendica']['FRIENDICA_PLATFORM'] == "Friendica": - #print("Servidor Friendica") - usuaris_friendica = 0 - total_friendica_users = total_friendica_users + usuaris_friendica - usuaris = usuaris_friendica - instancies = 0 - #print(server, usuaris_friendica) - #print("\n") + print("Servidor Friendica") + soft = "friendica" + friendica_users = 0 + total_friendica_users = total_friendica_users + friendica_users + users = friendica_users + print(server, friendica_users) + print("\n") elif px_nodeinfo.ok == True and check_pixelfed == True: - #print("Servidor Pixelfed") - usuaris_pixelfed = px_nodeinfo.json()['usage']['users']['total'] - total_pixelfed_users = total_pixelfed_users + usuaris_pixelfed - usuaris = usuaris_pixelfed - #print(server, usuaris_pixelfed) - #print("\n") + print("Servidor Pixelfed") + soft = "pixelfed" + pixelfed_users = px_nodeinfo.json()['usage']['users']['total'] + total_pixelfed_users = total_pixelfed_users + pixelfed_users + users = pixelfed_users + print(server, pixelfed_users) + print("\n") elif px_nodeinfo.ok == True and check_gnusocial2 == True: - #print("Servidor GNU Social 2.x") + print("Servidor GNU Social 2.x") + soft = "gnusocialv2" gs_users = px_nodeinfo.json()['usage']['users']['total'] if gs_users == 0: - gs_users = px_nodeinfo.json()['usage']['users']['activeHalfyear'] + gs_users = px_nodeinfo.json()['usage']['users']['activeHalfyear'] total_gs_users = total_gs_users + gs_users - usuaris = gs_users - #print(server, gs_users) - #print("\n") + users = gs_users + print(server, gs_users) + print("\n") elif wf_nodeinfo.ok == True and check_writefreely == True: - #print("Servidor WriteFreely") - usuaris_writefreely = wf_nodeinfo.json()['usage']['users']['total'] - total_writefreely_users = total_writefreely_users + usuaris_writefreely - usuaris = usuaris_writefreely - #print(server, usuaris_writefreely) - #print("\n") + print("Servidor WriteFreely") + soft = "writefreely" + writefreely_users = wf_nodeinfo.json()['usage']['users']['total'] + total_writefreely_users = total_writefreely_users + writefreely_users + users = writefreely_users + print(server, writefreely_users) + print("\n") else: @@ -508,7 +550,7 @@ def getserver(server): if api_reply == 1 and others_nodeinfo.ok or gs_nodeinfo.ok or nodeinfo.ok or statusnet.ok or px_nodeinfo.ok or wf_nodeinfo.ok: - insert_sql = "INSERT INTO fediverse(server, users, updated_at) VALUES(%s,%s,%s) ON CONFLICT DO NOTHING" + insert_sql = "INSERT INTO fediverse(server, users, updated_at, software) VALUES(%s,%s,%s,%s) ON CONFLICT DO NOTHING" conn = None try: @@ -517,9 +559,9 @@ def getserver(server): cur = conn.cursor() - cur.execute(insert_sql, (server, usuaris, now)) + cur.execute(insert_sql, (server, users, now, soft)) - cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s) where server=(%s)", (usuaris, now, server)) + cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s) where server=(%s)", (users, now, soft, server)) conn.commit() @@ -528,7 +570,7 @@ def getserver(server): except (Exception, psycopg2.DatabaseError) as error: - #print(error) + print(error) return finally: @@ -548,7 +590,7 @@ def getserver(server): except requests.exceptions.ChunkedEncodingError as chunkerr: - #print(server) + print(server) pass except requests.exceptions.ConnectionError as errc: @@ -615,37 +657,36 @@ mast_users = 0 total_mast_users = 0 gs_users = 0 total_gs_users = 0 -usuaris_zap = 0 +zap_users = 0 total_zap_users = 0 -usuaris_plume = 0 +plume_users = 0 total_plume_users = 0 -usuaris_hubzilla = 0 +hubzilla_users = 0 total_hubzilla_users = 0 -usuaris_misskey = 0 +misskey_users = 0 total_misskey_users = 0 -usuaris_prismo = 0 +prismo_users = 0 total_prismo_users = 0 -usuaris_osada = 0 +osada_users = 0 total_osada_users = 0 -usuaris_gpolis = 0 +gpolis_users = 0 total_gpolis_users = 0 -usuaris_ggg = 0 +ggg_users = 0 total_ggg_users = 0 -usuaris_squs = 0 +squs_users = 0 total_squs_users = 0 peertube_users = 0 total_peertube_users = 0 diaspora_users = 0 total_diaspora_users = 0 -usuaris_friendica = 0 +friendica_users = 0 total_friendica_users = 0 -usuaris_pixelfed = 0 +pixelfed_users = 0 total_pixelfed_users = 0 -usuaris_writefreely = 0 +writefreely_users = 0 total_writefreely_users = 0 ravenvale_users = 0 total_ravenvale_users = 0 -usuaris_total = 0 total_servers = 0 total_users = 0 @@ -682,11 +723,44 @@ finally: conn.close() ########################################################################### +# multiprocessing! -nprocs = multiprocessing.cpu_count() -pool = Pool(processes=nprocs) -for _ in tqdm.tqdm(pool.imap_unordered(getserver, world_servers), total=len(world_servers)): - pass +m = Manager() +q = m.Queue() +z = zip(world_servers) + +serv_number = len(world_servers) + +pool_tuple = [(x, q) for x in z] +with Pool(processes=64) as pool: + pool.starmap(getserver, pool_tuple) + +########################################################################### +# delete not alive servers from fediverse table + +try: + + conn = None + + conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") + + cur = conn.cursor() + + cur.execute("delete from fediverse where updated_at != (%s)", (now,)) + + conn.commit() + + cur.close() + +except (Exception, psycopg2.DatabaseError) as error: + + print(error) + +finally: + + if conn is not None: + + conn.close() ########################################################################### # get current total servers and users @@ -719,8 +793,8 @@ finally: conn.close() -######################################################################################### -# get last server and users values and calc evolution. Write current values to database +########################################################################### +# get last check values and write current total values select_sql = "select total_servers, total_users from totals order by datetime desc limit 1" insert_sql = "INSERT INTO totals(datetime, total_servers, total_users) VALUES(%s,%s,%s)" @@ -766,7 +840,7 @@ finally: conn.close() ################################################################################ -# escriure en la bbdd evo +# write evo values insert_sql = "INSERT INTO evo(datetime, servers, users) VALUES(%s,%s,%s)" @@ -794,23 +868,51 @@ finally: conn.close() +############################################################################## +# get world's last update datetime + +conn = None + +try: + + conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") + + cur = conn.cursor() + + cur.execute("select updated_at from world order by updated_at desc limit 1") + + row = cur.fetchone() + + last_update = row[0] + last_update = last_update.strftime('%m/%d/%Y, %H:%M:%S') + + cur.close() + +except (Exception, psycopg2.DatabaseError) as error: + + print(error) + +finally: + + if conn is not None: + + conn.close() + ############################################################################### # T O O T ! -toot_text = "#fediverse live servers stats" + " \n" +toot_text = "#fediverse alive servers stats" + " \n" toot_text += "\n" -if evo_servers > 0: - toot_text += "Live servers: " + str(total_servers) + " (+"+ str(evo_servers) + ") \n" -else: - toot_text += "Live servers: " + str(total_servers) + " ("+ str(evo_servers) + ") \n" +if evo_servers >= 0: + toot_text += "alive servers: " + str(total_servers) + " (+"+ str(evo_servers) + ") \n" +elif evo_servers < 0: + toot_text += "alive servers: " + str(total_servers) + " (-"+ str(evo_servers) + ") \n" +if evo_users >= 0: + toot_text += "total Users: " + str(total_users) + " (+"+ str(evo_users) + ") \n" +elif evo_users < 0: + toot_text += "total Users: " + str(total_users) + " (-"+ str(evo_users) + ") \n" toot_text += "\n" -if evo_users > 0: - toot_text += "Total Users: " + str(total_users) + " (+"+ str(evo_users) + ") \n" -else: - toot_text += "Total Users: " + str(total_users) + " ("+ str(evo_users) + ") \n" -toot_text += "\n" -toot_text += "Exec. time %s seconds" % (time.time() - start_time) - +toot_text += "updated at " + str(last_update) + " \n" print("Tooting...") print(toot_text)