spla/fediverse
1
1
Fork 1
Codi Incidències Sol·licituds Pull repo.project Llançaments Wiki Activitat

disabled tqdm, added get server's software

Explorar la Font
This commit is contained in:
spla 2020-05-19 23:33:55 +02:00
pare d2ec4522a4
commit 043ab501ff
S'han modificat 1 arxius amb 287 adicions i 185 eliminacions
Mostrar estadístiques Descarregar el fitxer de pedaç Descarregar el fitxer de diferències Expand all files Collapse all files

466
fediverse.py
Veure arxiu

@ -23,9 +23,7 @@ import psycopg2
from itertools import product
from multiprocessing import Pool, Lock, Process, Queue, current_process, Manager
import queue
import multiprocessing
import tqdm
from decimal import *
getcontext().prec = 2
@ -41,24 +39,24 @@ def is_json(myjson):
return False
return True
def getserver(server):
def getserver(server, x):
#server = server[0].rstrip('.').lower()
server = server[0].rstrip('.').lower()
if server.find(".") == -1:
#print(server + "is Not a real domain!")
print(server + " is Not a real domain!")
return
if server.find("@") != -1:
#print(server + "is Not a real domain!")
print(server + " is Not a real domain!")
return
if server.find("/") != -1:
#print("Removing substring after first / from " + server)
server = server.split("/", 1)[0]
print(server + " is Not a real domain!")
return
if server.find(":") != -1:
#print("Removing substring after first : from " + server)
server = server.split(":", 1)[0]
print(server + " is Not a real domain!")
return
global mastodont
global mastodon
global pleroma
global gnusocial
global zap
@ -76,44 +74,46 @@ def getserver(server):
global writefreely
global ravenvale
global diaspora
global dolphin
global pl_users
pl_users = 0
global total_pl_users
global mast_users
mast_users = 0
global total_mast_users
global gs_users
gs_users = 0
global total_gs_users
global usuaris_zap
zap_users = 0
global total_zap_users
global usuaris_plume
plume_users = 0
global total_plume_users
global usuaris_hubzilla
hubzilla_users = 0
global total_hubzilla_users
global usuaris_misskey
misskey_users = 0
global total_misskey_users
global usuaris_prismo
prismo_users = 0
global total_prismo_users
global usuaris_osada
osada_users = 0
global total_osada_users
global usuaris_gpolis
gpolis_users = 0
global total_gpolis_users
global usuaris_ggg
ggg_users = 0
global total_ggg_users
global usuaris_squs
squs_users = 0
global total_squs_users
global peertube_users
peertube_users = 0
global total_peertube_users
global diaspora_users
diaspora_users = 0
global total_diaspora_users
global usuaris_friendica
friendica_users = 0
global total_friendica_users
global usuaris_pixelfed
pixelfed_users = 0
global total_pixelfed_users
global usuaris_writefreely
writefreely_users = 0
global total_writefreely_users
global ravenvale_users
ravenvale_users = 0
global total_ravenvale_users
global usuaris_total
dolphin_users = 0
global total_dolphin_users
check_diaspora = False
check_peertube = False
@ -132,8 +132,9 @@ def getserver(server):
check_writefreely = False
check_raven = False
check_gnusocial2 = False
check_dolphin = False
usuaris = 0
users = 0
try:
@ -154,33 +155,39 @@ def getserver(server):
if nodeinfo.status_code == 200 and check_pleroma == True:
#print("Servidor Pleroma: ")
print("Servidor Pleroma: ")
soft = "pleroma"
pl_users = nodeinfo.json()['usage']['users']['total']
total_pl_users = total_pl_users + pl_users
usuaris = pl_users
#print(server, pl_users)
users = pl_users
print(server, pl_users)
else:
if type(res.json()) != int:
if res.json().get('stats') != None:
#print("Servidor Mastodon: ")
print("Servidor Mastodon: ")
soft = "mastodon"
mast_users = res.json()['stats']['user_count']
else:
mast_users = 0
soft = "mastodon"
else:
mast_users = 0
if mast_users != None:
if mast_users != None and mast_users < 999999:
total_mast_users = total_mast_users + mast_users
usuaris = mast_users
#print(server, mast_users)
users = mast_users
print(server, mast_users)
#print("\n")
print("\n")
insert_sql = "INSERT INTO fediverse(server, users, updated_at) VALUES(%s,%s,%s) ON CONFLICT DO NOTHING"
if users > 1000000:
return
insert_sql = "INSERT INTO fediverse(server, users, updated_at, software) VALUES(%s,%s,%s,%s) ON CONFLICT DO NOTHING"
conn = None
try:
@ -189,9 +196,9 @@ def getserver(server):
cur = conn.cursor()
cur.execute(insert_sql, (server, usuaris, now))
cur.execute(insert_sql, (server, users, now, soft))
cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s) where server=(%s)", (usuaris, now, server))
cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s) where server=(%s)", (users, now, soft, server))
conn.commit()
@ -292,9 +299,11 @@ def getserver(server):
check_squs = True
elif soft == "friendica" or soft == "Friendica":
check_friendica = True
elif soft == "dolphin":
check_dolphin = True
if is_json(wf_nodeinfo.text) == True and soft == "":
if wf_nodeinfo.json().get('error') != None and wf_nodeinfo.json().get('status') == None:
#if wf_nodeinfo.json().get('error') != None and wf_nodeinfo.json().get('status') == None:
soft = wf_nodeinfo.json()['software']['name']
if soft == "writefreely":
check_writefreely = True
@ -322,185 +331,218 @@ def getserver(server):
pass
if check_diaspora == False and check_peertube == False and check_zap == False and check_plume == False and check_hubzilla == False and check_misskey == False and check_prismo == False and check_osada == False and check_groundpolis == False and check_ganggo == False and check_squs == False and check_gnusocial == False and check_friendica == False and check_pixelfed == False and check_writefreely == False and check_raven == False and check_gnusocial2 == False:
if check_diaspora == False and check_peertube == False and check_zap == False and check_plume == False and check_hubzilla == False and check_misskey == False and check_prismo == False and check_osada == False and check_groundpolis == False and check_ganggo == False and check_squs == False and check_gnusocial == False and check_friendica == False and check_pixelfed == False and check_writefreely == False and check_raven == False and check_gnusocial2 == False and check_dolphin == False:
api_reply = 0
pass
elif check_diaspora == True or check_peertube == True or check_zap == True or check_plume == True or check_hubzilla == True or check_misskey == True or check_prismo == True or check_osada == True or check_groundpolis == True or check_ganggo == True or check_squs == True or check_gnusocial == True or check_friendica == True or check_pixelfed == True or check_writefreely == True or check_raven == True or check_gnusocial2 == True:
elif check_diaspora == True or check_peertube == True or check_zap == True or check_plume == True or check_hubzilla == True or check_misskey == True or check_prismo == True or check_osada == True or check_groundpolis == True or check_ganggo == True or check_squs == True or check_gnusocial == True or check_friendica == True or check_pixelfed == True or check_writefreely == True or check_raven == True or check_gnusocial2 == True or check_dolphin == True:
api_reply = 1
if nodeinfo.ok == True and check_peertube == True:
#print("Servidor Peertube")
print("Servidor Peertube")
soft = "peertube"
peertube_users = nodeinfo.json()['usage']['users']['total']
total_peertube_users = total_peertube_users + peertube_users
usuaris = peertube_users
#print(server, peertube_users)
#print("\n")
users = peertube_users
print(server, peertube_users)
print("\n")
if nodeinfo.ok == True and check_diaspora == True:
#print("Servidor Diaspora")
print("Servidor Diaspora")
soft = "diaspora"
diaspora_users = nodeinfo.json()['usage']['users']['total']
total_diaspora_users = total_diaspora_users + diaspora_users
usuaris = diaspora_users
#print(server, diaspora_users)
#print("\n")
users = diaspora_users
print(server, diaspora_users)
print("\n")
elif nodeinfo.ok == True and check_raven == True:
#print("Servidor Ravenvale")
print("Servidor Ravenvale")
soft = "ravenvale"
ravenvale_users = nodeinfo.json()['usage']['users']['total']
total_ravenvale_users = total_ravenvale_users + ravenvale_users
usuaris = ravenvale_users
#print(server, ravenvale_users)
#print("\n")
users = ravenvale_users
print(server, ravenvale_users)
print("\n")
elif others_nodeinfo.ok == True and check_zap == True:
#print("Servidor Zap")
usuaris_zap = others_nodeinfo.json()['usage']['users']['total']
total_zap_users = total_zap_users + usuaris_zap
usuaris = usuaris_zap
#print(server, usuaris_zap)
#print("\n")
print("Servidor Zap")
soft = "zap"
zap_users = others_nodeinfo.json()['usage']['users']['total']
total_zap_users = total_zap_users + zap_users
users = zap_users
print(server, zap_users)
print("\n")
elif others_nodeinfo.ok == True and check_plume == True:
#print("Servidor Plume")
usuaris_plume = others_nodeinfo.json()['usage']['users']['total']
total_plume_users = total_plume_users + usuaris_plume
usuaris = usuaris_plume
#print(server, usuaris_plume)
#print("\n")
print("Servidor Plume")
soft = "plume"
plume_users = others_nodeinfo.json()['usage']['users']['total']
total_plume_users = total_plume_users + plume_users
users = plume_users
print(server, plume_users)
print("\n")
elif others_nodeinfo.ok == True and check_hubzilla == True:
#print("Servidor Hubzilla")
usuaris_hubzilla = others_nodeinfo.json()['usage']['users']['total']
total_hubzilla_users = total_hubzilla_users + usuaris_hubzilla
usuaris = usuaris_hubzilla
#print(server, usuaris_hubzilla)
#print("\n")
print("Servidor Hubzilla")
soft = "hubzilla"
hubzilla_users = others_nodeinfo.json()['usage']['users']['total']
total_hubzilla_users = total_hubzilla_users + hubzilla_users
users = hubzilla_users
print(server, hubzilla_users)
print("\n")
elif others_nodeinfo.ok == True and check_misskey == True:
#print("Servidor Misskey")
usuaris_misskey = 0
total_misskey_users = total_misskey_users + usuaris_misskey
usuaris = usuaris_misskey
#print(server, usuaris_misskey)
#print("\n")
print("Servidor Misskey")
soft = "misskey"
misskey_users = others_nodeinfo.json()['usage']['users']['total']
if misskey_users == 0:
misskey_users = others_nodeinfo.json()['usage']['users']['activeHalfyear']
total_misskey_users = total_misskey_users + misskey_users
users = misskey_users
print(server, misskey_users)
print("\n")
elif others_nodeinfo.ok == True and check_prismo == True:
#print("Servidor Prismo")
usuaris_prismo = others_nodeinfo.json()['usage']['users']['total']
total_prismo_users = total_prismo_users + usuaris_prismo
usuaris = usuaris_prismo
#print(server, usuaris_prismo)
#print("\n")
print("Servidor Prismo")
soft = "prismo"
prismo_users = others_nodeinfo.json()['usage']['users']['total']
total_prismo_users = total_prismo_users + prismo_users
users = prismo_users
print(server, prismo_users)
print("\n")
elif others_nodeinfo.ok == True and check_osada == True:
#print("Servidor Osada")
usuaris_osada = others_nodeinfo.json()['usage']['users']['total']
total_osada_users = total_osada_users + usuaris_osada
usuaris = usuaris_osada
#print(server, usuaris_osada)
#print("\n")
print("Servidor Osada")
soft = "osada"
osada_users = others_nodeinfo.json()['usage']['users']['total']
total_osada_users = total_osada_users + osada_users
users = osada_users
print(server, osada_users)
print("\n")
elif others_nodeinfo.ok == True and check_groundpolis == True:
#print("Servidor Groundpolis")
usuaris_gpolis = others_nodeinfo.json()['usage']['users']['total']
total_gpolis_users = total_gpolis_users + usuaris_gpolis
usuaris = usuaris_gpolis
#print(server, usuaris_gpolis)
#print("\n")
print("Servidor Groundpolis")
soft = "groundpolis"
gpolis_users = others_nodeinfo.json()['usage']['users']['total']
total_gpolis_users = total_gpolis_users + gpolis_users
users = gpolis_users
print(server, gpolis_users)
print("\n")
elif others_nodeinfo.ok == True and check_ganggo == True:
#print("Servidor Ganggo")
usuaris_ggg = others_nodeinfo.json()['usage']['users']['total']
total_ggg_users = total_ggg_users + usuaris_ggg
usuaris = usuaris_ggg
#print(server, usuaris_ggg)
#print("\n")
print("Servidor Ganggo")
soft = "ganggo"
ggg_users = others_nodeinfo.json()['usage']['users']['total']
total_ggg_users = total_ggg_users + ggg_users
users = ggg_users
print(server, ggg_users)
print("\n")
elif others_nodeinfo.ok == True and check_squs == True:
#print("Servidor Squs")
usuaris_squs = others_nodeinfo.json()['usage']['users']['total']
total_squs_users = total_squs_users + usuaris_squs
usuaris = usuaris_squs
#print(server, usuaris_squs)
#print("\n")
print("Servidor Squs")
soft = "squs"
squs_users = others_nodeinfo.json()['usage']['users']['total']
total_squs_users = total_squs_users + squs_users
users = squs_users
print(server, squs_users)
print("\n")
elif others_nodeinfo.ok == True and check_friendica == True:
#print("Servidor Friendica")
print("Servidor Friendica")
soft = "friendica"
if others_nodeinfo.json()['usage'] != []:
usuaris_friendica = others_nodeinfo.json()['usage']['users']['total']
friendica_users = others_nodeinfo.json()['usage']['users']['total']
else:
usuaris_friendica = 0
total_friendica_users = total_friendica_users + usuaris_friendica
usuaris = usuaris_friendica
#print(server, usuaris_friendica)
#print("\n")
friendica_users = 0
total_friendica_users = total_friendica_users + friendica_users
users = friendica_users
print(server, friendica_users)
print("\n")
check_friendica = False
elif others_nodeinfo.ok == True and check_dolphin == True:
print("Servidor Dolphin")
soft = "dolphin"
if others_nodeinfo.json()['usage'] != []:
dolphin_users = others_nodeinfo.json()['usage']['users']['total']
else:
dolphin_users = 0
total_dolphin_users = total_dolphin_users + dolphin_users
users = dolphin_users
print(server, dolphin_users)
print("\n")
check_dolphin = False
elif gs_nodeinfo.ok == True and check_gnusocial == True:
#print("Servidor GNU Social")
print("Servidor GNU Social")
soft = "gnusocial"
gs_users = gs_nodeinfo.json()['usage']['users']['total']
if gs_users == 0:
gs_users = gs_nodeinfo.json()['usage']['users']['activeHalfyear']
total_gs_users = total_gs_users + gs_users
usuaris = gs_users
#print(server, gs_users)
#print("\n")
users = gs_users
print(server, gs_users)
print("\n")
elif statusnet.ok == True and check_friendica == True: #statusnet.json()['site']['friendica']['FRIENDICA_PLATFORM'] == "Friendica":
#print("Servidor Friendica")
usuaris_friendica = 0
total_friendica_users = total_friendica_users + usuaris_friendica
usuaris = usuaris_friendica
instancies = 0
#print(server, usuaris_friendica)
#print("\n")
print("Servidor Friendica")
soft = "friendica"
friendica_users = 0
total_friendica_users = total_friendica_users + friendica_users
users = friendica_users
print(server, friendica_users)
print("\n")
elif px_nodeinfo.ok == True and check_pixelfed == True:
#print("Servidor Pixelfed")
usuaris_pixelfed = px_nodeinfo.json()['usage']['users']['total']
total_pixelfed_users = total_pixelfed_users + usuaris_pixelfed
usuaris = usuaris_pixelfed
#print(server, usuaris_pixelfed)
#print("\n")
print("Servidor Pixelfed")
soft = "pixelfed"
pixelfed_users = px_nodeinfo.json()['usage']['users']['total']
total_pixelfed_users = total_pixelfed_users + pixelfed_users
users = pixelfed_users
print(server, pixelfed_users)
print("\n")
elif px_nodeinfo.ok == True and check_gnusocial2 == True:
#print("Servidor GNU Social 2.x")
print("Servidor GNU Social 2.x")
soft = "gnusocialv2"
gs_users = px_nodeinfo.json()['usage']['users']['total']
if gs_users == 0:
gs_users = px_nodeinfo.json()['usage']['users']['activeHalfyear']
total_gs_users = total_gs_users + gs_users
usuaris = gs_users
#print(server, gs_users)
#print("\n")
users = gs_users
print(server, gs_users)
print("\n")
elif wf_nodeinfo.ok == True and check_writefreely == True:
#print("Servidor WriteFreely")
usuaris_writefreely = wf_nodeinfo.json()['usage']['users']['total']
total_writefreely_users = total_writefreely_users + usuaris_writefreely
usuaris = usuaris_writefreely
#print(server, usuaris_writefreely)
#print("\n")
print("Servidor WriteFreely")
soft = "writefreely"
writefreely_users = wf_nodeinfo.json()['usage']['users']['total']
total_writefreely_users = total_writefreely_users + writefreely_users
users = writefreely_users
print(server, writefreely_users)
print("\n")
else:
@ -508,7 +550,7 @@ def getserver(server):
if api_reply == 1 and others_nodeinfo.ok or gs_nodeinfo.ok or nodeinfo.ok or statusnet.ok or px_nodeinfo.ok or wf_nodeinfo.ok:
insert_sql = "INSERT INTO fediverse(server, users, updated_at) VALUES(%s,%s,%s) ON CONFLICT DO NOTHING"
insert_sql = "INSERT INTO fediverse(server, users, updated_at, software) VALUES(%s,%s,%s,%s) ON CONFLICT DO NOTHING"
conn = None
try:
@ -517,9 +559,9 @@ def getserver(server):
cur = conn.cursor()
cur.execute(insert_sql, (server, usuaris, now))
cur.execute(insert_sql, (server, users, now, soft))
cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s) where server=(%s)", (usuaris, now, server))
cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s) where server=(%s)", (users, now, soft, server))
conn.commit()
@ -528,7 +570,7 @@ def getserver(server):
except (Exception, psycopg2.DatabaseError) as error:
#print(error)
print(error)
return
finally:
@ -548,7 +590,7 @@ def getserver(server):
except requests.exceptions.ChunkedEncodingError as chunkerr:
#print(server)
print(server)
pass
except requests.exceptions.ConnectionError as errc:
@ -615,37 +657,36 @@ mast_users = 0
total_mast_users = 0
gs_users = 0
total_gs_users = 0
usuaris_zap = 0
zap_users = 0
total_zap_users = 0
usuaris_plume = 0
plume_users = 0
total_plume_users = 0
usuaris_hubzilla = 0
hubzilla_users = 0
total_hubzilla_users = 0
usuaris_misskey = 0
misskey_users = 0
total_misskey_users = 0
usuaris_prismo = 0
prismo_users = 0
total_prismo_users = 0
usuaris_osada = 0
osada_users = 0
total_osada_users = 0
usuaris_gpolis = 0
gpolis_users = 0
total_gpolis_users = 0
usuaris_ggg = 0
ggg_users = 0
total_ggg_users = 0
usuaris_squs = 0
squs_users = 0
total_squs_users = 0
peertube_users = 0
total_peertube_users = 0
diaspora_users = 0
total_diaspora_users = 0
usuaris_friendica = 0
friendica_users = 0
total_friendica_users = 0
usuaris_pixelfed = 0
pixelfed_users = 0
total_pixelfed_users = 0
usuaris_writefreely = 0
writefreely_users = 0
total_writefreely_users = 0
ravenvale_users = 0
total_ravenvale_users = 0
usuaris_total = 0
total_servers = 0
total_users = 0
@ -682,11 +723,44 @@ finally:
conn.close()
###########################################################################
# multiprocessing!
nprocs = multiprocessing.cpu_count()
pool = Pool(processes=nprocs)
for _ in tqdm.tqdm(pool.imap_unordered(getserver, world_servers), total=len(world_servers)):
pass
m = Manager()
q = m.Queue()
z = zip(world_servers)
serv_number = len(world_servers)
pool_tuple = [(x, q) for x in z]
with Pool(processes=64) as pool:
pool.starmap(getserver, pool_tuple)
###########################################################################
# delete not alive servers from fediverse table
try:
conn = None
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute("delete from fediverse where updated_at != (%s)", (now,))
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
###########################################################################
# get current total servers and users
@ -719,8 +793,8 @@ finally:
conn.close()
#########################################################################################
# get last server and users values and calc evolution. Write current values to database
###########################################################################
# get last check values and write current total values
select_sql = "select total_servers, total_users from totals order by datetime desc limit 1"
insert_sql = "INSERT INTO totals(datetime, total_servers, total_users) VALUES(%s,%s,%s)"
@ -766,7 +840,7 @@ finally:
conn.close()
################################################################################
# escriure en la bbdd evo
# write evo values
insert_sql = "INSERT INTO evo(datetime, servers, users) VALUES(%s,%s,%s)"
@ -794,23 +868,51 @@ finally:
conn.close()
##############################################################################
# get world's last update datetime
conn = None
try:
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute("select updated_at from world order by updated_at desc limit 1")
row = cur.fetchone()
last_update = row[0]
last_update = last_update.strftime('%m/%d/%Y, %H:%M:%S')
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
###############################################################################
# T O O T !
toot_text = "#fediverse live servers stats" + " \n"
toot_text = "#fediverse alive servers stats" + " \n"
toot_text += "\n"
if evo_servers > 0:
toot_text += "Live servers: " + str(total_servers) + " (+"+ str(evo_servers) + ") \n"
else:
toot_text += "Live servers: " + str(total_servers) + " ("+ str(evo_servers) + ") \n"
if evo_servers >= 0:
toot_text += "alive servers: " + str(total_servers) + " (+"+ str(evo_servers) + ") \n"
elif evo_servers < 0:
toot_text += "alive servers: " + str(total_servers) + " (-"+ str(evo_servers) + ") \n"
if evo_users >= 0:
toot_text += "total Users: " + str(total_users) + " (+"+ str(evo_users) + ") \n"
elif evo_users < 0:
toot_text += "total Users: " + str(total_users) + " (-"+ str(evo_users) + ") \n"
toot_text += "\n"
if evo_users > 0:
toot_text += "Total Users: " + str(total_users) + " (+"+ str(evo_users) + ") \n"
else:
toot_text += "Total Users: " + str(total_users) + " ("+ str(evo_users) + ") \n"
toot_text += "\n"
toot_text += "Exec. time %s seconds" % (time.time() - start_time)
toot_text += "updated at " + str(last_update) + " \n"
print("Tooting...")
print(toot_text)