First version, beta stage
This commit is contained in:
spla
pare
6ddccb33bd
commit
bf284b8f69
S'han modificat 2 arxius amb 997 adicions i 0 eliminacions
817
fediverse.py
Normal file
817
fediverse.py
Normal file
|
|
@ -0,0 +1,817 @@
|
||||||
|
#!/usr/bin/env python
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
start_time = time.time()
|
||||||
|
import urllib3
|
||||||
|
from urllib3 import exceptions
|
||||||
|
from datetime import datetime
|
||||||
|
import pytz
|
||||||
|
from subprocess import call
|
||||||
|
from mastodon import Mastodon
|
||||||
|
import threading
|
||||||
|
import os
|
||||||
|
import json
|
||||||
|
import signal
|
||||||
|
import sys
|
||||||
|
import os.path
|
||||||
|
import requests
|
||||||
|
from requests import exceptions
|
||||||
|
import operator
|
||||||
|
import calendar
|
||||||
|
import psycopg2
|
||||||
|
from itertools import product
|
||||||
|
|
||||||
|
from multiprocessing import Pool, Lock, Process, Queue, current_process, Manager
|
||||||
|
import queue
|
||||||
|
import multiprocessing
|
||||||
|
import tqdm
|
||||||
|
|
||||||
|
from decimal import *
|
||||||
|
getcontext().prec = 2
|
||||||
|
|
||||||
|
###############################################################################
|
||||||
|
# INITIALISATION
|
||||||
|
###############################################################################
|
||||||
|
|
||||||
|
def is_json(myjson):
|
||||||
|
try:
|
||||||
|
json_object = json.loads(myjson)
|
||||||
|
except ValueError as e:
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
def getserver(server):
|
||||||
|
|
||||||
|
#server = server[0].rstrip('.').lower()
|
||||||
|
|
||||||
|
if server.find(".") == -1:
|
||||||
|
#print(server + "is Not a real domain!")
|
||||||
|
return
|
||||||
|
if server.find("@") != -1:
|
||||||
|
#print(server + "is Not a real domain!")
|
||||||
|
return
|
||||||
|
if server.find("/") != -1:
|
||||||
|
#print("Removing substring after first / from " + server)
|
||||||
|
server = server.split("/", 1)[0]
|
||||||
|
if server.find(":") != -1:
|
||||||
|
#print("Removing substring after first : from " + server)
|
||||||
|
server = server.split(":", 1)[0]
|
||||||
|
|
||||||
|
global mastodont
|
||||||
|
global pleroma
|
||||||
|
global gnusocial
|
||||||
|
global zap
|
||||||
|
global plume
|
||||||
|
global hubzilla
|
||||||
|
global misskey
|
||||||
|
global prismo
|
||||||
|
global osada
|
||||||
|
global groundpolis
|
||||||
|
global ganggo
|
||||||
|
global squs
|
||||||
|
global peertube
|
||||||
|
global friendica
|
||||||
|
global pixelfed
|
||||||
|
global writefreely
|
||||||
|
global ravenvale
|
||||||
|
global diaspora
|
||||||
|
|
||||||
|
global pl_users
|
||||||
|
global total_pl_users
|
||||||
|
global mast_users
|
||||||
|
global total_mast_users
|
||||||
|
global gs_users
|
||||||
|
global total_gs_users
|
||||||
|
global usuaris_zap
|
||||||
|
global total_zap_users
|
||||||
|
global usuaris_plume
|
||||||
|
global total_plume_users
|
||||||
|
global usuaris_hubzilla
|
||||||
|
global total_hubzilla_users
|
||||||
|
global usuaris_misskey
|
||||||
|
global total_misskey_users
|
||||||
|
global usuaris_prismo
|
||||||
|
global total_prismo_users
|
||||||
|
global usuaris_osada
|
||||||
|
global total_osada_users
|
||||||
|
global usuaris_gpolis
|
||||||
|
global total_gpolis_users
|
||||||
|
global usuaris_ggg
|
||||||
|
global total_ggg_users
|
||||||
|
global usuaris_squs
|
||||||
|
global total_squs_users
|
||||||
|
global peertube_users
|
||||||
|
global total_peertube_users
|
||||||
|
global diaspora_users
|
||||||
|
global total_diaspora_users
|
||||||
|
global usuaris_friendica
|
||||||
|
global total_friendica_users
|
||||||
|
global usuaris_pixelfed
|
||||||
|
global total_pixelfed_users
|
||||||
|
global usuaris_writefreely
|
||||||
|
global total_writefreely_users
|
||||||
|
global ravenvale_users
|
||||||
|
global total_ravenvale_users
|
||||||
|
global usuaris_total
|
||||||
|
|
||||||
|
check_diaspora = False
|
||||||
|
check_peertube = False
|
||||||
|
check_zap = False
|
||||||
|
check_plume = False
|
||||||
|
check_hubzilla = False
|
||||||
|
check_misskey = False
|
||||||
|
check_prismo = False
|
||||||
|
check_osada = False
|
||||||
|
check_groundpolis = False
|
||||||
|
check_ganggo = False
|
||||||
|
check_squs = False
|
||||||
|
check_gnusocial = False
|
||||||
|
check_friendica = False
|
||||||
|
check_pixelfed = False
|
||||||
|
check_writefreely = False
|
||||||
|
check_raven = False
|
||||||
|
check_gnusocial2 = False
|
||||||
|
|
||||||
|
usuaris = 0
|
||||||
|
|
||||||
|
try:
|
||||||
|
|
||||||
|
res = requests.get('https://' + server + '/api/v1/instance?',timeout=3)
|
||||||
|
res_friendica = requests.get('https://' + server + '/nodeinfo/2.0?',timeout=3)
|
||||||
|
check_pix = "compatible; Pixelfed" in res.text
|
||||||
|
check_ravenvale = "ravenvale" in res.text
|
||||||
|
check_friendica = "friendica" in res_friendica.text
|
||||||
|
if check_friendica == False:
|
||||||
|
check_friendica = "Friendica" in res_friendica.text
|
||||||
|
|
||||||
|
if (res.ok) and check_pix == False and check_ravenvale == False and check_friendica == False:
|
||||||
|
|
||||||
|
if is_json(res.text) == True and server in res.text:
|
||||||
|
|
||||||
|
nodeinfo = requests.get('https://' + server + '/nodeinfo/2.0.json?',timeout=3)
|
||||||
|
check_pleroma = "pleroma" in nodeinfo.text.lower()
|
||||||
|
|
||||||
|
if nodeinfo.status_code == 200 and check_pleroma == True:
|
||||||
|
|
||||||
|
#print("Servidor Pleroma: ")
|
||||||
|
pl_users = nodeinfo.json()['usage']['users']['total']
|
||||||
|
total_pl_users = total_pl_users + pl_users
|
||||||
|
usuaris = pl_users
|
||||||
|
#print(server, pl_users)
|
||||||
|
|
||||||
|
else:
|
||||||
|
|
||||||
|
if type(res.json()) != int:
|
||||||
|
|
||||||
|
if res.json().get('stats') != None:
|
||||||
|
#print("Servidor Mastodon: ")
|
||||||
|
mast_users = res.json()['stats']['user_count']
|
||||||
|
else:
|
||||||
|
mast_users = 0
|
||||||
|
else:
|
||||||
|
|
||||||
|
mast_users = 0
|
||||||
|
|
||||||
|
if mast_users != None:
|
||||||
|
total_mast_users = total_mast_users + mast_users
|
||||||
|
usuaris = mast_users
|
||||||
|
#print(server, mast_users)
|
||||||
|
|
||||||
|
#print("\n")
|
||||||
|
|
||||||
|
insert_sql = "INSERT INTO fediverse(server, users, updated_at) VALUES(%s,%s,%s) ON CONFLICT DO NOTHING"
|
||||||
|
conn = None
|
||||||
|
|
||||||
|
try:
|
||||||
|
|
||||||
|
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||||
|
|
||||||
|
cur = conn.cursor()
|
||||||
|
|
||||||
|
cur.execute(insert_sql, (server, usuaris, now))
|
||||||
|
|
||||||
|
cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s) where server=(%s)", (usuaris, now, server))
|
||||||
|
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
cur.close()
|
||||||
|
|
||||||
|
except (Exception, psycopg2.DatabaseError) as error:
|
||||||
|
print(error)
|
||||||
|
|
||||||
|
finally:
|
||||||
|
|
||||||
|
if conn is not None:
|
||||||
|
conn.close()
|
||||||
|
return
|
||||||
|
|
||||||
|
else:
|
||||||
|
|
||||||
|
return
|
||||||
|
|
||||||
|
except urllib3.exceptions.ProtocolError as protoerr:
|
||||||
|
|
||||||
|
pass
|
||||||
|
|
||||||
|
except requests.exceptions.ChunkedEncodingError as chunkerr:
|
||||||
|
|
||||||
|
pass
|
||||||
|
|
||||||
|
except KeyError as e:
|
||||||
|
|
||||||
|
pass
|
||||||
|
|
||||||
|
except ValueError as verr:
|
||||||
|
|
||||||
|
pass
|
||||||
|
|
||||||
|
except requests.exceptions.SSLError as errssl:
|
||||||
|
|
||||||
|
pass
|
||||||
|
|
||||||
|
except requests.exceptions.HTTPError as errh:
|
||||||
|
|
||||||
|
pass
|
||||||
|
|
||||||
|
except requests.exceptions.ConnectionError as errc:
|
||||||
|
|
||||||
|
pass
|
||||||
|
|
||||||
|
except requests.exceptions.Timeout as errt:
|
||||||
|
|
||||||
|
pass
|
||||||
|
|
||||||
|
except requests.exceptions.RequestException as err:
|
||||||
|
|
||||||
|
pass
|
||||||
|
|
||||||
|
else:
|
||||||
|
|
||||||
|
try:
|
||||||
|
|
||||||
|
soft = ""
|
||||||
|
others_nodeinfo = requests.get('https://' + server + '/nodeinfo/2.0?',timeout=3)
|
||||||
|
gs_nodeinfo = requests.get('https://' + server + '/main/nodeinfo/2.0?',timeout=3)
|
||||||
|
nodeinfo = requests.get('https://' + server + '/nodeinfo/2.0.json?',timeout=3)
|
||||||
|
statusnet = requests.get('https://' + server + '/api/statusnet/config?',timeout=3)
|
||||||
|
px_nodeinfo = requests.get('https://' + server + '/api/nodeinfo/2.0.json?',timeout=3)
|
||||||
|
wf_nodeinfo = requests.get('https://' + server + '/api/nodeinfo?',timeout=3)
|
||||||
|
|
||||||
|
try:
|
||||||
|
|
||||||
|
if is_json(nodeinfo.text) == True:
|
||||||
|
if nodeinfo.json() != None:
|
||||||
|
soft = nodeinfo.json()['software']['name']
|
||||||
|
if soft == "peertube":
|
||||||
|
check_peertube = True
|
||||||
|
elif soft == "prismo":
|
||||||
|
check_prismo = True
|
||||||
|
elif soft == "zap":
|
||||||
|
check_zap = True
|
||||||
|
elif soft == "osada":
|
||||||
|
check_osada = True
|
||||||
|
elif soft == "ravenvale":
|
||||||
|
check_raven = True
|
||||||
|
elif soft == "diaspora":
|
||||||
|
check_diaspora = True
|
||||||
|
|
||||||
|
if is_json(others_nodeinfo.text) == True:
|
||||||
|
soft = others_nodeinfo.json()['software']['name']
|
||||||
|
if soft == "plume":
|
||||||
|
check_plume = True
|
||||||
|
elif soft == "hubzilla":
|
||||||
|
check_hubzilla = True
|
||||||
|
elif soft == "misskey":
|
||||||
|
check_misskey = True
|
||||||
|
elif soft == "groundpolis":
|
||||||
|
check_groundpolis = True
|
||||||
|
elif soft == "ganggo":
|
||||||
|
check_ganggo = True
|
||||||
|
elif soft == "squs":
|
||||||
|
check_squs = True
|
||||||
|
elif soft == "friendica" or soft == "Friendica":
|
||||||
|
check_friendica = True
|
||||||
|
|
||||||
|
if is_json(wf_nodeinfo.text) == True and soft == "":
|
||||||
|
if wf_nodeinfo.json().get('error') != None and wf_nodeinfo.json().get('status') == None:
|
||||||
|
soft = wf_nodeinfo.json()['software']['name']
|
||||||
|
if soft == "writefreely":
|
||||||
|
check_writefreely = True
|
||||||
|
|
||||||
|
if is_json(gs_nodeinfo.text) == True:
|
||||||
|
soft = gs_nodeinfo.json()['software']['name']
|
||||||
|
if soft == "gnusocial":
|
||||||
|
check_gnusocial = True
|
||||||
|
|
||||||
|
if is_json(statusnet.text) == True and soft == "":
|
||||||
|
if soft != "zap" and soft != "osada":
|
||||||
|
soft = statusnet.json()['site']['friendica']['FRIENDICA_PLATFORM']
|
||||||
|
if soft == "Friendica":
|
||||||
|
check_friendica = True
|
||||||
|
|
||||||
|
if is_json(px_nodeinfo.text) == True and soft == "":
|
||||||
|
if soft != "Friendica" and soft != "hubzilla":
|
||||||
|
soft = px_nodeinfo.json()['software']['name']
|
||||||
|
if soft == "pixelfed":
|
||||||
|
check_pixelfed = True
|
||||||
|
if soft == "gnusocial":
|
||||||
|
check_gnusocial2 = True
|
||||||
|
|
||||||
|
except :
|
||||||
|
|
||||||
|
pass
|
||||||
|
|
||||||
|
if check_diaspora == False and check_peertube == False and check_zap == False and check_plume == False and check_hubzilla == False and check_misskey == False and check_prismo == False and check_osada == False and check_groundpolis == False and check_ganggo == False and check_squs == False and check_gnusocial == False and check_friendica == False and check_pixelfed == False and check_writefreely == False and check_raven == False and check_gnusocial2 == False:
|
||||||
|
|
||||||
|
api_reply = 0
|
||||||
|
pass
|
||||||
|
|
||||||
|
elif check_diaspora == True or check_peertube == True or check_zap == True or check_plume == True or check_hubzilla == True or check_misskey == True or check_prismo == True or check_osada == True or check_groundpolis == True or check_ganggo == True or check_squs == True or check_gnusocial == True or check_friendica == True or check_pixelfed == True or check_writefreely == True or check_raven == True or check_gnusocial2 == True:
|
||||||
|
|
||||||
|
api_reply = 1
|
||||||
|
|
||||||
|
if nodeinfo.ok == True and check_peertube == True:
|
||||||
|
|
||||||
|
#print("Servidor Peertube")
|
||||||
|
peertube_users = nodeinfo.json()['usage']['users']['total']
|
||||||
|
total_peertube_users = total_peertube_users + peertube_users
|
||||||
|
usuaris = peertube_users
|
||||||
|
#print(server, peertube_users)
|
||||||
|
#print("\n")
|
||||||
|
|
||||||
|
if nodeinfo.ok == True and check_diaspora == True:
|
||||||
|
|
||||||
|
#print("Servidor Diaspora")
|
||||||
|
diaspora_users = nodeinfo.json()['usage']['users']['total']
|
||||||
|
total_diaspora_users = total_diaspora_users + diaspora_users
|
||||||
|
usuaris = diaspora_users
|
||||||
|
#print(server, diaspora_users)
|
||||||
|
#print("\n")
|
||||||
|
|
||||||
|
elif nodeinfo.ok == True and check_raven == True:
|
||||||
|
|
||||||
|
#print("Servidor Ravenvale")
|
||||||
|
ravenvale_users = nodeinfo.json()['usage']['users']['total']
|
||||||
|
total_ravenvale_users = total_ravenvale_users + ravenvale_users
|
||||||
|
usuaris = ravenvale_users
|
||||||
|
#print(server, ravenvale_users)
|
||||||
|
#print("\n")
|
||||||
|
|
||||||
|
elif others_nodeinfo.ok == True and check_zap == True:
|
||||||
|
|
||||||
|
#print("Servidor Zap")
|
||||||
|
usuaris_zap = others_nodeinfo.json()['usage']['users']['total']
|
||||||
|
total_zap_users = total_zap_users + usuaris_zap
|
||||||
|
usuaris = usuaris_zap
|
||||||
|
#print(server, usuaris_zap)
|
||||||
|
#print("\n")
|
||||||
|
|
||||||
|
elif others_nodeinfo.ok == True and check_plume == True:
|
||||||
|
|
||||||
|
#print("Servidor Plume")
|
||||||
|
usuaris_plume = others_nodeinfo.json()['usage']['users']['total']
|
||||||
|
total_plume_users = total_plume_users + usuaris_plume
|
||||||
|
usuaris = usuaris_plume
|
||||||
|
#print(server, usuaris_plume)
|
||||||
|
#print("\n")
|
||||||
|
|
||||||
|
elif others_nodeinfo.ok == True and check_hubzilla == True:
|
||||||
|
|
||||||
|
#print("Servidor Hubzilla")
|
||||||
|
usuaris_hubzilla = others_nodeinfo.json()['usage']['users']['total']
|
||||||
|
total_hubzilla_users = total_hubzilla_users + usuaris_hubzilla
|
||||||
|
usuaris = usuaris_hubzilla
|
||||||
|
#print(server, usuaris_hubzilla)
|
||||||
|
#print("\n")
|
||||||
|
|
||||||
|
elif others_nodeinfo.ok == True and check_misskey == True:
|
||||||
|
|
||||||
|
#print("Servidor Misskey")
|
||||||
|
usuaris_misskey = 0
|
||||||
|
total_misskey_users = total_misskey_users + usuaris_misskey
|
||||||
|
usuaris = usuaris_misskey
|
||||||
|
#print(server, usuaris_misskey)
|
||||||
|
#print("\n")
|
||||||
|
|
||||||
|
elif others_nodeinfo.ok == True and check_prismo == True:
|
||||||
|
|
||||||
|
#print("Servidor Prismo")
|
||||||
|
usuaris_prismo = others_nodeinfo.json()['usage']['users']['total']
|
||||||
|
total_prismo_users = total_prismo_users + usuaris_prismo
|
||||||
|
usuaris = usuaris_prismo
|
||||||
|
#print(server, usuaris_prismo)
|
||||||
|
#print("\n")
|
||||||
|
|
||||||
|
elif others_nodeinfo.ok == True and check_osada == True:
|
||||||
|
|
||||||
|
#print("Servidor Osada")
|
||||||
|
usuaris_osada = others_nodeinfo.json()['usage']['users']['total']
|
||||||
|
total_osada_users = total_osada_users + usuaris_osada
|
||||||
|
usuaris = usuaris_osada
|
||||||
|
#print(server, usuaris_osada)
|
||||||
|
#print("\n")
|
||||||
|
|
||||||
|
elif others_nodeinfo.ok == True and check_groundpolis == True:
|
||||||
|
|
||||||
|
#print("Servidor Groundpolis")
|
||||||
|
usuaris_gpolis = others_nodeinfo.json()['usage']['users']['total']
|
||||||
|
total_gpolis_users = total_gpolis_users + usuaris_gpolis
|
||||||
|
usuaris = usuaris_gpolis
|
||||||
|
#print(server, usuaris_gpolis)
|
||||||
|
#print("\n")
|
||||||
|
|
||||||
|
elif others_nodeinfo.ok == True and check_ganggo == True:
|
||||||
|
|
||||||
|
#print("Servidor Ganggo")
|
||||||
|
usuaris_ggg = others_nodeinfo.json()['usage']['users']['total']
|
||||||
|
total_ggg_users = total_ggg_users + usuaris_ggg
|
||||||
|
usuaris = usuaris_ggg
|
||||||
|
#print(server, usuaris_ggg)
|
||||||
|
#print("\n")
|
||||||
|
|
||||||
|
elif others_nodeinfo.ok == True and check_squs == True:
|
||||||
|
|
||||||
|
#print("Servidor Squs")
|
||||||
|
usuaris_squs = others_nodeinfo.json()['usage']['users']['total']
|
||||||
|
total_squs_users = total_squs_users + usuaris_squs
|
||||||
|
usuaris = usuaris_squs
|
||||||
|
#print(server, usuaris_squs)
|
||||||
|
#print("\n")
|
||||||
|
|
||||||
|
elif others_nodeinfo.ok == True and check_friendica == True:
|
||||||
|
|
||||||
|
#print("Servidor Friendica")
|
||||||
|
if others_nodeinfo.json()['usage'] != []:
|
||||||
|
usuaris_friendica = others_nodeinfo.json()['usage']['users']['total']
|
||||||
|
else:
|
||||||
|
usuaris_friendica = 0
|
||||||
|
total_friendica_users = total_friendica_users + usuaris_friendica
|
||||||
|
usuaris = usuaris_friendica
|
||||||
|
#print(server, usuaris_friendica)
|
||||||
|
#print("\n")
|
||||||
|
check_friendica = False
|
||||||
|
|
||||||
|
elif gs_nodeinfo.ok == True and check_gnusocial == True:
|
||||||
|
|
||||||
|
#print("Servidor GNU Social")
|
||||||
|
gs_users = gs_nodeinfo.json()['usage']['users']['total']
|
||||||
|
if gs_users == 0:
|
||||||
|
gs_users = gs_nodeinfo.json()['usage']['users']['activeHalfyear']
|
||||||
|
total_gs_users = total_gs_users + gs_users
|
||||||
|
usuaris = gs_users
|
||||||
|
#print(server, gs_users)
|
||||||
|
#print("\n")
|
||||||
|
|
||||||
|
elif statusnet.ok == True and check_friendica == True: #statusnet.json()['site']['friendica']['FRIENDICA_PLATFORM'] == "Friendica":
|
||||||
|
|
||||||
|
#print("Servidor Friendica")
|
||||||
|
usuaris_friendica = 0
|
||||||
|
total_friendica_users = total_friendica_users + usuaris_friendica
|
||||||
|
usuaris = usuaris_friendica
|
||||||
|
instancies = 0
|
||||||
|
#print(server, usuaris_friendica)
|
||||||
|
#print("\n")
|
||||||
|
|
||||||
|
elif px_nodeinfo.ok == True and check_pixelfed == True:
|
||||||
|
|
||||||
|
#print("Servidor Pixelfed")
|
||||||
|
usuaris_pixelfed = px_nodeinfo.json()['usage']['users']['total']
|
||||||
|
total_pixelfed_users = total_pixelfed_users + usuaris_pixelfed
|
||||||
|
usuaris = usuaris_pixelfed
|
||||||
|
#print(server, usuaris_pixelfed)
|
||||||
|
#print("\n")
|
||||||
|
|
||||||
|
elif px_nodeinfo.ok == True and check_gnusocial2 == True:
|
||||||
|
|
||||||
|
#print("Servidor GNU Social 2.x")
|
||||||
|
gs_users = px_nodeinfo.json()['usage']['users']['total']
|
||||||
|
if gs_users == 0:
|
||||||
|
gs_users = px_nodeinfo.json()['usage']['users']['activeHalfyear']
|
||||||
|
total_gs_users = total_gs_users + gs_users
|
||||||
|
usuaris = gs_users
|
||||||
|
#print(server, gs_users)
|
||||||
|
#print("\n")
|
||||||
|
|
||||||
|
elif wf_nodeinfo.ok == True and check_writefreely == True:
|
||||||
|
|
||||||
|
#print("Servidor WriteFreely")
|
||||||
|
usuaris_writefreely = wf_nodeinfo.json()['usage']['users']['total']
|
||||||
|
total_writefreely_users = total_writefreely_users + usuaris_writefreely
|
||||||
|
usuaris = usuaris_writefreely
|
||||||
|
#print(server, usuaris_writefreely)
|
||||||
|
#print("\n")
|
||||||
|
|
||||||
|
else:
|
||||||
|
|
||||||
|
api_reply = 0
|
||||||
|
|
||||||
|
if api_reply == 1 and others_nodeinfo.ok or gs_nodeinfo.ok or nodeinfo.ok or statusnet.ok or px_nodeinfo.ok or wf_nodeinfo.ok:
|
||||||
|
|
||||||
|
insert_sql = "INSERT INTO fediverse(server, users, updated_at) VALUES(%s,%s,%s) ON CONFLICT DO NOTHING"
|
||||||
|
conn = None
|
||||||
|
|
||||||
|
try:
|
||||||
|
|
||||||
|
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||||
|
|
||||||
|
cur = conn.cursor()
|
||||||
|
|
||||||
|
cur.execute(insert_sql, (server, usuaris, now))
|
||||||
|
|
||||||
|
cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s) where server=(%s)", (usuaris, now, server))
|
||||||
|
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
cur.close()
|
||||||
|
return
|
||||||
|
|
||||||
|
except (Exception, psycopg2.DatabaseError) as error:
|
||||||
|
|
||||||
|
#print(error)
|
||||||
|
return
|
||||||
|
|
||||||
|
finally:
|
||||||
|
|
||||||
|
if conn is not None:
|
||||||
|
|
||||||
|
conn.close()
|
||||||
|
return
|
||||||
|
|
||||||
|
except KeyError as e:
|
||||||
|
|
||||||
|
pass
|
||||||
|
|
||||||
|
except ValueError as verr:
|
||||||
|
|
||||||
|
pass
|
||||||
|
|
||||||
|
except requests.exceptions.ChunkedEncodingError as chunkerr:
|
||||||
|
|
||||||
|
#print(server)
|
||||||
|
pass
|
||||||
|
|
||||||
|
except requests.exceptions.ConnectionError as errc:
|
||||||
|
|
||||||
|
pass
|
||||||
|
|
||||||
|
except requests.exceptions.Timeout as errt:
|
||||||
|
|
||||||
|
pass
|
||||||
|
|
||||||
|
except requests.exceptions.ConnectionError as errc:
|
||||||
|
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Returns the parameter from the specified file
|
||||||
|
def get_parameter( parameter, file_path ):
|
||||||
|
# Check if secrets file exists
|
||||||
|
if not os.path.isfile(file_path):
|
||||||
|
print("File %s not found, exiting."%file_path)
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
|
# Find parameter in file
|
||||||
|
with open( file_path ) as f:
|
||||||
|
for line in f:
|
||||||
|
if line.startswith( parameter ):
|
||||||
|
return line.replace(parameter + ":", "").strip()
|
||||||
|
|
||||||
|
# Cannot find parameter, exit
|
||||||
|
print(file_path + " Missing parameter %s "%parameter)
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
|
# Load secrets from secrets file
|
||||||
|
secrets_filepath = "secrets/secrets.txt"
|
||||||
|
uc_client_id = get_parameter("uc_client_id", secrets_filepath)
|
||||||
|
uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
|
||||||
|
uc_access_token = get_parameter("uc_access_token", secrets_filepath)
|
||||||
|
|
||||||
|
# Load configuration from config file
|
||||||
|
config_filepath = "config.txt"
|
||||||
|
mastodon_hostname = get_parameter("mastodon_hostname", config_filepath) # E.g., mastodon.social
|
||||||
|
|
||||||
|
# Load database config from db_config file
|
||||||
|
db_config_filepath = "config/db_config.txt"
|
||||||
|
fediverse_db = get_parameter("fediverse_db", db_config_filepath)
|
||||||
|
fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath)
|
||||||
|
|
||||||
|
# Initialise Mastodon API
|
||||||
|
mastodon = Mastodon(
|
||||||
|
client_id = uc_client_id,
|
||||||
|
client_secret = uc_client_secret,
|
||||||
|
access_token = uc_access_token,
|
||||||
|
api_base_url = 'https://' + mastodon_hostname,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Initialise access headers
|
||||||
|
headers={ 'Authorization': 'Bearer %s'%uc_access_token }
|
||||||
|
|
||||||
|
###############################################################################
|
||||||
|
# main
|
||||||
|
|
||||||
|
pl_users = 0
|
||||||
|
total_pl_users = 0
|
||||||
|
mast_users = 0
|
||||||
|
total_mast_users = 0
|
||||||
|
gs_users = 0
|
||||||
|
total_gs_users = 0
|
||||||
|
usuaris_zap = 0
|
||||||
|
total_zap_users = 0
|
||||||
|
usuaris_plume = 0
|
||||||
|
total_plume_users = 0
|
||||||
|
usuaris_hubzilla = 0
|
||||||
|
total_hubzilla_users = 0
|
||||||
|
usuaris_misskey = 0
|
||||||
|
total_misskey_users = 0
|
||||||
|
usuaris_prismo = 0
|
||||||
|
total_prismo_users = 0
|
||||||
|
usuaris_osada = 0
|
||||||
|
total_osada_users = 0
|
||||||
|
usuaris_gpolis = 0
|
||||||
|
total_gpolis_users = 0
|
||||||
|
usuaris_ggg = 0
|
||||||
|
total_ggg_users = 0
|
||||||
|
usuaris_squs = 0
|
||||||
|
total_squs_users = 0
|
||||||
|
peertube_users = 0
|
||||||
|
total_peertube_users = 0
|
||||||
|
diaspora_users = 0
|
||||||
|
total_diaspora_users = 0
|
||||||
|
usuaris_friendica = 0
|
||||||
|
total_friendica_users = 0
|
||||||
|
usuaris_pixelfed = 0
|
||||||
|
total_pixelfed_users = 0
|
||||||
|
usuaris_writefreely = 0
|
||||||
|
total_writefreely_users = 0
|
||||||
|
ravenvale_users = 0
|
||||||
|
total_ravenvale_users = 0
|
||||||
|
usuaris_total = 0
|
||||||
|
|
||||||
|
total_servers = 0
|
||||||
|
total_users = 0
|
||||||
|
|
||||||
|
now = datetime.now()
|
||||||
|
|
||||||
|
try:
|
||||||
|
|
||||||
|
conn = None
|
||||||
|
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||||
|
|
||||||
|
cur = conn.cursor()
|
||||||
|
|
||||||
|
### get world servers list
|
||||||
|
|
||||||
|
cur.execute("select server from world")
|
||||||
|
|
||||||
|
world_servers = []
|
||||||
|
|
||||||
|
for row in cur:
|
||||||
|
|
||||||
|
world_servers.append(row[0])
|
||||||
|
|
||||||
|
cur.close()
|
||||||
|
|
||||||
|
except (Exception, psycopg2.DatabaseError) as error:
|
||||||
|
|
||||||
|
print(error)
|
||||||
|
|
||||||
|
finally:
|
||||||
|
|
||||||
|
if conn is not None:
|
||||||
|
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
###########################################################################
|
||||||
|
|
||||||
|
nprocs = multiprocessing.cpu_count()
|
||||||
|
pool = Pool(processes=nprocs)
|
||||||
|
for _ in tqdm.tqdm(pool.imap_unordered(getserver, world_servers), total=len(world_servers)):
|
||||||
|
pass
|
||||||
|
|
||||||
|
###########################################################################
|
||||||
|
# get current total servers and users
|
||||||
|
|
||||||
|
select_sql = "select count(server), sum(users) from fediverse"
|
||||||
|
|
||||||
|
try:
|
||||||
|
|
||||||
|
conn = None
|
||||||
|
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||||
|
|
||||||
|
cur = conn.cursor()
|
||||||
|
|
||||||
|
cur.execute(select_sql)
|
||||||
|
|
||||||
|
row = cur.fetchone()
|
||||||
|
|
||||||
|
total_servers = row[0]
|
||||||
|
total_users = row[1]
|
||||||
|
|
||||||
|
cur.close()
|
||||||
|
|
||||||
|
except (Exception, psycopg2.DatabaseError) as error:
|
||||||
|
|
||||||
|
print(error)
|
||||||
|
|
||||||
|
finally:
|
||||||
|
|
||||||
|
if conn is not None:
|
||||||
|
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
#########################################################################################
|
||||||
|
# get last server and users values and calc evolution. Write current values to database
|
||||||
|
|
||||||
|
select_sql = "select total_servers, total_users from totals order by datetime desc limit 1"
|
||||||
|
insert_sql = "INSERT INTO totals(datetime, total_servers, total_users) VALUES(%s,%s,%s)"
|
||||||
|
|
||||||
|
try:
|
||||||
|
|
||||||
|
conn = None
|
||||||
|
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||||
|
|
||||||
|
cur = conn.cursor()
|
||||||
|
|
||||||
|
cur.execute(select_sql)
|
||||||
|
|
||||||
|
row = cur.fetchone()
|
||||||
|
|
||||||
|
if row != None:
|
||||||
|
|
||||||
|
servers_before = row[0]
|
||||||
|
users_before = row[1]
|
||||||
|
|
||||||
|
else:
|
||||||
|
|
||||||
|
servers_before = 0
|
||||||
|
users_before = 0
|
||||||
|
|
||||||
|
cur.execute(insert_sql, (now, total_servers, total_users))
|
||||||
|
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
cur.close()
|
||||||
|
|
||||||
|
evo_servers = total_servers - servers_before
|
||||||
|
evo_users = total_users - users_before
|
||||||
|
|
||||||
|
except (Exception, psycopg2.DatabaseError) as error:
|
||||||
|
|
||||||
|
print(error)
|
||||||
|
|
||||||
|
finally:
|
||||||
|
|
||||||
|
if conn is not None:
|
||||||
|
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
################################################################################
|
||||||
|
# escriure en la bbdd evo
|
||||||
|
|
||||||
|
insert_sql = "INSERT INTO evo(datetime, servers, users) VALUES(%s,%s,%s)"
|
||||||
|
|
||||||
|
conn = None
|
||||||
|
|
||||||
|
try:
|
||||||
|
|
||||||
|
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||||
|
|
||||||
|
cur = conn.cursor()
|
||||||
|
|
||||||
|
cur.execute(insert_sql, (now, evo_servers, evo_users))
|
||||||
|
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
cur.close()
|
||||||
|
|
||||||
|
except (Exception, psycopg2.DatabaseError) as error:
|
||||||
|
|
||||||
|
print(error)
|
||||||
|
|
||||||
|
finally:
|
||||||
|
|
||||||
|
if conn is not None:
|
||||||
|
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
###############################################################################
|
||||||
|
# T O O T !
|
||||||
|
|
||||||
|
toot_text = "#fediverse live servers stats" + " \n"
|
||||||
|
toot_text += "\n"
|
||||||
|
if evo_servers > 0:
|
||||||
|
toot_text += "Live servers: " + str(total_servers) + " (+"+ str(evo_servers) + ") \n"
|
||||||
|
else:
|
||||||
|
toot_text += "Live servers: " + str(total_servers) + " ("+ str(evo_servers) + ") \n"
|
||||||
|
toot_text += "\n"
|
||||||
|
if evo_users > 0:
|
||||||
|
toot_text += "Total Users: " + str(total_users) + " (+"+ str(evo_users) + ") \n"
|
||||||
|
else:
|
||||||
|
toot_text += "Total Users: " + str(total_users) + " ("+ str(evo_users) + ") \n"
|
||||||
|
toot_text += "\n"
|
||||||
|
toot_text += "Exec. time %s seconds" % (time.time() - start_time)
|
||||||
|
|
||||||
|
print("Tooting...")
|
||||||
|
print(toot_text)
|
||||||
|
|
||||||
|
mastodon.status_post(toot_text, in_reply_to_id=None, )
|
||||||
|
|
||||||
180
getworld.py
Normal file
180
getworld.py
Normal file
|
|
@ -0,0 +1,180 @@
|
||||||
|
#!/usr/bin/env python
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
import time
|
||||||
|
start_time = time.time()
|
||||||
|
from six.moves import urllib
|
||||||
|
from datetime import datetime
|
||||||
|
from subprocess import call
|
||||||
|
from mastodon import Mastodon
|
||||||
|
import threading
|
||||||
|
import os
|
||||||
|
import json
|
||||||
|
import signal
|
||||||
|
import sys
|
||||||
|
import os.path
|
||||||
|
import requests
|
||||||
|
import operator
|
||||||
|
import calendar
|
||||||
|
import psycopg2
|
||||||
|
from itertools import product
|
||||||
|
|
||||||
|
from multiprocessing import Pool, Lock, Process, Queue, current_process
|
||||||
|
import queue # imported for using queue.Empty exception
|
||||||
|
import multiprocessing
|
||||||
|
|
||||||
|
from decimal import *
|
||||||
|
getcontext().prec = 2
|
||||||
|
|
||||||
|
###############################################################################
|
||||||
|
# INITIALISATION
|
||||||
|
###############################################################################
|
||||||
|
|
||||||
|
def is_json(myjson):
|
||||||
|
try:
|
||||||
|
json_object = json.loads(myjson)
|
||||||
|
except ValueError as e:
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
def getpeers(server):
|
||||||
|
|
||||||
|
global updated_at
|
||||||
|
|
||||||
|
try:
|
||||||
|
|
||||||
|
res = requests.get('https://' + server + '/api/v1/instance/peers?',timeout=3)
|
||||||
|
|
||||||
|
if (res.ok):
|
||||||
|
|
||||||
|
if server.find(".") != -1 and server.find("@") == -1:
|
||||||
|
server_peers = res.json()
|
||||||
|
print("Server: " + server + ", " + "federated with " + str(len(server_peers)) + " servers")
|
||||||
|
else:
|
||||||
|
print("Server " + str(server) + " is not a domain")
|
||||||
|
|
||||||
|
i = 0
|
||||||
|
while i < len(server_peers) and server.find(".") != -1 and server.find("@") == -1:
|
||||||
|
|
||||||
|
saved_at = datetime.now()
|
||||||
|
insert_sql = "INSERT INTO world(server, federated_with, updated_at, saved_at) VALUES(%s,%s,%s,%s) ON CONFLICT DO NOTHING"
|
||||||
|
conn = None
|
||||||
|
try:
|
||||||
|
|
||||||
|
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||||
|
|
||||||
|
cur = conn.cursor()
|
||||||
|
|
||||||
|
cur.execute(insert_sql, (server_peers[i], server, updated_at, saved_at,))
|
||||||
|
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
cur.close()
|
||||||
|
|
||||||
|
except (Exception, psycopg2.DatabaseError) as error:
|
||||||
|
|
||||||
|
print(error)
|
||||||
|
|
||||||
|
finally:
|
||||||
|
|
||||||
|
if conn is not None:
|
||||||
|
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
i += 1
|
||||||
|
|
||||||
|
except KeyError as e:
|
||||||
|
|
||||||
|
pass
|
||||||
|
return
|
||||||
|
|
||||||
|
except ValueError as verr:
|
||||||
|
|
||||||
|
pass
|
||||||
|
return
|
||||||
|
|
||||||
|
except requests.exceptions.SSLError as errssl:
|
||||||
|
|
||||||
|
pass
|
||||||
|
return
|
||||||
|
|
||||||
|
except requests.exceptions.HTTPError as errh:
|
||||||
|
|
||||||
|
pass
|
||||||
|
return
|
||||||
|
|
||||||
|
except requests.exceptions.ConnectionError as errc:
|
||||||
|
|
||||||
|
pass
|
||||||
|
return
|
||||||
|
|
||||||
|
except requests.exceptions.Timeout as errt:
|
||||||
|
|
||||||
|
pass
|
||||||
|
return
|
||||||
|
|
||||||
|
except requests.exceptions.RequestException as err:
|
||||||
|
|
||||||
|
pass
|
||||||
|
return
|
||||||
|
|
||||||
|
# Returns the parameter from the specified file
|
||||||
|
def get_parameter( parameter, file_path ):
|
||||||
|
# Check if secrets file exists
|
||||||
|
if not os.path.isfile(file_path):
|
||||||
|
print("File %s not found, exiting."%file_path)
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
|
# Find parameter in file
|
||||||
|
with open( file_path ) as f:
|
||||||
|
for line in f:
|
||||||
|
if line.startswith( parameter ):
|
||||||
|
return line.replace(parameter + ":", "").strip()
|
||||||
|
|
||||||
|
# Cannot find parameter, exit
|
||||||
|
print(file_path + " Missing parameter %s "%parameter)
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
|
# Load secrets from secrets file
|
||||||
|
secrets_filepath = "secrets/secrets.txt"
|
||||||
|
uc_client_id = get_parameter("uc_client_id", secrets_filepath)
|
||||||
|
uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
|
||||||
|
uc_access_token = get_parameter("uc_access_token", secrets_filepath)
|
||||||
|
|
||||||
|
# Load configuration from config file
|
||||||
|
config_filepath = "config.txt"
|
||||||
|
mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
|
||||||
|
|
||||||
|
# Load database config from db_config file
|
||||||
|
db_config_filepath = "config/db_config.txt"
|
||||||
|
fediverse_db = get_parameter("fediverse_db", db_config_filepath)
|
||||||
|
fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath)
|
||||||
|
|
||||||
|
# Initialise Mastodon API
|
||||||
|
mastodon = Mastodon(
|
||||||
|
client_id = uc_client_id,
|
||||||
|
client_secret = uc_client_secret,
|
||||||
|
access_token = uc_access_token,
|
||||||
|
api_base_url = 'https://' + mastodon_hostname,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Initialise access headers
|
||||||
|
headers={ 'Authorization': 'Bearer %s'%uc_access_token }
|
||||||
|
|
||||||
|
###############################################################################
|
||||||
|
# get current datetime and Mastodon hostname peers
|
||||||
|
###############################################################################
|
||||||
|
|
||||||
|
updated_at = datetime.now()
|
||||||
|
|
||||||
|
getpeers(mastodon_hostname)
|
||||||
|
self_peers = mastodon.instance_peers()
|
||||||
|
|
||||||
|
###########################################################################
|
||||||
|
|
||||||
|
nprocs = multiprocessing.cpu_count()
|
||||||
|
with multiprocessing.Pool(processes=nprocs) as pool:
|
||||||
|
results = pool.starmap(getpeers, product(self_peers))
|
||||||
|
|
||||||
|
exec_time = str(round((time.time() - start_time), 2))
|
||||||
|
print(exec_time)
|
||||||
Loading…
Referencia en una nova incidència