728 líneas
20 KiB
Python
728 líneas
20 KiB
Python
#!/usr/bin/env python
|
|
# -*- coding: utf-8 -*-
|
|
|
|
import time
|
|
start_time = time.time()
|
|
import urllib3
|
|
from urllib3 import exceptions
|
|
from datetime import datetime
|
|
from subprocess import call
|
|
from mastodon import Mastodon
|
|
import threading
|
|
import os
|
|
import json
|
|
import signal
|
|
import sys
|
|
import os.path
|
|
import requests
|
|
from requests import exceptions
|
|
import operator
|
|
import calendar
|
|
import psycopg2
|
|
from itertools import product
|
|
|
|
from multiprocessing import Pool, Lock, Process, Queue, current_process, Manager
|
|
import multiprocessing
|
|
|
|
import aiohttp
|
|
import aiodns
|
|
import asyncio
|
|
from aiohttp import ClientError, ClientSession, ClientConnectionError, ClientConnectorError, ClientSSLError, ClientConnectorSSLError, ServerTimeoutError
|
|
from asyncio import TimeoutError
|
|
import socket
|
|
from socket import gaierror, gethostbyname
|
|
|
|
from decimal import *
|
|
getcontext().prec = 2
|
|
|
|
apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?', '/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?']
|
|
|
|
client_exceptions = (
|
|
aiohttp.ClientResponseError,
|
|
aiohttp.ClientConnectionError,
|
|
aiohttp.ClientConnectorError,
|
|
aiohttp.ClientError,
|
|
asyncio.TimeoutError,
|
|
socket.gaierror,
|
|
)
|
|
|
|
now = datetime.now()
|
|
|
|
###############################################################################
|
|
# INITIALISATION
|
|
###############################################################################
|
|
|
|
def is_json(myjson):
|
|
try:
|
|
json_object = json.loads(myjson)
|
|
except ValueError as e:
|
|
return False
|
|
return True
|
|
|
|
def alive_server(server, x):
|
|
|
|
server = server[0].rstrip('.').lower()
|
|
|
|
conn = None
|
|
|
|
try:
|
|
|
|
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
cur.execute("select alive, software, users_api from fediverse where server=(%s)", (server,))
|
|
|
|
row = cur.fetchone()
|
|
|
|
if row != None:
|
|
|
|
was_alive = row[0]
|
|
serv_soft = row[1]
|
|
serv_api = row[2]
|
|
|
|
cur.close()
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
conn.close()
|
|
|
|
alive = False
|
|
|
|
try:
|
|
|
|
data = requests.get('https://' + server + serv_api, timeout=3)
|
|
|
|
if serv_soft == "mastodon":
|
|
if serv_api == '/nodeinfo/2.0?':
|
|
try:
|
|
users = data.json()['usage']['users']['total']
|
|
alive = True
|
|
except:
|
|
users = 0
|
|
if serv_api == '/nodeinfo/2.0.json?':
|
|
try:
|
|
users = data.json()['usage']['users']['total']
|
|
alive = True
|
|
except:
|
|
users = 0
|
|
elif serv_api == '/api/v1/instance?':
|
|
try:
|
|
users = data.json()['stats']['user_count']
|
|
alive = True
|
|
except:
|
|
users = 0
|
|
if serv_soft == "pleroma" or serv_soft == "diaspora" or serv_soft == "peertube" or serv_soft == "pixelfed" or serv_soft == "hubzilla" or serv_soft == "writefreely" or serv_soft == "friendica":
|
|
try:
|
|
users = data.json()['usage']['users']['total']
|
|
alive = True
|
|
except:
|
|
users = 0
|
|
if serv_soft == "gnusocialv2" or serv_soft == "gnusocial":
|
|
try:
|
|
users = data.json()['usage']['users']['total']
|
|
if users == 0:
|
|
users = data.json()['usage']['users']['activeHalfyear']
|
|
alive = True
|
|
except:
|
|
users = 0
|
|
if serv_soft == "plume" or serv_soft == 'red' or serv_soft == "misskey" or serv_soft == "zap" or serv_soft == "prismo" or serv_soft == "ravenvale" or serv_soft == "osada" or serv_soft == "groundpolis":
|
|
try:
|
|
users = data.json()['usage']['users']['total']
|
|
alive = True
|
|
except:
|
|
users = 0
|
|
if serv_soft == "ganggo" or serv_soft == "squs" or serv_soft == "dolphin":
|
|
try:
|
|
users = data.json()['usage']['users']['total']
|
|
alive = True
|
|
except:
|
|
users = 0
|
|
|
|
print("Server " + str(server) + " (" + serv_soft + ") is alive!")
|
|
|
|
insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api) VALUES(%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
|
|
conn = None
|
|
|
|
try:
|
|
|
|
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
cur.execute(insert_sql, (server, users, now, serv_soft, alive, serv_api))
|
|
|
|
cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s) where server=(%s)", (users, now, serv_soft, alive, serv_api, server))
|
|
|
|
cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
|
|
|
|
conn.commit()
|
|
|
|
cur.close()
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
conn.close()
|
|
|
|
except urllib3.exceptions.ProtocolError as protoerr:
|
|
|
|
print("Server " + server + " is dead :-(")
|
|
alive = False
|
|
pass
|
|
|
|
except requests.exceptions.ChunkedEncodingError as chunkerr:
|
|
|
|
print("Server " + server + " is dead :-(")
|
|
alive = False
|
|
pass
|
|
|
|
except KeyError as e:
|
|
|
|
print("Server " + server + " is dead :-(")
|
|
alive = False
|
|
pass
|
|
|
|
except ValueError as verr:
|
|
|
|
print("Server " + server + " is dead :-(")
|
|
alive = False
|
|
pass
|
|
|
|
except requests.exceptions.SSLError as errssl:
|
|
|
|
print("Server " + server + " is dead :-(")
|
|
alive = False
|
|
pass
|
|
|
|
except requests.exceptions.HTTPError as errh:
|
|
|
|
print("Server " + server + " is dead :-(")
|
|
alive = False
|
|
pass
|
|
|
|
except requests.exceptions.ConnectionError as errc:
|
|
|
|
print("Server " + server + " is dead :-(")
|
|
alive = False
|
|
pass
|
|
|
|
except requests.exceptions.Timeout as errt:
|
|
|
|
print("Server " + server + " is dead :-(")
|
|
alive = False
|
|
pass
|
|
|
|
except requests.exceptions.RequestException as err:
|
|
|
|
print("Server " + server + " is dead :-(")
|
|
alive = False
|
|
pass
|
|
|
|
if alive == False:
|
|
|
|
conn = None
|
|
|
|
try:
|
|
|
|
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="",
|
|
host="/var/run/postgresql", port="5432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s) where server=(%s)", (now, alive, server))
|
|
|
|
cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
|
|
|
|
conn.commit()
|
|
|
|
cur.close()
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
conn.close()
|
|
|
|
def write_api(server, software, users, alive, api):
|
|
|
|
insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api) VALUES(%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
|
|
conn = None
|
|
|
|
try:
|
|
|
|
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
cur.execute(insert_sql, (server, now, software, users, alive, api))
|
|
|
|
cur.execute("UPDATE fediverse SET updated_at=(%s), software=(%s), users=(%s), alive=(%s), users_api=(%s) where server=(%s)", (now, software, users, alive, api, server))
|
|
|
|
cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
|
|
|
|
conn.commit()
|
|
|
|
cur.close()
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
async def getsoft(server):
|
|
|
|
try:
|
|
|
|
socket.gethostbyname(server)
|
|
|
|
except socket.gaierror:
|
|
|
|
return
|
|
|
|
soft = ''
|
|
|
|
url = 'https://' + server
|
|
|
|
timeout = aiohttp.ClientTimeout(total=3)
|
|
async with aiohttp.ClientSession(timeout=timeout) as session:
|
|
for api in apis:
|
|
try:
|
|
async with session.get(url+api) as response:
|
|
if response.status == 200:
|
|
try:
|
|
response_json = await response.json()
|
|
except:
|
|
pass
|
|
except aiohttp.ClientConnectorError as err:
|
|
pass
|
|
else:
|
|
if response.status == 200 and api != '/api/v1/instance?':
|
|
try:
|
|
soft = response_json['software']['name']
|
|
soft = soft.lower()
|
|
users = response_json['usage']['users']['total']
|
|
alive = True
|
|
write_api(server, soft, users, alive, api)
|
|
print("Server " + server + " is alive!")
|
|
return
|
|
except:
|
|
pass
|
|
if response.status == 200 and soft == '' and api == "/api/v1/instance?":
|
|
soft = 'mastodon'
|
|
users = response_json['stats']['user_count']
|
|
alive = True
|
|
write_api(server, soft, users, alive, api)
|
|
print("Server " + server + " is alive!")
|
|
|
|
def getserver(server, x):
|
|
|
|
server = server[0].rstrip('.').lower()
|
|
|
|
if server.find(".") == -1:
|
|
return
|
|
if server.find("@") != -1:
|
|
return
|
|
if server.find("/") != -1:
|
|
return
|
|
if server.find(":") != -1:
|
|
return
|
|
|
|
try:
|
|
|
|
loop = asyncio.get_event_loop()
|
|
coroutines = [getsoft(server)]
|
|
soft = loop.run_until_complete(asyncio.gather(*coroutines, return_exceptions=True))
|
|
|
|
except:
|
|
|
|
pass
|
|
|
|
# Returns the parameter from the specified file
|
|
def get_parameter( parameter, file_path ):
|
|
# Check if secrets file exists
|
|
if not os.path.isfile(file_path):
|
|
print("File %s not found, exiting."%file_path)
|
|
sys.exit(0)
|
|
|
|
# Find parameter in file
|
|
with open( file_path ) as f:
|
|
for line in f:
|
|
if line.startswith( parameter ):
|
|
return line.replace(parameter + ":", "").strip()
|
|
|
|
# Cannot find parameter, exit
|
|
print(file_path + " Missing parameter %s "%parameter)
|
|
sys.exit(0)
|
|
|
|
# Load secrets from secrets file
|
|
secrets_filepath = "secrets/secrets.txt"
|
|
uc_client_id = get_parameter("uc_client_id", secrets_filepath)
|
|
uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
|
|
uc_access_token = get_parameter("uc_access_token", secrets_filepath)
|
|
|
|
# Load configuration from config file
|
|
config_filepath = "config/config.txt"
|
|
mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
|
|
|
|
# Load database config from db_config file
|
|
db_config_filepath = "config/db_config.txt"
|
|
fediverse_db = get_parameter("fediverse_db", db_config_filepath)
|
|
fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath)
|
|
|
|
# Initialise Mastodon API
|
|
mastodon = Mastodon(
|
|
client_id = uc_client_id,
|
|
client_secret = uc_client_secret,
|
|
access_token = uc_access_token,
|
|
api_base_url = 'https://' + mastodon_hostname,
|
|
)
|
|
|
|
# Initialise access headers
|
|
headers={ 'Authorization': 'Bearer %s'%uc_access_token }
|
|
|
|
###############################################################################
|
|
# main
|
|
|
|
if __name__ == '__main__':
|
|
|
|
total_servers = 0
|
|
total_users = 0
|
|
|
|
############################################################################
|
|
# set all world servers's checked column to False
|
|
|
|
try:
|
|
|
|
conn = None
|
|
|
|
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
cur.execute("UPDATE world SET checked='f'")
|
|
|
|
conn.commit()
|
|
|
|
cur.close()
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
############################################################################
|
|
# get last check servers from fediverse DB
|
|
|
|
|
|
alive_servers = []
|
|
|
|
try:
|
|
|
|
conn = None
|
|
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
### get world servers list
|
|
|
|
cur.execute("select server from world where server in (select server from fediverse where users_api != '')")
|
|
|
|
alive_servers = []
|
|
|
|
for row in cur:
|
|
|
|
alive_servers.append(row[0])
|
|
|
|
cur.close()
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
###########################################################################
|
|
# multiprocessing!
|
|
|
|
m = Manager()
|
|
q = m.Queue()
|
|
z = zip(alive_servers)
|
|
|
|
serv_number = len(alive_servers)
|
|
|
|
pool_tuple = [(x, q) for x in z]
|
|
with Pool(processes=64) as pool:
|
|
pool.starmap(alive_server, pool_tuple)
|
|
|
|
###########################################################################
|
|
|
|
print("Getting the remaining servers from world")
|
|
|
|
world_servers = []
|
|
|
|
try:
|
|
|
|
conn = None
|
|
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
### get world servers list
|
|
|
|
cur.execute("select server from world where checked='f'")
|
|
|
|
for row in cur:
|
|
|
|
world_servers.append(row[0])
|
|
|
|
cur.close()
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
###########################################################################
|
|
# multiprocessing!
|
|
|
|
m = Manager()
|
|
q = m.Queue()
|
|
z = zip(world_servers)
|
|
|
|
serv_number = len(world_servers)
|
|
|
|
pool_tuple = [(x, q) for x in z]
|
|
with Pool(processes=64) as pool:
|
|
pool.starmap(getserver, pool_tuple)
|
|
|
|
###########################################################################
|
|
# delete not alive servers from fediverse table
|
|
|
|
try:
|
|
|
|
conn = None
|
|
|
|
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
cur.execute("delete from fediverse where not alive")
|
|
|
|
conn.commit()
|
|
|
|
cur.close()
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
###########################################################################
|
|
# get current total servers and users, get users from every software
|
|
|
|
gettotals_sql = "select count(server), sum(users) from fediverse where alive"
|
|
get_soft_totals_sql = "select software, sum(users) as users, count(server) as servers from fediverse where users != 0 and alive group by software order by users desc"
|
|
soft_total_project = []
|
|
soft_total_users = []
|
|
soft_total_servers = []
|
|
|
|
try:
|
|
|
|
conn = None
|
|
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
cur.execute(gettotals_sql)
|
|
|
|
row = cur.fetchone()
|
|
|
|
total_servers = row[0]
|
|
total_users = row[1]
|
|
|
|
cur.execute(get_soft_totals_sql)
|
|
rows = cur.fetchall()
|
|
for row in rows:
|
|
soft_total_project.append(row[0])
|
|
soft_total_users.append(row[1])
|
|
soft_total_servers.append(row[2])
|
|
cur.close()
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
###########################################################################
|
|
# get last check values and write current total ones
|
|
|
|
select_sql = "select total_servers, total_users from totals order by datetime desc limit 1"
|
|
insert_sql = "INSERT INTO totals(datetime, total_servers, total_users) VALUES(%s,%s,%s)"
|
|
|
|
try:
|
|
|
|
conn = None
|
|
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
cur.execute(select_sql)
|
|
|
|
row = cur.fetchone()
|
|
|
|
if row != None:
|
|
|
|
servers_before = row[0]
|
|
users_before = row[1]
|
|
|
|
else:
|
|
|
|
servers_before = 0
|
|
users_before = 0
|
|
|
|
cur.execute(insert_sql, (now, total_servers, total_users))
|
|
|
|
conn.commit()
|
|
|
|
cur.close()
|
|
|
|
evo_servers = total_servers - servers_before
|
|
evo_users = total_users - users_before
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
################################################################################
|
|
# write evo values
|
|
|
|
insert_sql = "INSERT INTO evo(datetime, servers, users) VALUES(%s,%s,%s)"
|
|
|
|
conn = None
|
|
|
|
try:
|
|
|
|
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
cur.execute(insert_sql, (now, evo_servers, evo_users))
|
|
|
|
conn.commit()
|
|
|
|
cur.close()
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
##############################################################################
|
|
# get world's last update datetime
|
|
|
|
conn = None
|
|
|
|
try:
|
|
|
|
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
cur.execute("select updated_at from world order by updated_at desc limit 1")
|
|
|
|
row = cur.fetchone()
|
|
|
|
last_update = row[0]
|
|
last_update = last_update.strftime('%m/%d/%Y, %H:%M:%S')
|
|
|
|
cur.close()
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
###############################################################################
|
|
# T O O T !
|
|
|
|
toot_text = "#fediverse alive servers stats" + " \n"
|
|
toot_text += "\n"
|
|
if evo_servers >= 0:
|
|
toot_text += "alive servers: " + str(total_servers) + " (+"+ str(evo_servers) + ") \n"
|
|
elif evo_servers < 0:
|
|
toot_text += "alive servers: " + str(total_servers) + " ("+ str(evo_servers) + ") \n"
|
|
if evo_users >= 0:
|
|
toot_text += "total users: " + str(total_users) + " (+"+ str(evo_users) + ") \n"
|
|
elif evo_users < 0:
|
|
toot_text += "total users: " + str(total_users) + " ("+ str(evo_users) + ") \n"
|
|
toot_text += "\n"
|
|
toot_text += "top five (soft: users servers):" + " \n"
|
|
toot_text += "\n"
|
|
toot_text += '{:>0}: {:>5} {:>5}'.format(str(soft_total_project[0]), str(soft_total_users[0]), str(soft_total_servers[0])) + " \n"
|
|
toot_text += '{:>0}: {:>11} {:>8}'.format(str(soft_total_project[1]), str(soft_total_users[1]), str(soft_total_servers[1])) + " \n"
|
|
toot_text += '{:>0}: {:>12} {:>7}'.format(str(soft_total_project[2]), str(soft_total_users[2]), str(soft_total_servers[2])) + " \n"
|
|
toot_text += '{:>0}: {:>13} {:>7}'.format(str(soft_total_project[3]), str(soft_total_users[3]), str(soft_total_servers[3])) + " \n"
|
|
toot_text += '{:>0}: {:>14} {:>7}'.format(str(soft_total_project[4]), str(soft_total_users[4]), str(soft_total_servers[4])) + " \n"
|
|
toot_text += "\n"
|
|
toot_text += "updated at " + str(last_update) + " \n"
|
|
print("Tooting...")
|
|
print(toot_text)
|
|
|
|
mastodon.status_post(toot_text, in_reply_to_id=None, )
|