Improved servers nodeinfo detection

This commit is contained in:
spla 2022-03-02 22:26:05 +01:00
pare 36d7ff62b1
commit 0b52e0090d
S'han modificat 2 arxius amb 417 adicions i 483 eliminacions

Veure arxiu

@ -1,3 +1,5 @@
from multiprocessing import set_start_method
from multiprocessing import get_context
import time import time
import urllib3 import urllib3
from datetime import datetime from datetime import datetime
@ -7,11 +9,10 @@ import json
import sys import sys
import os.path import os.path
import requests import requests
import psycopg2
from itertools import product
import multiprocessing
import aiohttp import aiohttp
import asyncio import asyncio
import psycopg2
from itertools import product
import socket import socket
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
import pdb import pdb
@ -19,18 +20,6 @@ plt.style.use('seaborn')
start_time = time.time() start_time = time.time()
apis = ['/nodeinfo/2.0?',
'/nodeinfo/2.0.json?',
'/main/nodeinfo/2.0?',
'/api/statusnet/config?',
'/api/nodeinfo/2.0.json?',
'/api/nodeinfo?',
'/api/v1/instance?',
'/wp-json/nodeinfo/2.0?',
'/api/v1/instance/nodeinfo/2.0?',
'/.well-known/x-nodeinfo2?'
]
client_exceptions = ( client_exceptions = (
aiohttp.ClientResponseError, aiohttp.ClientResponseError,
aiohttp.ClientConnectionError, aiohttp.ClientConnectionError,
@ -40,121 +29,91 @@ client_exceptions = (
socket.gaierror, socket.gaierror,
) )
def is_json(myjson): class Server:
try: name = 'Server'
json_object = json.loads(myjson)
except ValueError as e:
print(e)
return False
return True
def get_alive_servers(server): def __init_(self, server, software, users, alive, api, soft_version):
serv_api = '' self.server = server
serv_soft = '' self.software = software
soft_version = '' self.users = users
self.alive = alive
self.api = api
self.version = self.soft_version
try: def get_alive_servers(self, *args):
conn = None fediverse_db, fediverse_db_user = db_config()
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432") now = datetime.now()
cur = conn.cursor() if len(args) != 0:
cur.execute("select alive, software, users_api, version, first_checked_at, downs from fediverse where server=(%s)", (server,)) self.server = args[0]
row = cur.fetchone()
if row is not None:
was_alive = row[0]
serv_soft = row[1]
serv_api = row[2]
soft_version = row[3]
first_checked_at = row[4]
downs_qty = row[5]
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
alive = False
try: try:
user_agent = {'User-agent': 'Mozilla/5.0'} conn = None
data = requests.get('https://' + server + serv_api, headers = user_agent, timeout=3) conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor()
cur.execute("select alive, software, users_api, version, first_checked_at, downs from fediverse where server=(%s)", (self.server,))
row = cur.fetchone()
if row is not None:
was_alive = row[0]
self.software = row[1]
self.api = row[2]
self.soft_version = row[3]
first_checked_at = row[4]
downs_qty = row[5]
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
alive = False
try:
user_agent = {'User-agent': 'Mozilla/5.0'}
data = requests.get('https://' + self.server + self.api, headers = user_agent, timeout=3)
if serv_soft == "mastodon":
if serv_api == '/nodeinfo/2.0?':
try:
users = data.json()['usage']['users']['total']
soft_version = data.json()['software']['version']
alive = True
except:
users = 0
soft_version = ""
if serv_api == '/nodeinfo/2.0.json?':
try:
users = data.json()['usage']['users']['total']
soft_version = data.json()['software']['version']
alive = True
except:
users = 0
soft_version = ""
elif serv_api == '/api/v1/instance?':
try:
users = data.json()['stats']['user_count']
soft_version = data.json()['version']
alive = True
except:
users = 0
soft_version = ""
if serv_soft == "pleroma" or serv_soft == "diaspora" or serv_soft == "peertube" or serv_soft == "pixelfed" or serv_soft == "hubzilla" or serv_soft == "writefreely" or serv_soft == "friendica":
try: try:
users = data.json()['usage']['users']['total']
soft_version = data.json()['software']['version'] self.users = data.json()['usage']['users']['total']
alive = True
except: if self.users == 0:
users = 0
soft_version = "" self.users = data.json()['usage']['users']['activeHalfyear']
if serv_soft == "gnusocialv2" or serv_soft == "gnusocial":
try: if self.software == 'socialhome':
users = data.json()['usage']['users']['total']
if users == 0: self.soft_version = data.json()['server']['version']
users = data.json()['usage']['users']['activeHalfyear']
soft_version = data.json()['software']['version'] else:
alive = True
except: self.soft_version = data.json()['software']['version']
users = 0
soft_version = "" if self.software == "wordpress" and "activitypub" in data.json()['protocols']:
if serv_soft == "plume" or serv_soft == 'red' or serv_soft == "misskey" or serv_soft == "zap" or serv_soft == "prismo" or serv_soft == "ravenvale" or serv_soft == "osada" or serv_soft == "groundpolis":
try:
users = data.json()['usage']['users']['total']
soft_version = data.json()['software']['version']
alive = True
except:
users = 0
soft_version = ""
if serv_soft == "ganggo" or serv_soft == "squs" or serv_soft == "dolphin" or serv_soft == "lemmy" or serv_soft == "wordpress":
try:
users = data.json()['usage']['users']['total']
soft_version = data.json()['software']['version']
if serv_soft == "wordpress" and "activitypub" in data.json()['protocols']:
alive = True alive = True
elif serv_soft == "wordpress" and "activitypub" not in data.json()['protocols']: elif self.software == "wordpress" and "activitypub" not in data.json()['protocols']:
alive = False alive = False
@ -164,297 +123,173 @@ def get_alive_servers(server):
except: except:
users = 0 self.users = 0
soft_version = ""
if serv_soft == 'funkwhale': self.soft_version = ""
try: else:
users = data.json()['usage']['users']['total']
soft_version = data.json()['software']['version']
alive = True
except:
users = 0
soft_version = ""
if serv_soft == 'socialhome': if self.api == '/api/v1/instance?':
try: try:
users = data.json()['usage']['users']['total'] self.users = data.json()['stats']['user_count']
soft_version = data.json()['server']['version'] self.soft_version = data.json()['version']
alive = True alive = True
except: except:
users = 0 self.users = 0
soft_version = "" self.soft_version = ""
if alive: if alive:
if downs_qty != None:
downs = downs_qty
else:
downs = 0
if self.soft_version != "" and self.soft_version is not None:
print(f'\n** Server {self.server} ({self.software} {self.soft_version}) is alive! **')
else:
print(f'\n** Server {self.server} ({self.software}) is alive! **')
insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version, first_checked_at, last_checked_at, downs) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
conn = None
try:
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor()
cur.execute(insert_sql, (self.server, self.users, now, self.software, alive, self.api, self.soft_version, now, now, downs))
if first_checked_at != None:
cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (self.users, now, self.software, alive, self.api, self.soft_version, now, downs, self.server))
else:
cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), first_checked_at=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (self.users, now, self.software, alive, self.api, self.soft_version, now, now, downs, self.server))
cur.execute("UPDATE world SET checked='t' where server=(%s)", (self.server,))
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
except urllib3.exceptions.ProtocolError as protoerr:
print_dead(self.server)
pass
except requests.exceptions.ChunkedEncodingError as chunkerr:
print_dead(self.server)
pass
except KeyError as e:
print_dead(self.server)
pass
except ValueError as verr:
print_dead(self.server)
pass
except requests.exceptions.SSLError as errssl:
print_dead(self.server)
pass
except requests.exceptions.HTTPError as errh:
print_dead(self.server)
pass
except requests.exceptions.ConnectionError as errc:
print_dead(self.server)
pass
except requests.exceptions.Timeout as errt:
print_dead(self.server)
pass
except requests.exceptions.RequestException as err:
print_dead(self.server)
pass
except socket.gaierror as gai_error:
print_dead(self.server)
pass
if not alive:
if downs_qty != None: if downs_qty != None:
downs = downs_qty downs = downs_qty + 1
else: else:
downs = 0 downs = 1
if soft_version != "" and soft_version is not None:
print("Server " + str(server) + " (" + serv_soft + " " + soft_version + ") is alive!")
else:
print("Server " + str(server) + " (" + serv_soft + ") is alive!")
insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version, first_checked_at, last_checked_at, downs) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
conn = None conn = None
try: try:
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432") conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="",
host="/var/run/postgresql", port="5432")
cur = conn.cursor() cur = conn.cursor()
cur.execute(insert_sql, (server, users, now, serv_soft, alive, serv_api, soft_version, now, now, downs))
if first_checked_at != None: if first_checked_at != None:
cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (users, now, serv_soft, alive, serv_api, soft_version, now, downs, server)) cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), first_checked_at=(%s), downs=(%s) where server=(%s)", (now, alive, now, downs, self.server))
else: else:
cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), first_checked_at=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (users, now, serv_soft, alive, serv_api, soft_version, now, now, downs, server)) cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), downs=(%s) where server=(%s)", (now, alive, downs, self.server))
cur.execute("UPDATE world SET checked='t' where server=(%s)", (self.server,))
cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
conn.commit() conn.commit()
cur.close() cur.close()
except (Exception, psycopg2.DatabaseError) as error: except (Exception, psycopg2.DatabaseError) as error:
print(error) print(error)
finally: finally:
if conn is not None: if conn is not None:
conn.close() conn.close()
except urllib3.exceptions.ProtocolError as protoerr: def print_dead(server):
print(protoerr) print(f'\nServer {server} is dead :-(')
print("Server " + server + " is dead :-(")
alive = False
pass
except requests.exceptions.ChunkedEncodingError as chunkerr:
print(chunkerr)
print("Server " + server + " is dead :-(")
alive = False
pass
except KeyError as e:
print(e)
print("Server " + server + " is dead :-(")
alive = False
pass
except ValueError as verr:
print(verr)
print("Server " + server + " is dead :-(")
alive = False
pass
except requests.exceptions.SSLError as errssl:
print(errssl)
print("Server " + server + " is dead :-(")
alive = False
pass
except requests.exceptions.HTTPError as errh:
print(errh)
print("Server " + server + " is dead :-(")
alive = False
pass
except requests.exceptions.ConnectionError as errc:
print(errc)
print("Server " + server + " is dead :-(")
alive = False
pass
except requests.exceptions.Timeout as errt:
print(errt)
print("Server " + server + " is dead :-(")
alive = False
pass
except requests.exceptions.RequestException as err:
print(err)
print("Server " + server + " is dead :-(")
alive = False
pass
except socket.gaierror as gai_error:
print(gai_error)
pass
if alive is False:
if downs_qty != None:
downs = downs_qty + 1
else:
downs = 1
conn = None
try:
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="",
host="/var/run/postgresql", port="5432")
cur = conn.cursor()
if first_checked_at != None:
cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), first_checked_at=(%s), downs=(%s) where server=(%s)", (now, alive, now, downs, server))
else:
cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), downs=(%s) where server=(%s)", (now, alive, downs, server))
cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
def write_api(server, software, users, alive, api, soft_version):
insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
conn = None
try:
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor()
cur.execute(insert_sql, (server, now, software, users, alive, api, soft_version))
cur.execute("UPDATE fediverse SET updated_at=(%s), software=(%s), users=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)", (now, software, users, alive, api, soft_version, server))
cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
async def getsoft(server):
try:
socket.gethostbyname(server)
except socket.gaierror:
pass
return
soft = ''
url = 'https://' + server
user_agent = {'User-agent': 'Mozilla/5.0'}
timeout = aiohttp.ClientTimeout(total=3)
async with aiohttp.ClientSession(headers=user_agent, timeout=timeout) as session:
for api in apis:
try:
async with session.get(url+api) as response:
if response.status == 200:
try:
response_json = await response.json()
except:
pass
except aiohttp.ClientConnectorError as err:
print(err)
pass
else:
if response.status == 200 and api != '/api/v1/instance?':
try:
soft = response_json['software']['name']
soft = soft.lower()
soft_version = response_json['software']['version']
users = response_json['usage']['users']['total']
if users > 1000000:
return
alive = True
write_api(server, soft, users, alive, api, soft_version)
print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
return
except:
pass
if response.status == 200 and soft == '' and api == "/api/v1/instance?":
soft = 'mastodon'
users = response_json['stats']['user_count']
soft_version = response_json['version']
if users > 1000000:
return
alive = True
write_api(server, soft, users, alive, api)
print("Server " + server + " (" + soft + ") is alive!")
def getserver(server, x):
server = server[0].rstrip('.').lower()
if server.find(".") == -1:
return
if server.find("@") != -1:
return
if server.find("/") != -1:
return
if server.find(":") != -1:
return
try:
loop = asyncio.get_event_loop()
coroutines = [getsoft(server)]
soft = loop.run_until_complete(asyncio.gather(*coroutines, return_exceptions=True))
except:
pass
def set_world_servers_check_to_false(): def set_world_servers_check_to_false():
@ -562,7 +397,6 @@ def usage():
print('usage: python ' + sys.argv[0] + ' --multi' + ' (multiprocessing, fast)') print('usage: python ' + sys.argv[0] + ' --multi' + ' (multiprocessing, fast)')
print('usage: python ' + sys.argv[0] + ' --mono' + ' (one process, slow)') print('usage: python ' + sys.argv[0] + ' --mono' + ' (one process, slow)')
# Returns the parameter from the specified file
def get_parameter(parameter, file_path): def get_parameter(parameter, file_path):
# Check if secrets file exists # Check if secrets file exists
@ -594,8 +428,8 @@ if __name__ == '__main__':
elif len(sys.argv) == 2: elif len(sys.argv) == 2:
if sys.argv[1] == '--multi': if sys.argv[1] == '--multi':
now = datetime.now() set_start_method("spawn")
mastodon, mastodon_hostname = mastodon() mastodon, mastodon_hostname = mastodon()
@ -608,12 +442,16 @@ if __name__ == '__main__':
alive_servers = get_last_checked_servers() alive_servers = get_last_checked_servers()
getservers = Server()
########################################################################### ###########################################################################
# multiprocessing! # multiprocessing!
nprocs = multiprocessing.cpu_count() with get_context("spawn").Pool(processes=32) as pool:
with multiprocessing.Pool(processes=64) as pool:
results = pool.starmap(get_alive_servers, product(alive_servers)) res = pool.starmap(getservers.get_alive_servers, product(alive_servers))
pool.close()
pool.join()
elif sys.argv[1] == '--mono': elif sys.argv[1] == '--mono':
@ -624,17 +462,22 @@ if __name__ == '__main__':
fediverse_db, fediverse_db_user = db_config() fediverse_db, fediverse_db_user = db_config()
total_servers = 0 total_servers = 0
total_users = 0 total_users = 0
set_world_servers_check_to_false() set_world_servers_check_to_false()
alive_servers = get_last_checked_servers() alive_servers = get_last_checked_servers()
getservers = Server()
i = 0 i = 0
while i < len(alive_servers): while i < len(alive_servers):
get_alive_servers(alive_servers[i]) getservers.server = alive_servers[i]
getservers.get_alive_servers()
i += 1 i += 1
@ -645,6 +488,8 @@ if __name__ == '__main__':
########################################################################### ###########################################################################
# get current total servers and users, get users from every software # get current total servers and users, get users from every software
now = datetime.now()
gettotals_sql = "select count(server), sum(users) from fediverse where alive" gettotals_sql = "select count(server), sum(users) from fediverse where alive"
get_soft_totals_sql = "select software, sum(users) as users, count(server) as servers from fediverse where users != 0 and alive group by software order by users desc" get_soft_totals_sql = "select software, sum(users) as users, count(server) as servers from fediverse where users != 0 and alive group by software order by users desc"
soft_total_project = [] soft_total_project = []
@ -948,4 +793,4 @@ if __name__ == '__main__':
users_image_id = mastodon.media_post('users.png', "image/png", description='users graph').id users_image_id = mastodon.media_post('users.png', "image/png", description='users graph').id
mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, users_image_id}) #mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, users_image_id})

Veure arxiu

@ -1,3 +1,6 @@
from multiprocessing import set_start_method
from multiprocessing import get_context
from itertools import product
import time import time
from datetime import datetime from datetime import datetime
import os import os
@ -5,22 +8,10 @@ import json
import sys import sys
import os.path import os.path
import psycopg2 import psycopg2
from multiprocessing import Pool, Manager
import aiohttp import aiohttp
import asyncio import asyncio
import socket import socket
import pdb
apis = ['/nodeinfo/2.0?',
'/nodeinfo/2.0.json?',
'/main/nodeinfo/2.0?',
'/api/statusnet/config?',
'/api/nodeinfo/2.0.json?',
'/api/nodeinfo?',
'/api/v1/instance?',
'/wp-json/nodeinfo/2.0?',
'/api/v1/instance/nodeinfo/2.0?',
'/.well-known/x-nodeinfo2?'
]
client_exceptions = ( client_exceptions = (
aiohttp.ClientResponseError, aiohttp.ClientResponseError,
@ -31,28 +22,24 @@ client_exceptions = (
socket.gaierror, socket.gaierror,
) )
def is_json(myjson):
try:
json_object = json.loads(myjson)
except ValueError as e:
return False
return True
def write_api(server, software, users, alive, api, soft_version): def write_api(server, software, users, alive, api, soft_version):
now = datetime.now()
fediverse_db, fediverse_db_user = get_db_config()
insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING" insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
conn = None conn = None
try: try:
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
port="5432")
cur = conn.cursor() cur = conn.cursor()
print(f'Writing {server} nodeinfo data...')
cur.execute(insert_sql, (server, now, software, users, alive, api, soft_version)) cur.execute(insert_sql, (server, now, software, users, alive, api, soft_version))
cur.execute( cur.execute(
@ -73,16 +60,20 @@ def write_api(server, software, users, alive, api, soft_version):
finally: finally:
if conn is not None: if conn is not None:
conn.close() conn.close()
async def getsoft(server): async def getsoft(server):
fediverse_db, fediverse_db_user = get_db_config()
try: try:
socket.gethostbyname(server) socket.gethostbyname(server)
except socket.gaierror: except socket.gaierror as g_error:
print(f'Server {server} error: {g_error}')
pass pass
return return
@ -93,71 +84,134 @@ async def getsoft(server):
user_agent = {'User-agent': 'Mozilla/5.0'} user_agent = {'User-agent': 'Mozilla/5.0'}
timeout = aiohttp.ClientTimeout(total=3) timeout = aiohttp.ClientTimeout(total=3)
async with aiohttp.ClientSession(timeout=timeout, headers=user_agent) as session: async with aiohttp.ClientSession(timeout=timeout, headers=user_agent) as session:
for api in apis:
try: try:
async with session.get(url + '/.well-known/nodeinfo') as response:
if response.status == 200:
try:
response_json = await response.json()
nodeinfo = response_json['links'][0]['href'].replace(f'https://{server}','')
except:
pass
else:
print(f'Server {server} not responding: {response.status}')
pass
async with session.get(url + nodeinfo) as nodeinfo_response:
if nodeinfo_response.status == 200:
async with session.get(url + api) as response:
if response.status == 200:
try: try:
response_json = await response.json()
nodeinfo_json = await nodeinfo_response.json()
except: except:
pass pass
except aiohttp.ClientConnectorError as err: else:
pass print(f"Server {server}'s nodeinfo not responding: {response.status}")
pass
except aiohttp.ClientConnectorError as cc_err:
pass
except aiohttp.client_exceptions.ClientConnectorSSLError as ccssl_as:
pass
else:
if nodeinfo_response.status == 200 and nodeinfo != '/api/v1/instance?':
if nodeinfo != '/.well-known/x-nodeinfo2?':
try:
soft = nodeinfo_json['software']['name']
soft = soft.lower()
soft_version = nodeinfo_json['software']['version']
users = nodeinfo_json['usage']['users']['total']
if users > 1000000:
return
alive = True
write_api(server, soft, users, alive, nodeinfo, soft_version)
print('*********************************************************************')
print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
print('*********************************************************************')
return
except:
pass
else:
try:
soft = nodeinfo_json['server']['software']
soft = soft.lower()
soft_version = nodeinfo_json['server']['version']
users = nodeinfo_json['usage']['users']['total']
if users > 1000000:
return
alive = True
if soft == 'socialhome':
write_api(server, soft, users, alive, api, soft_version)
print('*********************************************************************')
print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
print('*********************************************************************')
return
except:
pass
if nodeinfo_response.status == 200 and soft == '' and nodeinfo == "/api/v1/instance?":
soft = 'mastodon'
users = nodeinfo_json['stats']['user_count']
soft_version = nodeinfo_json['version']
if users > 1000000:
return
alive = True
write_api(server, soft, users, alive, api)
print('*********************************************************************')
print("Server " + server + " (" + soft + ") is alive!")
print('*********************************************************************')
else: else:
if response.status == 200 and api != '/api/v1/instance?': print(f'Server {server} is dead')
if api != '/.well-known/x-nodeinfo2?': def getserver(server, *args):
try: if len(args) != 0:
soft = response_json['software']['name']
soft = soft.lower()
soft_version = response_json['software']['version']
users = response_json['usage']['users']['total']
if users > 1000000:
return
alive = True
write_api(server, soft, users, alive, api, soft_version)
print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
return
except:
pass
else:
try: server = server[0].rstrip('.').lower()
soft = response_json['server']['software']
soft = soft.lower()
soft_version = response_json['server']['version']
users = response_json['usage']['users']['total']
if users > 1000000:
return
alive = True
if soft == 'socialhome':
write_api(server, soft, users, alive, api, soft_version)
print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
return
except:
pass
if response.status == 200 and soft == '' and api == "/api/v1/instance?":
soft = 'mastodon'
users = response_json['stats']['user_count']
soft_version = response_json['version']
if users > 1000000:
return
alive = True
write_api(server, soft, users, alive, api)
print("Server " + server + " (" + soft + ") is alive!")
def getserver(server, x):
server = server[0].rstrip('.').lower()
if server.find(".") == -1: if server.find(".") == -1:
return return
@ -178,7 +232,46 @@ def getserver(server, x):
pass pass
def get_world_servers():
world_servers = []
try:
conn = None
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor()
# get world servers list
cur.execute("select server from world where checked='f'")
rows = cur.fetchall()
for row in rows:
world_servers.append(row[0])
cur.close()
print("Remaining servers: " + str(len(world_servers)))
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
return world_servers
def get_parameter(parameter, file_path): def get_parameter(parameter, file_path):
# Check if secrets file exists # Check if secrets file exists
if not os.path.isfile(file_path): if not os.path.isfile(file_path):
print("File %s not found, exiting." % file_path) print("File %s not found, exiting." % file_path)
@ -194,6 +287,10 @@ def get_parameter(parameter, file_path):
print(file_path + " Missing parameter %s " % parameter) print(file_path + " Missing parameter %s " % parameter)
sys.exit(0) sys.exit(0)
def usage():
print('usage: python ' + sys.argv[0] + ' --multi' + ' (multiprocessing, fast)')
print('usage: python ' + sys.argv[0] + ' --mono' + ' (one process, slow)')
def get_config(): def get_config():
@ -214,59 +311,51 @@ def get_db_config():
# main # main
if __name__ == '__main__': if __name__ == '__main__':
now = datetime.now()
start_time = time.time()
mastodon_hostname = get_config() # usage modes
fediverse_db, fediverse_db_user = get_db_config() if len(sys.argv) == 1:
world_servers = [] usage()
try: elif len(sys.argv) == 2:
conn = None if sys.argv[1] == '--multi':
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432") now = datetime.now()
cur = conn.cursor() start_time = time.time()
# get world servers list mastodon_hostname = get_config()
cur.execute("select server from world where checked='f'") fediverse_db, fediverse_db_user = get_db_config()
rows = cur.fetchall() world_servers = get_world_servers()
for row in rows:
world_servers.append(row[0])
cur.close() with get_context("spawn").Pool(processes=32) as pool:
print("Remaining servers: " + str(len(world_servers))) res = pool.starmap(getserver, product(world_servers))
except (Exception, psycopg2.DatabaseError) as error: pool.close()
print(error) pool.join()
finally: print('Done.')
if conn is not None: elif sys.argv[1] == '--mono':
conn.close() now = datetime.now()
########################################################################### start_time = time.time()
# multiprocessing!
m = Manager() mastodon_hostname = get_config()
q = m.Queue()
z = zip(world_servers)
serv_number = len(world_servers) fediverse_db, fediverse_db_user = get_db_config()
pool_tuple = [(x, q) for x in z] world_servers = get_world_servers()
with Pool(processes=64) as pool:
pool.starmap(getserver, pool_tuple)
print('Done.') for server in world_servers:
getserver(server)
print('Done.')