Improved servers nodeinfo detection

This commit is contained in:
spla 2022-03-02 22:26:05 +01:00
pare 36d7ff62b1
commit 0b52e0090d
S'han modificat 2 arxius amb 417 adicions i 483 eliminacions

Veure arxiu

@ -1,3 +1,5 @@
from multiprocessing import set_start_method
from multiprocessing import get_context
import time
import urllib3
from datetime import datetime
@ -7,11 +9,10 @@ import json
import sys
import os.path
import requests
import psycopg2
from itertools import product
import multiprocessing
import aiohttp
import asyncio
import psycopg2
from itertools import product
import socket
import matplotlib.pyplot as plt
import pdb
@ -19,18 +20,6 @@ plt.style.use('seaborn')
start_time = time.time()
apis = ['/nodeinfo/2.0?',
'/nodeinfo/2.0.json?',
'/main/nodeinfo/2.0?',
'/api/statusnet/config?',
'/api/nodeinfo/2.0.json?',
'/api/nodeinfo?',
'/api/v1/instance?',
'/wp-json/nodeinfo/2.0?',
'/api/v1/instance/nodeinfo/2.0?',
'/.well-known/x-nodeinfo2?'
]
client_exceptions = (
aiohttp.ClientResponseError,
aiohttp.ClientConnectionError,
@ -40,121 +29,91 @@ client_exceptions = (
socket.gaierror,
)
def is_json(myjson):
class Server:
try:
json_object = json.loads(myjson)
except ValueError as e:
print(e)
return False
return True
name = 'Server'
def get_alive_servers(server):
def __init_(self, server, software, users, alive, api, soft_version):
serv_api = ''
serv_soft = ''
soft_version = ''
self.server = server
self.software = software
self.users = users
self.alive = alive
self.api = api
self.version = self.soft_version
try:
def get_alive_servers(self, *args):
conn = None
fediverse_db, fediverse_db_user = db_config()
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
now = datetime.now()
cur = conn.cursor()
if len(args) != 0:
cur.execute("select alive, software, users_api, version, first_checked_at, downs from fediverse where server=(%s)", (server,))
row = cur.fetchone()
if row is not None:
was_alive = row[0]
serv_soft = row[1]
serv_api = row[2]
soft_version = row[3]
first_checked_at = row[4]
downs_qty = row[5]
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
alive = False
self.server = args[0]
try:
user_agent = {'User-agent': 'Mozilla/5.0'}
conn = None
data = requests.get('https://' + server + serv_api, headers = user_agent, timeout=3)
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor()
cur.execute("select alive, software, users_api, version, first_checked_at, downs from fediverse where server=(%s)", (self.server,))
row = cur.fetchone()
if row is not None:
was_alive = row[0]
self.software = row[1]
self.api = row[2]
self.soft_version = row[3]
first_checked_at = row[4]
downs_qty = row[5]
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
alive = False
try:
user_agent = {'User-agent': 'Mozilla/5.0'}
data = requests.get('https://' + self.server + self.api, headers = user_agent, timeout=3)
if serv_soft == "mastodon":
if serv_api == '/nodeinfo/2.0?':
try:
users = data.json()['usage']['users']['total']
soft_version = data.json()['software']['version']
alive = True
except:
users = 0
soft_version = ""
if serv_api == '/nodeinfo/2.0.json?':
try:
users = data.json()['usage']['users']['total']
soft_version = data.json()['software']['version']
alive = True
except:
users = 0
soft_version = ""
elif serv_api == '/api/v1/instance?':
try:
users = data.json()['stats']['user_count']
soft_version = data.json()['version']
alive = True
except:
users = 0
soft_version = ""
if serv_soft == "pleroma" or serv_soft == "diaspora" or serv_soft == "peertube" or serv_soft == "pixelfed" or serv_soft == "hubzilla" or serv_soft == "writefreely" or serv_soft == "friendica":
try:
users = data.json()['usage']['users']['total']
soft_version = data.json()['software']['version']
alive = True
except:
users = 0
soft_version = ""
if serv_soft == "gnusocialv2" or serv_soft == "gnusocial":
try:
users = data.json()['usage']['users']['total']
if users == 0:
users = data.json()['usage']['users']['activeHalfyear']
soft_version = data.json()['software']['version']
alive = True
except:
users = 0
soft_version = ""
if serv_soft == "plume" or serv_soft == 'red' or serv_soft == "misskey" or serv_soft == "zap" or serv_soft == "prismo" or serv_soft == "ravenvale" or serv_soft == "osada" or serv_soft == "groundpolis":
try:
users = data.json()['usage']['users']['total']
soft_version = data.json()['software']['version']
alive = True
except:
users = 0
soft_version = ""
if serv_soft == "ganggo" or serv_soft == "squs" or serv_soft == "dolphin" or serv_soft == "lemmy" or serv_soft == "wordpress":
try:
users = data.json()['usage']['users']['total']
soft_version = data.json()['software']['version']
if serv_soft == "wordpress" and "activitypub" in data.json()['protocols']:
self.users = data.json()['usage']['users']['total']
if self.users == 0:
self.users = data.json()['usage']['users']['activeHalfyear']
if self.software == 'socialhome':
self.soft_version = data.json()['server']['version']
else:
self.soft_version = data.json()['software']['version']
if self.software == "wordpress" and "activitypub" in data.json()['protocols']:
alive = True
elif serv_soft == "wordpress" and "activitypub" not in data.json()['protocols']:
elif self.software == "wordpress" and "activitypub" not in data.json()['protocols']:
alive = False
@ -164,297 +123,173 @@ def get_alive_servers(server):
except:
users = 0
soft_version = ""
self.users = 0
if serv_soft == 'funkwhale':
self.soft_version = ""
try:
users = data.json()['usage']['users']['total']
soft_version = data.json()['software']['version']
alive = True
except:
users = 0
soft_version = ""
else:
if serv_soft == 'socialhome':
if self.api == '/api/v1/instance?':
try:
users = data.json()['usage']['users']['total']
soft_version = data.json()['server']['version']
alive = True
except:
users = 0
soft_version = ""
try:
self.users = data.json()['stats']['user_count']
self.soft_version = data.json()['version']
alive = True
except:
self.users = 0
self.soft_version = ""
if alive:
if alive:
if downs_qty != None:
downs = downs_qty
else:
downs = 0
if self.soft_version != "" and self.soft_version is not None:
print(f'\n** Server {self.server} ({self.software} {self.soft_version}) is alive! **')
else:
print(f'\n** Server {self.server} ({self.software}) is alive! **')
insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version, first_checked_at, last_checked_at, downs) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
conn = None
try:
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor()
cur.execute(insert_sql, (self.server, self.users, now, self.software, alive, self.api, self.soft_version, now, now, downs))
if first_checked_at != None:
cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (self.users, now, self.software, alive, self.api, self.soft_version, now, downs, self.server))
else:
cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), first_checked_at=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (self.users, now, self.software, alive, self.api, self.soft_version, now, now, downs, self.server))
cur.execute("UPDATE world SET checked='t' where server=(%s)", (self.server,))
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
except urllib3.exceptions.ProtocolError as protoerr:
print_dead(self.server)
pass
except requests.exceptions.ChunkedEncodingError as chunkerr:
print_dead(self.server)
pass
except KeyError as e:
print_dead(self.server)
pass
except ValueError as verr:
print_dead(self.server)
pass
except requests.exceptions.SSLError as errssl:
print_dead(self.server)
pass
except requests.exceptions.HTTPError as errh:
print_dead(self.server)
pass
except requests.exceptions.ConnectionError as errc:
print_dead(self.server)
pass
except requests.exceptions.Timeout as errt:
print_dead(self.server)
pass
except requests.exceptions.RequestException as err:
print_dead(self.server)
pass
except socket.gaierror as gai_error:
print_dead(self.server)
pass
if not alive:
if downs_qty != None:
downs = downs_qty
downs = downs_qty + 1
else:
downs = 0
if soft_version != "" and soft_version is not None:
print("Server " + str(server) + " (" + serv_soft + " " + soft_version + ") is alive!")
else:
print("Server " + str(server) + " (" + serv_soft + ") is alive!")
insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version, first_checked_at, last_checked_at, downs) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
downs = 1
conn = None
try:
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="",
host="/var/run/postgresql", port="5432")
cur = conn.cursor()
cur.execute(insert_sql, (server, users, now, serv_soft, alive, serv_api, soft_version, now, now, downs))
if first_checked_at != None:
cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (users, now, serv_soft, alive, serv_api, soft_version, now, downs, server))
cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), first_checked_at=(%s), downs=(%s) where server=(%s)", (now, alive, now, downs, self.server))
else:
cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), first_checked_at=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (users, now, serv_soft, alive, serv_api, soft_version, now, now, downs, server))
cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), downs=(%s) where server=(%s)", (now, alive, downs, self.server))
cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
cur.execute("UPDATE world SET checked='t' where server=(%s)", (self.server,))
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
except urllib3.exceptions.ProtocolError as protoerr:
def print_dead(server):
print(protoerr)
print("Server " + server + " is dead :-(")
alive = False
pass
except requests.exceptions.ChunkedEncodingError as chunkerr:
print(chunkerr)
print("Server " + server + " is dead :-(")
alive = False
pass
except KeyError as e:
print(e)
print("Server " + server + " is dead :-(")
alive = False
pass
except ValueError as verr:
print(verr)
print("Server " + server + " is dead :-(")
alive = False
pass
except requests.exceptions.SSLError as errssl:
print(errssl)
print("Server " + server + " is dead :-(")
alive = False
pass
except requests.exceptions.HTTPError as errh:
print(errh)
print("Server " + server + " is dead :-(")
alive = False
pass
except requests.exceptions.ConnectionError as errc:
print(errc)
print("Server " + server + " is dead :-(")
alive = False
pass
except requests.exceptions.Timeout as errt:
print(errt)
print("Server " + server + " is dead :-(")
alive = False
pass
except requests.exceptions.RequestException as err:
print(err)
print("Server " + server + " is dead :-(")
alive = False
pass
except socket.gaierror as gai_error:
print(gai_error)
pass
if alive is False:
if downs_qty != None:
downs = downs_qty + 1
else:
downs = 1
conn = None
try:
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="",
host="/var/run/postgresql", port="5432")
cur = conn.cursor()
if first_checked_at != None:
cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), first_checked_at=(%s), downs=(%s) where server=(%s)", (now, alive, now, downs, server))
else:
cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), downs=(%s) where server=(%s)", (now, alive, downs, server))
cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
def write_api(server, software, users, alive, api, soft_version):
insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
conn = None
try:
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor()
cur.execute(insert_sql, (server, now, software, users, alive, api, soft_version))
cur.execute("UPDATE fediverse SET updated_at=(%s), software=(%s), users=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)", (now, software, users, alive, api, soft_version, server))
cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
async def getsoft(server):
try:
socket.gethostbyname(server)
except socket.gaierror:
pass
return
soft = ''
url = 'https://' + server
user_agent = {'User-agent': 'Mozilla/5.0'}
timeout = aiohttp.ClientTimeout(total=3)
async with aiohttp.ClientSession(headers=user_agent, timeout=timeout) as session:
for api in apis:
try:
async with session.get(url+api) as response:
if response.status == 200:
try:
response_json = await response.json()
except:
pass
except aiohttp.ClientConnectorError as err:
print(err)
pass
else:
if response.status == 200 and api != '/api/v1/instance?':
try:
soft = response_json['software']['name']
soft = soft.lower()
soft_version = response_json['software']['version']
users = response_json['usage']['users']['total']
if users > 1000000:
return
alive = True
write_api(server, soft, users, alive, api, soft_version)
print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
return
except:
pass
if response.status == 200 and soft == '' and api == "/api/v1/instance?":
soft = 'mastodon'
users = response_json['stats']['user_count']
soft_version = response_json['version']
if users > 1000000:
return
alive = True
write_api(server, soft, users, alive, api)
print("Server " + server + " (" + soft + ") is alive!")
def getserver(server, x):
server = server[0].rstrip('.').lower()
if server.find(".") == -1:
return
if server.find("@") != -1:
return
if server.find("/") != -1:
return
if server.find(":") != -1:
return
try:
loop = asyncio.get_event_loop()
coroutines = [getsoft(server)]
soft = loop.run_until_complete(asyncio.gather(*coroutines, return_exceptions=True))
except:
pass
print(f'\nServer {server} is dead :-(')
def set_world_servers_check_to_false():
@ -562,7 +397,6 @@ def usage():
print('usage: python ' + sys.argv[0] + ' --multi' + ' (multiprocessing, fast)')
print('usage: python ' + sys.argv[0] + ' --mono' + ' (one process, slow)')
# Returns the parameter from the specified file
def get_parameter(parameter, file_path):
# Check if secrets file exists
@ -595,7 +429,7 @@ if __name__ == '__main__':
if sys.argv[1] == '--multi':
now = datetime.now()
set_start_method("spawn")
mastodon, mastodon_hostname = mastodon()
@ -608,12 +442,16 @@ if __name__ == '__main__':
alive_servers = get_last_checked_servers()
getservers = Server()
###########################################################################
# multiprocessing!
nprocs = multiprocessing.cpu_count()
with multiprocessing.Pool(processes=64) as pool:
results = pool.starmap(get_alive_servers, product(alive_servers))
with get_context("spawn").Pool(processes=32) as pool:
res = pool.starmap(getservers.get_alive_servers, product(alive_servers))
pool.close()
pool.join()
elif sys.argv[1] == '--mono':
@ -624,17 +462,22 @@ if __name__ == '__main__':
fediverse_db, fediverse_db_user = db_config()
total_servers = 0
total_users = 0
set_world_servers_check_to_false()
alive_servers = get_last_checked_servers()
getservers = Server()
i = 0
while i < len(alive_servers):
get_alive_servers(alive_servers[i])
getservers.server = alive_servers[i]
getservers.get_alive_servers()
i += 1
@ -645,6 +488,8 @@ if __name__ == '__main__':
###########################################################################
# get current total servers and users, get users from every software
now = datetime.now()
gettotals_sql = "select count(server), sum(users) from fediverse where alive"
get_soft_totals_sql = "select software, sum(users) as users, count(server) as servers from fediverse where users != 0 and alive group by software order by users desc"
soft_total_project = []
@ -948,4 +793,4 @@ if __name__ == '__main__':
users_image_id = mastodon.media_post('users.png', "image/png", description='users graph').id
mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, users_image_id})
#mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, users_image_id})

Veure arxiu

@ -1,3 +1,6 @@
from multiprocessing import set_start_method
from multiprocessing import get_context
from itertools import product
import time
from datetime import datetime
import os
@ -5,22 +8,10 @@ import json
import sys
import os.path
import psycopg2
from multiprocessing import Pool, Manager
import aiohttp
import asyncio
import socket
apis = ['/nodeinfo/2.0?',
'/nodeinfo/2.0.json?',
'/main/nodeinfo/2.0?',
'/api/statusnet/config?',
'/api/nodeinfo/2.0.json?',
'/api/nodeinfo?',
'/api/v1/instance?',
'/wp-json/nodeinfo/2.0?',
'/api/v1/instance/nodeinfo/2.0?',
'/.well-known/x-nodeinfo2?'
]
import pdb
client_exceptions = (
aiohttp.ClientResponseError,
@ -31,28 +22,24 @@ client_exceptions = (
socket.gaierror,
)
def is_json(myjson):
try:
json_object = json.loads(myjson)
except ValueError as e:
return False
return True
def write_api(server, software, users, alive, api, soft_version):
now = datetime.now()
fediverse_db, fediverse_db_user = get_db_config()
insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
conn = None
try:
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql",
port="5432")
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor()
print(f'Writing {server} nodeinfo data...')
cur.execute(insert_sql, (server, now, software, users, alive, api, soft_version))
cur.execute(
@ -73,16 +60,20 @@ def write_api(server, software, users, alive, api, soft_version):
finally:
if conn is not None:
conn.close()
async def getsoft(server):
fediverse_db, fediverse_db_user = get_db_config()
try:
socket.gethostbyname(server)
except socket.gaierror:
except socket.gaierror as g_error:
print(f'Server {server} error: {g_error}')
pass
return
@ -93,71 +84,134 @@ async def getsoft(server):
user_agent = {'User-agent': 'Mozilla/5.0'}
timeout = aiohttp.ClientTimeout(total=3)
async with aiohttp.ClientSession(timeout=timeout, headers=user_agent) as session:
for api in apis:
try:
try:
async with session.get(url + '/.well-known/nodeinfo') as response:
if response.status == 200:
try:
response_json = await response.json()
nodeinfo = response_json['links'][0]['href'].replace(f'https://{server}','')
except:
pass
else:
print(f'Server {server} not responding: {response.status}')
pass
async with session.get(url + nodeinfo) as nodeinfo_response:
if nodeinfo_response.status == 200:
async with session.get(url + api) as response:
if response.status == 200:
try:
response_json = await response.json()
nodeinfo_json = await nodeinfo_response.json()
except:
pass
except aiohttp.ClientConnectorError as err:
else:
pass
print(f"Server {server}'s nodeinfo not responding: {response.status}")
pass
except aiohttp.ClientConnectorError as cc_err:
pass
except aiohttp.client_exceptions.ClientConnectorSSLError as ccssl_as:
pass
else:
if nodeinfo_response.status == 200 and nodeinfo != '/api/v1/instance?':
if nodeinfo != '/.well-known/x-nodeinfo2?':
try:
soft = nodeinfo_json['software']['name']
soft = soft.lower()
soft_version = nodeinfo_json['software']['version']
users = nodeinfo_json['usage']['users']['total']
if users > 1000000:
return
alive = True
write_api(server, soft, users, alive, nodeinfo, soft_version)
print('*********************************************************************')
print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
print('*********************************************************************')
return
except:
pass
else:
try:
soft = nodeinfo_json['server']['software']
soft = soft.lower()
soft_version = nodeinfo_json['server']['version']
users = nodeinfo_json['usage']['users']['total']
if users > 1000000:
return
alive = True
if soft == 'socialhome':
write_api(server, soft, users, alive, api, soft_version)
print('*********************************************************************')
print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
print('*********************************************************************')
return
except:
pass
if nodeinfo_response.status == 200 and soft == '' and nodeinfo == "/api/v1/instance?":
soft = 'mastodon'
users = nodeinfo_json['stats']['user_count']
soft_version = nodeinfo_json['version']
if users > 1000000:
return
alive = True
write_api(server, soft, users, alive, api)
print('*********************************************************************')
print("Server " + server + " (" + soft + ") is alive!")
print('*********************************************************************')
else:
if response.status == 200 and api != '/api/v1/instance?':
print(f'Server {server} is dead')
if api != '/.well-known/x-nodeinfo2?':
def getserver(server, *args):
try:
soft = response_json['software']['name']
soft = soft.lower()
soft_version = response_json['software']['version']
users = response_json['usage']['users']['total']
if users > 1000000:
return
alive = True
write_api(server, soft, users, alive, api, soft_version)
print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
return
except:
pass
else:
if len(args) != 0:
try:
soft = response_json['server']['software']
soft = soft.lower()
soft_version = response_json['server']['version']
users = response_json['usage']['users']['total']
if users > 1000000:
return
alive = True
if soft == 'socialhome':
write_api(server, soft, users, alive, api, soft_version)
print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
return
except:
pass
if response.status == 200 and soft == '' and api == "/api/v1/instance?":
soft = 'mastodon'
users = response_json['stats']['user_count']
soft_version = response_json['version']
if users > 1000000:
return
alive = True
write_api(server, soft, users, alive, api)
print("Server " + server + " (" + soft + ") is alive!")
def getserver(server, x):
server = server[0].rstrip('.').lower()
server = server[0].rstrip('.').lower()
if server.find(".") == -1:
return
@ -178,49 +232,7 @@ def getserver(server, x):
pass
def get_parameter(parameter, file_path):
# Check if secrets file exists
if not os.path.isfile(file_path):
print("File %s not found, exiting." % file_path)
sys.exit(0)
# Find parameter in file
with open(file_path) as f:
for line in f:
if line.startswith(parameter):
return line.replace(parameter + ":", "").strip()
# Cannot find parameter, exit
print(file_path + " Missing parameter %s " % parameter)
sys.exit(0)
def get_config():
# Load configuration from config file
config_filepath = "config/config.txt"
mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
return mastodon_hostname
def get_db_config():
# Load database config from db_config file
db_config_filepath = "config/db_config.txt"
fediverse_db = get_parameter("fediverse_db", db_config_filepath)
fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath)
return (fediverse_db, fediverse_db_user)
###############################################################################
# main
if __name__ == '__main__':
now = datetime.now()
start_time = time.time()
mastodon_hostname = get_config()
fediverse_db, fediverse_db_user = get_db_config()
def get_world_servers():
world_servers = []
@ -256,17 +268,94 @@ if __name__ == '__main__':
conn.close()
###########################################################################
# multiprocessing!
return world_servers
m = Manager()
q = m.Queue()
z = zip(world_servers)
def get_parameter(parameter, file_path):
serv_number = len(world_servers)
# Check if secrets file exists
if not os.path.isfile(file_path):
print("File %s not found, exiting." % file_path)
sys.exit(0)
pool_tuple = [(x, q) for x in z]
with Pool(processes=64) as pool:
pool.starmap(getserver, pool_tuple)
# Find parameter in file
with open(file_path) as f:
for line in f:
if line.startswith(parameter):
return line.replace(parameter + ":", "").strip()
print('Done.')
# Cannot find parameter, exit
print(file_path + " Missing parameter %s " % parameter)
sys.exit(0)
def usage():
print('usage: python ' + sys.argv[0] + ' --multi' + ' (multiprocessing, fast)')
print('usage: python ' + sys.argv[0] + ' --mono' + ' (one process, slow)')
def get_config():
# Load configuration from config file
config_filepath = "config/config.txt"
mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
return mastodon_hostname
def get_db_config():
# Load database config from db_config file
db_config_filepath = "config/db_config.txt"
fediverse_db = get_parameter("fediverse_db", db_config_filepath)
fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath)
return (fediverse_db, fediverse_db_user)
###############################################################################
# main
if __name__ == '__main__':
# usage modes
if len(sys.argv) == 1:
usage()
elif len(sys.argv) == 2:
if sys.argv[1] == '--multi':
now = datetime.now()
start_time = time.time()
mastodon_hostname = get_config()
fediverse_db, fediverse_db_user = get_db_config()
world_servers = get_world_servers()
with get_context("spawn").Pool(processes=32) as pool:
res = pool.starmap(getserver, product(world_servers))
pool.close()
pool.join()
print('Done.')
elif sys.argv[1] == '--mono':
now = datetime.now()
start_time = time.time()
mastodon_hostname = get_config()
fediverse_db, fediverse_db_user = get_db_config()
world_servers = get_world_servers()
for server in world_servers:
getserver(server)
print('Done.')