Improved servers nodeinfo detection

master
spla 9 months ago
parent 36d7ff62b1
commit 0b52e0090d
  1. 485
      fediverse.py
  2. 297
      fetchservers.py

@ -1,3 +1,5 @@
from multiprocessing import set_start_method
from multiprocessing import get_context
import time
import urllib3
from datetime import datetime
@ -7,11 +9,10 @@ import json
import sys
import os.path
import requests
import psycopg2
from itertools import product
import multiprocessing
import aiohttp
import asyncio
import psycopg2
from itertools import product
import socket
import matplotlib.pyplot as plt
import pdb
@ -19,18 +20,6 @@ plt.style.use('seaborn')
start_time = time.time()
apis = ['/nodeinfo/2.0?',
'/nodeinfo/2.0.json?',
'/main/nodeinfo/2.0?',
'/api/statusnet/config?',
'/api/nodeinfo/2.0.json?',
'/api/nodeinfo?',
'/api/v1/instance?',
'/wp-json/nodeinfo/2.0?',
'/api/v1/instance/nodeinfo/2.0?',
'/.well-known/x-nodeinfo2?'
]
client_exceptions = (
aiohttp.ClientResponseError,
aiohttp.ClientConnectionError,
@ -40,421 +29,267 @@ client_exceptions = (
socket.gaierror,
)
def is_json(myjson):
class Server:
try:
json_object = json.loads(myjson)
except ValueError as e:
print(e)
return False
return True
name = 'Server'
def get_alive_servers(server):
def __init_(self, server, software, users, alive, api, soft_version):
serv_api = ''
serv_soft = ''
soft_version = ''
self.server = server
self.software = software
self.users = users
self.alive = alive
self.api = api
self.version = self.soft_version
try:
def get_alive_servers(self, *args):
conn = None
fediverse_db, fediverse_db_user = db_config()
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
now = datetime.now()
cur = conn.cursor()
if len(args) != 0:
cur.execute("select alive, software, users_api, version, first_checked_at, downs from fediverse where server=(%s)", (server,))
self.server = args[0]
row = cur.fetchone()
if row is not None:
was_alive = row[0]
serv_soft = row[1]
serv_api = row[2]
soft_version = row[3]
first_checked_at = row[4]
downs_qty = row[5]
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
try:
finally:
conn = None
if conn is not None:
conn.close()
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
alive = False
cur = conn.cursor()
try:
cur.execute("select alive, software, users_api, version, first_checked_at, downs from fediverse where server=(%s)", (self.server,))
user_agent = {'User-agent': 'Mozilla/5.0'}
row = cur.fetchone()
data = requests.get('https://' + server + serv_api, headers = user_agent, timeout=3)
if row is not None:
if serv_soft == "mastodon":
if serv_api == '/nodeinfo/2.0?':
try:
users = data.json()['usage']['users']['total']
soft_version = data.json()['software']['version']
alive = True
except:
users = 0
soft_version = ""
if serv_api == '/nodeinfo/2.0.json?':
try:
users = data.json()['usage']['users']['total']
soft_version = data.json()['software']['version']
alive = True
except:
users = 0
soft_version = ""
elif serv_api == '/api/v1/instance?':
try:
users = data.json()['stats']['user_count']
soft_version = data.json()['version']
alive = True
except:
users = 0
soft_version = ""
if serv_soft == "pleroma" or serv_soft == "diaspora" or serv_soft == "peertube" or serv_soft == "pixelfed" or serv_soft == "hubzilla" or serv_soft == "writefreely" or serv_soft == "friendica":
try:
users = data.json()['usage']['users']['total']
soft_version = data.json()['software']['version']
alive = True
except:
users = 0
soft_version = ""
if serv_soft == "gnusocialv2" or serv_soft == "gnusocial":
try:
users = data.json()['usage']['users']['total']
if users == 0:
users = data.json()['usage']['users']['activeHalfyear']
soft_version = data.json()['software']['version']
alive = True
except:
users = 0
soft_version = ""
if serv_soft == "plume" or serv_soft == 'red' or serv_soft == "misskey" or serv_soft == "zap" or serv_soft == "prismo" or serv_soft == "ravenvale" or serv_soft == "osada" or serv_soft == "groundpolis":
try:
users = data.json()['usage']['users']['total']
soft_version = data.json()['software']['version']
alive = True
except:
users = 0
soft_version = ""
if serv_soft == "ganggo" or serv_soft == "squs" or serv_soft == "dolphin" or serv_soft == "lemmy" or serv_soft == "wordpress":
try:
users = data.json()['usage']['users']['total']
soft_version = data.json()['software']['version']
if serv_soft == "wordpress" and "activitypub" in data.json()['protocols']:
was_alive = row[0]
self.software = row[1]
self.api = row[2]
self.soft_version = row[3]
first_checked_at = row[4]
downs_qty = row[5]
alive = True
cur.close()
elif serv_soft == "wordpress" and "activitypub" not in data.json()['protocols']:
except (Exception, psycopg2.DatabaseError) as error:
alive = False
print(error)
else:
finally:
alive = True
if conn is not None:
except:
conn.close()
users = 0
soft_version = ""
alive = False
if serv_soft == 'funkwhale':
try:
try:
users = data.json()['usage']['users']['total']
soft_version = data.json()['software']['version']
alive = True
except:
users = 0
soft_version = ""
user_agent = {'User-agent': 'Mozilla/5.0'}
if serv_soft == 'socialhome':
data = requests.get('https://' + self.server + self.api, headers = user_agent, timeout=3)
try:
users = data.json()['usage']['users']['total']
soft_version = data.json()['server']['version']
alive = True
except:
users = 0
soft_version = ""
if alive:
self.users = data.json()['usage']['users']['total']
if downs_qty != None:
if self.users == 0:
downs = downs_qty
self.users = data.json()['usage']['users']['activeHalfyear']
else:
if self.software == 'socialhome':
downs = 0
if soft_version != "" and soft_version is not None:
print("Server " + str(server) + " (" + serv_soft + " " + soft_version + ") is alive!")
else:
print("Server " + str(server) + " (" + serv_soft + ") is alive!")
self.soft_version = data.json()['server']['version']
insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version, first_checked_at, last_checked_at, downs) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
else:
conn = None
self.soft_version = data.json()['software']['version']
try:
if self.software == "wordpress" and "activitypub" in data.json()['protocols']:
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
alive = True
cur = conn.cursor()
elif self.software == "wordpress" and "activitypub" not in data.json()['protocols']:
cur.execute(insert_sql, (server, users, now, serv_soft, alive, serv_api, soft_version, now, now, downs))
alive = False
if first_checked_at != None:
else:
cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (users, now, serv_soft, alive, serv_api, soft_version, now, downs, server))
alive = True
else:
except:
cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), first_checked_at=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (users, now, serv_soft, alive, serv_api, soft_version, now, now, downs, server))
self.users = 0
self.soft_version = ""
cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
else:
conn.commit()
if self.api == '/api/v1/instance?':
cur.close()
try:
self.users = data.json()['stats']['user_count']
self.soft_version = data.json()['version']
alive = True
except:
self.users = 0
self.soft_version = ""
except (Exception, psycopg2.DatabaseError) as error:
print(error)
if alive:
finally:
if downs_qty != None:
if conn is not None:
conn.close()
downs = downs_qty
except urllib3.exceptions.ProtocolError as protoerr:
else:
print(protoerr)
print("Server " + server + " is dead :-(")
alive = False
pass
downs = 0
except requests.exceptions.ChunkedEncodingError as chunkerr:
if self.soft_version != "" and self.soft_version is not None:
print(chunkerr)
print("Server " + server + " is dead :-(")
alive = False
pass
print(f'\n** Server {self.server} ({self.software} {self.soft_version}) is alive! **')
except KeyError as e:
else:
print(e)
print("Server " + server + " is dead :-(")
alive = False
pass
print(f'\n** Server {self.server} ({self.software}) is alive! **')
except ValueError as verr:
insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version, first_checked_at, last_checked_at, downs) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
print(verr)
print("Server " + server + " is dead :-(")
alive = False
pass
conn = None
except requests.exceptions.SSLError as errssl:
try:
print(errssl)
print("Server " + server + " is dead :-(")
alive = False
pass
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
except requests.exceptions.HTTPError as errh:
cur = conn.cursor()
print(errh)
print("Server " + server + " is dead :-(")
alive = False
pass
cur.execute(insert_sql, (self.server, self.users, now, self.software, alive, self.api, self.soft_version, now, now, downs))
except requests.exceptions.ConnectionError as errc:
if first_checked_at != None:
print(errc)
print("Server " + server + " is dead :-(")
alive = False
pass
cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (self.users, now, self.software, alive, self.api, self.soft_version, now, downs, self.server))
except requests.exceptions.Timeout as errt:
else:
print(errt)
print("Server " + server + " is dead :-(")
alive = False
pass
cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), first_checked_at=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (self.users, now, self.software, alive, self.api, self.soft_version, now, now, downs, self.server))
except requests.exceptions.RequestException as err:
print(err)
print("Server " + server + " is dead :-(")
alive = False
pass
cur.execute("UPDATE world SET checked='t' where server=(%s)", (self.server,))
except socket.gaierror as gai_error:
conn.commit()
print(gai_error)
pass
cur.close()
if alive is False:
except (Exception, psycopg2.DatabaseError) as error:
if downs_qty != None:
print(error)
downs = downs_qty + 1
finally:
else:
if conn is not None:
downs = 1
conn.close()
conn = None
except urllib3.exceptions.ProtocolError as protoerr:
try:
print_dead(self.server)
pass
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="",
host="/var/run/postgresql", port="5432")
except requests.exceptions.ChunkedEncodingError as chunkerr:
cur = conn.cursor()
print_dead(self.server)
pass
if first_checked_at != None:
except KeyError as e:
cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), first_checked_at=(%s), downs=(%s) where server=(%s)", (now, alive, now, downs, server))
print_dead(self.server)
pass
else:
except ValueError as verr:
cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), downs=(%s) where server=(%s)", (now, alive, downs, server))
print_dead(self.server)
pass
cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
except requests.exceptions.SSLError as errssl:
conn.commit()
print_dead(self.server)
pass
cur.close()
except requests.exceptions.HTTPError as errh:
except (Exception, psycopg2.DatabaseError) as error:
print(error)
print_dead(self.server)
pass
finally:
except requests.exceptions.ConnectionError as errc:
if conn is not None:
conn.close()
print_dead(self.server)
pass
def write_api(server, software, users, alive, api, soft_version):
except requests.exceptions.Timeout as errt:
insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
conn = None
print_dead(self.server)
pass
try:
except requests.exceptions.RequestException as err:
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
print_dead(self.server)
pass
cur = conn.cursor()
except socket.gaierror as gai_error:
cur.execute(insert_sql, (server, now, software, users, alive, api, soft_version))
print_dead(self.server)
pass
cur.execute("UPDATE fediverse SET updated_at=(%s), software=(%s), users=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)", (now, software, users, alive, api, soft_version, server))
if not alive:
cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
if downs_qty != None:
conn.commit()
downs = downs_qty + 1
cur.close()
else:
except (Exception, psycopg2.DatabaseError) as error:
downs = 1
print(error)
conn = None
finally:
try:
if conn is not None:
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="",
host="/var/run/postgresql", port="5432")
conn.close()
cur = conn.cursor()
async def getsoft(server):
if first_checked_at != None:
try:
cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), first_checked_at=(%s), downs=(%s) where server=(%s)", (now, alive, now, downs, self.server))
socket.gethostbyname(server)
else:
except socket.gaierror:
cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), downs=(%s) where server=(%s)", (now, alive, downs, self.server))
pass
return
cur.execute("UPDATE world SET checked='t' where server=(%s)", (self.server,))
soft = ''
conn.commit()
url = 'https://' + server
cur.close()
user_agent = {'User-agent': 'Mozilla/5.0'}
except (Exception, psycopg2.DatabaseError) as error:
timeout = aiohttp.ClientTimeout(total=3)
print(error)
async with aiohttp.ClientSession(headers=user_agent, timeout=timeout) as session:
for api in apis:
try:
async with session.get(url+api) as response:
if response.status == 200:
try:
response_json = await response.json()
except:
pass
except aiohttp.ClientConnectorError as err:
print(err)
pass
else:
if response.status == 200 and api != '/api/v1/instance?':
try:
soft = response_json['software']['name']
soft = soft.lower()
soft_version = response_json['software']['version']
users = response_json['usage']['users']['total']
if users > 1000000:
return
alive = True
write_api(server, soft, users, alive, api, soft_version)
print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
return
except:
pass
if response.status == 200 and soft == '' and api == "/api/v1/instance?":
soft = 'mastodon'
users = response_json['stats']['user_count']
soft_version = response_json['version']
if users > 1000000:
return
alive = True
write_api(server, soft, users, alive, api)
print("Server " + server + " (" + soft + ") is alive!")
def getserver(server, x):
server = server[0].rstrip('.').lower()
if server.find(".") == -1:
return
if server.find("@") != -1:
return
if server.find("/") != -1:
return
if server.find(":") != -1:
return
finally:
try:
if conn is not None:
loop = asyncio.get_event_loop()
coroutines = [getsoft(server)]
soft = loop.run_until_complete(asyncio.gather(*coroutines, return_exceptions=True))
conn.close()
except:
def print_dead(server):
pass
print(f'\nServer {server} is dead :-(')
def set_world_servers_check_to_false():
@ -562,7 +397,6 @@ def usage():
print('usage: python ' + sys.argv[0] + ' --multi' + ' (multiprocessing, fast)')
print('usage: python ' + sys.argv[0] + ' --mono' + ' (one process, slow)')
# Returns the parameter from the specified file
def get_parameter(parameter, file_path):
# Check if secrets file exists
@ -594,8 +428,8 @@ if __name__ == '__main__':
elif len(sys.argv) == 2:
if sys.argv[1] == '--multi':
now = datetime.now()
set_start_method("spawn")
mastodon, mastodon_hostname = mastodon()
@ -608,12 +442,16 @@ if __name__ == '__main__':
alive_servers = get_last_checked_servers()
getservers = Server()
###########################################################################
# multiprocessing!
nprocs = multiprocessing.cpu_count()
with multiprocessing.Pool(processes=64) as pool:
results = pool.starmap(get_alive_servers, product(alive_servers))
with get_context("spawn").Pool(processes=32) as pool:
res = pool.starmap(getservers.get_alive_servers, product(alive_servers))
pool.close()
pool.join()
elif sys.argv[1] == '--mono':
@ -624,17 +462,22 @@ if __name__ == '__main__':
fediverse_db, fediverse_db_user = db_config()
total_servers = 0
total_users = 0
set_world_servers_check_to_false()
alive_servers = get_last_checked_servers()
getservers = Server()
i = 0
while i < len(alive_servers):
get_alive_servers(alive_servers[i])
getservers.server = alive_servers[i]
getservers.get_alive_servers()
i += 1
@ -645,6 +488,8 @@ if __name__ == '__main__':
###########################################################################
# get current total servers and users, get users from every software
now = datetime.now()
gettotals_sql = "select count(server), sum(users) from fediverse where alive"
get_soft_totals_sql = "select software, sum(users) as users, count(server) as servers from fediverse where users != 0 and alive group by software order by users desc"
soft_total_project = []
@ -948,4 +793,4 @@ if __name__ == '__main__':
users_image_id = mastodon.media_post('users.png', "image/png", description='users graph').id
mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, users_image_id})
#mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, users_image_id})

@ -1,3 +1,6 @@
from multiprocessing import set_start_method
from multiprocessing import get_context
from itertools import product
import time
from datetime import datetime
import os
@ -5,22 +8,10 @@ import json
import sys
import os.path
import psycopg2
from multiprocessing import Pool, Manager
import aiohttp
import asyncio
import socket
apis = ['/nodeinfo/2.0?',
'/nodeinfo/2.0.json?',
'/main/nodeinfo/2.0?',
'/api/statusnet/config?',
'/api/nodeinfo/2.0.json?',
'/api/nodeinfo?',
'/api/v1/instance?',
'/wp-json/nodeinfo/2.0?',
'/api/v1/instance/nodeinfo/2.0?',
'/.well-known/x-nodeinfo2?'
]
import pdb
client_exceptions = (
aiohttp.ClientResponseError,
@ -31,16 +22,11 @@ client_exceptions = (
socket.gaierror,
)
def write_api(server, software, users, alive, api, soft_version):
def is_json(myjson):
try:
json_object = json.loads(myjson)
except ValueError as e:
return False
return True
now = datetime.now()
def write_api(server, software, users, alive, api, soft_version):
fediverse_db, fediverse_db_user = get_db_config()
insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
@ -48,11 +34,12 @@ def write_api(server, software, users, alive, api, soft_version):
try:
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql",
port="5432")
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor()
print(f'Writing {server} nodeinfo data...')
cur.execute(insert_sql, (server, now, software, users, alive, api, soft_version))
cur.execute(
@ -73,16 +60,20 @@ def write_api(server, software, users, alive, api, soft_version):
finally:
if conn is not None:
conn.close()
async def getsoft(server):
fediverse_db, fediverse_db_user = get_db_config()
try:
socket.gethostbyname(server)
except socket.gaierror:
except socket.gaierror as g_error:
print(f'Server {server} error: {g_error}')
pass
return
@ -93,71 +84,134 @@ async def getsoft(server):
user_agent = {'User-agent': 'Mozilla/5.0'}
timeout = aiohttp.ClientTimeout(total=3)
async with aiohttp.ClientSession(timeout=timeout, headers=user_agent) as session:
for api in apis:
try:
try:
async with session.get(url + api) as response:
if response.status == 200:
try:
response_json = await response.json()
except:
pass
async with session.get(url + '/.well-known/nodeinfo') as response:
except aiohttp.ClientConnectorError as err:
if response.status == 200:
pass
try:
else:
response_json = await response.json()
nodeinfo = response_json['links'][0]['href'].replace(f'https://{server}','')
except:
pass
else:
if response.status == 200 and api != '/api/v1/instance?':
print(f'Server {server} not responding: {response.status}')
if api != '/.well-known/x-nodeinfo2?':
pass
async with session.get(url + nodeinfo) as nodeinfo_response:
if nodeinfo_response.status == 200:
try:
soft = response_json['software']['name']
soft = soft.lower()
soft_version = response_json['software']['version']
users = response_json['usage']['users']['total']
if users > 1000000:
return
alive = True
write_api(server, soft, users, alive, api, soft_version)
print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
return
nodeinfo_json = await nodeinfo_response.json()
except:
pass
else:
try:
soft = response_json['server']['software']
soft = soft.lower()
soft_version = response_json['server']['version']
users = response_json['usage']['users']['total']
if users > 1000000:
return
alive = True
if soft == 'socialhome':
write_api(server, soft, users, alive, api, soft_version)
print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
print(f"Server {server}'s nodeinfo not responding: {response.status}")
pass
except aiohttp.ClientConnectorError as cc_err:
pass
except aiohttp.client_exceptions.ClientConnectorSSLError as ccssl_as:
pass
else:
if nodeinfo_response.status == 200 and nodeinfo != '/api/v1/instance?':
if nodeinfo != '/.well-known/x-nodeinfo2?':
try:
soft = nodeinfo_json['software']['name']
soft = soft.lower()
soft_version = nodeinfo_json['software']['version']
users = nodeinfo_json['usage']['users']['total']
if users > 1000000:
return
except:
pass
alive = True
write_api(server, soft, users, alive, nodeinfo, soft_version)
print('*********************************************************************')
print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
print('*********************************************************************')
if response.status == 200 and soft == '' and api == "/api/v1/instance?":
soft = 'mastodon'
users = response_json['stats']['user_count']
soft_version = response_json['version']
if users > 1000000:
return
alive = True
write_api(server, soft, users, alive, api)
print("Server " + server + " (" + soft + ") is alive!")
def getserver(server, x):
except:
pass
else:
try:
soft = nodeinfo_json['server']['software']
soft = soft.lower()
soft_version = nodeinfo_json['server']['version']
users = nodeinfo_json['usage']['users']['total']
if users > 1000000:
return
alive = True
if soft == 'socialhome':
write_api(server, soft, users, alive, api, soft_version)
print('*********************************************************************')
print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
print('*********************************************************************')
return
except:
pass
if nodeinfo_response.status == 200 and soft == '' and nodeinfo == "/api/v1/instance?":
soft = 'mastodon'
users = nodeinfo_json['stats']['user_count']
soft_version = nodeinfo_json['version']
if users > 1000000:
return
alive = True
write_api(server, soft, users, alive, api)
print('*********************************************************************')
print("Server " + server + " (" + soft + ") is alive!")
print('*********************************************************************')
else:
print(f'Server {server} is dead')
def getserver(server, *args):
if len(args) != 0:
server = server[0].rstrip('.').lower()
server = server[0].rstrip('.').lower()
if server.find(".") == -1:
return
@ -178,7 +232,46 @@ def getserver(server, x):
pass
def get_world_servers():
world_servers = []
try:
conn = None
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor()
# get world servers list
cur.execute("select server from world where checked='f'")
rows = cur.fetchall()
for row in rows:
world_servers.append(row[0])
cur.close()
print("Remaining servers: " + str(len(world_servers)))
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
return world_servers
def get_parameter(parameter, file_path):
# Check if secrets file exists
if not os.path.isfile(file_path):
print("File %s not found, exiting." % file_path)
@ -194,6 +287,10 @@ def get_parameter(parameter, file_path):
print(file_path + " Missing parameter %s " % parameter)
sys.exit(0)
def usage():
print('usage: python ' + sys.argv[0] + ' --multi' + ' (multiprocessing, fast)')
print('usage: python ' + sys.argv[0] + ' --mono' + ' (one process, slow)')
def get_config():
@ -214,59 +311,51 @@ def get_db_config():
# main
if __name__ == '__main__':
now = datetime.now()
start_time = time.time()
mastodon_hostname = get_config()
# usage modes
fediverse_db, fediverse_db_user = get_db_config()
if len(sys.argv) == 1:
world_servers = []
usage()
try:
elif len(sys.argv) == 2:
conn = None
if sys.argv[1] == '--multi':
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
now = datetime.now()
cur = conn.cursor()
start_time = time.time()
# get world servers list
mastodon_hostname = get_config()
cur.execute("select server from world where checked='f'")
fediverse_db, fediverse_db_user = get_db_config()
rows = cur.fetchall()
for row in rows:
world_servers.append(row[0])
world_servers = get_world_servers()
cur.close()
with get_context("spawn").Pool(processes=32) as pool:
print("Remaining servers: " + str(len(world_servers)))
res = pool.starmap(getserver, product(world_servers))
except (Exception, psycopg2.DatabaseError) as error:
pool.close()
print(error)
pool.join()
finally:
print('Done.')
if conn is not None:
elif sys.argv[1] == '--mono':
conn.close()
now = datetime.now()
start_time = time.time()
mastodon_hostname = get_config()
###########################################################################
# multiprocessing!
fediverse_db, fediverse_db_user = get_db_config()
m = Manager()
q = m.Queue()
z = zip(world_servers)
world_servers = get_world_servers()
serv_number = len(world_servers)
for server in world_servers:
pool_tuple = [(x, q) for x in z]
with Pool(processes=64) as pool:
pool.starmap(getserver, pool_tuple)
getserver(server)
print('Done.')
print('Done.')

Loading…
Cancel
Save