forkeado de spla/fediverse
Improved servers nodeinfo detection
This commit is contained in:
pare
36d7ff62b1
commit
0b52e0090d
S'han modificat 2 arxius amb 417 adicions i 483 eliminacions
591
fediverse.py
591
fediverse.py
|
@ -1,3 +1,5 @@
|
||||||
|
from multiprocessing import set_start_method
|
||||||
|
from multiprocessing import get_context
|
||||||
import time
|
import time
|
||||||
import urllib3
|
import urllib3
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
@ -7,11 +9,10 @@ import json
|
||||||
import sys
|
import sys
|
||||||
import os.path
|
import os.path
|
||||||
import requests
|
import requests
|
||||||
import psycopg2
|
|
||||||
from itertools import product
|
|
||||||
import multiprocessing
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
import asyncio
|
import asyncio
|
||||||
|
import psycopg2
|
||||||
|
from itertools import product
|
||||||
import socket
|
import socket
|
||||||
import matplotlib.pyplot as plt
|
import matplotlib.pyplot as plt
|
||||||
import pdb
|
import pdb
|
||||||
|
@ -19,18 +20,6 @@ plt.style.use('seaborn')
|
||||||
|
|
||||||
start_time = time.time()
|
start_time = time.time()
|
||||||
|
|
||||||
apis = ['/nodeinfo/2.0?',
|
|
||||||
'/nodeinfo/2.0.json?',
|
|
||||||
'/main/nodeinfo/2.0?',
|
|
||||||
'/api/statusnet/config?',
|
|
||||||
'/api/nodeinfo/2.0.json?',
|
|
||||||
'/api/nodeinfo?',
|
|
||||||
'/api/v1/instance?',
|
|
||||||
'/wp-json/nodeinfo/2.0?',
|
|
||||||
'/api/v1/instance/nodeinfo/2.0?',
|
|
||||||
'/.well-known/x-nodeinfo2?'
|
|
||||||
]
|
|
||||||
|
|
||||||
client_exceptions = (
|
client_exceptions = (
|
||||||
aiohttp.ClientResponseError,
|
aiohttp.ClientResponseError,
|
||||||
aiohttp.ClientConnectionError,
|
aiohttp.ClientConnectionError,
|
||||||
|
@ -40,121 +29,91 @@ client_exceptions = (
|
||||||
socket.gaierror,
|
socket.gaierror,
|
||||||
)
|
)
|
||||||
|
|
||||||
def is_json(myjson):
|
class Server:
|
||||||
|
|
||||||
try:
|
name = 'Server'
|
||||||
json_object = json.loads(myjson)
|
|
||||||
except ValueError as e:
|
|
||||||
print(e)
|
|
||||||
return False
|
|
||||||
return True
|
|
||||||
|
|
||||||
def get_alive_servers(server):
|
def __init_(self, server, software, users, alive, api, soft_version):
|
||||||
|
|
||||||
serv_api = ''
|
self.server = server
|
||||||
serv_soft = ''
|
self.software = software
|
||||||
soft_version = ''
|
self.users = users
|
||||||
|
self.alive = alive
|
||||||
|
self.api = api
|
||||||
|
self.version = self.soft_version
|
||||||
|
|
||||||
try:
|
def get_alive_servers(self, *args):
|
||||||
|
|
||||||
conn = None
|
fediverse_db, fediverse_db_user = db_config()
|
||||||
|
|
||||||
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
|
now = datetime.now()
|
||||||
|
|
||||||
cur = conn.cursor()
|
if len(args) != 0:
|
||||||
|
|
||||||
cur.execute("select alive, software, users_api, version, first_checked_at, downs from fediverse where server=(%s)", (server,))
|
self.server = args[0]
|
||||||
|
|
||||||
row = cur.fetchone()
|
|
||||||
|
|
||||||
if row is not None:
|
|
||||||
|
|
||||||
was_alive = row[0]
|
|
||||||
serv_soft = row[1]
|
|
||||||
serv_api = row[2]
|
|
||||||
soft_version = row[3]
|
|
||||||
first_checked_at = row[4]
|
|
||||||
downs_qty = row[5]
|
|
||||||
|
|
||||||
cur.close()
|
|
||||||
|
|
||||||
except (Exception, psycopg2.DatabaseError) as error:
|
|
||||||
|
|
||||||
print(error)
|
|
||||||
|
|
||||||
finally:
|
|
||||||
|
|
||||||
if conn is not None:
|
|
||||||
conn.close()
|
|
||||||
|
|
||||||
alive = False
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|
||||||
user_agent = {'User-agent': 'Mozilla/5.0'}
|
conn = None
|
||||||
|
|
||||||
data = requests.get('https://' + server + serv_api, headers = user_agent, timeout=3)
|
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
|
||||||
|
|
||||||
|
cur = conn.cursor()
|
||||||
|
|
||||||
|
cur.execute("select alive, software, users_api, version, first_checked_at, downs from fediverse where server=(%s)", (self.server,))
|
||||||
|
|
||||||
|
row = cur.fetchone()
|
||||||
|
|
||||||
|
if row is not None:
|
||||||
|
|
||||||
|
was_alive = row[0]
|
||||||
|
self.software = row[1]
|
||||||
|
self.api = row[2]
|
||||||
|
self.soft_version = row[3]
|
||||||
|
first_checked_at = row[4]
|
||||||
|
downs_qty = row[5]
|
||||||
|
|
||||||
|
cur.close()
|
||||||
|
|
||||||
|
except (Exception, psycopg2.DatabaseError) as error:
|
||||||
|
|
||||||
|
print(error)
|
||||||
|
|
||||||
|
finally:
|
||||||
|
|
||||||
|
if conn is not None:
|
||||||
|
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
alive = False
|
||||||
|
|
||||||
|
try:
|
||||||
|
|
||||||
|
user_agent = {'User-agent': 'Mozilla/5.0'}
|
||||||
|
|
||||||
|
data = requests.get('https://' + self.server + self.api, headers = user_agent, timeout=3)
|
||||||
|
|
||||||
if serv_soft == "mastodon":
|
|
||||||
if serv_api == '/nodeinfo/2.0?':
|
|
||||||
try:
|
|
||||||
users = data.json()['usage']['users']['total']
|
|
||||||
soft_version = data.json()['software']['version']
|
|
||||||
alive = True
|
|
||||||
except:
|
|
||||||
users = 0
|
|
||||||
soft_version = ""
|
|
||||||
if serv_api == '/nodeinfo/2.0.json?':
|
|
||||||
try:
|
|
||||||
users = data.json()['usage']['users']['total']
|
|
||||||
soft_version = data.json()['software']['version']
|
|
||||||
alive = True
|
|
||||||
except:
|
|
||||||
users = 0
|
|
||||||
soft_version = ""
|
|
||||||
elif serv_api == '/api/v1/instance?':
|
|
||||||
try:
|
|
||||||
users = data.json()['stats']['user_count']
|
|
||||||
soft_version = data.json()['version']
|
|
||||||
alive = True
|
|
||||||
except:
|
|
||||||
users = 0
|
|
||||||
soft_version = ""
|
|
||||||
if serv_soft == "pleroma" or serv_soft == "diaspora" or serv_soft == "peertube" or serv_soft == "pixelfed" or serv_soft == "hubzilla" or serv_soft == "writefreely" or serv_soft == "friendica":
|
|
||||||
try:
|
try:
|
||||||
users = data.json()['usage']['users']['total']
|
|
||||||
soft_version = data.json()['software']['version']
|
self.users = data.json()['usage']['users']['total']
|
||||||
alive = True
|
|
||||||
except:
|
if self.users == 0:
|
||||||
users = 0
|
|
||||||
soft_version = ""
|
self.users = data.json()['usage']['users']['activeHalfyear']
|
||||||
if serv_soft == "gnusocialv2" or serv_soft == "gnusocial":
|
|
||||||
try:
|
if self.software == 'socialhome':
|
||||||
users = data.json()['usage']['users']['total']
|
|
||||||
if users == 0:
|
self.soft_version = data.json()['server']['version']
|
||||||
users = data.json()['usage']['users']['activeHalfyear']
|
|
||||||
soft_version = data.json()['software']['version']
|
else:
|
||||||
alive = True
|
|
||||||
except:
|
self.soft_version = data.json()['software']['version']
|
||||||
users = 0
|
|
||||||
soft_version = ""
|
if self.software == "wordpress" and "activitypub" in data.json()['protocols']:
|
||||||
if serv_soft == "plume" or serv_soft == 'red' or serv_soft == "misskey" or serv_soft == "zap" or serv_soft == "prismo" or serv_soft == "ravenvale" or serv_soft == "osada" or serv_soft == "groundpolis":
|
|
||||||
try:
|
|
||||||
users = data.json()['usage']['users']['total']
|
|
||||||
soft_version = data.json()['software']['version']
|
|
||||||
alive = True
|
|
||||||
except:
|
|
||||||
users = 0
|
|
||||||
soft_version = ""
|
|
||||||
if serv_soft == "ganggo" or serv_soft == "squs" or serv_soft == "dolphin" or serv_soft == "lemmy" or serv_soft == "wordpress":
|
|
||||||
try:
|
|
||||||
users = data.json()['usage']['users']['total']
|
|
||||||
soft_version = data.json()['software']['version']
|
|
||||||
if serv_soft == "wordpress" and "activitypub" in data.json()['protocols']:
|
|
||||||
|
|
||||||
alive = True
|
alive = True
|
||||||
|
|
||||||
elif serv_soft == "wordpress" and "activitypub" not in data.json()['protocols']:
|
elif self.software == "wordpress" and "activitypub" not in data.json()['protocols']:
|
||||||
|
|
||||||
alive = False
|
alive = False
|
||||||
|
|
||||||
|
@ -164,297 +123,173 @@ def get_alive_servers(server):
|
||||||
|
|
||||||
except:
|
except:
|
||||||
|
|
||||||
users = 0
|
self.users = 0
|
||||||
soft_version = ""
|
|
||||||
|
|
||||||
if serv_soft == 'funkwhale':
|
self.soft_version = ""
|
||||||
|
|
||||||
try:
|
else:
|
||||||
users = data.json()['usage']['users']['total']
|
|
||||||
soft_version = data.json()['software']['version']
|
|
||||||
alive = True
|
|
||||||
except:
|
|
||||||
users = 0
|
|
||||||
soft_version = ""
|
|
||||||
|
|
||||||
if serv_soft == 'socialhome':
|
if self.api == '/api/v1/instance?':
|
||||||
|
|
||||||
try:
|
try:
|
||||||
users = data.json()['usage']['users']['total']
|
self.users = data.json()['stats']['user_count']
|
||||||
soft_version = data.json()['server']['version']
|
self.soft_version = data.json()['version']
|
||||||
alive = True
|
alive = True
|
||||||
except:
|
except:
|
||||||
users = 0
|
self.users = 0
|
||||||
soft_version = ""
|
self.soft_version = ""
|
||||||
|
|
||||||
if alive:
|
if alive:
|
||||||
|
|
||||||
|
if downs_qty != None:
|
||||||
|
|
||||||
|
downs = downs_qty
|
||||||
|
|
||||||
|
else:
|
||||||
|
|
||||||
|
downs = 0
|
||||||
|
|
||||||
|
if self.soft_version != "" and self.soft_version is not None:
|
||||||
|
|
||||||
|
print(f'\n** Server {self.server} ({self.software} {self.soft_version}) is alive! **')
|
||||||
|
|
||||||
|
else:
|
||||||
|
|
||||||
|
print(f'\n** Server {self.server} ({self.software}) is alive! **')
|
||||||
|
|
||||||
|
insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version, first_checked_at, last_checked_at, downs) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
|
||||||
|
|
||||||
|
conn = None
|
||||||
|
|
||||||
|
try:
|
||||||
|
|
||||||
|
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
|
||||||
|
|
||||||
|
cur = conn.cursor()
|
||||||
|
|
||||||
|
cur.execute(insert_sql, (self.server, self.users, now, self.software, alive, self.api, self.soft_version, now, now, downs))
|
||||||
|
|
||||||
|
if first_checked_at != None:
|
||||||
|
|
||||||
|
cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (self.users, now, self.software, alive, self.api, self.soft_version, now, downs, self.server))
|
||||||
|
|
||||||
|
else:
|
||||||
|
|
||||||
|
cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), first_checked_at=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (self.users, now, self.software, alive, self.api, self.soft_version, now, now, downs, self.server))
|
||||||
|
|
||||||
|
|
||||||
|
cur.execute("UPDATE world SET checked='t' where server=(%s)", (self.server,))
|
||||||
|
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
cur.close()
|
||||||
|
|
||||||
|
except (Exception, psycopg2.DatabaseError) as error:
|
||||||
|
|
||||||
|
print(error)
|
||||||
|
|
||||||
|
finally:
|
||||||
|
|
||||||
|
if conn is not None:
|
||||||
|
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
except urllib3.exceptions.ProtocolError as protoerr:
|
||||||
|
|
||||||
|
print_dead(self.server)
|
||||||
|
pass
|
||||||
|
|
||||||
|
except requests.exceptions.ChunkedEncodingError as chunkerr:
|
||||||
|
|
||||||
|
print_dead(self.server)
|
||||||
|
pass
|
||||||
|
|
||||||
|
except KeyError as e:
|
||||||
|
|
||||||
|
print_dead(self.server)
|
||||||
|
pass
|
||||||
|
|
||||||
|
except ValueError as verr:
|
||||||
|
|
||||||
|
print_dead(self.server)
|
||||||
|
pass
|
||||||
|
|
||||||
|
except requests.exceptions.SSLError as errssl:
|
||||||
|
|
||||||
|
print_dead(self.server)
|
||||||
|
pass
|
||||||
|
|
||||||
|
except requests.exceptions.HTTPError as errh:
|
||||||
|
|
||||||
|
print_dead(self.server)
|
||||||
|
pass
|
||||||
|
|
||||||
|
except requests.exceptions.ConnectionError as errc:
|
||||||
|
|
||||||
|
print_dead(self.server)
|
||||||
|
pass
|
||||||
|
|
||||||
|
except requests.exceptions.Timeout as errt:
|
||||||
|
|
||||||
|
print_dead(self.server)
|
||||||
|
pass
|
||||||
|
|
||||||
|
except requests.exceptions.RequestException as err:
|
||||||
|
|
||||||
|
print_dead(self.server)
|
||||||
|
pass
|
||||||
|
|
||||||
|
except socket.gaierror as gai_error:
|
||||||
|
|
||||||
|
print_dead(self.server)
|
||||||
|
pass
|
||||||
|
|
||||||
|
if not alive:
|
||||||
|
|
||||||
if downs_qty != None:
|
if downs_qty != None:
|
||||||
|
|
||||||
downs = downs_qty
|
downs = downs_qty + 1
|
||||||
|
|
||||||
else:
|
else:
|
||||||
|
|
||||||
downs = 0
|
downs = 1
|
||||||
|
|
||||||
if soft_version != "" and soft_version is not None:
|
|
||||||
print("Server " + str(server) + " (" + serv_soft + " " + soft_version + ") is alive!")
|
|
||||||
else:
|
|
||||||
print("Server " + str(server) + " (" + serv_soft + ") is alive!")
|
|
||||||
|
|
||||||
insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version, first_checked_at, last_checked_at, downs) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
|
|
||||||
|
|
||||||
conn = None
|
conn = None
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|
||||||
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
|
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="",
|
||||||
|
host="/var/run/postgresql", port="5432")
|
||||||
|
|
||||||
cur = conn.cursor()
|
cur = conn.cursor()
|
||||||
|
|
||||||
cur.execute(insert_sql, (server, users, now, serv_soft, alive, serv_api, soft_version, now, now, downs))
|
|
||||||
|
|
||||||
if first_checked_at != None:
|
if first_checked_at != None:
|
||||||
|
|
||||||
cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (users, now, serv_soft, alive, serv_api, soft_version, now, downs, server))
|
cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), first_checked_at=(%s), downs=(%s) where server=(%s)", (now, alive, now, downs, self.server))
|
||||||
|
|
||||||
else:
|
else:
|
||||||
|
|
||||||
cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), first_checked_at=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (users, now, serv_soft, alive, serv_api, soft_version, now, now, downs, server))
|
cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), downs=(%s) where server=(%s)", (now, alive, downs, self.server))
|
||||||
|
|
||||||
|
cur.execute("UPDATE world SET checked='t' where server=(%s)", (self.server,))
|
||||||
cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
|
|
||||||
|
|
||||||
conn.commit()
|
conn.commit()
|
||||||
|
|
||||||
cur.close()
|
cur.close()
|
||||||
|
|
||||||
except (Exception, psycopg2.DatabaseError) as error:
|
except (Exception, psycopg2.DatabaseError) as error:
|
||||||
|
|
||||||
print(error)
|
print(error)
|
||||||
|
|
||||||
finally:
|
finally:
|
||||||
|
|
||||||
if conn is not None:
|
if conn is not None:
|
||||||
|
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
except urllib3.exceptions.ProtocolError as protoerr:
|
def print_dead(server):
|
||||||
|
|
||||||
print(protoerr)
|
print(f'\nServer {server} is dead :-(')
|
||||||
print("Server " + server + " is dead :-(")
|
|
||||||
alive = False
|
|
||||||
pass
|
|
||||||
|
|
||||||
except requests.exceptions.ChunkedEncodingError as chunkerr:
|
|
||||||
|
|
||||||
print(chunkerr)
|
|
||||||
print("Server " + server + " is dead :-(")
|
|
||||||
alive = False
|
|
||||||
pass
|
|
||||||
|
|
||||||
except KeyError as e:
|
|
||||||
|
|
||||||
print(e)
|
|
||||||
print("Server " + server + " is dead :-(")
|
|
||||||
alive = False
|
|
||||||
pass
|
|
||||||
|
|
||||||
except ValueError as verr:
|
|
||||||
|
|
||||||
print(verr)
|
|
||||||
print("Server " + server + " is dead :-(")
|
|
||||||
alive = False
|
|
||||||
pass
|
|
||||||
|
|
||||||
except requests.exceptions.SSLError as errssl:
|
|
||||||
|
|
||||||
print(errssl)
|
|
||||||
print("Server " + server + " is dead :-(")
|
|
||||||
alive = False
|
|
||||||
pass
|
|
||||||
|
|
||||||
except requests.exceptions.HTTPError as errh:
|
|
||||||
|
|
||||||
print(errh)
|
|
||||||
print("Server " + server + " is dead :-(")
|
|
||||||
alive = False
|
|
||||||
pass
|
|
||||||
|
|
||||||
except requests.exceptions.ConnectionError as errc:
|
|
||||||
|
|
||||||
print(errc)
|
|
||||||
print("Server " + server + " is dead :-(")
|
|
||||||
alive = False
|
|
||||||
pass
|
|
||||||
|
|
||||||
except requests.exceptions.Timeout as errt:
|
|
||||||
|
|
||||||
print(errt)
|
|
||||||
print("Server " + server + " is dead :-(")
|
|
||||||
alive = False
|
|
||||||
pass
|
|
||||||
|
|
||||||
except requests.exceptions.RequestException as err:
|
|
||||||
|
|
||||||
print(err)
|
|
||||||
print("Server " + server + " is dead :-(")
|
|
||||||
alive = False
|
|
||||||
pass
|
|
||||||
|
|
||||||
except socket.gaierror as gai_error:
|
|
||||||
|
|
||||||
print(gai_error)
|
|
||||||
pass
|
|
||||||
|
|
||||||
if alive is False:
|
|
||||||
|
|
||||||
if downs_qty != None:
|
|
||||||
|
|
||||||
downs = downs_qty + 1
|
|
||||||
|
|
||||||
else:
|
|
||||||
|
|
||||||
downs = 1
|
|
||||||
|
|
||||||
conn = None
|
|
||||||
|
|
||||||
try:
|
|
||||||
|
|
||||||
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="",
|
|
||||||
host="/var/run/postgresql", port="5432")
|
|
||||||
|
|
||||||
cur = conn.cursor()
|
|
||||||
|
|
||||||
if first_checked_at != None:
|
|
||||||
|
|
||||||
cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), first_checked_at=(%s), downs=(%s) where server=(%s)", (now, alive, now, downs, server))
|
|
||||||
|
|
||||||
else:
|
|
||||||
|
|
||||||
cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), downs=(%s) where server=(%s)", (now, alive, downs, server))
|
|
||||||
|
|
||||||
cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
|
|
||||||
|
|
||||||
conn.commit()
|
|
||||||
|
|
||||||
cur.close()
|
|
||||||
|
|
||||||
except (Exception, psycopg2.DatabaseError) as error:
|
|
||||||
print(error)
|
|
||||||
|
|
||||||
finally:
|
|
||||||
|
|
||||||
if conn is not None:
|
|
||||||
conn.close()
|
|
||||||
|
|
||||||
def write_api(server, software, users, alive, api, soft_version):
|
|
||||||
|
|
||||||
insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
|
|
||||||
conn = None
|
|
||||||
|
|
||||||
try:
|
|
||||||
|
|
||||||
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
|
|
||||||
|
|
||||||
cur = conn.cursor()
|
|
||||||
|
|
||||||
cur.execute(insert_sql, (server, now, software, users, alive, api, soft_version))
|
|
||||||
|
|
||||||
cur.execute("UPDATE fediverse SET updated_at=(%s), software=(%s), users=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)", (now, software, users, alive, api, soft_version, server))
|
|
||||||
|
|
||||||
cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
|
|
||||||
|
|
||||||
conn.commit()
|
|
||||||
|
|
||||||
cur.close()
|
|
||||||
|
|
||||||
except (Exception, psycopg2.DatabaseError) as error:
|
|
||||||
|
|
||||||
print(error)
|
|
||||||
|
|
||||||
finally:
|
|
||||||
|
|
||||||
if conn is not None:
|
|
||||||
|
|
||||||
conn.close()
|
|
||||||
|
|
||||||
async def getsoft(server):
|
|
||||||
|
|
||||||
try:
|
|
||||||
|
|
||||||
socket.gethostbyname(server)
|
|
||||||
|
|
||||||
except socket.gaierror:
|
|
||||||
|
|
||||||
pass
|
|
||||||
return
|
|
||||||
|
|
||||||
soft = ''
|
|
||||||
|
|
||||||
url = 'https://' + server
|
|
||||||
|
|
||||||
user_agent = {'User-agent': 'Mozilla/5.0'}
|
|
||||||
|
|
||||||
timeout = aiohttp.ClientTimeout(total=3)
|
|
||||||
|
|
||||||
async with aiohttp.ClientSession(headers=user_agent, timeout=timeout) as session:
|
|
||||||
for api in apis:
|
|
||||||
try:
|
|
||||||
async with session.get(url+api) as response:
|
|
||||||
if response.status == 200:
|
|
||||||
try:
|
|
||||||
response_json = await response.json()
|
|
||||||
except:
|
|
||||||
pass
|
|
||||||
except aiohttp.ClientConnectorError as err:
|
|
||||||
print(err)
|
|
||||||
pass
|
|
||||||
else:
|
|
||||||
if response.status == 200 and api != '/api/v1/instance?':
|
|
||||||
try:
|
|
||||||
soft = response_json['software']['name']
|
|
||||||
soft = soft.lower()
|
|
||||||
soft_version = response_json['software']['version']
|
|
||||||
users = response_json['usage']['users']['total']
|
|
||||||
if users > 1000000:
|
|
||||||
return
|
|
||||||
alive = True
|
|
||||||
write_api(server, soft, users, alive, api, soft_version)
|
|
||||||
print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
|
|
||||||
return
|
|
||||||
except:
|
|
||||||
pass
|
|
||||||
if response.status == 200 and soft == '' and api == "/api/v1/instance?":
|
|
||||||
soft = 'mastodon'
|
|
||||||
users = response_json['stats']['user_count']
|
|
||||||
soft_version = response_json['version']
|
|
||||||
if users > 1000000:
|
|
||||||
return
|
|
||||||
alive = True
|
|
||||||
write_api(server, soft, users, alive, api)
|
|
||||||
print("Server " + server + " (" + soft + ") is alive!")
|
|
||||||
|
|
||||||
def getserver(server, x):
|
|
||||||
|
|
||||||
server = server[0].rstrip('.').lower()
|
|
||||||
|
|
||||||
if server.find(".") == -1:
|
|
||||||
return
|
|
||||||
if server.find("@") != -1:
|
|
||||||
return
|
|
||||||
if server.find("/") != -1:
|
|
||||||
return
|
|
||||||
if server.find(":") != -1:
|
|
||||||
return
|
|
||||||
|
|
||||||
try:
|
|
||||||
|
|
||||||
loop = asyncio.get_event_loop()
|
|
||||||
coroutines = [getsoft(server)]
|
|
||||||
soft = loop.run_until_complete(asyncio.gather(*coroutines, return_exceptions=True))
|
|
||||||
|
|
||||||
except:
|
|
||||||
|
|
||||||
pass
|
|
||||||
|
|
||||||
def set_world_servers_check_to_false():
|
def set_world_servers_check_to_false():
|
||||||
|
|
||||||
|
@ -562,7 +397,6 @@ def usage():
|
||||||
print('usage: python ' + sys.argv[0] + ' --multi' + ' (multiprocessing, fast)')
|
print('usage: python ' + sys.argv[0] + ' --multi' + ' (multiprocessing, fast)')
|
||||||
print('usage: python ' + sys.argv[0] + ' --mono' + ' (one process, slow)')
|
print('usage: python ' + sys.argv[0] + ' --mono' + ' (one process, slow)')
|
||||||
|
|
||||||
# Returns the parameter from the specified file
|
|
||||||
def get_parameter(parameter, file_path):
|
def get_parameter(parameter, file_path):
|
||||||
|
|
||||||
# Check if secrets file exists
|
# Check if secrets file exists
|
||||||
|
@ -594,8 +428,8 @@ if __name__ == '__main__':
|
||||||
elif len(sys.argv) == 2:
|
elif len(sys.argv) == 2:
|
||||||
|
|
||||||
if sys.argv[1] == '--multi':
|
if sys.argv[1] == '--multi':
|
||||||
|
|
||||||
now = datetime.now()
|
set_start_method("spawn")
|
||||||
|
|
||||||
mastodon, mastodon_hostname = mastodon()
|
mastodon, mastodon_hostname = mastodon()
|
||||||
|
|
||||||
|
@ -608,12 +442,16 @@ if __name__ == '__main__':
|
||||||
|
|
||||||
alive_servers = get_last_checked_servers()
|
alive_servers = get_last_checked_servers()
|
||||||
|
|
||||||
|
getservers = Server()
|
||||||
|
|
||||||
###########################################################################
|
###########################################################################
|
||||||
# multiprocessing!
|
# multiprocessing!
|
||||||
|
|
||||||
nprocs = multiprocessing.cpu_count()
|
with get_context("spawn").Pool(processes=32) as pool:
|
||||||
with multiprocessing.Pool(processes=64) as pool:
|
|
||||||
results = pool.starmap(get_alive_servers, product(alive_servers))
|
res = pool.starmap(getservers.get_alive_servers, product(alive_servers))
|
||||||
|
pool.close()
|
||||||
|
pool.join()
|
||||||
|
|
||||||
elif sys.argv[1] == '--mono':
|
elif sys.argv[1] == '--mono':
|
||||||
|
|
||||||
|
@ -624,17 +462,22 @@ if __name__ == '__main__':
|
||||||
fediverse_db, fediverse_db_user = db_config()
|
fediverse_db, fediverse_db_user = db_config()
|
||||||
|
|
||||||
total_servers = 0
|
total_servers = 0
|
||||||
|
|
||||||
total_users = 0
|
total_users = 0
|
||||||
|
|
||||||
set_world_servers_check_to_false()
|
set_world_servers_check_to_false()
|
||||||
|
|
||||||
alive_servers = get_last_checked_servers()
|
alive_servers = get_last_checked_servers()
|
||||||
|
|
||||||
|
getservers = Server()
|
||||||
|
|
||||||
i = 0
|
i = 0
|
||||||
|
|
||||||
while i < len(alive_servers):
|
while i < len(alive_servers):
|
||||||
|
|
||||||
get_alive_servers(alive_servers[i])
|
getservers.server = alive_servers[i]
|
||||||
|
|
||||||
|
getservers.get_alive_servers()
|
||||||
|
|
||||||
i += 1
|
i += 1
|
||||||
|
|
||||||
|
@ -645,6 +488,8 @@ if __name__ == '__main__':
|
||||||
###########################################################################
|
###########################################################################
|
||||||
# get current total servers and users, get users from every software
|
# get current total servers and users, get users from every software
|
||||||
|
|
||||||
|
now = datetime.now()
|
||||||
|
|
||||||
gettotals_sql = "select count(server), sum(users) from fediverse where alive"
|
gettotals_sql = "select count(server), sum(users) from fediverse where alive"
|
||||||
get_soft_totals_sql = "select software, sum(users) as users, count(server) as servers from fediverse where users != 0 and alive group by software order by users desc"
|
get_soft_totals_sql = "select software, sum(users) as users, count(server) as servers from fediverse where users != 0 and alive group by software order by users desc"
|
||||||
soft_total_project = []
|
soft_total_project = []
|
||||||
|
@ -948,4 +793,4 @@ if __name__ == '__main__':
|
||||||
|
|
||||||
users_image_id = mastodon.media_post('users.png', "image/png", description='users graph').id
|
users_image_id = mastodon.media_post('users.png', "image/png", description='users graph').id
|
||||||
|
|
||||||
mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, users_image_id})
|
#mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, users_image_id})
|
||||||
|
|
309
fetchservers.py
309
fetchservers.py
|
@ -1,3 +1,6 @@
|
||||||
|
from multiprocessing import set_start_method
|
||||||
|
from multiprocessing import get_context
|
||||||
|
from itertools import product
|
||||||
import time
|
import time
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
import os
|
import os
|
||||||
|
@ -5,22 +8,10 @@ import json
|
||||||
import sys
|
import sys
|
||||||
import os.path
|
import os.path
|
||||||
import psycopg2
|
import psycopg2
|
||||||
from multiprocessing import Pool, Manager
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
import asyncio
|
import asyncio
|
||||||
import socket
|
import socket
|
||||||
|
import pdb
|
||||||
apis = ['/nodeinfo/2.0?',
|
|
||||||
'/nodeinfo/2.0.json?',
|
|
||||||
'/main/nodeinfo/2.0?',
|
|
||||||
'/api/statusnet/config?',
|
|
||||||
'/api/nodeinfo/2.0.json?',
|
|
||||||
'/api/nodeinfo?',
|
|
||||||
'/api/v1/instance?',
|
|
||||||
'/wp-json/nodeinfo/2.0?',
|
|
||||||
'/api/v1/instance/nodeinfo/2.0?',
|
|
||||||
'/.well-known/x-nodeinfo2?'
|
|
||||||
]
|
|
||||||
|
|
||||||
client_exceptions = (
|
client_exceptions = (
|
||||||
aiohttp.ClientResponseError,
|
aiohttp.ClientResponseError,
|
||||||
|
@ -31,28 +22,24 @@ client_exceptions = (
|
||||||
socket.gaierror,
|
socket.gaierror,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def is_json(myjson):
|
|
||||||
|
|
||||||
try:
|
|
||||||
json_object = json.loads(myjson)
|
|
||||||
except ValueError as e:
|
|
||||||
return False
|
|
||||||
return True
|
|
||||||
|
|
||||||
def write_api(server, software, users, alive, api, soft_version):
|
def write_api(server, software, users, alive, api, soft_version):
|
||||||
|
|
||||||
|
now = datetime.now()
|
||||||
|
|
||||||
|
fediverse_db, fediverse_db_user = get_db_config()
|
||||||
|
|
||||||
insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
|
insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
|
||||||
|
|
||||||
conn = None
|
conn = None
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|
||||||
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql",
|
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
|
||||||
port="5432")
|
|
||||||
|
|
||||||
cur = conn.cursor()
|
cur = conn.cursor()
|
||||||
|
|
||||||
|
print(f'Writing {server} nodeinfo data...')
|
||||||
|
|
||||||
cur.execute(insert_sql, (server, now, software, users, alive, api, soft_version))
|
cur.execute(insert_sql, (server, now, software, users, alive, api, soft_version))
|
||||||
|
|
||||||
cur.execute(
|
cur.execute(
|
||||||
|
@ -73,16 +60,20 @@ def write_api(server, software, users, alive, api, soft_version):
|
||||||
finally:
|
finally:
|
||||||
|
|
||||||
if conn is not None:
|
if conn is not None:
|
||||||
|
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
async def getsoft(server):
|
async def getsoft(server):
|
||||||
|
|
||||||
|
fediverse_db, fediverse_db_user = get_db_config()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|
||||||
socket.gethostbyname(server)
|
socket.gethostbyname(server)
|
||||||
|
|
||||||
except socket.gaierror:
|
except socket.gaierror as g_error:
|
||||||
|
|
||||||
|
print(f'Server {server} error: {g_error}')
|
||||||
pass
|
pass
|
||||||
return
|
return
|
||||||
|
|
||||||
|
@ -93,71 +84,134 @@ async def getsoft(server):
|
||||||
user_agent = {'User-agent': 'Mozilla/5.0'}
|
user_agent = {'User-agent': 'Mozilla/5.0'}
|
||||||
|
|
||||||
timeout = aiohttp.ClientTimeout(total=3)
|
timeout = aiohttp.ClientTimeout(total=3)
|
||||||
|
|
||||||
async with aiohttp.ClientSession(timeout=timeout, headers=user_agent) as session:
|
async with aiohttp.ClientSession(timeout=timeout, headers=user_agent) as session:
|
||||||
for api in apis:
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|
||||||
|
async with session.get(url + '/.well-known/nodeinfo') as response:
|
||||||
|
|
||||||
|
if response.status == 200:
|
||||||
|
|
||||||
|
try:
|
||||||
|
|
||||||
|
response_json = await response.json()
|
||||||
|
|
||||||
|
nodeinfo = response_json['links'][0]['href'].replace(f'https://{server}','')
|
||||||
|
|
||||||
|
except:
|
||||||
|
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
|
||||||
|
print(f'Server {server} not responding: {response.status}')
|
||||||
|
|
||||||
|
pass
|
||||||
|
|
||||||
|
async with session.get(url + nodeinfo) as nodeinfo_response:
|
||||||
|
|
||||||
|
if nodeinfo_response.status == 200:
|
||||||
|
|
||||||
async with session.get(url + api) as response:
|
|
||||||
if response.status == 200:
|
|
||||||
try:
|
try:
|
||||||
response_json = await response.json()
|
|
||||||
|
nodeinfo_json = await nodeinfo_response.json()
|
||||||
|
|
||||||
except:
|
except:
|
||||||
|
|
||||||
pass
|
pass
|
||||||
|
|
||||||
except aiohttp.ClientConnectorError as err:
|
else:
|
||||||
|
|
||||||
pass
|
print(f"Server {server}'s nodeinfo not responding: {response.status}")
|
||||||
|
|
||||||
|
pass
|
||||||
|
|
||||||
|
except aiohttp.ClientConnectorError as cc_err:
|
||||||
|
|
||||||
|
pass
|
||||||
|
|
||||||
|
except aiohttp.client_exceptions.ClientConnectorSSLError as ccssl_as:
|
||||||
|
|
||||||
|
pass
|
||||||
|
|
||||||
|
else:
|
||||||
|
|
||||||
|
if nodeinfo_response.status == 200 and nodeinfo != '/api/v1/instance?':
|
||||||
|
|
||||||
|
if nodeinfo != '/.well-known/x-nodeinfo2?':
|
||||||
|
|
||||||
|
try:
|
||||||
|
|
||||||
|
soft = nodeinfo_json['software']['name']
|
||||||
|
soft = soft.lower()
|
||||||
|
soft_version = nodeinfo_json['software']['version']
|
||||||
|
users = nodeinfo_json['usage']['users']['total']
|
||||||
|
if users > 1000000:
|
||||||
|
return
|
||||||
|
alive = True
|
||||||
|
|
||||||
|
write_api(server, soft, users, alive, nodeinfo, soft_version)
|
||||||
|
|
||||||
|
print('*********************************************************************')
|
||||||
|
print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
|
||||||
|
print('*********************************************************************')
|
||||||
|
|
||||||
|
return
|
||||||
|
|
||||||
|
except:
|
||||||
|
|
||||||
|
pass
|
||||||
|
|
||||||
|
else:
|
||||||
|
|
||||||
|
try:
|
||||||
|
|
||||||
|
soft = nodeinfo_json['server']['software']
|
||||||
|
soft = soft.lower()
|
||||||
|
soft_version = nodeinfo_json['server']['version']
|
||||||
|
users = nodeinfo_json['usage']['users']['total']
|
||||||
|
if users > 1000000:
|
||||||
|
return
|
||||||
|
alive = True
|
||||||
|
|
||||||
|
if soft == 'socialhome':
|
||||||
|
|
||||||
|
write_api(server, soft, users, alive, api, soft_version)
|
||||||
|
|
||||||
|
print('*********************************************************************')
|
||||||
|
print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
|
||||||
|
print('*********************************************************************')
|
||||||
|
|
||||||
|
return
|
||||||
|
|
||||||
|
except:
|
||||||
|
|
||||||
|
pass
|
||||||
|
|
||||||
|
if nodeinfo_response.status == 200 and soft == '' and nodeinfo == "/api/v1/instance?":
|
||||||
|
|
||||||
|
soft = 'mastodon'
|
||||||
|
users = nodeinfo_json['stats']['user_count']
|
||||||
|
soft_version = nodeinfo_json['version']
|
||||||
|
if users > 1000000:
|
||||||
|
return
|
||||||
|
alive = True
|
||||||
|
|
||||||
|
write_api(server, soft, users, alive, api)
|
||||||
|
|
||||||
|
print('*********************************************************************')
|
||||||
|
print("Server " + server + " (" + soft + ") is alive!")
|
||||||
|
print('*********************************************************************')
|
||||||
|
|
||||||
else:
|
else:
|
||||||
|
|
||||||
if response.status == 200 and api != '/api/v1/instance?':
|
print(f'Server {server} is dead')
|
||||||
|
|
||||||
if api != '/.well-known/x-nodeinfo2?':
|
def getserver(server, *args):
|
||||||
|
|
||||||
try:
|
if len(args) != 0:
|
||||||
soft = response_json['software']['name']
|
|
||||||
soft = soft.lower()
|
|
||||||
soft_version = response_json['software']['version']
|
|
||||||
users = response_json['usage']['users']['total']
|
|
||||||
if users > 1000000:
|
|
||||||
return
|
|
||||||
alive = True
|
|
||||||
write_api(server, soft, users, alive, api, soft_version)
|
|
||||||
print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
|
|
||||||
return
|
|
||||||
except:
|
|
||||||
pass
|
|
||||||
else:
|
|
||||||
|
|
||||||
try:
|
server = server[0].rstrip('.').lower()
|
||||||
soft = response_json['server']['software']
|
|
||||||
soft = soft.lower()
|
|
||||||
soft_version = response_json['server']['version']
|
|
||||||
users = response_json['usage']['users']['total']
|
|
||||||
if users > 1000000:
|
|
||||||
return
|
|
||||||
alive = True
|
|
||||||
if soft == 'socialhome':
|
|
||||||
write_api(server, soft, users, alive, api, soft_version)
|
|
||||||
print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
|
|
||||||
return
|
|
||||||
except:
|
|
||||||
pass
|
|
||||||
|
|
||||||
if response.status == 200 and soft == '' and api == "/api/v1/instance?":
|
|
||||||
soft = 'mastodon'
|
|
||||||
users = response_json['stats']['user_count']
|
|
||||||
soft_version = response_json['version']
|
|
||||||
if users > 1000000:
|
|
||||||
return
|
|
||||||
alive = True
|
|
||||||
write_api(server, soft, users, alive, api)
|
|
||||||
print("Server " + server + " (" + soft + ") is alive!")
|
|
||||||
|
|
||||||
def getserver(server, x):
|
|
||||||
|
|
||||||
server = server[0].rstrip('.').lower()
|
|
||||||
|
|
||||||
if server.find(".") == -1:
|
if server.find(".") == -1:
|
||||||
return
|
return
|
||||||
|
@ -178,7 +232,46 @@ def getserver(server, x):
|
||||||
|
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
def get_world_servers():
|
||||||
|
|
||||||
|
world_servers = []
|
||||||
|
|
||||||
|
try:
|
||||||
|
|
||||||
|
conn = None
|
||||||
|
|
||||||
|
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
|
||||||
|
|
||||||
|
cur = conn.cursor()
|
||||||
|
|
||||||
|
# get world servers list
|
||||||
|
|
||||||
|
cur.execute("select server from world where checked='f'")
|
||||||
|
|
||||||
|
rows = cur.fetchall()
|
||||||
|
|
||||||
|
for row in rows:
|
||||||
|
|
||||||
|
world_servers.append(row[0])
|
||||||
|
|
||||||
|
cur.close()
|
||||||
|
|
||||||
|
print("Remaining servers: " + str(len(world_servers)))
|
||||||
|
|
||||||
|
except (Exception, psycopg2.DatabaseError) as error:
|
||||||
|
|
||||||
|
print(error)
|
||||||
|
|
||||||
|
finally:
|
||||||
|
|
||||||
|
if conn is not None:
|
||||||
|
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
return world_servers
|
||||||
|
|
||||||
def get_parameter(parameter, file_path):
|
def get_parameter(parameter, file_path):
|
||||||
|
|
||||||
# Check if secrets file exists
|
# Check if secrets file exists
|
||||||
if not os.path.isfile(file_path):
|
if not os.path.isfile(file_path):
|
||||||
print("File %s not found, exiting." % file_path)
|
print("File %s not found, exiting." % file_path)
|
||||||
|
@ -194,6 +287,10 @@ def get_parameter(parameter, file_path):
|
||||||
print(file_path + " Missing parameter %s " % parameter)
|
print(file_path + " Missing parameter %s " % parameter)
|
||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
|
|
||||||
|
def usage():
|
||||||
|
|
||||||
|
print('usage: python ' + sys.argv[0] + ' --multi' + ' (multiprocessing, fast)')
|
||||||
|
print('usage: python ' + sys.argv[0] + ' --mono' + ' (one process, slow)')
|
||||||
|
|
||||||
def get_config():
|
def get_config():
|
||||||
|
|
||||||
|
@ -214,59 +311,51 @@ def get_db_config():
|
||||||
# main
|
# main
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
|
||||||
now = datetime.now()
|
|
||||||
start_time = time.time()
|
|
||||||
|
|
||||||
mastodon_hostname = get_config()
|
# usage modes
|
||||||
|
|
||||||
fediverse_db, fediverse_db_user = get_db_config()
|
if len(sys.argv) == 1:
|
||||||
|
|
||||||
world_servers = []
|
usage()
|
||||||
|
|
||||||
try:
|
elif len(sys.argv) == 2:
|
||||||
|
|
||||||
conn = None
|
if sys.argv[1] == '--multi':
|
||||||
|
|
||||||
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
|
now = datetime.now()
|
||||||
|
|
||||||
cur = conn.cursor()
|
start_time = time.time()
|
||||||
|
|
||||||
# get world servers list
|
mastodon_hostname = get_config()
|
||||||
|
|
||||||
cur.execute("select server from world where checked='f'")
|
fediverse_db, fediverse_db_user = get_db_config()
|
||||||
|
|
||||||
rows = cur.fetchall()
|
world_servers = get_world_servers()
|
||||||
|
|
||||||
for row in rows:
|
|
||||||
|
|
||||||
world_servers.append(row[0])
|
|
||||||
|
|
||||||
cur.close()
|
with get_context("spawn").Pool(processes=32) as pool:
|
||||||
|
|
||||||
print("Remaining servers: " + str(len(world_servers)))
|
res = pool.starmap(getserver, product(world_servers))
|
||||||
|
|
||||||
except (Exception, psycopg2.DatabaseError) as error:
|
pool.close()
|
||||||
|
|
||||||
print(error)
|
pool.join()
|
||||||
|
|
||||||
finally:
|
print('Done.')
|
||||||
|
|
||||||
if conn is not None:
|
elif sys.argv[1] == '--mono':
|
||||||
|
|
||||||
conn.close()
|
now = datetime.now()
|
||||||
|
|
||||||
###########################################################################
|
start_time = time.time()
|
||||||
# multiprocessing!
|
|
||||||
|
|
||||||
m = Manager()
|
mastodon_hostname = get_config()
|
||||||
q = m.Queue()
|
|
||||||
z = zip(world_servers)
|
|
||||||
|
|
||||||
serv_number = len(world_servers)
|
fediverse_db, fediverse_db_user = get_db_config()
|
||||||
|
|
||||||
pool_tuple = [(x, q) for x in z]
|
world_servers = get_world_servers()
|
||||||
with Pool(processes=64) as pool:
|
|
||||||
pool.starmap(getserver, pool_tuple)
|
|
||||||
|
|
||||||
print('Done.')
|
for server in world_servers:
|
||||||
|
|
||||||
|
getserver(server)
|
||||||
|
|
||||||
|
print('Done.')
|
||||||
|
|
Loading…
Referencia en una nova incidència