Browse Source

Added aiohttp support for asynchronous peers requests

master
spla 1 year ago
parent
commit
2283983227
  1. 141
      getworld.py

141
getworld.py

@ -20,15 +20,19 @@ import psycopg2
from itertools import product
from multiprocessing import Pool, Lock, Process, Queue, current_process
import queue # imported for using queue.Empty exception
import queue
import multiprocessing
from decimal import *
getcontext().prec = 2
import aiohttp
import aiodns
import asyncio
from aiohttp import ClientError, ClientSession, ClientConnectionError, ClientConnectorError, ClientSSLError, ClientConnectorSSLError, ServerTimeoutError
from asyncio import TimeoutError
import socket
from socket import gaierror, gethostbyname
###############################################################################
# INITIALISATION
###############################################################################
updated_at = datetime.now()
peers_api = '/api/v1/instance/peers?'
def is_json(myjson):
try:
@ -37,86 +41,88 @@ def is_json(myjson):
return False
return True
def getpeers(server):
def getserver(server):
global updated_at
if server.find(".") == -1:
return
if server.find("@") != -1:
return
if server.find("/") != -1:
return
if server.find(":") != -1:
return
try:
res = requests.get('https://' + server + '/api/v1/instance/peers?',timeout=3)
try:
if (res.ok):
loop = asyncio.get_event_loop()
coroutines = [getpeers(server)]
loop.run_until_complete(asyncio.gather(*coroutines, return_exceptions=True))
if server.find(".") != -1 and server.find("@") == -1:
server_peers = res.json()
print("Server: " + server + ", " + "federated with " + str(len(server_peers)) + " servers")
else:
print("Server " + str(server) + " is not a domain")
except:
i = 0
while i < len(server_peers) and server.find(".") != -1 and server.find("@") == -1:
saved_at = datetime.now()
insert_sql = "INSERT INTO world(server, federated_with, updated_at, saved_at) VALUES(%s,%s,%s,%s) ON CONFLICT DO NOTHING"
conn = None
try:
pass
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
async def getpeers(server):
cur = conn.cursor()
try:
cur.execute(insert_sql, (server_peers[i], server, updated_at, saved_at,))
socket.gethostbyname(server)
conn.commit()
except socket.gaierror:
cur.close()
return
except (Exception, psycopg2.DatabaseError) as error:
url = 'https://' + server
print(error)
finally:
timeout = aiohttp.ClientTimeout(total=3)
async with aiohttp.ClientSession(timeout=timeout) as session:
try:
async with session.get(url+peers_api) as response:
if response.status == 200:
try:
response_json = await response.json()
print("Server: " + server + ", " + "federated with " + str(len(response_json)) + " servers")
i = 0
while i < len(response_json):
if conn is not None:
saved_at = datetime.now()
insert_sql = "INSERT INTO world(server, federated_with, updated_at, saved_at) VALUES(%s,%s,%s,%s) ON CONFLICT DO NOTHING"
conn = None
try:
conn.close()
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
i += 1
cur = conn.cursor()
except KeyError as e:
cur.execute(insert_sql, (response_json[i], server, updated_at, saved_at,))
pass
return
conn.commit()
except ValueError as verr:
cur.close()
pass
return
except (Exception, psycopg2.DatabaseError) as error:
except requests.exceptions.SSLError as errssl:
print(error)
pass
return
finally:
except requests.exceptions.HTTPError as errh:
if conn is not None:
pass
return
conn.close()
except requests.exceptions.ConnectionError as errc:
i += 1
pass
return
except:
except requests.exceptions.Timeout as errt:
pass
pass
return
except aiohttp.ClientConnectorError as err:
except requests.exceptions.RequestException as err:
pass
pass
return
###############################################################################
# INITIALISATION
###############################################################################
# Returns the parameter from the specified file
def get_parameter( parameter, file_path ):
@ -162,19 +168,18 @@ mastodon = Mastodon(
headers={ 'Authorization': 'Bearer %s'%uc_access_token }
###############################################################################
# get current datetime and Mastodon hostname peers
###############################################################################
# main
updated_at = datetime.now()
if __name__ == '__main__':
getpeers(mastodon_hostname)
self_peers = mastodon.instance_peers()
getserver(mastodon_hostname)
self_peers = mastodon.instance_peers()
###########################################################################
###########################################################################
nprocs = multiprocessing.cpu_count()
with multiprocessing.Pool(processes=nprocs) as pool:
results = pool.starmap(getpeers, product(self_peers))
nprocs = multiprocessing.cpu_count()
with multiprocessing.Pool(processes=nprocs) as pool:
results = pool.starmap(getserver, product(self_peers))
exec_time = str(round((time.time() - start_time), 2))
print(exec_time)
exec_time = str(round((time.time() - start_time), 2))
print(exec_time)
Loading…
Cancel
Save