fediverse/getpeers.py

178 líneas
4,3 KiB
Python
Original Vista normal Històric

2021-05-29 12:44:12 +02:00
import os
import sys
import time
from datetime import datetime
import requests
import json
import psycopg2
import ray
import pdb
ray.init(num_cpus = 32) # Specify this system CPUs.
def write_server(server, federated_with):
2021-05-29 12:44:12 +02:00
2022-03-24 13:36:59 +01:00
insert_sql = "INSERT INTO world(server, federated_with, updated_at, saved_at, checked) VALUES(%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
2021-05-29 12:44:12 +02:00
conn = None
2021-05-29 12:44:12 +02:00
try:
2021-05-29 12:44:12 +02:00
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
2021-05-29 12:44:12 +02:00
cur = conn.cursor()
2021-05-29 12:44:12 +02:00
2022-03-24 13:36:59 +01:00
cur.execute(insert_sql, (server, federated_with, now, now, 'f'))
2021-05-29 12:44:12 +02:00
print(f'writing {server} to world database')
2021-05-29 12:44:12 +02:00
conn.commit()
2021-05-29 12:44:12 +02:00
cur.close()
2021-05-29 12:44:12 +02:00
except (Exception, psycopg2.DatabaseError) as error:
2021-05-29 12:44:12 +02:00
print(error)
2021-05-29 12:44:12 +02:00
finally:
2021-05-29 12:44:12 +02:00
if conn is not None:
2021-05-29 12:44:12 +02:00
conn.close()
2021-05-29 12:44:12 +02:00
@ray.remote
2021-05-29 12:44:12 +02:00
def get_peers(peer):
2021-05-29 12:44:12 +02:00
try:
user_agent = {'User-agent': "fediverse's stats (fediverse@mastodont.cat)"}
2022-05-11 21:08:44 +02:00
domain_res = requests.get('https://' + peer + '/api/v1/instance', headers = user_agent, timeout=3)
2022-05-11 21:08:44 +02:00
domain_res_json = domain_res.json()
2021-05-29 12:44:12 +02:00
2022-05-11 21:08:44 +02:00
if domain_res.status_code == 200:
2021-05-29 12:44:12 +02:00
2022-05-11 21:08:44 +02:00
domain_uri = domain_res_json['uri'].replace('https://', '')
2021-05-29 12:44:12 +02:00
2022-05-11 21:08:44 +02:00
if domain_uri != peer:
2022-05-11 21:08:44 +02:00
print(f'{peer} is an aliased domain of {domain_uri}!')
2021-05-29 12:44:12 +02:00
2022-05-11 21:08:44 +02:00
else:
2022-05-11 21:08:44 +02:00
response = requests.get('https://' + peer + peers_api, headers = user_agent, timeout=3)
response_json = response.json()
if response.status_code == 200:
try:
print(f"Server: {peer}, federated with {str(len(response_json))} servers")
for peer_peer in response_json:
write_server(peer_peer, peer)
except:
pass
2021-05-29 12:44:12 +02:00
except:
pass
def save_time(program, start, finish):
insert_sql = "INSERT INTO execution_time(program, start, finish) VALUES(%s,%s,%s) ON CONFLICT DO NOTHING"
conn = None
try:
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute(insert_sql, (program, start, finish,))
cur.execute("UPDATE execution_time SET start=(%s), finish=(%s) where program=(%s)", (start, finish, program))
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
2021-05-29 12:44:12 +02:00
def get_parameter( parameter, file_path ):
2021-05-29 12:44:12 +02:00
# Check if secrets file exists
if not os.path.isfile(file_path):
print("File %s not found, exiting."%file_path)
sys.exit(0)
# Find parameter in file
with open( file_path ) as f:
for line in f:
if line.startswith( parameter ):
return line.replace(parameter + ":", "").strip()
# Cannot find parameter, exit
print(file_path + " Missing parameter %s "%parameter)
sys.exit(0)
###############################################################################
# main
if __name__ == '__main__':
now = datetime.now()
2021-05-29 12:44:12 +02:00
peers_api = '/api/v1/instance/peers?'
2021-05-29 12:44:12 +02:00
# Load configuration from config file
config_filepath = "config/config.txt"
mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
# Load database config from db_config file
db_config_filepath = "config/db_config.txt"
fediverse_db = get_parameter("fediverse_db", db_config_filepath)
fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath)
user_agent = {'User-agent': "fediverse's stats (fediverse@mastodont.cat)"}
res = requests.get('https://' + mastodon_hostname + peers_api, headers = user_agent, timeout=3)
2021-05-29 12:44:12 +02:00
hostname_peers = res.json()
start = datetime.now()
2021-05-29 12:44:12 +02:00
program = 'getpeers'
finish = start
2021-05-29 12:44:12 +02:00
save_time(program, start, finish)
2021-05-29 12:44:12 +02:00
for peer in hostname_peers:
write_server(peer, mastodon_hostname)
results = ray.get([get_peers.remote(server) for server in hostname_peers])
finish = datetime.now()
print(f"duration = {finish - start}.\nprocessed servers: {len(results)}")
save_time(program, start, finish)
2021-05-29 12:44:12 +02:00