#!/usr/bin/env python # -*- coding: utf-8 -*- import time start_time = time.time() from six.moves import urllib from datetime import datetime from subprocess import call from mastodon import Mastodon import threading import os import json import signal import sys import os.path import requests import operator import calendar import psycopg2 from itertools import product from multiprocessing import Pool, Lock, Process, Queue, current_process import queue # imported for using queue.Empty exception import multiprocessing from decimal import * getcontext().prec = 2 ############################################################################### # INITIALISATION ############################################################################### def is_json(myjson): try: json_object = json.loads(myjson) except ValueError as e: return False return True def getpeers(server): global updated_at try: res = requests.get('https://' + server + '/api/v1/instance/peers?',timeout=3) if (res.ok): if server.find(".") != -1 and server.find("@") == -1: server_peers = res.json() print("Server: " + server + ", " + "federated with " + str(len(server_peers)) + " servers") else: print("Server " + str(server) + " is not a domain") i = 0 while i < len(server_peers) and server.find(".") != -1 and server.find("@") == -1: saved_at = datetime.now() insert_sql = "INSERT INTO world(server, federated_with, updated_at, saved_at) VALUES(%s,%s,%s,%s) ON CONFLICT DO NOTHING" conn = None try: conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") cur = conn.cursor() cur.execute(insert_sql, (server_peers[i], server, updated_at, saved_at,)) conn.commit() cur.close() except (Exception, psycopg2.DatabaseError) as error: print(error) finally: if conn is not None: conn.close() i += 1 except KeyError as e: pass return except ValueError as verr: pass return except requests.exceptions.SSLError as errssl: pass return except requests.exceptions.HTTPError as errh: pass return except requests.exceptions.ConnectionError as errc: pass return except requests.exceptions.Timeout as errt: pass return except requests.exceptions.RequestException as err: pass return # Returns the parameter from the specified file def get_parameter( parameter, file_path ): # Check if secrets file exists if not os.path.isfile(file_path): print("File %s not found, exiting."%file_path) sys.exit(0) # Find parameter in file with open( file_path ) as f: for line in f: if line.startswith( parameter ): return line.replace(parameter + ":", "").strip() # Cannot find parameter, exit print(file_path + " Missing parameter %s "%parameter) sys.exit(0) # Load secrets from secrets file secrets_filepath = "secrets/secrets.txt" uc_client_id = get_parameter("uc_client_id", secrets_filepath) uc_client_secret = get_parameter("uc_client_secret", secrets_filepath) uc_access_token = get_parameter("uc_access_token", secrets_filepath) # Load configuration from config file config_filepath = "config/config.txt" mastodon_hostname = get_parameter("mastodon_hostname", config_filepath) # Load database config from db_config file db_config_filepath = "config/db_config.txt" fediverse_db = get_parameter("fediverse_db", db_config_filepath) fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath) # Initialise Mastodon API mastodon = Mastodon( client_id = uc_client_id, client_secret = uc_client_secret, access_token = uc_access_token, api_base_url = 'https://' + mastodon_hostname, ) # Initialise access headers headers={ 'Authorization': 'Bearer %s'%uc_access_token } ############################################################################### # get current datetime and Mastodon hostname peers ############################################################################### updated_at = datetime.now() getpeers(mastodon_hostname) self_peers = mastodon.instance_peers() ########################################################################### nprocs = multiprocessing.cpu_count() with multiprocessing.Pool(processes=nprocs) as pool: results = pool.starmap(getpeers, product(self_peers)) exec_time = str(round((time.time() - start_time), 2)) print(exec_time)