forkeado de spla/fediverse
180 líneas
4,6 KiB
Python
180 líneas
4,6 KiB
Python
#!/usr/bin/env python
|
|
# -*- coding: utf-8 -*-
|
|
|
|
import time
|
|
start_time = time.time()
|
|
from six.moves import urllib
|
|
from datetime import datetime
|
|
from subprocess import call
|
|
from mastodon import Mastodon
|
|
import threading
|
|
import os
|
|
import json
|
|
import signal
|
|
import sys
|
|
import os.path
|
|
import requests
|
|
import operator
|
|
import calendar
|
|
import psycopg2
|
|
from itertools import product
|
|
|
|
from multiprocessing import Pool, Lock, Process, Queue, current_process
|
|
import queue # imported for using queue.Empty exception
|
|
import multiprocessing
|
|
|
|
from decimal import *
|
|
getcontext().prec = 2
|
|
|
|
###############################################################################
|
|
# INITIALISATION
|
|
###############################################################################
|
|
|
|
def is_json(myjson):
|
|
try:
|
|
json_object = json.loads(myjson)
|
|
except ValueError as e:
|
|
return False
|
|
return True
|
|
|
|
def getpeers(server):
|
|
|
|
global updated_at
|
|
|
|
try:
|
|
|
|
res = requests.get('https://' + server + '/api/v1/instance/peers?',timeout=3)
|
|
|
|
if (res.ok):
|
|
|
|
if server.find(".") != -1 and server.find("@") == -1:
|
|
server_peers = res.json()
|
|
print("Server: " + server + ", " + "federated with " + str(len(server_peers)) + " servers")
|
|
else:
|
|
print("Server " + str(server) + " is not a domain")
|
|
|
|
i = 0
|
|
while i < len(server_peers) and server.find(".") != -1 and server.find("@") == -1:
|
|
|
|
saved_at = datetime.now()
|
|
insert_sql = "INSERT INTO world(server, federated_with, updated_at, saved_at) VALUES(%s,%s,%s,%s) ON CONFLICT DO NOTHING"
|
|
conn = None
|
|
try:
|
|
|
|
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
cur.execute(insert_sql, (server_peers[i], server, updated_at, saved_at,))
|
|
|
|
conn.commit()
|
|
|
|
cur.close()
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
i += 1
|
|
|
|
except KeyError as e:
|
|
|
|
pass
|
|
return
|
|
|
|
except ValueError as verr:
|
|
|
|
pass
|
|
return
|
|
|
|
except requests.exceptions.SSLError as errssl:
|
|
|
|
pass
|
|
return
|
|
|
|
except requests.exceptions.HTTPError as errh:
|
|
|
|
pass
|
|
return
|
|
|
|
except requests.exceptions.ConnectionError as errc:
|
|
|
|
pass
|
|
return
|
|
|
|
except requests.exceptions.Timeout as errt:
|
|
|
|
pass
|
|
return
|
|
|
|
except requests.exceptions.RequestException as err:
|
|
|
|
pass
|
|
return
|
|
|
|
# Returns the parameter from the specified file
|
|
def get_parameter( parameter, file_path ):
|
|
# Check if secrets file exists
|
|
if not os.path.isfile(file_path):
|
|
print("File %s not found, exiting."%file_path)
|
|
sys.exit(0)
|
|
|
|
# Find parameter in file
|
|
with open( file_path ) as f:
|
|
for line in f:
|
|
if line.startswith( parameter ):
|
|
return line.replace(parameter + ":", "").strip()
|
|
|
|
# Cannot find parameter, exit
|
|
print(file_path + " Missing parameter %s "%parameter)
|
|
sys.exit(0)
|
|
|
|
# Load secrets from secrets file
|
|
secrets_filepath = "secrets/secrets.txt"
|
|
uc_client_id = get_parameter("uc_client_id", secrets_filepath)
|
|
uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
|
|
uc_access_token = get_parameter("uc_access_token", secrets_filepath)
|
|
|
|
# Load configuration from config file
|
|
config_filepath = "config/config.txt"
|
|
mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
|
|
|
|
# Load database config from db_config file
|
|
db_config_filepath = "config/db_config.txt"
|
|
fediverse_db = get_parameter("fediverse_db", db_config_filepath)
|
|
fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath)
|
|
|
|
# Initialise Mastodon API
|
|
mastodon = Mastodon(
|
|
client_id = uc_client_id,
|
|
client_secret = uc_client_secret,
|
|
access_token = uc_access_token,
|
|
api_base_url = 'https://' + mastodon_hostname,
|
|
)
|
|
|
|
# Initialise access headers
|
|
headers={ 'Authorization': 'Bearer %s'%uc_access_token }
|
|
|
|
###############################################################################
|
|
# get current datetime and Mastodon hostname peers
|
|
###############################################################################
|
|
|
|
updated_at = datetime.now()
|
|
|
|
getpeers(mastodon_hostname)
|
|
self_peers = mastodon.instance_peers()
|
|
|
|
###########################################################################
|
|
|
|
nprocs = multiprocessing.cpu_count()
|
|
with multiprocessing.Pool(processes=nprocs) as pool:
|
|
results = pool.starmap(getpeers, product(self_peers))
|
|
|
|
exec_time = str(round((time.time() - start_time), 2))
|
|
print(exec_time)
|