This commit is contained in:
spla 2023-04-13 18:01:18 +02:00
pare 1281d4bf8e
commit c535f37a99
S'han modificat 8 arxius amb 1316 adicions i 990 eliminacions

675
app/libraries/database.py Normal file
Veure arxiu

@ -0,0 +1,675 @@
import os
import sys
import psycopg2
from psycopg2 import sql
from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
import uuid
from datetime import datetime
import pytz
import pdb
tz = pytz.timezone('Europe/Madrid')
class Database():
name = 'fediverse database library'
def __init__(self, config_file=None, blocker_db=None, blocker_db_user=None, blocker_db_user_password=None):
self.config_file = "config/db_config.txt"
self.blocker_db = self.__get_parameter("blocker_db", self.config_file)
self.blocker_db_user = self.__get_parameter("blocker_db_user", self.config_file)
self.blocker_db_user_password = self.__get_parameter("blocker_db_user_password", self.config_file)
db_setup = self.__check_dbsetup(self)
if not db_setup:
self.blocker_db = input("\nMau database name: ")
self.blocker_db_user = input("\nMau database user: ")
self.blocker_db_user_password = input("\nMau database user password: ")
self.__createdb(self)
self.__create_config(self)
self.__write_config(self)
def get_nodeinfo_endpoint(self, server):
try:
conn = None
conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = self.blocker_db_user_password, host = "/var/run/postgresql", port = "6432")
cur = conn.cursor()
cur.execute("select server, api from servers where server=(%s)", (server,))
row = cur.fetchone()
if row != None:
server = row[0]
api = row[1]
else:
server = ''
api = ''
cur.close()
return (server, api)
except (Exception, psycopg2.DatabaseError) as error:
sys.exit(error)
finally:
if conn is not None:
conn.close()
def get_servers(self, software):
servers_list = []
try:
conn = None
conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute("select server from servers where software=(%s)", (software,))
rows = cur.fetchall()
for row in rows:
servers_list.append(row[0])
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
sys.exit(error)
finally:
if conn is not None:
conn.close()
return (servers_list)
def save_time(self, program, start, finish):
insert_sql = "INSERT INTO execution_time(program, start, finish) VALUES(%s,%s,%s) ON CONFLICT DO NOTHING"
conn = None
try:
conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = self.blocker_db_user_password, host = "/var/run/postgresql", port = "6432")
cur = conn.cursor()
cur.execute(insert_sql, (program, start, finish,))
cur.execute("UPDATE execution_time SET start=(%s), finish=(%s) where program=(%s)", (start, finish, program))
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
def check_peer(self, server):
found = False
select_server_sql = 'select server from servers where server=(%s)'
select_dead_server_sql = 'select server from deadservers where server=(%s)'
try:
conn = None
conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432")
cur = conn.cursor()
# check server
cur.execute(select_server_sql, (server,))
row = cur.fetchone()
if row != None:
found = True
cur.execute(select_dead_server_sql, (server,))
row = cur.fetchone()
if row != None:
found = True
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
return found
def get_not_updated_servers(self):
not_updated_servers = []
try:
conn = None
conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432")
cur = conn.cursor()
# get saved servers list
cur.execute("select server from servers where updated_at is null")
rows = cur.fetchall()
for row in rows:
not_updated_servers.append(row[0])
cur.close()
print("Not updated servers: " + str(len(not_updated_servers)))
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
return not_updated_servers
def get_saved_servers(self):
saved_servers = []
try:
conn = None
conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432")
cur = conn.cursor()
# get saved servers list
cur.execute("select server from servers where api is not null")
rows = cur.fetchall()
for row in rows:
saved_servers.append(row[0])
cur.close()
print("Saved servers: " + str(len(saved_servers)))
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
return saved_servers
def get_dead_servers(self):
dead_servers = []
try:
conn = None
conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432")
cur = conn.cursor()
# get dead servers list
cur.execute("select server from deadservers")
rows = cur.fetchall()
for row in rows:
dead_servers.append(row[0])
cur.close()
print("Dead servers: " + str(len(dead_servers)))
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
return dead_servers
def soft_totals(self):
# get current total servers and users, get users from every software
now = datetime.now()
gettotals_sql = "select count(server), sum(users), sum(mau) from mau where alive"
get_soft_totals_sql = "select software, sum(users) as users, sum(mau) as mau, count(server) as servers from mau where users != 0 and mau is not null and alive group by software order by mau desc"
soft_total_project = []
soft_total_users = []
soft_total_mau = []
soft_total_servers = []
try:
conn = None
conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432")
cur = conn.cursor()
cur.execute(gettotals_sql)
row = cur.fetchone()
total_servers = row[0]
total_users = row[1]
total_mau = row[2]
cur.execute(get_soft_totals_sql)
rows = cur.fetchall()
for row in rows:
soft_total_project.append(row[0])
soft_total_users.append(row[1])
soft_total_mau.append(row[2])
soft_total_servers.append(row[3])
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
return (soft_total_project, soft_total_users, soft_total_mau, soft_total_servers, total_servers, total_users, total_mau)
def get_last_checked_servers(self):
# get last checked servers from fediverse DB
alive_servers = []
try:
conn = None
conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432")
cur = conn.cursor()
# get world servers list
cur.execute("select server from world where server in (select server from fediverse where users_api != '')")
rows = cur.fetchall()
for row in rows:
alive_servers.append(row[0])
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
return alive_servers
def write_alive_server(self, server, software, version, users, mau, alive):
now = datetime.now()
insert_sql = "INSERT INTO mau(server, software, version, users, mau, alive, updated_at) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
conn = None
try:
conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432")
cur = conn.cursor()
cur.execute(insert_sql, (server, software, version, users, mau, alive, now))
cur.execute("UPDATE mau SET software=(%s), version=(%s), users=(%s), mau=(%s), alive=(%s), updated_at=(%s) where server=(%s)", (software, version, users, mau, alive, now, server))
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
def write_not_alive_server(self, server):
now = datetime.now()
update_sql = "UPDATE mau set alive='f', updated_at=(%s) where server=(%s)"
conn = None
try:
conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432")
cur = conn.cursor()
cur.execute(update_sql, (now, server))
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
def write_server(self, server, api, software):
now = datetime.now()
insert_sql = "INSERT INTO servers(server, api, software, created_at) VALUES(%s, %s, %s, %s) ON CONFLICT DO NOTHING"
conn = None
try:
conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432")
cur = conn.cursor()
cur.execute(insert_sql, (server, api, software, now))
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
def update_peer(self, server):
now = datetime.now()
update_sql = "UPDATE servers set updated_at=(%s) where server=(%s)"
conn = None
try:
conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432")
cur = conn.cursor()
cur.execute(update_sql, (now, server))
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
def update_server(self, server, api):
now = datetime.now()
update_sql = "UPDATE servers set server=(%s), api=(%s), updated_at=(%s) where server=(%s)"
conn = None
try:
conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password = self.blocker_db_user_password, host="/var/run/postgresql", port="6432")
cur = conn.cursor()
cur.execute(update_sql, (server, api, now, server))
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
@staticmethod
def __check_dbsetup(self):
db_setup = False
try:
conn = None
conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = self.blocker_db_user_password, host = "/var/run/postgresql", port = "6432")
db_setup = True
except (Exception, psycopg2.DatabaseError) as error:
print(error)
return db_setup
@staticmethod
def __createdb(self):
conn = None
try:
conn = psycopg2.connect(dbname='postgres',
user=self.blocker_db_user, host='',
password=self.blocker_db_user_password)
conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
cur = conn.cursor()
print(f"Creating database {self.blocker_db}. Please wait...")
cur.execute(sql.SQL("CREATE DATABASE {}").format(
sql.Identifier(self.blocker_db))
)
print(f"Database {self.blocker_db} created!\n")
self.__dbtables_schemes(self)
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
@staticmethod
def __dbtables_schemes(self):
db = self.blocker_db
table = "servers"
sql = "create table "+table+" (server varchar(200) PRIMARY KEY, api varchar(50), software varchar(50), created_at timestamptz, updated_at timestamptz, downs int)"
self.__create_table(self, table, sql)
@staticmethod
def __create_table(self, table, sql):
conn = None
try:
conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = self.blocker_db_user_password, host = "/var/run/postgresql", port = "6432")
cur = conn.cursor()
print(f"Creating table {table}")
cur.execute(sql)
conn.commit()
print(f"Table {table} created!\n")
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
def __get_parameter(self, parameter, config_file):
if not os.path.isfile(config_file):
print(f"File {config_file} not found..")
return
with open( config_file ) as f:
for line in f:
if line.startswith( parameter ):
return line.replace(parameter + ":", "").strip()
print(f"{config_file} Missing parameter {parameter}")
sys.exit(0)
@staticmethod
def __create_config(self):
if not os.path.exists('config'):
os.makedirs('config')
if not os.path.exists(self.config_file):
print(self.config_file + " created!")
with open(self.config_file, 'w'): pass
@staticmethod
def __write_config(self):
with open(self.config_file, 'a') as the_file:
the_file.write(f'blocker_db: {self.blocker_db}\nblocker_db_user: {self.blocker_db_user}\nblocker_db_user_password: {self.blocker_db_user_password}')
print(f"adding parameters to {self.config_file}\n")

Veure arxiu

@ -0,0 +1,114 @@
import os
import sys
from app.libraries.setup import Setup
from mastodon import Mastodon
from mastodon.Mastodon import MastodonNotFoundError, MastodonNetworkError, MastodonReadTimeout, MastodonAPIError, MastodonUnauthorizedError, MastodonIllegalArgumentError
import requests
import pdb
class DomainBlocks():
name = 'Domain blocks for Mastodon'
def __init__(self, mastodon_hostname=None, domain_block_api=None, session=None):
self.domain_blocks_api = '/api/v1/admin/domain_blocks'
setup = Setup()
self.mastodon_hostname = setup.mastodon_hostname
self.mastodon_app_token = { 'Authorization': 'Bearer %s'%setup.mastodon_app_token }
if session:
self.session = session
else:
self.session = requests.Session()
def domain_blocks_create(self, server, priv_comment, pub_comment):
data = {
'domain': server,
'severity': 'suspend',
'reject_media': 'true',
'reject_reports': 'true',
'private_comment': priv_comment,
'public_comment': pub_comment,
'obfuscate': 'true',
}
endpoint = f'https://{self.mastodon_hostname}/{self.domain_blocks_api}'
response = self.__api_request('POST', endpoint, data)
if response.ok:
print(f"Done, {server} is now suspended")
else:
pass
def __api_request(self, method, endpoint, data={}):
response = None
try:
kwargs = dict(data=data)
response = self.session.request(method, url = endpoint, headers = self.mastodon_app_token, **kwargs)
except Exception as e:
raise MastodonNetworkError(f"Could not complete request: {e}")
if response is None:
raise MastodonIllegalArgumentError("Illegal request.")
if not response.ok:
try:
if isinstance(response, dict) and 'error' in response:
error_msg = response['error']
elif isinstance(response, str):
error_msg = response
else:
error_msg = None
except ValueError:
error_msg = None
if response.status_code == 404:
ex_type = MastodonNotFoundError
if not error_msg:
error_msg = 'Endpoint not found.'
# this is for compatibility with older versions
# which raised MastodonAPIError('Endpoint not found.')
# on any 404
elif response.status_code == 401:
ex_type = MastodonUnauthorizedError
elif response.status_code == 422:
return response
elif response.status_code == 500:
ex_type = MastodonInternalServerError
elif response.status_code == 502:
ex_type = MastodonBadGatewayError
elif response.status_code == 503:
ex_type = MastodonServiceUnavailableError
elif response.status_code == 504:
ex_type = MastodonGatewayTimeoutError
elif response.status_code >= 500 and \
response.status_code <= 511:
ex_type = MastodonServerError
else:
ex_type = MastodonAPIError
raise ex_type(
'Mastodon API returned error',
response.status_code,
response.reason,
error_msg)
else:
return response

70
app/libraries/nodeinfo.py Normal file
Veure arxiu

@ -0,0 +1,70 @@
import time
from datetime import datetime
import os
import json
import sys
import os.path
import requests
import urllib3
import socket
from app.libraries.database import Database
from app.libraries.setup import Setup
import pdb
def is_json(myjson):
try:
json_object = json.loads(myjson)
except ValueError as e:
return False
return True
class Nodeinfo():
name = "Query server nodeinfo data"
def __init__(self, server=None, db=None, setup=None, soft=None):
self.server = server
self.db = Database()
self.setup = Setup()
self.soft = ''
def getdata(self, server, api):
if server.find(".") == -1:
return
if server.find("@") != -1:
return
if server.find("/") != -1:
return
if server.find(":") != -1:
return
url = 'https://' + server
self.soft = ''
try:
response = requests.get(url + api, headers = self.setup.user_agent, timeout=3)
if response.status_code == 200:
try:
self.soft = response.json()['software']['name']
self.soft = self.soft.lower()
print(f"Server {server} ({self.soft} is alive!")
print('*********************************************************************')
except:
pass
except:
pass
return (server, self.soft)

264
app/libraries/peers.py Normal file
Veure arxiu

@ -0,0 +1,264 @@
import time
from datetime import datetime
import os
import json
import sys
import os.path
from app.libraries.setup import Setup
from app.libraries.database import Database
import requests
import urllib3
import socket
class Peers():
name = "Get peers from server"
def __init__(self, server=None, db=None, setup=None):
self.server = server
self.db = Database()
self.setup = Setup()
def getpeers(self, server):
try:
res = requests.get(f'https://{server}/{self.setup.peers_api}, headers = self.setup.user_agent, timeout=3')
peers = res.json()
if isinstance(peers, list):
self.db.update_peer(server)
return peers
else:
return None
except requests.exceptions.SSLError as errssl:
pass
except requests.exceptions.HTTPError as errh:
pass
except requests.exceptions.ConnectionError as errc:
pass
except requests.exceptions.ReadTimeout as to_err:
pass
except requests.exceptions.TooManyRedirects as tmr_err:
pass
except urllib3.exceptions.LocationParseError as lp_err:
pass
except requests.exceptions.InvalidURL as iu_err:
pass
except requests.exceptions.ChunkedEncodingError as chunk_err:
print(f'ChunkedEncodingError! {server}')
pass
except requests.exceptions.JSONDecodeError as jsondec_err:
print(jsondec_err)
pass
print('*** Not peers api!')
return None
def getapi(self, server):
if server.find(".") == -1:
return
if server.find("@") != -1:
return
if server.find("/") != -1:
return
if server.find(":") != -1:
return
is_nodeinfo = False
nodeinfo = ''
url = 'https://' + server
try:
response = requests.get(url + '/.well-known/nodeinfo', headers = self.setup.user_agent, timeout=3)
if response.status_code == 200:
try:
response_json = response.json()
if len(response_json['links']) == 1:
start = [pos for pos, char in enumerate(response_json['links'][0]['href']) if char == '/'][1]
end = [pos for pos, char in enumerate(response_json['links'][0]['href']) if char == '/'][2]
server = response_json['links'][0]['href'][start+1:end]
server_idx = response_json['links'][0]['href'].index(server)
nodeinfo = response_json['links'][0]['href'][server_idx:].replace(server, '')
elif len(response_json['links']) == 2:
start = [pos for pos, char in enumerate(response_json['links'][1]['href']) if char == '/'][1]
end = [pos for pos, char in enumerate(response_json['links'][1]['href']) if char == '/'][2]
server = response_json['links'][1]['href'][start+1:end]
server_idx = response_json['links'][1]['href'].index(server)
nodeinfo = response_json['links'][1]['href'][server_idx:].replace(server, '')
except:
pass
except requests.exceptions.ChunkedEncodingError as chunk_err:
pass
except requests.exceptions.InvalidSchema as invalid_err:
pass
except requests.exceptions.SSLError as errssl:
pass
except requests.exceptions.HTTPError as errh:
pass
except requests.exceptions.ConnectionError as errc:
pass
except requests.exceptions.ReadTimeout as to_err:
pass
except requests.exceptions.TooManyRedirects as tmr_err:
pass
except urllib3.exceptions.LocationParseError as lp_err:
pass
except requests.exceptions.InvalidURL as iu_err:
pass
except requests.exceptions.ChunkedEncodingError as chunk_err:
print(f'ChunkedEncodingError! {server}')
pass
return nodeinfo
def updateapi(self, server):
if server.find(".") == -1:
return
if server.find("@") != -1:
return
if server.find("/") != -1:
return
if server.find(":") != -1:
return
nodeinfo = ''
url = 'https://' + server
try:
response = requests.get(url + '/.well-known/nodeinfo', headers = self.setup.user_agent, timeout=3)
if response.status_code == 200:
try:
response_json = response.json()
if len(response_json['links']) == 1:
start = [pos for pos, char in enumerate(response_json['links'][0]['href']) if char == '/'][1]
end = [pos for pos, char in enumerate(response_json['links'][0]['href']) if char == '/'][2]
server = response_json['links'][0]['href'][start+1:end]
server_idx = response_json['links'][0]['href'].index(server)
nodeinfo = response_json['links'][0]['href'][server_idx:].replace(server, '')
elif len(response_json['links']) == 2:
start = [pos for pos, char in enumerate(response_json['links'][1]['href']) if char == '/'][1]
end = [pos for pos, char in enumerate(response_json['links'][1]['href']) if char == '/'][2]
server = response_json['links'][1]['href'][start+1:end]
server_idx = response_json['links'][1]['href'].index(server)
nodeinfo = response_json['links'][1]['href'][server_idx:].replace(server, '')
except:
pass
except requests.exceptions.SSLError as errssl:
pass
except requests.exceptions.HTTPError as errh:
pass
except requests.exceptions.ConnectionError as errc:
pass
except requests.exceptions.ReadTimeout as to_err:
pass
except requests.exceptions.TooManyRedirects as tmr_err:
pass
except urllib3.exceptions.LocationParseError as lp_err:
pass
except requests.exceptions.InvalidURL as iu_err:
pass
except requests.exceptions.ChunkedEncodingError as chunk_err:
print(f'ChunkedEncodingError! {server}')
pass
return server, nodeinfo

157
app/libraries/setup.py Normal file
Veure arxiu

@ -0,0 +1,157 @@
import os
import sys
from datetime import datetime
import pytz
from mastodon import Mastodon
from mastodon.Mastodon import MastodonMalformedEventError, MastodonNetworkError, MastodonReadTimeout, MastodonAPIError, MastodonIllegalArgumentError
import pdb
class Setup():
name = 'fediverse setup'
def __init__(self, config_file=None, mastodon_hostname=None, peers_api=None, user_agent=None, secrets_filepath=None, mastodon_app_token=None):
self.config_file = "config/config.txt"
self.mastodon_hostname = self.__get_parameter("mastodon_hostname", self.config_file)
self.peers_api = '/api/v1/instance/peers?'
self.user_agent = {'User-agent': "fediverse's stats (fediverse@mastodont.cat)"}
self.secrets_filepath = 'secrets/secrets.txt'
is_setup = self.__check_mastodon_setup(self)
if is_setup:
self.mastodon_app_token = self.__get_mastodon_parameter("mastodon_app_token", self.secrets_filepath)
else:
self.mastodon_app_token = self.mastodon_setup(self)
@staticmethod
def __check_mastodon_setup(self):
is_setup = False
if not os.path.isfile(self.secrets_filepath):
print(f"File {self.secrets_filepath} not found, running setup.")
else:
is_setup = True
return is_setup
@staticmethod
def mastodon_setup(self):
if not os.path.exists('secrets'):
os.makedirs('secrets')
self.mastodon_user = input("Mastodon user login? ")
self.mastodon_password = input("Mastodon user password? ")
self.app_name = 'fediverse'
self.mastodon_app_token = self.mastodon_log_in()
if not os.path.exists(self.secrets_filepath):
with open(self.secrets_filepath, 'w'): pass
print(f"{self.secrets_filepath} created!")
with open(self.secrets_filepath, 'a') as the_file:
print("Writing Mastodon parameters to " + self.secrets_filepath)
the_file.write(f'mastodon_app_token: {self.mastodon_app_token}')
return self.mastodon_app_token
def mastodon_log_in(self):
token = ''
try:
response = Mastodon.create_app(
self.app_name,
scopes=["read","write", "admin:read", "admin:write"],
to_file=None,
api_base_url=self.mastodon_hostname
)
client_id = response[0]
client_secret = response[1]
mastodon = Mastodon(client_id = client_id, client_secret = client_secret, api_base_url = self.mastodon_hostname)
token = mastodon.log_in(
self.mastodon_user,
self.mastodon_password,
scopes = ["read", "write", "admin:read", "admin:write"],
to_file = None
)
print('Log in succesful!')
except MastodonIllegalArgumentError as i_error:
sys.stdout.write(f'\n{str(i_error)}\n')
except MastodonNetworkError as n_error:
sys.stdout.write(f'\n{str(n_error)}\n')
except MastodonReadTimeout as r_error:
sys.stdout.write(f'\n{str(r_error)}\n')
except MastodonAPIError as a_error:
sys.stdout.write(f'\n{str(a_error)}\n')
finally:
return token
def __get_parameter(self, parameter, config_file):
if not os.path.isfile(config_file):
print(f"File {config_file} not found..")
self.mastodon_hostname = input("\nMastodon hostname: ")
self.__create_config(self)
self.__write_config(self)
with open( self.config_file ) as f:
for line in f:
if line.startswith( parameter ):
return line.replace(parameter + ":", "").strip()
def __get_mastodon_parameter(self, parameter, secrets_filepath):
if not os.path.isfile(secrets_filepath):
print(f"File {secrets_filepath} not found..")
self.sign_in()
with open( self.secrets_filepath ) as f:
for line in f:
if line.startswith( parameter ):
return line.replace(parameter + ":", "").strip()
@staticmethod
def __create_config(self):
if not os.path.exists('config'):
os.makedirs('config')
if not os.path.exists(self.config_file):
print(self.config_file + " created!")
with open(self.config_file, 'w'): pass
@staticmethod
def __write_config(self):
with open(self.config_file, 'a') as the_file:
the_file.write(f'mastodon_hostname: {self.mastodon_hostname}')
print(f"adding parameters to {self.config_file}\n")

Veure arxiu

@ -1,487 +1,25 @@
import os import os
import sys import sys
from app.libraries.setup import Setup
from app.libraries.database import Database
from app.libraries.domainblocks import DomainBlocks
from mastodon import Mastodon from mastodon import Mastodon
from mastodon.Mastodon import MastodonNotFoundError, MastodonNetworkError, MastodonReadTimeout, MastodonAPIError, MastodonUnauthorizedError, MastodonIllegalArgumentError
import psycopg2
from psycopg2 import sql
from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
import getpass
import fileinput,re
import requests
class DomainBlocks():
name = 'Domain blocks for Mastodon'
def __init__(self, mastodon_hostname=None, domain_block_api=None, session=None):
self.domain_blocks_api = '/api/v1/admin/domain_blocks'
self.config_file = "config/config.txt"
self.secrets_file = "secrets/secrets.txt"
is_setup = self.__check_setup(self.secrets_file)
if is_setup:
self.__uc_client_id = self.__get_parameter("uc_client_id", self.secrets_file)
self.__uc_client_secret = self.__get_parameter("uc_client_secret", self.secrets_file)
self.__uc_access_token = self.__get_parameter("uc_access_token", self.secrets_file)
self.blocker_db, self.blocker_db_user = self.db_config()
self.mastodon, self.mastodon_hostname, self.headers = self.log_in()
else:
while(True):
logged_in, self.mastodon, self.mastodon_hostname = self.setup()
if not logged_in:
print("\nLog in failed! Try again.\n")
else:
break
db_setup = self.__check_dbsetup(self)
if not db_setup:
self.__createdb(self)
if session:
self.session = session
else:
self.session = requests.Session()
def get_servers(self, software):
servers_list = []
try:
conn = None
conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute("select server from blocker where software=(%s)", (software,))
rows = cur.fetchall()
for row in rows:
servers_list.append(row[0])
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
sys.exit(error)
finally:
if conn is not None:
conn.close()
return (servers_list)
def log_in(self):
uc_client_id = self.__get_parameter("uc_client_id", self.secrets_file)
uc_client_secret = self.__get_parameter("uc_client_secret", self.secrets_file)
uc_access_token = self.__get_parameter("uc_access_token", self.secrets_file)
self.mastodon_hostname = self.__get_parameter("mastodon_hostname", self.config_file)
mastodon = Mastodon(
client_id = uc_client_id,
client_secret = uc_client_secret,
access_token = uc_access_token,
api_base_url = 'https://' + self.mastodon_hostname,
)
headers={ 'Authorization': 'Bearer %s'%uc_access_token }
return (mastodon, self.mastodon_hostname, headers)
def domain_blocks_create(self, server, priv_comment, pub_comment):
data = {
'domain': server,
'severity': 'suspend',
'reject_media': 'true',
'reject_reports': 'true',
'private_comment': priv_comment,
'public_comment': pub_comment,
'obfuscate': 'true',
}
endpoint = f'https://{self.mastodon_hostname}/{self.domain_blocks_api}'
response = self.__api_request('POST', endpoint, data)
if response.ok:
print(f"Done, {server} is now suspended")
else:
pass
@staticmethod
def __check_setup(file_path):
is_setup = False
if not os.path.isfile(file_path):
print(f"File {file_path} not found, running setup.")
return
else:
is_setup = True
return is_setup
def setup(self):
logged_in = False
try:
self.mastodon_hostname = input("Enter Mastodon hostname (or 'q' to exit): ")
if self.mastodon_hostname == 'q':
sys.exit("Bye")
user_name = input("Bot user name, ex. john? ")
user_password = getpass.getpass("Bot password? ")
app_name = input("Bot App name? ")
self.blocker_db = input("Blocker's database name: ")
self.blocker_db_user = input("blocker's database user: ")
client_id, client_secret = Mastodon.create_app(
app_name,
scopes = ["read", "write", "admin:read", "admin:write"],
to_file="app_clientcred.txt",
api_base_url=self.mastodon_hostname
)
mastodon = Mastodon(client_id = "app_clientcred.txt", api_base_url = self.mastodon_hostname)
mastodon.log_in(
user_name,
user_password,
scopes = ["read", "write", "admin:read", "admin:write"],
to_file = "app_usercred.txt"
)
if os.path.isfile("app_usercred.txt"):
print(f"Log in succesful!")
logged_in = True
if not os.path.exists('secrets'):
os.makedirs('secrets')
if not os.path.exists(self.secrets_file):
with open(self.secrets_file, 'w'): pass
print(f"{self.secrets_file} created!")
with open(self.secrets_file, 'a') as the_file:
print(f"Writing secrets parameter names to {self.secrets_file}")
the_file.write('uc_client_id: \n'+'uc_client_secret: \n'+'uc_access_token: \n')
client_path = 'app_clientcred.txt'
with open(client_path) as fp:
line = fp.readline()
cnt = 1
while line:
if cnt == 1:
print(f"Writing client id to {self.secrets_file}")
self.__modify_file(self, self.secrets_file, "uc_client_id: ", value=line.rstrip())
elif cnt == 2:
print(f"Writing client secret to {self.secrets_file}")
self.__modify_file(self, self.secrets_file, "uc_client_secret: ", value=line.rstrip())
line = fp.readline()
cnt += 1
token_path = 'app_usercred.txt'
with open(token_path) as fp:
line = fp.readline()
print(f"Writing access token to {self.secrets_file}")
self.__modify_file(self, self.secrets_file, "uc_access_token: ", value=line.rstrip())
if os.path.exists("app_clientcred.txt"):
print("Removing app_clientcred.txt temp file..")
os.remove("app_clientcred.txt")
if os.path.exists("app_usercred.txt"):
print("Removing app_usercred.txt temp file..")
os.remove("app_usercred.txt")
self.__create_config(self)
self.__write_config(self)
print("Secrets setup done!\n")
with open(self.config_file, 'a') as the_file:
print(f"Writing Mastodon hostname parameter to {self.config_file}")
the_file.write(f'blocker_db: {self.blocker_db}\n')
the_file.write(f'blocker_db_user: {self.blocker_db_user}\n')
print("Blocker setup done!\n")
except MastodonIllegalArgumentError as i_error:
sys.stdout.write(f'\n{str(i_error)}\n')
except MastodonNetworkError as n_error:
sys.stdout.write(f'\n{str(n_error)}\n')
except MastodonReadTimeout as r_error:
sys.stdout.write(f'\n{str(r_error)}\n')
except MastodonAPIError as a_error:
sys.stdout.write(f'\n{str(a_error)}\n')
return (logged_in, mastodon, self.mastodon_hostname)
def __api_request(self, method, endpoint, data={}):
response = None
try:
kwargs = dict(data=data)
response = self.session.request(method, url = endpoint, headers = self.headers, **kwargs)
except Exception as e:
raise MastodonNetworkError(f"Could not complete request: {e}")
if response is None:
raise MastodonIllegalArgumentError("Illegal request.")
if not response.ok:
try:
if isinstance(response, dict) and 'error' in response:
error_msg = response['error']
elif isinstance(response, str):
error_msg = response
else:
error_msg = None
except ValueError:
error_msg = None
if response.status_code == 404:
ex_type = MastodonNotFoundError
if not error_msg:
error_msg = 'Endpoint not found.'
# this is for compatibility with older versions
# which raised MastodonAPIError('Endpoint not found.')
# on any 404
elif response.status_code == 401:
ex_type = MastodonUnauthorizedError
elif response.status_code == 422:
return response
elif response.status_code == 500:
ex_type = MastodonInternalServerError
elif response.status_code == 502:
ex_type = MastodonBadGatewayError
elif response.status_code == 503:
ex_type = MastodonServiceUnavailableError
elif response.status_code == 504:
ex_type = MastodonGatewayTimeoutError
elif response.status_code >= 500 and \
response.status_code <= 511:
ex_type = MastodonServerError
else:
ex_type = MastodonAPIError
raise ex_type(
'Mastodon API returned error',
response.status_code,
response.reason,
error_msg)
else:
return response
def db_config(self):
self.blocker_db = self.__get_parameter("blocker_db", self.config_file)
self.blocker_db_user = self.__get_parameter("blocker_db_user", self.config_file)
return (self.blocker_db, self.blocker_db_user)
@staticmethod
def __check_dbsetup(self):
dbsetup = False
try:
conn = None
conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = "", host = "/var/run/postgresql", port = "5432")
dbsetup = True
except (Exception, psycopg2.DatabaseError) as error:
print(error)
return dbsetup
@staticmethod
def __createdb(self):
conn = None
try:
conn = psycopg2.connect(dbname='postgres',
user=self.blocker_db_user, host='',
password='')
conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
cur = conn.cursor()
print(f"Creating database {self.blocker_db}. Please wait...")
cur.execute(sql.SQL("CREATE DATABASE {}").format(
sql.Identifier(self.blocker_db))
)
print(f"Database {self.blocker_db} created!\n")
self.__dbtables_schemes(self)
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
@staticmethod
def __dbtables_schemes(self):
table = "blocker"
sql = "create table "+table+" (server varchar(200), users int, updated_at timestamptz, software varchar(50), alive boolean, users_api varchar(50), version varchar(100), first_checked_at timestamptz, last_checked_at timestamptz, downs int)"
self.__create_table(self, table, sql)
table = "execution_time"
sql = "create table "+table+" (program varchar(30), start timestamptz, finish timestamptz)"
self.__create_table(self, table, sql)
@staticmethod
def __create_table(self, table, sql):
conn = None
try:
conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
print(f"Creating table {table}")
cur.execute(sql)
conn.commit()
print(f"Table {table} created!\n")
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
def __get_parameter(self, parameter, config_file):
if not os.path.isfile(config_file):
print(f"File {config_file} not found, exiting.")
sys.exit(0)
with open( config_file ) as f:
for line in f:
if line.startswith( parameter ):
return line.replace(parameter + ":", "").strip()
print(f"{config_file} Missing parameter {parameter}")
sys.exit(0)
@staticmethod
def __modify_file(self, file_name, pattern,value=""):
fh=fileinput.input(file_name,inplace=True)
for line in fh:
replacement=pattern + value
line=re.sub(pattern,replacement,line)
sys.stdout.write(line)
fh.close()
@staticmethod
def __create_config(self):
if not os.path.exists('config'):
os.makedirs('config')
if not os.path.exists(self.config_file):
print(self.config_file + " created!")
with open('config/config.txt', 'w'): pass
@staticmethod
def __write_config(self):
with open(self.config_file, 'a') as the_file:
the_file.write(f'mastodon_hostname: {self.mastodon_hostname}\n')
print(f"adding parameter 'mastodon_hostname' to {self.config_file}\n")
###############################################################################
# main # main
if __name__ == '__main__': if __name__ == '__main__':
setup = Setup()
db = Database()
blocker = DomainBlocks() blocker = DomainBlocks()
mastodon = Mastodon(
access_token = setup.mastodon_app_token,
api_base_url= setup.mastodon_hostname
)
soft_list = 'software.txt' soft_list = 'software.txt'
soft_file = open(soft_list, 'r') soft_file = open(soft_list, 'r')
@ -500,11 +38,11 @@ if __name__ == '__main__':
print(f'checking software {software}...') print(f'checking software {software}...')
servers_list = blocker.get_servers(software) servers_list = db.get_servers(software)
for server in servers_list: for server in servers_list:
blocker.domain_blocks_create(server, private_comment, public_comment ) blocker.domain_blocks_create(server, private_comment, public_comment)

533
peers.py
Veure arxiu

@ -1,535 +1,42 @@
import os from app.libraries.setup import Setup
import sys from app.libraries.database import Database
import time from app.libraries.peers import Peers
from datetime import datetime from app.libraries.nodeinfo import Nodeinfo
import requests import ray
import urllib3
import json
import psycopg2
from psycopg2 import sql
from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
#import ray
import pdb
#ray.init(num_cpus = 32) # Specify this system CPUs. ray.init(num_cpus = 25) # Specify this system CPUs.
class Peers: @ray.remote
def peerapi(peer):
name = 'Mastodon server peers' api = peers.getapi(peer) # get peer nodeinfo's URL
def __init__(self, mastodon_hostname=None, peers_api=None): if api != '':
self.peers_api = '/api/v1/instance/peers'
self.config_path = "config/config.txt"
self.apis = ['/api/v1/instance?',
'/nodeinfo/2.0?',
'/nodeinfo/2.0.json?',
'/main/nodeinfo/2.0?',
'/api/statusnet/config?',
'/api/nodeinfo/2.0.json?',
'/api/nodeinfo?',
'/wp-json/nodeinfo/2.0?',
'/api/v1/instance/nodeinfo/2.0?',
'/.well-known/x-nodeinfo2?'
]
is_setup = self.__check_setup(self)
if is_setup:
self.mastodon_hostname = self.__get_parameter("mastodon_hostname", self.config_path)
self.blocker_db = self.__get_parameter("blocker_db", self.config_path)
self.blocker_db_user = self.__get_parameter("blocker_db_user", self.config_path)
else:
self.mastodon_hostname, self.blocker_db, self.blocker_db_user = self.__setup(self)
db_setup = self.__check_dbsetup(self)
if not db_setup:
self.__createdb(self)
def get_peers(self):
user_agent = {'User-agent': "fediverse's stats (fediverse@soc.catala.digital)"}
res = requests.get(f'https://{self.mastodon_hostname}{self.peers_api}', headers = user_agent, timeout=3)
peers = res.json()
return peers
def getsoft(self, server):
if server.find(".") == -1:
return
if server.find("@") != -1:
return
if server.find("/") != -1:
return
if server.find(":") != -1:
return
soft = ''
is_nodeinfo = False
url = 'https://' + server
user_agent = {'User-agent': "fediverse's stats (fediverse@soc.catala.digital)"}
try: try:
response = requests.get(url + '/.well-known/nodeinfo', headers = user_agent, timeout=3) server, software = nd.getdata(peer, api)
if response.status_code == 200: db.write_server(peer, api, software)
try: print(f'server: {peer}, api: {api}, software: {software}')
response_json = response.json()
nodeinfo = response_json['links'][0]['href'].replace(f'https://{server}','')
try:
nodeinfo_data = requests.get(url + nodeinfo, headers = user_agent, timeout=3)
if nodeinfo_data.status_code == 200:
nodeinfo_json = nodeinfo_data.json()
is_nodeinfo = True
else:
print(f"Server {server}'s nodeinfo not responding: error code {nodeinfo_data.status_code}")
except: except:
pass pass
except:
print(f'Server {server} not responding: error code {response.status_code}')
print('*********************************************************************')
pass
else:
for api in self.apis:
try:
response = requests.get(url + api, headers = user_agent, timeout=3)
if is_json(response.text):
nodeinfo_json = response.json()
if 'software' in nodeinfo_json:
nodeinfo = api
is_nodeinfo = True
break
elif 'title' in nodeinfo_json:
if nodeinfo_json['title'] == 'Zap':
nodeinfo = api
is_nodeinfo = True
soft = 'zap'
break
elif 'version' in nodeinfo_json:
nodeinfo = api
is_nodeinfo = True
break
except:
pass
except requests.exceptions.SSLError as errssl:
pass
except requests.exceptions.HTTPError as errh:
pass
except requests.exceptions.ConnectionError as errc:
pass
except requests.exceptions.ReadTimeout as to_err:
pass
except requests.exceptions.TooManyRedirects as tmr_err:
pass
except urllib3.exceptions.LocationParseError as lp_err:
pass
except requests.exceptions.InvalidURL as iu_err:
pass
except requests.exceptions.ChunkedEncodingError as chunk_err:
print(f'ChunkedEncodingError! {server}')
pass
except ray.exceptions.RaySystemError as ray_sys_error:
print(ray_sys_error)
pass
else:
if is_nodeinfo:
if nodeinfo != '/api/v1/instance?':
if nodeinfo != '/.well-known/x-nodeinfo2?':
try:
soft = nodeinfo_json['software']['name']
soft = soft.lower()
soft_version = nodeinfo_json['software']['version']
users = nodeinfo_json['usage']['users']['total']
alive = True
self.write_api(server, soft, users, alive, nodeinfo, soft_version)
print(f"Server {server} ({soft} {soft_version}) is alive!")
print('*********************************************************************')
return
except:
pass
else:
try:
soft = nodeinfo_json['server']['software']
soft = soft.lower()
soft_version = nodeinfo_json['server']['version']
users = nodeinfo_json['usage']['users']['total']
alive = True
if soft == 'socialhome':
self.write_api(server, soft, users, alive, nodeinfo, soft_version)
print('*********************************************************************')
print(f"Server {serve}r ({soft} {soft_version}) is alive!")
print('*********************************************************************')
return
except:
pass
if soft == '' and nodeinfo == "/api/v1/instance?":
soft = 'mastodon'
try:
users = nodeinfo_json['stats']['user_count']
if users > 1000000:
return
except:
users = 0
try:
soft_version = nodeinfo_json['version']
except:
soft_version = 'unknown'
alive = True
self.write_api(self, server, soft, users, alive, nodeinfo, soft_version)
print('*********************************************************************')
print(f"Server {server} ({soft}) is alive!")
elif soft == 'zap' and nodeinfo == "/api/v1/instance?":
soft = 'zap'
users = nodeinfo_json['stats']['user_count']
soft_version = nodeinfo_json['version']
alive = True
print(server, soft, users, alive, api)
print('*********************************************************************')
print(f"Server {server} ({soft}) is alive!")
else:
print(f'Server {server} is dead')
print('*********************************************************************')
def write_api(self, server, software, users, alive, api, soft_version):
insert_sql = "INSERT INTO blocker(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
conn = None
try:
conn = psycopg2.connect(database=self.blocker_db, user=self.blocker_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor()
print(f'Writing {server} nodeinfo data...')
cur.execute(insert_sql, (server, now, software, users, alive, api, soft_version))
cur.execute(
"UPDATE blocker SET updated_at=(%s), software=(%s), users=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)",
(now, software, users, alive, api, soft_version, server)
)
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
def save_time(self, program, start, finish):
insert_sql = "INSERT INTO execution_time(program, start, finish) VALUES(%s,%s,%s) ON CONFLICT DO NOTHING"
conn = None
try:
conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute(insert_sql, (program, start, finish,))
cur.execute("UPDATE execution_time SET start=(%s), finish=(%s) where program=(%s)", (start, finish, program))
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
@staticmethod
def __check_setup(self):
is_setup = False
if not os.path.isfile(self.config_path):
print(f"File {self.config_path} not found, running setup.")
else:
is_setup = True
return is_setup
@staticmethod
def __setup(self):
if not os.path.exists('config'):
os.makedirs('config')
self.mastodon_hostname = input("Mastodon hostname: ")
self.blocker_db = input("Blocker database name: ")
self.blocker_db_user = input("Blocker database user: ")
if not os.path.exists(self.config_path):
with open(self.config_path, 'w'): pass
print(f"{self.config_path} created!")
with open(self.config_path, 'a') as the_file:
print("Writing Blocker parameters to " + self.config_path)
the_file.write(f'mastodon_hostname: {self.mastodon_hostname}\n')
the_file.write(f'blocker_db: {self.blocker_db}\n')
the_file.write(f'blocker_db_user: {self.blocker_db_user}\n')
return (self.mastodon_hostname, self.blocker_db, self.blocker_db_user)
@staticmethod
def __check_dbsetup(self):
dbsetup = False
try:
conn = None
conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = "", host = "/var/run/postgresql", port = "5432")
dbsetup = True
except (Exception, psycopg2.DatabaseError) as error:
print(error)
return dbsetup
@staticmethod
def __createdb(self):
conn = None
try:
conn = psycopg2.connect(dbname='postgres',
user=self.blocker_db_user, host='',
password='')
conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
cur = conn.cursor()
print(f"Creating database {self.blocker_db}. Please wait...")
cur.execute(sql.SQL("CREATE DATABASE {}").format(
sql.Identifier(self.blocker_db))
)
print(f"Database {self.blocker_db} created!\n")
self.__dbtables_schemes(self)
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
@staticmethod
def __dbtables_schemes(self):
table = "blocker"
sql = f"""create table {table} (server varchar(200) PRIMARY KEY, users INT, updated_at timestamptz, software varchar(50), alive boolean, users_api varchar(50),
version varchar(100), first_checked_at timestamptz, last_checked_at timestamptz, downs int)"""
self.__create_table(self, table, sql)
table = "execution_time"
sql = "create table "+table+" (program varchar(30) PRIMARY KEY, start timestamptz, finish timestamptz)"
self.__create_table(self, table, sql)
@staticmethod
def __create_table(self, table, sql):
conn = None
try:
conn = psycopg2.connect(database = self.blocker_db, user = self.blocker_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
print(f"Creating table {table}")
cur.execute(sql)
conn.commit()
print(f"Table {table} created!\n")
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
@staticmethod
def __get_parameter(parameter, file_path ):
with open( file_path ) as f:
for line in f:
if line.startswith( parameter ):
return line.replace(parameter + ":", "").strip()
print(f'{file_path} Missing parameter {parameter}')
sys.exit(0)
###############################################################################
# main
if __name__ == '__main__': if __name__ == '__main__':
obj = Peers() db = Database()
peers = obj.get_peers() setup = Setup()
print(f"{obj.mastodon_hostname}'s peers: {len(peers)}") peers = Peers()
now = datetime.now() nd = Nodeinfo()
start = datetime.now() peers_list = peers.getpeers(setup.mastodon_hostname) #get peers from Mastodon host
program = obj.name
finish = start
obj.save_time(program, start, finish)
for peer in peers:
obj.getsoft(peer)
#results = ray.get([obj.getsoft.remote(server) for server in peers])
finish = datetime.now()
print(f"duration = {finish - start}.\nprocessed servers: {len(peers)}")
obj.save_time(program, start, finish)
if peers_list != None:
ray.get([peerapi.remote(peer) for peer in peers_list])

Veure arxiu

@ -1,3 +1,4 @@
requests requests
psycopg2-binary psycopg2-binary
Mastodonplus.py Mastodonplus.py
ray