Added sqlite3 support

This commit is contained in:
spla 2022-08-23 20:39:07 +02:00
pare d939c260f0
commit cc91bccb8f
S'han modificat 2 arxius amb 990 adicions i 0 eliminacions

481
sqlite-bs.py Normal file
Veure arxiu

@ -0,0 +1,481 @@
import os
import sys
from mastodon import Mastodon
from mastodon.Mastodon import MastodonNotFoundError, MastodonNetworkError, MastodonReadTimeout, MastodonAPIError, MastodonUnauthorizedError, MastodonIllegalArgumentError
import sqlite3
from sqlite3 import Error
import getpass
import fileinput,re
import requests
class DomainBlocks():
name = 'Domain blocks for Mastodon'
def __init__(self, mastodon_hostname=None, domain_block_api=None, session=None):
self.domain_blocks_api = '/api/v1/admin/domain_blocks'
self.config_file = "config/config.txt"
self.secrets_file = "secrets/secrets.txt"
is_setup = self.__check_setup(self.secrets_file)
if is_setup:
self.__uc_client_id = self.__get_parameter("uc_client_id", self.secrets_file)
self.__uc_client_secret = self.__get_parameter("uc_client_secret", self.secrets_file)
self.__uc_access_token = self.__get_parameter("uc_access_token", self.secrets_file)
self.sqlite_db = 'database/blocksoft.db'
self.mastodon, self.mastodon_hostname, self.headers = self.log_in()
else:
while(True):
logged_in, self.mastodon, self.mastodon_hostname = self.setup()
if not logged_in:
print("\nLog in failed! Try again.\n")
else:
db_setup = self.__check_dbsetup(self)
break
if not os.path.exists('database'):
os.makedirs('database')
self.__check_dbsetup(self)
if session:
self.session = session
else:
self.session = requests.Session()
def get_servers(self, software):
servers_list = []
try:
conn = None
conn = sqlite3.connect(self.sqlite_db)
cur = conn.cursor()
cur.execute("select server from blocker where software=?", (software,))
rows = cur.fetchall()
for row in rows:
servers_list.append(row[0])
cur.close()
except sqlite3.DatabaseError as db_error:
sys.exit(db_error)
finally:
if conn is not None:
conn.close()
return (servers_list)
def log_in(self):
uc_client_id = self.__get_parameter("uc_client_id", self.secrets_file)
uc_client_secret = self.__get_parameter("uc_client_secret", self.secrets_file)
uc_access_token = self.__get_parameter("uc_access_token", self.secrets_file)
self.mastodon_hostname = self.__get_parameter("mastodon_hostname", self.config_file)
mastodon = Mastodon(
client_id = uc_client_id,
client_secret = uc_client_secret,
access_token = uc_access_token,
api_base_url = 'https://' + self.mastodon_hostname,
)
headers={ 'Authorization': 'Bearer %s'%uc_access_token }
return (mastodon, self.mastodon_hostname, headers)
def domain_blocks_create(self, server):
data = {
'domain': server,
'severity': 'suspend',
'reject_media': 'true',
'reject_reports': 'true',
'obfuscate': 'true',
}
endpoint = f'https://{self.mastodon_hostname}/{self.domain_blocks_api}'
response = self.__api_request('POST', endpoint, data)
if response.ok:
print(f"Done, {server} is now suspended")
else:
pass
@staticmethod
def __check_setup(file_path):
is_setup = False
if not os.path.isfile(file_path):
print(f"File {file_path} not found, running setup.")
return
else:
is_setup = True
return is_setup
def setup(self):
logged_in = False
try:
self.mastodon_hostname = input("Enter Mastodon hostname (or 'q' to exit): ")
if self.mastodon_hostname == 'q':
sys.exit("Bye")
user_name = input("Bot user name, ex. john? ")
user_password = getpass.getpass("Bot password? ")
app_name = input("Bot App name? ")
client_id, client_secret = Mastodon.create_app(
app_name,
scopes = ["read", "write", "admin:read", "admin:write"],
to_file="app_clientcred.txt",
api_base_url=self.mastodon_hostname
)
mastodon = Mastodon(client_id = "app_clientcred.txt", api_base_url = self.mastodon_hostname)
mastodon.log_in(
user_name,
user_password,
scopes = ["read", "write", "admin:read", "admin:write"],
to_file = "app_usercred.txt"
)
if os.path.isfile("app_usercred.txt"):
print(f"Log in succesful!")
logged_in = True
if not os.path.exists('secrets'):
os.makedirs('secrets')
if not os.path.exists(self.secrets_file):
with open(self.secrets_file, 'w'): pass
print(f"{self.secrets_file} created!")
with open(self.secrets_file, 'a') as the_file:
print(f"Writing secrets parameter names to {self.secrets_file}")
the_file.write('uc_client_id: \n'+'uc_client_secret: \n'+'uc_access_token: \n')
client_path = 'app_clientcred.txt'
with open(client_path) as fp:
line = fp.readline()
cnt = 1
while line:
if cnt == 1:
print(f"Writing client id to {self.secrets_file}")
self.__modify_file(self, self.secrets_file, "uc_client_id: ", value=line.rstrip())
elif cnt == 2:
print(f"Writing client secret to {self.secrets_file}")
self.__modify_file(self, self.secrets_file, "uc_client_secret: ", value=line.rstrip())
line = fp.readline()
cnt += 1
token_path = 'app_usercred.txt'
with open(token_path) as fp:
line = fp.readline()
print(f"Writing access token to {self.secrets_file}")
self.__modify_file(self, self.secrets_file, "uc_access_token: ", value=line.rstrip())
if os.path.exists("app_clientcred.txt"):
print("Removing app_clientcred.txt temp file..")
os.remove("app_clientcred.txt")
if os.path.exists("app_usercred.txt"):
print("Removing app_usercred.txt temp file..")
os.remove("app_usercred.txt")
self.__create_config(self)
self.__write_config(self)
print("Secrets setup done!\n")
print("Blocker setup done!\n")
except MastodonIllegalArgumentError as i_error:
sys.stdout.write(f'\n{str(i_error)}\n')
except MastodonNetworkError as n_error:
sys.stdout.write(f'\n{str(n_error)}\n')
except MastodonReadTimeout as r_error:
sys.stdout.write(f'\n{str(r_error)}\n')
except MastodonAPIError as a_error:
sys.stdout.write(f'\n{str(a_error)}\n')
return (logged_in, mastodon, self.mastodon_hostname)
def __api_request(self, method, endpoint, data={}):
response = None
try:
kwargs = dict(data=data)
response = self.session.request(method, url = endpoint, headers = self.headers, **kwargs)
except Exception as e:
raise MastodonNetworkError(f"Could not complete request: {e}")
if response is None:
raise MastodonIllegalArgumentError("Illegal request.")
if not response.ok:
try:
if isinstance(response, dict) and 'error' in response:
error_msg = response['error']
elif isinstance(response, str):
error_msg = response
else:
error_msg = None
except ValueError:
error_msg = None
if response.status_code == 404:
ex_type = MastodonNotFoundError
if not error_msg:
error_msg = 'Endpoint not found.'
# this is for compatibility with older versions
# which raised MastodonAPIError('Endpoint not found.')
# on any 404
elif response.status_code == 401:
ex_type = MastodonUnauthorizedError
elif response.status_code == 422:
return response
elif response.status_code == 500:
ex_type = MastodonInternalServerError
elif response.status_code == 502:
ex_type = MastodonBadGatewayError
elif response.status_code == 503:
ex_type = MastodonServiceUnavailableError
elif response.status_code == 504:
ex_type = MastodonGatewayTimeoutError
elif response.status_code >= 500 and \
response.status_code <= 511:
ex_type = MastodonServerError
else:
ex_type = MastodonAPIError
raise ex_type(
'Mastodon API returned error',
response.status_code,
response.reason,
error_msg)
else:
return response
@staticmethod
def __check_dbsetup(self):
if not os.path.exists('database'):
os.makedirs('database')
dbsetup = False
try:
conn = None
conn = sqlite3.connect(self.sqlite_db)
dbsetup = True
self.__createdb(self)
except sqlite3.DatabaseError as db_error:
print(db_error)
return dbsetup
@staticmethod
def __createdb(self):
conn = None
try:
conn = sqlite3.connect(self.sqlite_db)
print(f"Database {self.sqlite_db} created!\n")
self.__dbtables_schemes(self)
except sqlite3.DatabaseError as db_error:
print(db_error)
finally:
if conn is not None:
conn.close()
@staticmethod
def __dbtables_schemes(self):
table = "blocker"
sql = "create table "+table+" (server varchar(200), users int, updated_at timestamptz, software varchar(50), alive boolean, users_api varchar(50), version varchar(100), first_checked_at timestamptz, last_checked_at timestamptz, downs int)"
self.__create_table(self, table, sql)
table = "execution_time"
sql = "create table "+table+" (program varchar(30), start timestamptz, finish timestamptz)"
self.__create_table(self, table, sql)
@staticmethod
def __create_table(self, table, sql):
conn = None
try:
conn = sqlite3.connect(self.sqlite_db)
cur = conn.cursor()
print(f"Creating table {table}")
cur.execute(sql)
conn.commit()
print(f"Table {table} created!\n")
except sqlite3.DatabaseError as db_error:
print(db_error)
finally:
if conn is not None:
conn.close()
def __get_parameter(self, parameter, config_file):
if not os.path.isfile(config_file):
print(f"File {config_file} not found, exiting.")
sys.exit(0)
with open( config_file ) as f:
for line in f:
if line.startswith( parameter ):
return line.replace(parameter + ":", "").strip()
print(f"{config_file} Missing parameter {parameter}")
sys.exit(0)
@staticmethod
def __modify_file(self, file_name, pattern,value=""):
fh=fileinput.input(file_name,inplace=True)
for line in fh:
replacement=pattern + value
line=re.sub(pattern,replacement,line)
sys.stdout.write(line)
fh.close()
@staticmethod
def __create_config(self):
if not os.path.exists('config'):
os.makedirs('config')
if not os.path.exists(self.config_file):
print(self.config_file + " created!")
with open('config/config.txt', 'w'): pass
@staticmethod
def __write_config(self):
with open(self.config_file, 'a') as the_file:
the_file.write(f'mastodon_hostname: {self.mastodon_hostname}\n')
print(f"adding parameter 'mastodon_hostname' to {self.config_file}\n")
###############################################################################
# main
if __name__ == '__main__':
blocker = DomainBlocks()
soft_list = 'software.txt'
soft_file = open(soft_list, 'r')
Lines = soft_file.readlines()
for software in Lines:
software = software.replace('\n', '')
print(f'checking software {software}...')
servers_list = blocker.get_servers(software)
for server in servers_list:
blocker.domain_blocks_create(server)

509
sqlite-peers.py Normal file
Veure arxiu

@ -0,0 +1,509 @@
import os
import sys
import time
from datetime import datetime
import requests
import urllib3
import json
import sqlite3
from sqlite3 import Error
class Peers:
name = 'Mastodon server peers'
def __init__(self, mastodon_hostname=None, peers_api=None):
self.peers_api = '/api/v1/instance/peers'
self.config_path = "config/config.txt"
self.apis = ['/api/v1/instance?',
'/nodeinfo/2.0?',
'/nodeinfo/2.0.json?',
'/main/nodeinfo/2.0?',
'/api/statusnet/config?',
'/api/nodeinfo/2.0.json?',
'/api/nodeinfo?',
'/wp-json/nodeinfo/2.0?',
'/api/v1/instance/nodeinfo/2.0?',
'/.well-known/x-nodeinfo2?'
]
is_setup = self.__check_setup(self)
if is_setup:
self.mastodon_hostname = self.__get_parameter("mastodon_hostname", self.config_path)
self.sqlite_db = 'database/blocksoft.db'
else:
self.mastodon_hostname = self.__setup(self)
self.__check_dbsetup(self)
if not os.path.exists('database'):
os.makedirs('database')
self.__check_dbsetup(self)
def get_peers(self):
user_agent = {'User-agent': "fediverse's stats (fediverse@soc.catala.digital)"}
res = requests.get(f'https://{self.mastodon_hostname}{self.peers_api}', headers = user_agent, timeout=3)
peers = res.json()
return peers
def getsoft(self, server):
if server.find(".") == -1:
return
if server.find("@") != -1:
return
if server.find("/") != -1:
return
if server.find(":") != -1:
return
soft = ''
is_nodeinfo = False
url = 'https://' + server
user_agent = {'User-agent': "fediverse's stats (fediverse@soc.catala.digital)"}
try:
response = requests.get(url + '/.well-known/nodeinfo', headers = user_agent, timeout=3)
if response.status_code == 200:
try:
response_json = response.json()
nodeinfo = response_json['links'][0]['href'].replace(f'https://{server}','')
try:
nodeinfo_data = requests.get(url + nodeinfo, headers = user_agent, timeout=3)
if nodeinfo_data.status_code == 200:
nodeinfo_json = nodeinfo_data.json()
is_nodeinfo = True
else:
print(f"Server {server}'s nodeinfo not responding: error code {nodeinfo_data.status_code}")
except:
pass
except:
print(f'Server {server} not responding: error code {response.status_code}')
print('*********************************************************************')
pass
else:
for api in self.apis:
try:
response = requests.get(url + api, headers = user_agent, timeout=3)
if is_json(response.text):
nodeinfo_json = response.json()
if 'software' in nodeinfo_json:
nodeinfo = api
is_nodeinfo = True
break
elif 'title' in nodeinfo_json:
if nodeinfo_json['title'] == 'Zap':
nodeinfo = api
is_nodeinfo = True
soft = 'zap'
break
elif 'version' in nodeinfo_json:
nodeinfo = api
is_nodeinfo = True
break
except:
pass
except requests.exceptions.SSLError as errssl:
pass
except requests.exceptions.HTTPError as errh:
pass
except requests.exceptions.ConnectionError as errc:
pass
except requests.exceptions.ReadTimeout as to_err:
pass
except requests.exceptions.TooManyRedirects as tmr_err:
pass
except urllib3.exceptions.LocationParseError as lp_err:
pass
except requests.exceptions.InvalidURL as iu_err:
pass
except requests.exceptions.ChunkedEncodingError as chunk_err:
print(f'ChunkedEncodingError! {server}')
pass
else:
if is_nodeinfo:
if nodeinfo != '/api/v1/instance?':
if nodeinfo != '/.well-known/x-nodeinfo2?':
try:
soft = nodeinfo_json['software']['name']
soft = soft.lower()
soft_version = nodeinfo_json['software']['version']
users = nodeinfo_json['usage']['users']['total']
alive = True
self.write_api(server, soft, users, alive, nodeinfo, soft_version)
print(f"Server {server} ({soft} {soft_version}) is alive!")
print('*********************************************************************')
return
except:
pass
else:
try:
soft = nodeinfo_json['server']['software']
soft = soft.lower()
soft_version = nodeinfo_json['server']['version']
users = nodeinfo_json['usage']['users']['total']
alive = True
if soft == 'socialhome':
self.write_api(server, soft, users, alive, nodeinfo, soft_version)
print('*********************************************************************')
print(f"Server {serve}r ({soft} {soft_version}) is alive!")
print('*********************************************************************')
return
except:
pass
if soft == '' and nodeinfo == "/api/v1/instance?":
soft = 'mastodon'
try:
users = nodeinfo_json['stats']['user_count']
if users > 1000000:
return
except:
users = 0
try:
soft_version = nodeinfo_json['version']
except:
soft_version = 'unknown'
alive = True
self.write_api(self, server, soft, users, alive, nodeinfo, soft_version)
print('*********************************************************************')
print(f"Server {server} ({soft}) is alive!")
elif soft == 'zap' and nodeinfo == "/api/v1/instance?":
soft = 'zap'
users = nodeinfo_json['stats']['user_count']
soft_version = nodeinfo_json['version']
alive = True
print(server, soft, users, alive, api)
print('*********************************************************************')
print(f"Server {server} ({soft}) is alive!")
else:
print(f'Server {server} is dead')
print('*********************************************************************')
def write_api(self, server, software, users, alive, api, soft_version):
insert_sql = "INSERT INTO blocker(server, updated_at, software, users, alive, users_api, version) VALUES(?,?,?,?,?,?,?) ON CONFLICT DO NOTHING"
conn = None
try:
conn = sqlite3.connect(self.sqlite_db)
cur = conn.cursor()
print(f'Writing {server} nodeinfo data...')
cur.execute(insert_sql, (server, now, software, users, alive, api, soft_version))
cur.execute(
"UPDATE blocker SET updated_at=?, software=?, users=?, alive=?, users_api=?, version=? where server=?",
(now, software, users, alive, api, soft_version, server)
)
conn.commit()
cur.close()
except sqlite3.DatabaseError as db_error:
print(db_error)
finally:
if conn is not None:
conn.close()
def save_time(self, program, start, finish):
insert_sql = "INSERT INTO execution_time(program, start, finish) VALUES(?,?,?) ON CONFLICT DO NOTHING"
conn = None
try:
conn = sqlite3.connect(self.sqlite_db)
cur = conn.cursor()
cur.execute(insert_sql, (program, start, finish,))
cur.execute("UPDATE execution_time SET start=?, finish=? where program=?", (start, finish, program))
conn.commit()
cur.close()
except sqlite3.DatabaseError as db_error:
print(db_error)
finally:
if conn is not None:
conn.close()
@staticmethod
def __check_setup(self):
is_setup = False
if not os.path.isfile(self.config_path):
print(f"File {self.config_path} not found, running setup.")
else:
is_setup = True
return is_setup
@staticmethod
def __setup(self):
if not os.path.exists('config'):
os.makedirs('config')
self.mastodon_hostname = input("Mastodon hostname: ")
if not os.path.exists(self.config_path):
with open(self.config_path, 'w'): pass
print(f"{self.config_path} created!")
with open(self.config_path, 'a') as the_file:
print("Writing Softblock parameters to " + self.config_path)
the_file.write(f'mastodon_hostname: {self.mastodon_hostname}\n')
return (self.mastodon_hostname)
@staticmethod
def __check_dbsetup(self):
dbsetup = False
try:
conn = None
conn = sqlite3.connect(self.sqlite_db)
dbsetup = True
except sqlite3.DatabaseError as db_error:
print(db_error)
return dbsetup
@staticmethod
def __createdb(self):
conn = None
try:
conn = sqlite3.connect(self.sqlite_db)
print(f"Database {self.sqlite_db} created!\n")
self.__dbtables_schemes(self)
except sqlite3.DatabaseError as db_error:
print(db_error)
finally:
if conn is not None:
conn.close()
@staticmethod
def __dbtables_schemes(self):
table = "blocker"
sql = f"""create table {table} (server varchar(200) PRIMARY KEY, users INT, updated_at timestamptz, software varchar(50), alive boolean, users_api varchar(50),
version varchar(100), first_checked_at timestamptz, last_checked_at timestamptz, downs int)"""
self.__create_table(self, table, sql)
table = "execution_time"
sql = "create table "+table+" (program varchar(30) PRIMARY KEY, start timestamptz, finish timestamptz)"
self.__create_table(self, table, sql)
@staticmethod
def __create_table(self, table, sql):
conn = None
try:
conn = sqlite3.connect(self.sqlite_db)
cur = conn.cursor()
print(f"Creating table {table}")
cur.execute(sql)
conn.commit()
print(f"Table {table} created!\n")
except sqlite3.DatabaseError as db_error:
print(db_error)
finally:
if conn is not None:
conn.close()
@staticmethod
def __get_parameter(parameter, file_path ):
with open( file_path ) as f:
for line in f:
if line.startswith( parameter ):
return line.replace(parameter + ":", "").strip()
print(f'{file_path} Missing parameter {parameter}')
sys.exit(0)
###############################################################################
# main
if __name__ == '__main__':
obj = Peers()
peers = obj.get_peers()
print(f"{obj.mastodon_hostname}'s peers: {len(peers)}")
now = datetime.now()
start = datetime.now()
program = obj.name
finish = start
obj.save_time(program, start, finish)
for peer in peers:
obj.getsoft(peer)
#results = ray.get([obj.getsoft.remote(server) for server in peers])
finish = datetime.now()
print(f"duration = {finish - start}.\nprocessed servers: {len(peers)}")
obj.save_time(program, start, finish)