705 líneas
19 KiB
Python
705 líneas
19 KiB
Python
import datetime
|
|
from datetime import date, datetime, timedelta
|
|
from mastodon import Mastodon, MastodonNetworkError, MastodonNotFoundError, MastodonInternalServerError
|
|
import time
|
|
import os
|
|
import json
|
|
import sys
|
|
import os.path
|
|
import operator
|
|
import psycopg2
|
|
from psycopg2 import sql
|
|
import requests
|
|
from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
|
|
|
|
class Spamcheck:
|
|
|
|
name = "Spamcheck for Mastodon social server"
|
|
|
|
def __init__(self, mastodon_hostname=None, mastodon_db=None, mastodon_db_user=None, spamcheck_db=None, spamcheck_db_user=None):
|
|
|
|
self.config_file = 'config/config.txt'
|
|
self.secrets_file = "secrets/secrets.txt"
|
|
|
|
is_setup = self.__check_setup(self)
|
|
|
|
if is_setup:
|
|
|
|
self.mastodon_hostname = self.__get_parameter("mastodon_hostname", self.config_file)
|
|
self.mastodon_db = self.__get_parameter("mastodon_db", self.config_file)
|
|
self.mastodon_db_user = self.__get_parameter("mastodon_db_user", self.config_file)
|
|
self.spamcheck_db = self.__get_parameter("spamcheck_db", self.config_file)
|
|
self.spamcheck_db_user = self.__get_parameter("spamcheck_db_user", self.config_file)
|
|
self.__uc_client_id = self.__get_parameter("uc_client_id", self.secrets_file)
|
|
self.__uc_client_secret = self.__get_parameter("uc_client_secret", self.secrets_file)
|
|
self.__uc_access_token = self.__get_parameter("uc_access_token", self.secrets_file)
|
|
|
|
else:
|
|
|
|
self.mastodon_hostname, self.mastodon_db, self.mastodon_db_user, self.spamcheck_db, self.spamcheck_db_user = self.__setup(self)
|
|
|
|
db_setup = self.__check_dbsetup(self)
|
|
|
|
if not db_setup:
|
|
|
|
self.__createdb(self)
|
|
|
|
def new_registers(self, created_at_lst=[], id_lst=[], email_lst=[], ip_lst=[]):
|
|
|
|
try:
|
|
|
|
conn = None
|
|
|
|
conn = psycopg2.connect(database = self.mastodon_db, user = self.mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
cur.execute("select users.created_at, users.id, users.email, users.sign_up_ip from users where users.created_at > now() - interval '70 days'")
|
|
|
|
rows = cur.fetchall()
|
|
|
|
for row in rows:
|
|
|
|
if row != None:
|
|
|
|
created_at_lst.append(row[0])
|
|
|
|
id_lst.append(row[1])
|
|
|
|
email_lst.append(row[2])
|
|
|
|
ip_lst.append(row[3])
|
|
|
|
cur.close()
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print (error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
return (created_at_lst, id_lst, email_lst, ip_lst)
|
|
|
|
def save_registers(self, created_at_lst, id_lst, email_lst, ip_lst):
|
|
|
|
insert_sql = 'INSERT INTO spamcheck(created_at, id, email, ip, tor_exit_node) VALUES(%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING'
|
|
|
|
i = 0
|
|
|
|
while i < len(id_lst):
|
|
|
|
is_tor_exit_node = self.__check_ip(self, ip_lst[i])
|
|
|
|
tor_exit_node = 't' if is_tor_exit_node == 't' else 'f'
|
|
|
|
conn = None
|
|
|
|
try:
|
|
|
|
conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
cur.execute(insert_sql, (created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i], tor_exit_node))
|
|
|
|
conn.commit()
|
|
|
|
cur.close()
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
print(created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i], tor_exit_node)
|
|
|
|
i += 1
|
|
|
|
def get_totals(self):
|
|
|
|
spamcheck_datetime_lst = []
|
|
|
|
spamcheck_registers_lst = []
|
|
|
|
select_sql = 'select date(created_at), count(ip) as registers from spamcheck group by date(created_at) order by date(created_at)'
|
|
|
|
conn = None
|
|
|
|
try:
|
|
|
|
conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
cur.execute(select_sql)
|
|
|
|
rows = cur.fetchall()
|
|
|
|
for row in rows:
|
|
|
|
spamcheck_datetime_lst.append(row[0])
|
|
|
|
spamcheck_registers_lst.append(row[1])
|
|
|
|
cur.close()
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
return (spamcheck_datetime_lst, spamcheck_registers_lst)
|
|
|
|
def write_totals(self, spamcheck_datetime_lst, spamcheck_registers_lst):
|
|
|
|
insert_sql = 'INSERT INTO totals(datetime, registers) VALUES(%s,%s) ON CONFLICT (datetime) DO UPDATE SET (datetime, registers) = (EXCLUDED.datetime, EXCLUDED.registers)'
|
|
|
|
first_date = spamcheck_datetime_lst[0]
|
|
|
|
last_date = spamcheck_datetime_lst[len(spamcheck_datetime_lst)-1]
|
|
|
|
i = 0
|
|
|
|
while i < len(spamcheck_datetime_lst):
|
|
|
|
conn = None
|
|
|
|
try:
|
|
|
|
conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
if first_date == spamcheck_datetime_lst[i]:
|
|
|
|
cur.execute(insert_sql, (spamcheck_datetime_lst[i], spamcheck_registers_lst[i]))
|
|
|
|
i += 1
|
|
|
|
else:
|
|
|
|
cur.execute(insert_sql, (first_date, '0'))
|
|
|
|
conn.commit()
|
|
|
|
cur.close()
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
first_date = first_date + timedelta(days=1)
|
|
|
|
if date.today() == last_date + timedelta(days=1):
|
|
|
|
insert_sql = 'INSERT INTO totals(datetime, registers) VALUES(%s,%s) ON CONFLICT (datetime) DO UPDATE SET (datetime, registers) = (EXCLUDED.datetime, EXCLUDED.registers)'
|
|
|
|
conn = None
|
|
|
|
try:
|
|
|
|
conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
cur.execute(insert_sql, (date.today(), '0'))
|
|
|
|
conn.commit()
|
|
|
|
cur.close()
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
def check_approval(self, user_id):
|
|
|
|
approved = False
|
|
|
|
try:
|
|
|
|
conn = None
|
|
|
|
conn = psycopg2.connect(database = self.mastodon_db, user = self.mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
cur.execute("select approved from users where id = (%s)", (user_id,))
|
|
|
|
row = cur.fetchone()
|
|
|
|
if row != None:
|
|
|
|
approved = row[0]
|
|
|
|
cur.close()
|
|
|
|
return approved
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print (error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
def get_tor(self):
|
|
|
|
tor_list = []
|
|
|
|
try:
|
|
|
|
conn = None
|
|
|
|
conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
cur.execute("select ip from spamcheck where tor_exit_node")
|
|
|
|
rows = cur.fetchall()
|
|
|
|
for row in rows:
|
|
|
|
tor_list.append(row[0])
|
|
|
|
cur.close()
|
|
|
|
return tor_list
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print (error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
def ip_blocks(self, ip):
|
|
|
|
'''
|
|
severity:
|
|
Limit sign-ups : 5000
|
|
Block sign-ups : 5500
|
|
Block access : 9999
|
|
'''
|
|
data = {
|
|
'ip': ip,
|
|
'severity': 5500,
|
|
'comment': 'Spam from this Tor exit node',
|
|
'expires_in': 8600,
|
|
}
|
|
endpoint = f'https://{self.mastodon_hostname}/api/v1/admin/ip_blocks'
|
|
|
|
response = self.api_request('POST', endpoint, data)
|
|
|
|
if response.ok:
|
|
|
|
data = response.json()
|
|
|
|
else:
|
|
|
|
pass
|
|
|
|
def api_request(self, method, endpoint, data={}):
|
|
|
|
session = requests.Session()
|
|
response = None
|
|
|
|
try:
|
|
|
|
kwargs = dict(data=data)
|
|
|
|
response = session.request(method, url = endpoint, headers = headers, **kwargs)
|
|
|
|
except Exception as e:
|
|
|
|
raise MastodonNetworkError(f"Could not complete request: {e}")
|
|
|
|
if response is None:
|
|
|
|
raise MastodonIllegalArgumentError("Illegal request.")
|
|
|
|
if not response.ok:
|
|
|
|
try:
|
|
if isinstance(response, dict) and 'error' in response:
|
|
error_msg = response['error']
|
|
elif isinstance(response, str):
|
|
error_msg = response
|
|
else:
|
|
error_msg = None
|
|
except ValueError:
|
|
error_msg = None
|
|
|
|
if response.status_code == 404:
|
|
ex_type = MastodonNotFoundError
|
|
if not error_msg:
|
|
error_msg = 'Endpoint not found.'
|
|
# this is for compatibility with older versions
|
|
# which raised MastodonAPIError('Endpoint not found.')
|
|
# on any 404
|
|
elif response.status_code == 401:
|
|
ex_type = MastodonUnauthorizedError
|
|
elif response.status_code == 422:
|
|
return response
|
|
elif response.status_code == 500:
|
|
ex_type = MastodonInternalServerError
|
|
elif response.status_code == 502:
|
|
ex_type = MastodonBadGatewayError
|
|
elif response.status_code == 503:
|
|
ex_type = MastodonServiceUnavailableError
|
|
elif response.status_code == 504:
|
|
ex_type = MastodonGatewayTimeoutError
|
|
elif response.status_code >= 500 and \
|
|
response.status_code <= 511:
|
|
ex_type = MastodonServerError
|
|
else:
|
|
ex_type = MastodonAPIError
|
|
|
|
raise ex_type(
|
|
'Mastodon API returned error',
|
|
response.status_code,
|
|
response.reason,
|
|
error_msg)
|
|
|
|
else:
|
|
|
|
return response
|
|
|
|
def blocked_ip_list(self):
|
|
|
|
blocked_ip_list = []
|
|
|
|
temp_ip_list = {}
|
|
|
|
i = 0
|
|
|
|
while True:
|
|
|
|
if i == 0:
|
|
|
|
temp_ip_list[i] = mastodon.admin_ip_blocks_list(limit=200)
|
|
|
|
if temp_ip_list[i] == []:
|
|
|
|
return blocked_ip_list
|
|
|
|
else:
|
|
|
|
temp_ip_list[i] = mastodon.fetch_next(temp_ip_list[i-1]._pagination_next)
|
|
|
|
dict_len = len(temp_ip_list[i])
|
|
|
|
ii = 0
|
|
|
|
while ii - dict_len:
|
|
|
|
blocked_ip_list.append(temp_ip_list[i][ii])
|
|
|
|
ii += 1
|
|
|
|
if len(temp_ip_list[i]) < 200:
|
|
|
|
return blocked_ip_list
|
|
|
|
i += 1
|
|
|
|
def ip_blocked(self, ip, blocked_list):
|
|
|
|
is_blocked = False
|
|
|
|
for block_ip_item in blocked_list:
|
|
|
|
if ip in block_ip_item.ip:
|
|
|
|
print(f'\n{ip} is already blocked')
|
|
|
|
is_blocked = True
|
|
|
|
return (is_blocked)
|
|
|
|
@staticmethod
|
|
def __check_ip(self, ip):
|
|
|
|
is_tor_exit_node = 'f'
|
|
|
|
if ip == None:
|
|
|
|
return
|
|
|
|
conn = None
|
|
|
|
try:
|
|
|
|
conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
cur.execute('select ip from torexit_ips where ip=(%s)', (ip,))
|
|
|
|
row = cur.fetchone()
|
|
|
|
if row != None:
|
|
|
|
is_tor_exit_node = 't'
|
|
|
|
cur.close()
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
return is_tor_exit_node
|
|
|
|
@staticmethod
|
|
def __check_setup(self):
|
|
|
|
is_setup = False
|
|
|
|
if not os.path.isfile(self.config_file):
|
|
print(f"File {self.config_file} not found, running setup.\n")
|
|
else:
|
|
is_setup = True
|
|
|
|
return is_setup
|
|
|
|
@staticmethod
|
|
def __setup(self):
|
|
|
|
if not os.path.exists('config'):
|
|
os.makedirs('config')
|
|
|
|
self.mastodon_hostname = input("Mastodon hostname, in ex. 'mastodon.social': ")
|
|
self.mastodon_db = input("Mastodon's database name: ")
|
|
self.mastodon_db_user = input("Mastodon's database user: ")
|
|
self.spamcheck_db = input("Spamcheck's database name: ")
|
|
self.spamcheck_db_user = input("Spamcheck's database user: ")
|
|
|
|
if not os.path.exists(self.config_file):
|
|
with open(self.config_file, 'w'): pass
|
|
print(f"\n{self.config_file} created!\n")
|
|
|
|
with open(self.config_file, 'a') as the_file:
|
|
print(f"Writing Mastodon hostname parameter to {self.config_file}")
|
|
the_file.write(f'mastodon_hostname: {self.mastodon_hostname}\n')
|
|
the_file.write(f'mastodon_db: {self.mastodon_db}\n')
|
|
the_file.write(f'mastodon_db_user: {self.mastodon_db_user}\n')
|
|
the_file.write(f'spamcheck_db: {self.spamcheck_db}\n')
|
|
the_file.write(f'spamcheck_db_user: {self.spamcheck_db_user}\n')
|
|
|
|
return (self.mastodon_hostname, self.mastodon_db, self.mastodon_db_user, self.spamcheck_db, self.spamcheck_db_user)
|
|
|
|
@staticmethod
|
|
def __check_dbsetup(self):
|
|
|
|
dbsetup = False
|
|
|
|
try:
|
|
|
|
conn = None
|
|
|
|
conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
|
|
|
dbsetup = True
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
return dbsetup
|
|
|
|
@staticmethod
|
|
def __createdb(self):
|
|
|
|
conn = None
|
|
|
|
try:
|
|
|
|
conn = psycopg2.connect(dbname='postgres',
|
|
user=self.spamcheck_db_user, host='',
|
|
password='')
|
|
|
|
conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
|
|
|
|
cur = conn.cursor()
|
|
|
|
print(f"Creating database {self.spamcheck_db}. Please wait...")
|
|
|
|
cur.execute(sql.SQL("CREATE DATABASE {}").format(
|
|
sql.Identifier(self.spamcheck_db))
|
|
)
|
|
print(f"Database {self.spamcheck_db} created!\n")
|
|
|
|
self.__dbtables_schemes(self)
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
@staticmethod
|
|
def __dbtables_schemes(self):
|
|
|
|
table = "spamcheck"
|
|
sql = "create table "+table+" (created_at timestamptz, id bigint PRIMARY KEY, email varchar(200), ip inet, tor_exit_node boolean)"
|
|
self.__create_table(self, table, sql)
|
|
|
|
table = "torexit_ips"
|
|
sql = "create table "+table+" (created_at timestamptz, ip inet PRIMARY KEY)"
|
|
self.__create_table(self, table, sql)
|
|
|
|
table = "totals"
|
|
sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, registers int)"
|
|
self.__create_table(self, table, sql)
|
|
|
|
@staticmethod
|
|
def __create_table(self, table, sql):
|
|
|
|
conn = None
|
|
|
|
try:
|
|
|
|
conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
|
cur = conn.cursor()
|
|
|
|
print(f"Creating table {table}")
|
|
cur.execute(sql)
|
|
|
|
conn.commit()
|
|
print(f"Table {table} created!\n")
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
def log_in(self):
|
|
|
|
uc_client_id = self.__get_parameter("uc_client_id", self.secrets_file)
|
|
uc_client_secret = self.__get_parameter("uc_client_secret", self.secrets_file)
|
|
uc_access_token = self.__get_parameter("uc_access_token", self.secrets_file)
|
|
|
|
self.mastodon_hostname = self.__get_parameter("mastodon_hostname", self.config_file)
|
|
|
|
mastodon = Mastodon(
|
|
client_id = uc_client_id,
|
|
client_secret = uc_client_secret,
|
|
access_token = uc_access_token,
|
|
api_base_url = 'https://' + self.mastodon_hostname,
|
|
)
|
|
|
|
headers={ 'Authorization': 'Bearer %s'%uc_access_token }
|
|
|
|
return (mastodon, self.mastodon_hostname, headers)
|
|
|
|
@staticmethod
|
|
def __get_parameter(parameter, file_path ):
|
|
|
|
with open( file_path ) as f:
|
|
for line in f:
|
|
if line.startswith( parameter ):
|
|
return line.replace(parameter + ":", "").strip()
|
|
|
|
print(f'{file_path} Missing parameter {parameter}')
|
|
sys.exit(0)
|
|
|
|
###############################################################################
|
|
# main
|
|
|
|
if __name__ == '__main__':
|
|
|
|
spamcheck = Spamcheck()
|
|
|
|
mastodon, mastodon_hostname, headers = spamcheck.log_in()
|
|
|
|
created_at_lst, id_lst, email_lst, ip_lst = spamcheck.new_registers()
|
|
|
|
spamcheck.save_registers(created_at_lst, id_lst, email_lst, ip_lst)
|
|
|
|
spamcheck_datetime_lst, spamcheck_registers_lst = spamcheck.get_totals()
|
|
|
|
spamcheck.write_totals(spamcheck_datetime_lst, spamcheck_registers_lst)
|
|
|
|
tor_list = spamcheck.get_tor()
|
|
|
|
print(f'\nTor IPs: {len(tor_list)}')
|
|
|
|
blocked_list = spamcheck.blocked_ip_list()
|
|
|
|
print(f'\nalready blocked IPs: {len(blocked_list)}')
|
|
|
|
for ip in tor_list:
|
|
|
|
is_blocked = spamcheck.ip_blocked(ip, blocked_list)
|
|
|
|
if not is_blocked:
|
|
|
|
print(f'\n** blocking IP {ip}')
|
|
|
|
severity = 'sign_up_requires_approval'
|
|
comment = 'Tor exit node'
|
|
expires_in = 86400
|
|
|
|
result = mastodon.admin_ip_blocks_create(ip, severity, comment, expires_in)
|
|
|
|
blocked_list.append(result)
|
|
|
|
print(f'\nalready blocked IPs: {len(blocked_list)}')
|
|
|
|
print(f'\nRate limit: {mastodon.ratelimit_remaining} of {mastodon.ratelimit_limit}\nNext reset: {datetime.fromtimestamp(mastodon.ratelimit_reset).isoformat()}\n')
|
|
|
|
if mastodon.ratelimit_remaining < 10:
|
|
|
|
print(f'Rate limit is lower than 10! Sleeping by 5 minutes...')
|
|
time.sleep(300)
|
|
|
|
time.sleep(1)
|