Check and store data from sign-ups registered on your Mastodon server last seven says.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
spamcheck/spamcheck.py

704 lines
19 KiB

import datetime
from datetime import date, datetime, timedelta
from mastodon import Mastodon, MastodonNetworkError, MastodonNotFoundError, MastodonInternalServerError
import time
import os
import json
import sys
import os.path
import operator
import psycopg2
from psycopg2 import sql
import requests
from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
class Spamcheck:
name = "Spamcheck for Mastodon social server"
def __init__(self, mastodon_hostname=None, mastodon_db=None, mastodon_db_user=None, spamcheck_db=None, spamcheck_db_user=None):
self.config_file = 'config/config.txt'
self.secrets_file = "secrets/secrets.txt"
is_setup = self.__check_setup(self)
if is_setup:
self.mastodon_hostname = self.__get_parameter("mastodon_hostname", self.config_file)
self.mastodon_db = self.__get_parameter("mastodon_db", self.config_file)
self.mastodon_db_user = self.__get_parameter("mastodon_db_user", self.config_file)
self.spamcheck_db = self.__get_parameter("spamcheck_db", self.config_file)
self.spamcheck_db_user = self.__get_parameter("spamcheck_db_user", self.config_file)
self.__uc_client_id = self.__get_parameter("uc_client_id", self.secrets_file)
self.__uc_client_secret = self.__get_parameter("uc_client_secret", self.secrets_file)
self.__uc_access_token = self.__get_parameter("uc_access_token", self.secrets_file)
else:
self.mastodon_hostname, self.mastodon_db, self.mastodon_db_user, self.spamcheck_db, self.spamcheck_db_user = self.__setup(self)
db_setup = self.__check_dbsetup(self)
if not db_setup:
self.__createdb(self)
def new_registers(self, created_at_lst=[], id_lst=[], email_lst=[], ip_lst=[]):
try:
conn = None
conn = psycopg2.connect(database = self.mastodon_db, user = self.mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute("select users.created_at, users.id, users.email, users.sign_up_ip from users where users.created_at > now() - interval '70 days'")
rows = cur.fetchall()
for row in rows:
if row != None:
created_at_lst.append(row[0])
id_lst.append(row[1])
email_lst.append(row[2])
ip_lst.append(row[3])
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print (error)
finally:
if conn is not None:
conn.close()
return (created_at_lst, id_lst, email_lst, ip_lst)
def save_registers(self, created_at_lst, id_lst, email_lst, ip_lst):
insert_sql = 'INSERT INTO spamcheck(created_at, id, email, ip, tor_exit_node) VALUES(%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING'
i = 0
while i < len(id_lst):
is_tor_exit_node = self.__check_ip(self, ip_lst[i])
tor_exit_node = 't' if is_tor_exit_node == 't' else 'f'
conn = None
try:
conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute(insert_sql, (created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i], tor_exit_node))
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
print(created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i], tor_exit_node)
i += 1
def get_totals(self):
spamcheck_datetime_lst = []
spamcheck_registers_lst = []
select_sql = 'select date(created_at), count(ip) as registers from spamcheck group by date(created_at) order by date(created_at)'
conn = None
try:
conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute(select_sql)
rows = cur.fetchall()
for row in rows:
spamcheck_datetime_lst.append(row[0])
spamcheck_registers_lst.append(row[1])
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
return (spamcheck_datetime_lst, spamcheck_registers_lst)
def write_totals(self, spamcheck_datetime_lst, spamcheck_registers_lst):
insert_sql = 'INSERT INTO totals(datetime, registers) VALUES(%s,%s) ON CONFLICT (datetime) DO UPDATE SET (datetime, registers) = (EXCLUDED.datetime, EXCLUDED.registers)'
first_date = spamcheck_datetime_lst[0]
last_date = spamcheck_datetime_lst[len(spamcheck_datetime_lst)-1]
i = 0
while i < len(spamcheck_datetime_lst):
conn = None
try:
conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
if first_date == spamcheck_datetime_lst[i]:
cur.execute(insert_sql, (spamcheck_datetime_lst[i], spamcheck_registers_lst[i]))
i += 1
else:
cur.execute(insert_sql, (first_date, '0'))
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
first_date = first_date + timedelta(days=1)
if date.today() == last_date + timedelta(days=1):
insert_sql = 'INSERT INTO totals(datetime, registers) VALUES(%s,%s) ON CONFLICT (datetime) DO UPDATE SET (datetime, registers) = (EXCLUDED.datetime, EXCLUDED.registers)'
conn = None
try:
conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute(insert_sql, (date.today(), '0'))
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
def check_approval(self, user_id):
approved = False
try:
conn = None
conn = psycopg2.connect(database = self.mastodon_db, user = self.mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute("select approved from users where id = (%s)", (user_id,))
row = cur.fetchone()
if row != None:
approved = row[0]
cur.close()
return approved
except (Exception, psycopg2.DatabaseError) as error:
print (error)
finally:
if conn is not None:
conn.close()
def get_tor(self):
tor_list = []
try:
conn = None
conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute("select ip from spamcheck where tor_exit_node")
rows = cur.fetchall()
for row in rows:
tor_list.append(row[0])
cur.close()
return tor_list
except (Exception, psycopg2.DatabaseError) as error:
print (error)
finally:
if conn is not None:
conn.close()
def ip_blocks(self, ip):
'''
severity:
Limit sign-ups : 5000
Block sign-ups : 5500
Block access : 9999
'''
data = {
'ip': ip,
'severity': 5500,
'comment': 'Spam from this Tor exit node',
'expires_in': 8600,
}
endpoint = f'https://{self.mastodon_hostname}/api/v1/admin/ip_blocks'
response = self.api_request('POST', endpoint, data)
if response.ok:
data = response.json()
else:
pass
def api_request(self, method, endpoint, data={}):
session = requests.Session()
response = None
try:
kwargs = dict(data=data)
response = session.request(method, url = endpoint, headers = headers, **kwargs)
except Exception as e:
raise MastodonNetworkError(f"Could not complete request: {e}")
if response is None:
raise MastodonIllegalArgumentError("Illegal request.")
if not response.ok:
try:
if isinstance(response, dict) and 'error' in response:
error_msg = response['error']
elif isinstance(response, str):
error_msg = response
else:
error_msg = None
except ValueError:
error_msg = None
if response.status_code == 404:
ex_type = MastodonNotFoundError
if not error_msg:
error_msg = 'Endpoint not found.'
# this is for compatibility with older versions
# which raised MastodonAPIError('Endpoint not found.')
# on any 404
elif response.status_code == 401:
ex_type = MastodonUnauthorizedError
elif response.status_code == 422:
return response
elif response.status_code == 500:
ex_type = MastodonInternalServerError
elif response.status_code == 502:
ex_type = MastodonBadGatewayError
elif response.status_code == 503:
ex_type = MastodonServiceUnavailableError
elif response.status_code == 504:
ex_type = MastodonGatewayTimeoutError
elif response.status_code >= 500 and \
response.status_code <= 511:
ex_type = MastodonServerError
else:
ex_type = MastodonAPIError
raise ex_type(
'Mastodon API returned error',
response.status_code,
response.reason,
error_msg)
else:
return response
def blocked_ip_list(self):
blocked_ip_list = []
temp_ip_list = {}
i = 0
while True:
if i == 0:
temp_ip_list[i] = mastodon.admin_ip_blocks_list(limit=200)
if temp_ip_list[i] == []:
return blocked_ip_list
else:
temp_ip_list[i] = mastodon.fetch_next(temp_ip_list[i-1]._pagination_next)
dict_len = len(temp_ip_list[i])
ii = 0
while ii - dict_len:
blocked_ip_list.append(temp_ip_list[i][ii])
ii += 1
if len(temp_ip_list[i]) < 200:
return blocked_ip_list
i += 1
def ip_blocked(self, ip, blocked_list):
is_blocked = False
for block_ip_item in blocked_list:
if ip in block_ip_item.ip:
print(f'\n{ip} is already blocked')
is_blocked = True
return (is_blocked)
@staticmethod
def __check_ip(self, ip):
is_tor_exit_node = 'f'
if ip == None:
return
conn = None
try:
conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute('select ip from torexit_ips where ip=(%s)', (ip,))
row = cur.fetchone()
if row != None:
is_tor_exit_node = 't'
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
return is_tor_exit_node
@staticmethod
def __check_setup(self):
is_setup = False
if not os.path.isfile(self.config_file):
print(f"File {self.config_file} not found, running setup.\n")
else:
is_setup = True
return is_setup
@staticmethod
def __setup(self):
if not os.path.exists('config'):
os.makedirs('config')
self.mastodon_hostname = input("Mastodon hostname, in ex. 'mastodon.social': ")
self.mastodon_db = input("Mastodon's database name: ")
self.mastodon_db_user = input("Mastodon's database user: ")
self.spamcheck_db = input("Spamcheck's database name: ")
self.spamcheck_db_user = input("Spamcheck's database user: ")
if not os.path.exists(self.config_file):
with open(self.config_file, 'w'): pass
print(f"\n{self.config_file} created!\n")
with open(self.config_file, 'a') as the_file:
print(f"Writing Mastodon hostname parameter to {self.config_file}")
the_file.write(f'mastodon_hostname: {self.mastodon_hostname}\n')
the_file.write(f'mastodon_db: {self.mastodon_db}\n')
the_file.write(f'mastodon_db_user: {self.mastodon_db_user}\n')
the_file.write(f'spamcheck_db: {self.spamcheck_db}\n')
the_file.write(f'spamcheck_db_user: {self.spamcheck_db_user}\n')
return (self.mastodon_hostname, self.mastodon_db, self.mastodon_db_user, self.spamcheck_db, self.spamcheck_db_user)
@staticmethod
def __check_dbsetup(self):
dbsetup = False
try:
conn = None
conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
dbsetup = True
except (Exception, psycopg2.DatabaseError) as error:
print(error)
return dbsetup
@staticmethod
def __createdb(self):
conn = None
try:
conn = psycopg2.connect(dbname='postgres',
user=self.spamcheck_db_user, host='',
password='')
conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
cur = conn.cursor()
print(f"Creating database {self.spamcheck_db}. Please wait...")
cur.execute(sql.SQL("CREATE DATABASE {}").format(
sql.Identifier(self.spamcheck_db))
)
print(f"Database {self.spamcheck_db} created!\n")
self.__dbtables_schemes(self)
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
@staticmethod
def __dbtables_schemes(self):
table = "spamcheck"
sql = "create table "+table+" (created_at timestamptz, id bigint PRIMARY KEY, email varchar(200), ip inet, tor_exit_node boolean)"
self.__create_table(self, table, sql)
table = "torexit_ips"
sql = "create table "+table+" (created_at timestamptz, ip inet PRIMARY KEY)"
self.__create_table(self, table, sql)
table = "totals"
sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, registers int)"
self.__create_table(self, table, sql)
@staticmethod
def __create_table(self, table, sql):
conn = None
try:
conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
print(f"Creating table {table}")
cur.execute(sql)
conn.commit()
print(f"Table {table} created!\n")
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
def log_in(self):
uc_client_id = self.__get_parameter("uc_client_id", self.secrets_file)
uc_client_secret = self.__get_parameter("uc_client_secret", self.secrets_file)
uc_access_token = self.__get_parameter("uc_access_token", self.secrets_file)
self.mastodon_hostname = self.__get_parameter("mastodon_hostname", self.config_file)
mastodon = Mastodon(
client_id = uc_client_id,
client_secret = uc_client_secret,
access_token = uc_access_token,
api_base_url = 'https://' + self.mastodon_hostname,
)
headers={ 'Authorization': 'Bearer %s'%uc_access_token }
return (mastodon, self.mastodon_hostname, headers)
@staticmethod
def __get_parameter(parameter, file_path ):
with open( file_path ) as f:
for line in f:
if line.startswith( parameter ):
return line.replace(parameter + ":", "").strip()
print(f'{file_path} Missing parameter {parameter}')
sys.exit(0)
###############################################################################
# main
if __name__ == '__main__':
spamcheck = Spamcheck()
mastodon, mastodon_hostname, headers = spamcheck.log_in()
created_at_lst, id_lst, email_lst, ip_lst = spamcheck.new_registers()
spamcheck.save_registers(created_at_lst, id_lst, email_lst, ip_lst)
spamcheck_datetime_lst, spamcheck_registers_lst = spamcheck.get_totals()
spamcheck.write_totals(spamcheck_datetime_lst, spamcheck_registers_lst)
tor_list = spamcheck.get_tor()
print(f'\nTor IPs: {len(tor_list)}')
blocked_list = spamcheck.blocked_ip_list()
print(f'\nalready blocked IPs: {len(blocked_list)}')
for ip in tor_list:
is_blocked = spamcheck.ip_blocked(ip, blocked_list)
if not is_blocked:
print(f'\n** blocking IP {ip}')
severity = 'sign_up_requires_approval'
comment = 'Tor exit node'
expires_in = 86400
result = mastodon.admin_ip_blocks_create(ip, severity, comment, expires_in)
blocked_list.append(result)
print(f'\nalready blocked IPs: {len(blocked_list)}')
print(f'\nRate limit: {mastodon.ratelimit_remaining} of {mastodon.ratelimit_limit}\nNext reset: {datetime.fromtimestamp(mastodon.ratelimit_reset).isoformat()}\n')
if mastodon.ratelimit_remaining < 10:
print(f'Rate limit is lower than 10! Sleeping by 5 minutes...')
time.sleep(300)
time.sleep(1)