spamcheck/spamcheck.py

import datetime
from datetime import date, datetime, timedelta
from mastodon import Mastodon, MastodonNetworkError, MastodonNotFoundError, MastodonInternalServerError
import time
import os
import json
import sys
import os.path
import operator
import psycopg2
from psycopg2 import sql
import requests
from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT

class Spamcheck:

    name = "Spamcheck for Mastodon social server"

    def __init__(self, mastodon_hostname=None, mastodon_db=None, mastodon_db_user=None, spamcheck_db=None, spamcheck_db_user=None):

        self.config_file = 'config/config.txt'
        self.secrets_file = "secrets/secrets.txt"

        is_setup = self.__check_setup(self)

        if is_setup:

            self.mastodon_hostname = self.__get_parameter("mastodon_hostname", self.config_file)
            self.mastodon_db = self.__get_parameter("mastodon_db", self.config_file)
            self.mastodon_db_user = self.__get_parameter("mastodon_db_user", self.config_file)
            self.spamcheck_db = self.__get_parameter("spamcheck_db", self.config_file)
            self.spamcheck_db_user = self.__get_parameter("spamcheck_db_user", self.config_file)
            self.__uc_client_id     = self.__get_parameter("uc_client_id",     self.secrets_file)
            self.__uc_client_secret = self.__get_parameter("uc_client_secret", self.secrets_file)
            self.__uc_access_token  = self.__get_parameter("uc_access_token",  self.secrets_file)

        else:

            self.mastodon_hostname, self.mastodon_db, self.mastodon_db_user, self.spamcheck_db, self.spamcheck_db_user = self.__setup(self)

        db_setup = self.__check_dbsetup(self)

        if not db_setup:

            self.__createdb(self)

    def new_registers(self, created_at_lst=[], id_lst=[], email_lst=[], ip_lst=[]):

        try:

            conn = None

            conn = psycopg2.connect(database = self.mastodon_db, user = self.mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432")

            cur = conn.cursor()

            cur.execute("select users.created_at, users.id, users.email, users.sign_up_ip from users where users.created_at > now() - interval '70 days'")

            rows = cur.fetchall()

            for row in rows:

                if row != None:

                    created_at_lst.append(row[0])

                    id_lst.append(row[1])

                    email_lst.append(row[2])

                    ip_lst.append(row[3])

            cur.close()

        except (Exception, psycopg2.DatabaseError) as error:

            print (error)

        finally:

            if conn is not None:

                conn.close()

        return (created_at_lst, id_lst, email_lst, ip_lst)

    def save_registers(self, created_at_lst, id_lst, email_lst, ip_lst):

        insert_sql = 'INSERT INTO spamcheck(created_at, id, email, ip, tor_exit_node) VALUES(%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING'
        
        i = 0

        while i < len(id_lst):

            is_tor_exit_node = self.__check_ip(self, ip_lst[i])

            tor_exit_node = 't' if is_tor_exit_node == 't' else 'f'

            conn = None

            try:

                conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")

                cur = conn.cursor()

                cur.execute(insert_sql, (created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i], tor_exit_node))

                conn.commit()

                cur.close()

            except (Exception, psycopg2.DatabaseError) as error:

                print(error)

            finally:

                if conn is not None:

                    conn.close()
            
            print(created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i], tor_exit_node)

            i += 1

    def get_totals(self):

        spamcheck_datetime_lst = []

        spamcheck_registers_lst = []

        select_sql = 'select date(created_at), count(ip) as registers from spamcheck group by date(created_at) order by date(created_at)'

        conn = None

        try:

            conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")

            cur = conn.cursor()

            cur.execute(select_sql)

            rows = cur.fetchall()

            for row in rows:

                spamcheck_datetime_lst.append(row[0])

                spamcheck_registers_lst.append(row[1])

            cur.close()

        except (Exception, psycopg2.DatabaseError) as error:

            print(error)

        finally:

            if conn is not None:

                conn.close()

        return (spamcheck_datetime_lst, spamcheck_registers_lst)

    def write_totals(self, spamcheck_datetime_lst, spamcheck_registers_lst):

        insert_sql = 'INSERT INTO totals(datetime, registers) VALUES(%s,%s) ON CONFLICT (datetime) DO UPDATE SET (datetime, registers) = (EXCLUDED.datetime, EXCLUDED.registers)'

        first_date = spamcheck_datetime_lst[0]

        last_date = spamcheck_datetime_lst[len(spamcheck_datetime_lst)-1]

        i = 0

        while i < len(spamcheck_datetime_lst):

            conn = None

            try:

                conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")

                cur = conn.cursor()

                if first_date == spamcheck_datetime_lst[i]:

                    cur.execute(insert_sql, (spamcheck_datetime_lst[i], spamcheck_registers_lst[i]))

                    i += 1

                else:

                    cur.execute(insert_sql, (first_date, '0'))

                conn.commit()

                cur.close()

            except (Exception, psycopg2.DatabaseError) as error:

                print(error)

            finally:

                if conn is not None:

                    conn.close()

            first_date = first_date + timedelta(days=1)

        if date.today() == last_date + timedelta(days=1):

            insert_sql = 'INSERT INTO totals(datetime, registers) VALUES(%s,%s) ON CONFLICT (datetime) DO UPDATE SET (datetime, registers) = (EXCLUDED.datetime, EXCLUDED.registers)'

            conn = None

            try:

                conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")

                cur = conn.cursor()

                cur.execute(insert_sql, (date.today(), '0'))

                conn.commit()

                cur.close()

            except (Exception, psycopg2.DatabaseError) as error:

                print(error)

            finally:

                if conn is not None:

                    conn.close()

    def check_approval(self, user_id):

        approved = False

        try:

            conn = None

            conn = psycopg2.connect(database = self.mastodon_db, user = self.mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432")

            cur = conn.cursor()

            cur.execute("select approved from users where id = (%s)", (user_id,))

            row = cur.fetchone()

            if row != None:

                approved = row[0]

            cur.close()

            return approved

        except (Exception, psycopg2.DatabaseError) as error:

            print (error)

        finally:

            if conn is not None:

                conn.close()

    def get_tor(self):

        tor_list = []

        try:

            conn = None

            conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")

            cur = conn.cursor()

            cur.execute("select ip from spamcheck where tor_exit_node")

            rows = cur.fetchall()

            for row in rows:

                tor_list.append(row[0])

            cur.close()

            return tor_list

        except (Exception, psycopg2.DatabaseError) as error:

            print (error)

        finally:

            if conn is not None:

                conn.close()

    def ip_blocks(self, ip):

        '''
        severity:
           Limit sign-ups : 5000
           Block sign-ups : 5500
           Block access   : 9999
        '''
        data = {
            'ip': ip,
            'severity': 5500,
            'comment': 'Spam from this Tor exit node',
            'expires_in': 8600,
            }
        endpoint = f'https://{self.mastodon_hostname}/api/v1/admin/ip_blocks'

        response = self.api_request('POST', endpoint, data)

        if response.ok:

            data = response.json()

        else:

            pass

    def api_request(self, method, endpoint, data={}):

        session = requests.Session()
        response = None

        try:

            kwargs = dict(data=data)

            response = session.request(method, url = endpoint, headers = headers, **kwargs)

        except Exception as e:

            raise MastodonNetworkError(f"Could not complete request: {e}")

        if response is None:

            raise MastodonIllegalArgumentError("Illegal request.")

        if not response.ok:

                try:
                    if isinstance(response, dict) and 'error' in response:
                        error_msg = response['error']
                    elif isinstance(response, str):
                        error_msg = response
                    else:
                        error_msg = None
                except ValueError:
                    error_msg = None

                if response.status_code == 404:
                    ex_type = MastodonNotFoundError
                    if not error_msg:
                        error_msg = 'Endpoint not found.'
                        # this is for compatibility with older versions
                        # which raised MastodonAPIError('Endpoint not found.')
                        # on any 404
                elif response.status_code == 401:
                    ex_type = MastodonUnauthorizedError
                elif response.status_code == 422:
                    return response
                elif response.status_code == 500:
                    ex_type = MastodonInternalServerError
                elif response.status_code == 502:
                    ex_type = MastodonBadGatewayError
                elif response.status_code == 503:
                    ex_type = MastodonServiceUnavailableError
                elif response.status_code == 504:
                    ex_type = MastodonGatewayTimeoutError
                elif response.status_code >= 500 and \
                    response.status_code <= 511:
                    ex_type = MastodonServerError
                else:
                    ex_type = MastodonAPIError

                raise ex_type(
                    'Mastodon API returned error',
                    response.status_code,
                    response.reason,
                    error_msg)

        else:

            return response

    def blocked_ip_list(self):

        blocked_ip_list = []

        temp_ip_list = {}

        i = 0

        while True:

            if i == 0:

                temp_ip_list[i] = mastodon.admin_ip_blocks_list(limit=200)

                if temp_ip_list[i] == []:

                    return blocked_ip_list

            else:

                temp_ip_list[i] = mastodon.fetch_next(temp_ip_list[i-1]._pagination_next)

            dict_len = len(temp_ip_list[i])

            ii = 0

            while ii - dict_len:

                blocked_ip_list.append(temp_ip_list[i][ii])

                ii += 1

            if len(temp_ip_list[i]) < 200:

                return blocked_ip_list

            i += 1

    def ip_blocked(self, ip, blocked_list):

        is_blocked = False

        for block_ip_item in blocked_list:

            if ip in block_ip_item.ip:

                print(f'\n{ip} is already blocked')

                is_blocked = True

                return (is_blocked)

    @staticmethod
    def __check_ip(self, ip):

        is_tor_exit_node = 'f'

        if ip == None:

            return

        conn = None

        try:

            conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")

            cur = conn.cursor()

            cur.execute('select ip from torexit_ips where ip=(%s)', (ip,))

            row = cur.fetchone()

            if row != None:

                is_tor_exit_node = 't'

            cur.close()

        except (Exception, psycopg2.DatabaseError) as error:

            print(error)

        finally:

            if conn is not None:

                conn.close()

        return is_tor_exit_node

    @staticmethod
    def __check_setup(self):

        is_setup = False

        if not os.path.isfile(self.config_file):
            print(f"File {self.config_file} not found, running setup.\n")
        else:
            is_setup = True

        return is_setup
    
    @staticmethod
    def __setup(self):

        if not os.path.exists('config'):
            os.makedirs('config')

        self.mastodon_hostname = input("Mastodon hostname, in ex. 'mastodon.social': ")
        self.mastodon_db = input("Mastodon's database name: ")
        self.mastodon_db_user = input("Mastodon's database user: ")
        self.spamcheck_db = input("Spamcheck's database name: ")
        self.spamcheck_db_user = input("Spamcheck's database user: ")

        if not os.path.exists(self.config_file):
            with open(self.config_file, 'w'): pass
            print(f"\n{self.config_file} created!\n")

        with open(self.config_file, 'a') as the_file:
            print(f"Writing Mastodon hostname parameter to {self.config_file}")
            the_file.write(f'mastodon_hostname: {self.mastodon_hostname}\n')
            the_file.write(f'mastodon_db: {self.mastodon_db}\n')
            the_file.write(f'mastodon_db_user: {self.mastodon_db_user}\n')
            the_file.write(f'spamcheck_db: {self.spamcheck_db}\n')
            the_file.write(f'spamcheck_db_user: {self.spamcheck_db_user}\n')

        return (self.mastodon_hostname, self.mastodon_db, self.mastodon_db_user, self.spamcheck_db, self.spamcheck_db_user)

    @staticmethod
    def __check_dbsetup(self):

        dbsetup = False

        try:

            conn = None

            conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")

            dbsetup = True

        except (Exception, psycopg2.DatabaseError) as error:

            print(error)

        return dbsetup

    @staticmethod
    def __createdb(self):

        conn = None

        try:

            conn = psycopg2.connect(dbname='postgres',
                user=self.spamcheck_db_user, host='',
                password='')

            conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)

            cur = conn.cursor()

            print(f"Creating database {self.spamcheck_db}. Please wait...")

            cur.execute(sql.SQL("CREATE DATABASE {}").format(
                sql.Identifier(self.spamcheck_db))
            )
            print(f"Database {self.spamcheck_db} created!\n")

            self.__dbtables_schemes(self)

        except (Exception, psycopg2.DatabaseError) as error:

            print(error)

        finally:

            if conn is not None:

                conn.close()

    @staticmethod
    def __dbtables_schemes(self):

        table = "spamcheck"
        sql = "create table "+table+" (created_at timestamptz, id bigint PRIMARY KEY, email varchar(200), ip inet, tor_exit_node boolean)"
        self.__create_table(self, table, sql)

        table = "torexit_ips"
        sql = "create table "+table+" (created_at timestamptz, ip inet PRIMARY KEY)"
        self.__create_table(self, table, sql)

        table = "totals"
        sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, registers int)"
        self.__create_table(self, table, sql)

    @staticmethod
    def __create_table(self, table, sql):

        conn = None

        try:

            conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
            cur = conn.cursor()

            print(f"Creating table {table}")
            cur.execute(sql)

            conn.commit()
            print(f"Table {table} created!\n")

        except (Exception, psycopg2.DatabaseError) as error:

            print(error)

        finally:

            if conn is not None:

                conn.close()

    def log_in(self):

        uc_client_id     = self.__get_parameter("uc_client_id",     self.secrets_file)
        uc_client_secret = self.__get_parameter("uc_client_secret", self.secrets_file)
        uc_access_token  = self.__get_parameter("uc_access_token",  self.secrets_file)

        self.mastodon_hostname = self.__get_parameter("mastodon_hostname", self.config_file)

        mastodon = Mastodon(
            client_id = uc_client_id,
            client_secret = uc_client_secret,
            access_token = uc_access_token,
            api_base_url = 'https://' + self.mastodon_hostname,
        )

        headers={ 'Authorization': 'Bearer %s'%uc_access_token }

        return (mastodon, self.mastodon_hostname, headers)

    @staticmethod
    def __get_parameter(parameter, file_path ):

        with open( file_path ) as f:
            for line in f:
                if line.startswith( parameter ):
                    return line.replace(parameter + ":", "").strip()

        print(f'{file_path} Missing parameter {parameter}')
        sys.exit(0)

###############################################################################
# main

if __name__ == '__main__':

    spamcheck = Spamcheck()

    mastodon, mastodon_hostname, headers = spamcheck.log_in()

    created_at_lst, id_lst, email_lst, ip_lst = spamcheck.new_registers()

    spamcheck.save_registers(created_at_lst, id_lst, email_lst, ip_lst)

    spamcheck_datetime_lst, spamcheck_registers_lst = spamcheck.get_totals()

    spamcheck.write_totals(spamcheck_datetime_lst, spamcheck_registers_lst)

    tor_list = spamcheck.get_tor()

    print(f'\nTor IPs: {len(tor_list)}')

    blocked_list = spamcheck.blocked_ip_list()

    print(f'\nalready blocked IPs: {len(blocked_list)}')

    for ip in tor_list:

        is_blocked = spamcheck.ip_blocked(ip, blocked_list)

        if not is_blocked:

            print(f'\n** blocking IP {ip}')

            severity = 'sign_up_requires_approval'
            comment = 'Tor exit node'
            expires_in = 86400

            result = mastodon.admin_ip_blocks_create(ip, severity, comment, expires_in)

            blocked_list.append(result)

            print(f'\nalready blocked IPs: {len(blocked_list)}')

            print(f'\nRate limit: {mastodon.ratelimit_remaining} of {mastodon.ratelimit_limit}\nNext reset: {datetime.fromtimestamp(mastodon.ratelimit_reset).isoformat()}\n')

            if mastodon.ratelimit_remaining < 10:

                print(f'Rate limit is lower than 10! Sleeping by 5 minutes...')
                time.sleep(300)

            time.sleep(1)