spamcheck/spamcheck.py

from datetime import date, datetime, timedelta
from mastodon import Mastodon
import time
import os
import json
import sys
import os.path
import operator
import psycopg2
from langdetect import detect
import requests
import pdb

def write_totals(spamcheck_datetime_lst, spamcheck_registers_lst):

    insert_sql = 'INSERT INTO totals(datetime, registers) VALUES(%s,%s) ON CONFLICT (datetime) DO UPDATE SET (datetime, registers) = (EXCLUDED.datetime, EXCLUDED.registers)'

    first_date = spamcheck_datetime_lst[0]

    last_date = spamcheck_datetime_lst[len(spamcheck_datetime_lst)-1]

    i = 0

    while i < len(spamcheck_datetime_lst):

        conn = None

        try:

            conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")

            cur = conn.cursor()

            if first_date == spamcheck_datetime_lst[i]:

                cur.execute(insert_sql, (spamcheck_datetime_lst[i], spamcheck_registers_lst[i]))

                i += 1

            else:

                cur.execute(insert_sql, (first_date, '0'))

            conn.commit()

            cur.close()

        except (Exception, psycopg2.DatabaseError) as error:

            print(error)

        finally:

            if conn is not None:

                conn.close()

        first_date = first_date + timedelta(days=1)

    if date.today() == last_date + timedelta(days=1):

        insert_sql = 'INSERT INTO totals(datetime, registers) VALUES(%s,%s) ON CONFLICT (datetime) DO UPDATE SET (datetime, registers) = (EXCLUDED.datetime, EXCLUDED.registers)'

        conn = None

        try:

            conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")

            cur = conn.cursor()

            cur.execute(insert_sql, (date.today(), '0'))

            conn.commit()

            cur.close()

        except (Exception, psycopg2.DatabaseError) as error:

            print(error)

        finally:

            if conn is not None:

                conn.close()

def get_totals():

    spamcheck_datetime_lst = []

    spamcheck_registers_lst = []

    select_sql = 'select date(created_at), count(ip) as registers from spamcheck group by date(created_at) order by date(created_at)'

    conn = None

    try:

        conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")

        cur = conn.cursor()

        cur.execute(select_sql)

        rows = cur.fetchall()

        for row in rows:

            spamcheck_datetime_lst.append(row[0])

            spamcheck_registers_lst.append(row[1])

        cur.close()

    except (Exception, psycopg2.DatabaseError) as error:

        print(error)

    finally:

        if conn is not None:

            conn.close()

    return (spamcheck_datetime_lst, spamcheck_registers_lst)

def check_ip(ip):

    is_tor_exit_node = 'f'

    conn = None

    try:

        conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")

        cur = conn.cursor()

        cur.execute('select ip from torexit_ips where ip=(%s)', (ip,))

        row = cur.fetchone()

        if row != None:

            is_tor_exit_node = 't'

        cur.close()

    except (Exception, psycopg2.DatabaseError) as error:

        print(error)

    finally:

        if conn is not None:

            conn.close()

    return is_tor_exit_node

def check_approval(user_id):

    approved = False

    try:

        conn = None

        conn = psycopg2.connect(database = mastodon_db, user = mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432")

        cur = conn.cursor()

        cur.execute("select approved from users where id = (%s)", (user_id,))

        row = cur.fetchone()

        if row != None:

            approved = row[0]

        cur.close()

        return approved

    except (Exception, psycopg2.DatabaseError) as error:

        print (error)

    finally:

        if conn is not None:

            conn.close()

def mastodon():

    # Load secrets from secrets file
    secrets_filepath = "secrets/secrets.txt"
    uc_client_id = get_parameter("uc_client_id",     secrets_filepath)
    uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
    uc_access_token = get_parameter("uc_access_token",  secrets_filepath)

    # Load configuration from config file
    config_filepath = "config/config.txt"
    mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)

    # Initialise Mastodon API
    mastodon = Mastodon(
        client_id=uc_client_id,
        client_secret=uc_client_secret,
        access_token=uc_access_token,
        api_base_url='https://' + mastodon_hostname,
    )

    # Initialise access headers
    headers = {'Authorization': 'Bearer %s'%uc_access_token}

    return (mastodon, mastodon_hostname)

def db_config():

    # Load db configuration from config file
    config_filepath = "config/db_config.txt"
    mastodon_db = get_parameter("mastodon_db", config_filepath)
    mastodon_db_user = get_parameter("mastodon_db_user", config_filepath)
    spamcheck_db = get_parameter("spamcheck_db", config_filepath)
    spamcheck_db_user = get_parameter("spamcheck_db_user", config_filepath)

    return (mastodon_db, mastodon_db_user, spamcheck_db, spamcheck_db_user)

# Returns the parameter from the specified file
def get_parameter( parameter, file_path ):
    # Check if secrets file exists
    if not os.path.isfile(file_path):
        print("File %s not found, exiting."%file_path)
        sys.exit(0)

    # Find parameter in file
    with open( file_path ) as f:
        for line in f:
            if line.startswith( parameter ):
                return line.replace(parameter + ":", "").strip()

    # Cannot find parameter, exit
    print(file_path + "  Missing parameter %s "%parameter)
    sys.exit(0)

###############################################################################
# main

if __name__ == '__main__':

    mastodon_db, mastodon_db_user, spamcheck_db, spamcheck_db_user = db_config()

    mastodon, mastodon_hostname = mastodon()

    ###############################################################################
    # check new registering
    ###############################################################################
    
    created_at_lst = []

    id_lst = []

    email_lst = []

    ip_lst = []

    text_lst = []
    
    try:

        conn = None

        conn = psycopg2.connect(database = mastodon_db, user = mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432")

        cur = conn.cursor()

        cur.execute("select users.created_at, users.id, users.email, users.sign_up_ip, user_invite_requests.text from users, user_invite_requests where not users.approved and users.id = user_invite_requests.user_id")

        rows = cur.fetchall()

        for row in rows:

            if row != None:

                created_at_lst.append(row[0])

                id_lst.append(row[1])

                email_lst.append(row[2])

                ip_lst.append(row[3])

                text_lst.append(row[4])

        cur.close()

    except (Exception, psycopg2.DatabaseError) as error:

        print (error)

    finally:

        if conn is not None:

            conn.close()

    ###############################################################################

    insert_sql = 'INSERT INTO spamcheck(created_at, id, email, ip, text, tor_exit_node) VALUES(%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING'
    
    i = 0

    while i < len(id_lst):

        if len(text_lst[i]) > 1:

            text_lang = detect(text_lst[i])

        else:

            text_lang = 'en'

        if text_lang != 'ca' or len(text_lst) == 1:

            is_tor_exit_node = check_ip(ip_lst[i])

            tor_exit_node = 't' if is_tor_exit_node == 't' else 'f'

            text_length = 200 if len(text_lst[i]) > 200 else len(text_lst[i])

            reason_text = '{0}...'.format(text_lst[i][:text_length].rsplit(' ', 1)[0])

            conn = None

            try:

                conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")

                cur = conn.cursor()

                cur.execute(insert_sql, (created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i], reason_text, tor_exit_node))

                conn.commit()

                cur.close()

            except (Exception, psycopg2.DatabaseError) as error:

                print(error)

            finally:

                if conn is not None:

                    conn.close()

            print(created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i], text_lst[i], tor_exit_node)

        i = i + 1

        ##########################################################################

        approved_users_id_lst = []

        conn = None

        try:

            conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")

            cur = conn.cursor()

            cur.execute('select id from spamcheck')

            rows = cur.fetchall()

            for row in rows:

                approved_users_id_lst.append(row)

            cur.close()

        except (Exception, psycopg2.DatabaseError) as error:

            print(error)

        finally:

            if conn is not None:

                conn.close()

        for user_id in approved_users_id_lst:

            approved = check_approval(user_id)

            if approved:

                conn = None

                try:

                    conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")

                    cur = conn.cursor()

                    cur.execute('delete from spamcheck where id=(%s)', (user_id,))

                    conn.commit()

                    cur.close()

                except (Exception, psycopg2.DatabaseError) as error:

                    print(error)

                finally:

                    if conn is not None:

                        conn.close()

    spamcheck_datetime_lst, spamcheck_registers_lst = get_totals()

    write_totals(spamcheck_datetime_lst, spamcheck_registers_lst)