diff --git a/README.md b/README.md index 5f15f88..0dcb646 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,6 @@ # Mastodon's spamcheck -Check and store data from spam bots trying to register to your Mastodon server. -You need to configure your Mastodon server like this: - -- Registration mode -> Approval required for sign up -- Require new users to enter a reason to join -> activated +Check and store data from sign-ups registered on your Mastodon server. ### Dependencies @@ -18,9 +14,9 @@ Within Python Virtual Environment: 1. Run `pip install -r requirements.txt` to install needed Python libraries. -2. Run `python db-setup.py` to setup and create new Postgresql database and needed tables in it. +2. Run `python torips.py` to write Tor exit nodes IPs to new created database. You need to get the torbulkexitlist from [here](https://check.torproject.org/exit-addresses) -3. Run `python torips.py` to write Tor exit nodes IPs to database. You need to get the torbulkexitlist from [here](https://blog.torproject.org/changes-tor-exit-list-service/) +3. Run `python spamcheck.py` for the first time to configure it. 4. Use your favourite scheduling method to set `python spamcheck.py` and `python torips.py` to run regularly. diff --git a/db-setup.py b/db-setup.py deleted file mode 100644 index 1d5dc89..0000000 --- a/db-setup.py +++ /dev/null @@ -1,170 +0,0 @@ -#import getpass -import os -import sys -import psycopg2 -from psycopg2 import sql -from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT - -# Returns the parameter from the specified file -def get_parameter( parameter, file_path ): - # Check if secrets file exists - if not os.path.isfile(file_path): - print("File %s not found, asking."%file_path) - write_parameter( parameter, file_path ) - #sys.exit(0) - - # Find parameter in file - with open( file_path ) as f: - for line in f: - if line.startswith( parameter ): - return line.replace(parameter + ":", "").strip() - - # Cannot find parameter, exit - print(file_path + " Missing parameter %s "%parameter) - sys.exit(0) - -def write_parameter( parameter, file_path ): - if not os.path.exists('config'): - os.makedirs('config') - print("Setting up spamcheck parameters...") - print("\n") - spamcheck_db = input("spamcheck db name: ") - spamcheck_db_user = input("spamcheck db user: ") - mastodon_db = input("Mastodon db name: ") - mastodon_db_user = input("Mastodon db user: ") - - - with open(file_path, "w") as text_file: - print("spamcheck_db: {}".format(spamcheck_db), file=text_file) - print("spamcheck_db_user: {}".format(spamcheck_db_user), file=text_file) - print("mastodon_db: {}".format(mastodon_db), file=text_file) - print("mastodon_db_user: {}".format(mastodon_db_user), file=text_file) - -def create_table(db, db_user, table, sql): - - conn = None - - try: - - conn = psycopg2.connect(database = db, user = db_user, password = "", host = "/var/run/postgresql", port = "5432") - cur = conn.cursor() - - print("Creating table.. "+table) - # Create the table in PostgreSQL database - cur.execute(sql) - - conn.commit() - print("Table "+table+" created!") - print("\n") - - except (Exception, psycopg2.DatabaseError) as error: - - print(error) - - finally: - - if conn is not None: - - conn.close() - -############################################################################### -# main - -if __name__ == '__main__': - - # Load configuration from config file - config_filepath = "config/db_config.txt" - spamcheck_db = get_parameter("spamcheck_db", config_filepath) - spamcheck_db_user = get_parameter("spamcheck_db_user", config_filepath) - mastodon_db = get_parameter("mastodon_db", config_filepath) - mastodon_db_user = get_parameter("mastodon_db_user", config_filepath) - - ############################################################ - # create database - ############################################################ - - conn = None - - try: - - conn = psycopg2.connect(dbname='postgres', - user=spamcheck_db_user, host='', - password='') - - conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) - - cur = conn.cursor() - - print("Creating database " + spamcheck_db + ". Please wait...") - - cur.execute(sql.SQL("CREATE DATABASE {}").format( - sql.Identifier(spamcheck_db)) - ) - print("Database " + spamcheck_db + " created!") - - except (Exception, psycopg2.DatabaseError) as error: - - print(error) - - finally: - - if conn is not None: - - conn.close() - - ############################################################ - - try: - - conn = None - - conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432") - - except (Exception, psycopg2.DatabaseError) as error: - - print(error) - - # Load configuration from config file - os.remove("config/db_config.txt") - - print("Exiting. Run db-setup again with right parameters") - sys.exit(0) - - finally: - - if conn is not None: - - conn.close() - - print("\n") - print("spamcheck parameters saved to db-config.txt!") - print("\n") - - ############################################################ - # Create needed tables - ############################################################ - - print("Creating table...") - - db = spamcheck_db - db_user = spamcheck_db_user - - table = "spamcheck" - sql = "create table "+table+" (created_at timestamptz, id bigint PRIMARY KEY, email varchar(200), ip inet, text varchar(200), tor_exit_node boolean)" - create_table(db, db_user, table, sql) - - table = "torexit_ips" - sql = "create table "+table+" (created_at timestamptz, ip inet PRIMARY KEY)" - create_table(db, db_user, table, sql) - - table = "totals" - sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, registers int)" - create_table(db, db_user, table, sql) - - - - ############################################################ - - print("Done!") - print("Now you can run setup.py!") - print("\n") diff --git a/requirements.txt b/requirements.txt index 64b2eb5..b24e524 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,2 @@ -Mastodon.py>=1.5.1 -wheel>=0.36.2 -psycopg2>=2.8.6 +psycopg2-binary +requests diff --git a/spamcheck.py b/spamcheck.py index 84666a2..15ec037 100644 --- a/spamcheck.py +++ b/spamcheck.py @@ -1,5 +1,5 @@ from datetime import date, datetime, timedelta -from mastodon import Mastodon +#from mastodon import Mastodon import time import os import json @@ -7,341 +7,186 @@ import sys import os.path import operator import psycopg2 -from langdetect import detect -import requests -import pdb +from psycopg2 import sql +from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT -def write_totals(spamcheck_datetime_lst, spamcheck_registers_lst): +class Spamcheck: - insert_sql = 'INSERT INTO totals(datetime, registers) VALUES(%s,%s) ON CONFLICT (datetime) DO UPDATE SET (datetime, registers) = (EXCLUDED.datetime, EXCLUDED.registers)' + name = "Spamcheck for Mastodon social servers" - first_date = spamcheck_datetime_lst[0] + def __init__(self, mastodon_hostname=None, mastodon_db=None, mastodon_db_user=None, spamcheck_db=None, spamcheck_db_user=None): - last_date = spamcheck_datetime_lst[len(spamcheck_datetime_lst)-1] + self.config_file = 'config/config.txt' - i = 0 + is_setup = self.__check_setup(self) - while i < len(spamcheck_datetime_lst): - - conn = None - - try: - - conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432") - - cur = conn.cursor() - - if first_date == spamcheck_datetime_lst[i]: - - cur.execute(insert_sql, (spamcheck_datetime_lst[i], spamcheck_registers_lst[i])) - - i += 1 - - else: - - cur.execute(insert_sql, (first_date, '0')) - - conn.commit() - - cur.close() - - except (Exception, psycopg2.DatabaseError) as error: - - print(error) - - finally: - - if conn is not None: - - conn.close() - - first_date = first_date + timedelta(days=1) - - if date.today() == last_date + timedelta(days=1): - - insert_sql = 'INSERT INTO totals(datetime, registers) VALUES(%s,%s) ON CONFLICT (datetime) DO UPDATE SET (datetime, registers) = (EXCLUDED.datetime, EXCLUDED.registers)' - - conn = None - - try: - - conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432") - - cur = conn.cursor() - - cur.execute(insert_sql, (date.today(), '0')) - - conn.commit() - - cur.close() - - except (Exception, psycopg2.DatabaseError) as error: - - print(error) - - finally: - - if conn is not None: - - conn.close() - -def get_totals(): - - spamcheck_datetime_lst = [] - - spamcheck_registers_lst = [] - - select_sql = 'select date(created_at), count(ip) as registers from spamcheck group by date(created_at) order by date(created_at)' - - conn = None - - try: - - conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432") - - cur = conn.cursor() - - cur.execute(select_sql) - - rows = cur.fetchall() - - for row in rows: - - spamcheck_datetime_lst.append(row[0]) - - spamcheck_registers_lst.append(row[1]) - - cur.close() - - except (Exception, psycopg2.DatabaseError) as error: - - print(error) - - finally: - - if conn is not None: - - conn.close() - - return (spamcheck_datetime_lst, spamcheck_registers_lst) - -def check_ip(ip): - - is_tor_exit_node = 'f' - - conn = None - - try: - - conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432") - - cur = conn.cursor() - - cur.execute('select ip from torexit_ips where ip=(%s)', (ip,)) - - row = cur.fetchone() - - if row != None: - - is_tor_exit_node = 't' - - cur.close() - - except (Exception, psycopg2.DatabaseError) as error: - - print(error) - - finally: - - if conn is not None: - - conn.close() - - return is_tor_exit_node - -def check_approval(user_id): - - approved = False - - try: - - conn = None - - conn = psycopg2.connect(database = mastodon_db, user = mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432") - - cur = conn.cursor() - - cur.execute("select approved from users where id = (%s)", (user_id,)) - - row = cur.fetchone() - - if row != None: - - approved = row[0] - - cur.close() - - return approved - - except (Exception, psycopg2.DatabaseError) as error: - - print (error) - - finally: - - if conn is not None: - - conn.close() - -def mastodon(): - - # Load secrets from secrets file - secrets_filepath = "secrets/secrets.txt" - uc_client_id = get_parameter("uc_client_id", secrets_filepath) - uc_client_secret = get_parameter("uc_client_secret", secrets_filepath) - uc_access_token = get_parameter("uc_access_token", secrets_filepath) - - # Load configuration from config file - config_filepath = "config/config.txt" - mastodon_hostname = get_parameter("mastodon_hostname", config_filepath) - - # Initialise Mastodon API - mastodon = Mastodon( - client_id=uc_client_id, - client_secret=uc_client_secret, - access_token=uc_access_token, - api_base_url='https://' + mastodon_hostname, - ) - - # Initialise access headers - headers = {'Authorization': 'Bearer %s'%uc_access_token} - - return (mastodon, mastodon_hostname) - -def db_config(): - - # Load db configuration from config file - config_filepath = "config/db_config.txt" - mastodon_db = get_parameter("mastodon_db", config_filepath) - mastodon_db_user = get_parameter("mastodon_db_user", config_filepath) - spamcheck_db = get_parameter("spamcheck_db", config_filepath) - spamcheck_db_user = get_parameter("spamcheck_db_user", config_filepath) - - return (mastodon_db, mastodon_db_user, spamcheck_db, spamcheck_db_user) - -# Returns the parameter from the specified file -def get_parameter( parameter, file_path ): - # Check if secrets file exists - if not os.path.isfile(file_path): - print("File %s not found, exiting."%file_path) - sys.exit(0) - - # Find parameter in file - with open( file_path ) as f: - for line in f: - if line.startswith( parameter ): - return line.replace(parameter + ":", "").strip() - - # Cannot find parameter, exit - print(file_path + " Missing parameter %s "%parameter) - sys.exit(0) - -############################################################################### -# main - -if __name__ == '__main__': - - mastodon_db, mastodon_db_user, spamcheck_db, spamcheck_db_user = db_config() - - mastodon, mastodon_hostname = mastodon() - - ############################################################################### - # check new registering - ############################################################################### - - created_at_lst = [] - - id_lst = [] - - email_lst = [] - - ip_lst = [] - - text_lst = [] - - try: - - conn = None - - conn = psycopg2.connect(database = mastodon_db, user = mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432") - - cur = conn.cursor() - - cur.execute("select users.created_at, users.id, users.email, users.sign_up_ip, user_invite_requests.text from users, user_invite_requests where not users.approved and users.id = user_invite_requests.user_id") - - rows = cur.fetchall() - - for row in rows: - - if row != None: - - created_at_lst.append(row[0]) - - id_lst.append(row[1]) - - email_lst.append(row[2]) - - ip_lst.append(row[3]) - - text_lst.append(row[4]) - - cur.close() - - except (Exception, psycopg2.DatabaseError) as error: - - print (error) - - finally: - - if conn is not None: - - conn.close() - - ############################################################################### - - insert_sql = 'INSERT INTO spamcheck(created_at, id, email, ip, text, tor_exit_node) VALUES(%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING' - - i = 0 - - while i < len(id_lst): - - if len(text_lst[i]) > 1: - - text_lang = detect(text_lst[i]) + if is_setup: + self.mastodon_hostname = self.__get_parameter("mastodon_hostname", self.config_file) + self.mastodon_db = self.__get_parameter("mastodon_db", self.config_file) + self.mastodon_db_user = self.__get_parameter("mastodon_db_user", self.config_file) + self.spamcheck_db = self.__get_parameter("spamcheck_db", self.config_file) + self.spamcheck_db_user = self.__get_parameter("spamcheck_db_user", self.config_file) + else: - text_lang = 'en' + self.mastodon_hostname, self.mastodon_db, self.mastodon_db_user, self.spamcheck_db, self.spamcheck_db_user = self.__setup(self) - if text_lang != 'ca' or len(text_lst) == 1: + db_setup = self.__check_dbsetup(self) - is_tor_exit_node = check_ip(ip_lst[i]) + if not db_setup: + + self.__createdb(self) + + def new_registers(self, created_at_lst=[], id_lst=[], email_lst=[], ip_lst=[]): + + try: + + conn = None + + conn = psycopg2.connect(database = self.mastodon_db, user = self.mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432") + + cur = conn.cursor() + + cur.execute("select users.created_at, users.id, users.email, users.sign_up_ip from users where users.approved and users.created_at > now() - interval '7 days'") + + rows = cur.fetchall() + + for row in rows: + + if row != None: + + created_at_lst.append(row[0]) + + id_lst.append(row[1]) + + email_lst.append(row[2]) + + ip_lst.append(row[3]) + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print (error) + + finally: + + if conn is not None: + + conn.close() + + return (created_at_lst, id_lst, email_lst, ip_lst) + + def save_registers(self, created_at_lst, id_lst, email_lst, ip_lst): + + insert_sql = 'INSERT INTO spamcheck(created_at, id, email, ip, tor_exit_node) VALUES(%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING' + + i = 0 + + while i < len(id_lst): + + is_tor_exit_node = self.__check_ip(self, ip_lst[i]) tor_exit_node = 't' if is_tor_exit_node == 't' else 'f' - text_length = 200 if len(text_lst[i]) > 200 else len(text_lst[i]) - - reason_text = '{0}...'.format(text_lst[i][:text_length].rsplit(' ', 1)[0]) - conn = None try: - conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432") + conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432") cur = conn.cursor() - cur.execute(insert_sql, (created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i], reason_text, tor_exit_node)) + cur.execute(insert_sql, (created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i], tor_exit_node)) + + conn.commit() + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + print(created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i], tor_exit_node) + + i += 1 + + def get_totals(self): + + spamcheck_datetime_lst = [] + + spamcheck_registers_lst = [] + + select_sql = 'select date(created_at), count(ip) as registers from spamcheck group by date(created_at) order by date(created_at)' + + conn = None + + try: + + conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432") + + cur = conn.cursor() + + cur.execute(select_sql) + + rows = cur.fetchall() + + for row in rows: + + spamcheck_datetime_lst.append(row[0]) + + spamcheck_registers_lst.append(row[1]) + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + return (spamcheck_datetime_lst, spamcheck_registers_lst) + + def write_totals(self, spamcheck_datetime_lst, spamcheck_registers_lst): + + insert_sql = 'INSERT INTO totals(datetime, registers) VALUES(%s,%s) ON CONFLICT (datetime) DO UPDATE SET (datetime, registers) = (EXCLUDED.datetime, EXCLUDED.registers)' + + first_date = spamcheck_datetime_lst[0] + + last_date = spamcheck_datetime_lst[len(spamcheck_datetime_lst)-1] + + i = 0 + + while i < len(spamcheck_datetime_lst): + + conn = None + + try: + + conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432") + + cur = conn.cursor() + + if first_date == spamcheck_datetime_lst[i]: + + cur.execute(insert_sql, (spamcheck_datetime_lst[i], spamcheck_registers_lst[i])) + + i += 1 + + else: + + cur.execute(insert_sql, (first_date, '0')) conn.commit() @@ -357,29 +202,94 @@ if __name__ == '__main__': conn.close() - print(created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i], text_lst[i], tor_exit_node) + first_date = first_date + timedelta(days=1) - i = i + 1 + if date.today() == last_date + timedelta(days=1): - ########################################################################## + insert_sql = 'INSERT INTO totals(datetime, registers) VALUES(%s,%s) ON CONFLICT (datetime) DO UPDATE SET (datetime, registers) = (EXCLUDED.datetime, EXCLUDED.registers)' - approved_users_id_lst = [] + conn = None + + try: + + conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432") + + cur = conn.cursor() + + cur.execute(insert_sql, (date.today(), '0')) + + conn.commit() + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + def check_approval(self, user_id): + + approved = False + + try: + + conn = None + + conn = psycopg2.connect(database = self.mastodon_db, user = self.mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432") + + cur = conn.cursor() + + cur.execute("select approved from users where id = (%s)", (user_id,)) + + row = cur.fetchone() + + if row != None: + + approved = row[0] + + cur.close() + + return approved + + except (Exception, psycopg2.DatabaseError) as error: + + print (error) + + finally: + + if conn is not None: + + conn.close() + + @staticmethod + def __check_ip(self, ip): + + is_tor_exit_node = 'f' + + if ip == None: + + return conn = None try: - conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432") + conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432") cur = conn.cursor() - cur.execute('select id from spamcheck') + cur.execute('select ip from torexit_ips where ip=(%s)', (ip,)) - rows = cur.fetchall() + row = cur.fetchone() - for row in rows: + if row != None: - approved_users_id_lst.append(row) + is_tor_exit_node = 't' cur.close() @@ -393,36 +303,162 @@ if __name__ == '__main__': conn.close() - for user_id in approved_users_id_lst: + return is_tor_exit_node - approved = check_approval(user_id) + @staticmethod + def __check_setup(self): - if approved: + is_setup = False - conn = None + if not os.path.isfile(self.config_file): + print(f"File {self.config_file} not found, running setup.\n") + else: + is_setup = True - try: + return is_setup + + @staticmethod + def __setup(self): - conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432") + if not os.path.exists('config'): + os.makedirs('config') - cur = conn.cursor() + self.mastodon_hostname = input("Mastodon hostname, in ex. 'mastodon.social': ") + self.mastodon_db = input("Mastodon's database name: ") + self.mastodon_db_user = input("Mastodon's database user: ") + self.spamcheck_db = input("Spamcheck's database name: ") + self.spamcheck_db_user = input("Spamcheck's database user: ") - cur.execute('delete from spamcheck where id=(%s)', (user_id,)) + if not os.path.exists(self.config_file): + with open(self.config_file, 'w'): pass + print(f"\n{self.config_file} created!\n") - conn.commit() + with open(self.config_file, 'a') as the_file: + print(f"Writing Mastodon hostname parameter to {self.config_file}") + the_file.write(f'mastodon_hostname: {self.mastodon_hostname}\n') + the_file.write(f'mastodon_db: {self.mastodon_db}\n') + the_file.write(f'mastodon_db_user: {self.mastodon_db_user}\n') + the_file.write(f'spamcheck_db: {self.spamcheck_db}\n') + the_file.write(f'spamcheck_db_user: {self.spamcheck_db_user}\n') - cur.close() + return (self.mastodon_hostname, self.mastodon_db, self.mastodon_db_user, self.spamcheck_db, self.spamcheck_db_user) - except (Exception, psycopg2.DatabaseError) as error: + @staticmethod + def __check_dbsetup(self): - print(error) + dbsetup = False - finally: + try: - if conn is not None: + conn = None - conn.close() + conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432") - spamcheck_datetime_lst, spamcheck_registers_lst = get_totals() + dbsetup = True - write_totals(spamcheck_datetime_lst, spamcheck_registers_lst) + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + return dbsetup + + @staticmethod + def __createdb(self): + + conn = None + + try: + + conn = psycopg2.connect(dbname='postgres', + user=self.spamcheck_db_user, host='', + password='') + + conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) + + cur = conn.cursor() + + print(f"Creating database {self.spamcheck_db}. Please wait...") + + cur.execute(sql.SQL("CREATE DATABASE {}").format( + sql.Identifier(self.spamcheck_db)) + ) + print(f"Database {self.spamcheck_db} created!\n") + + self.__dbtables_schemes(self) + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + @staticmethod + def __dbtables_schemes(self): + + table = "spamcheck" + sql = "create table "+table+" (created_at timestamptz, id bigint PRIMARY KEY, email varchar(200), ip inet, tor_exit_node boolean)" + self.__create_table(self, table, sql) + + table = "torexit_ips" + sql = "create table "+table+" (created_at timestamptz, ip inet PRIMARY KEY)" + self.__create_table(self, table, sql) + + table = "totals" + sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, registers int)" + self.__create_table(self, table, sql) + + @staticmethod + def __create_table(self, table, sql): + + conn = None + + try: + + conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432") + cur = conn.cursor() + + print(f"Creating table {table}") + cur.execute(sql) + + conn.commit() + print(f"Table {table} created!\n") + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + @staticmethod + def __get_parameter(parameter, file_path ): + + with open( file_path ) as f: + for line in f: + if line.startswith( parameter ): + return line.replace(parameter + ":", "").strip() + + print(f'{file_path} Missing parameter {parameter}') + sys.exit(0) + +############################################################################### +# main + +if __name__ == '__main__': + + spamcheck = Spamcheck() + + created_at_lst, id_lst, email_lst, ip_lst = spamcheck.new_registers() + + spamcheck.save_registers(created_at_lst, id_lst, email_lst, ip_lst) + + spamcheck_datetime_lst, spamcheck_registers_lst = spamcheck.get_totals() + + spamcheck.write_totals(spamcheck_datetime_lst, spamcheck_registers_lst) diff --git a/torips.py b/torips.py index 0e3f6c5..d2e5299 100644 --- a/torips.py +++ b/torips.py @@ -2,8 +2,8 @@ import os import datetime import psycopg2 import requests +import re import sys -import pdb def insert_tor_ip(tor_ip): @@ -43,7 +43,7 @@ def get_torbulk_list(): user_agent = {'User-agent': 'Mozilla/5.0'} - response = requests.get('https://check.torproject.org/torbulkexitlist', headers = user_agent, timeout=3) + response = requests.get('https://check.torproject.org/exit-addresses', headers = user_agent, timeout=3) if response.status_code == 200: @@ -68,7 +68,7 @@ def get_torbulk_list(): def db_config(): # Load db configuration from config file - config_filepath = "config/db_config.txt" + config_filepath = "config/config.txt" spamcheck_db = get_parameter("spamcheck_db", config_filepath) spamcheck_db_user = get_parameter("spamcheck_db_user", config_filepath) @@ -111,9 +111,12 @@ if __name__ == '__main__': line = fp.readline().rstrip('\n') - if line != '': + if 'ExitAddress' in line: - insert_tor_ip(line) + pattern = re.compile(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})') + + ip = pattern.search(line)[0] + insert_tor_ip(ip) cnt += 1