From db242058997189d7442fdb50933fa7e913933be8 Mon Sep 17 00:00:00 2001 From: spla Date: Mon, 15 Aug 2022 16:55:31 +0200 Subject: [PATCH] Refactored to Class --- README.md | 10 +- db-setup.py | 170 ----------------- requirements.txt | 5 +- spamcheck.py | 482 +++++++++++++++++++++++++---------------------- torips.py | 13 +- 5 files changed, 272 insertions(+), 408 deletions(-) delete mode 100644 db-setup.py diff --git a/README.md b/README.md index 5f15f88..0dcb646 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,6 @@ # Mastodon's spamcheck -Check and store data from spam bots trying to register to your Mastodon server. -You need to configure your Mastodon server like this: - -- Registration mode -> Approval required for sign up -- Require new users to enter a reason to join -> activated +Check and store data from sign-ups registered on your Mastodon server. ### Dependencies @@ -18,9 +14,9 @@ Within Python Virtual Environment: 1. Run `pip install -r requirements.txt` to install needed Python libraries. -2. Run `python db-setup.py` to setup and create new Postgresql database and needed tables in it. +2. Run `python torips.py` to write Tor exit nodes IPs to new created database. You need to get the torbulkexitlist from [here](https://check.torproject.org/exit-addresses) -3. Run `python torips.py` to write Tor exit nodes IPs to database. You need to get the torbulkexitlist from [here](https://blog.torproject.org/changes-tor-exit-list-service/) +3. Run `python spamcheck.py` for the first time to configure it. 4. Use your favourite scheduling method to set `python spamcheck.py` and `python torips.py` to run regularly. diff --git a/db-setup.py b/db-setup.py deleted file mode 100644 index 1d5dc89..0000000 --- a/db-setup.py +++ /dev/null @@ -1,170 +0,0 @@ -#import getpass -import os -import sys -import psycopg2 -from psycopg2 import sql -from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT - -# Returns the parameter from the specified file -def get_parameter( parameter, file_path ): - # Check if secrets file exists - if not os.path.isfile(file_path): - print("File %s not found, asking."%file_path) - write_parameter( parameter, file_path ) - #sys.exit(0) - - # Find parameter in file - with open( file_path ) as f: - for line in f: - if line.startswith( parameter ): - return line.replace(parameter + ":", "").strip() - - # Cannot find parameter, exit - print(file_path + " Missing parameter %s "%parameter) - sys.exit(0) - -def write_parameter( parameter, file_path ): - if not os.path.exists('config'): - os.makedirs('config') - print("Setting up spamcheck parameters...") - print("\n") - spamcheck_db = input("spamcheck db name: ") - spamcheck_db_user = input("spamcheck db user: ") - mastodon_db = input("Mastodon db name: ") - mastodon_db_user = input("Mastodon db user: ") - - - with open(file_path, "w") as text_file: - print("spamcheck_db: {}".format(spamcheck_db), file=text_file) - print("spamcheck_db_user: {}".format(spamcheck_db_user), file=text_file) - print("mastodon_db: {}".format(mastodon_db), file=text_file) - print("mastodon_db_user: {}".format(mastodon_db_user), file=text_file) - -def create_table(db, db_user, table, sql): - - conn = None - - try: - - conn = psycopg2.connect(database = db, user = db_user, password = "", host = "/var/run/postgresql", port = "5432") - cur = conn.cursor() - - print("Creating table.. "+table) - # Create the table in PostgreSQL database - cur.execute(sql) - - conn.commit() - print("Table "+table+" created!") - print("\n") - - except (Exception, psycopg2.DatabaseError) as error: - - print(error) - - finally: - - if conn is not None: - - conn.close() - -############################################################################### -# main - -if __name__ == '__main__': - - # Load configuration from config file - config_filepath = "config/db_config.txt" - spamcheck_db = get_parameter("spamcheck_db", config_filepath) - spamcheck_db_user = get_parameter("spamcheck_db_user", config_filepath) - mastodon_db = get_parameter("mastodon_db", config_filepath) - mastodon_db_user = get_parameter("mastodon_db_user", config_filepath) - - ############################################################ - # create database - ############################################################ - - conn = None - - try: - - conn = psycopg2.connect(dbname='postgres', - user=spamcheck_db_user, host='', - password='') - - conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) - - cur = conn.cursor() - - print("Creating database " + spamcheck_db + ". Please wait...") - - cur.execute(sql.SQL("CREATE DATABASE {}").format( - sql.Identifier(spamcheck_db)) - ) - print("Database " + spamcheck_db + " created!") - - except (Exception, psycopg2.DatabaseError) as error: - - print(error) - - finally: - - if conn is not None: - - conn.close() - - ############################################################ - - try: - - conn = None - - conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432") - - except (Exception, psycopg2.DatabaseError) as error: - - print(error) - - # Load configuration from config file - os.remove("config/db_config.txt") - - print("Exiting. Run db-setup again with right parameters") - sys.exit(0) - - finally: - - if conn is not None: - - conn.close() - - print("\n") - print("spamcheck parameters saved to db-config.txt!") - print("\n") - - ############################################################ - # Create needed tables - ############################################################ - - print("Creating table...") - - db = spamcheck_db - db_user = spamcheck_db_user - - table = "spamcheck" - sql = "create table "+table+" (created_at timestamptz, id bigint PRIMARY KEY, email varchar(200), ip inet, text varchar(200), tor_exit_node boolean)" - create_table(db, db_user, table, sql) - - table = "torexit_ips" - sql = "create table "+table+" (created_at timestamptz, ip inet PRIMARY KEY)" - create_table(db, db_user, table, sql) - - table = "totals" - sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, registers int)" - create_table(db, db_user, table, sql) - - - - ############################################################ - - print("Done!") - print("Now you can run setup.py!") - print("\n") diff --git a/requirements.txt b/requirements.txt index 64b2eb5..b24e524 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,2 @@ -Mastodon.py>=1.5.1 -wheel>=0.36.2 -psycopg2>=2.8.6 +psycopg2-binary +requests diff --git a/spamcheck.py b/spamcheck.py index 84666a2..15ec037 100644 --- a/spamcheck.py +++ b/spamcheck.py @@ -1,5 +1,5 @@ from datetime import date, datetime, timedelta -from mastodon import Mastodon +#from mastodon import Mastodon import time import os import json @@ -7,47 +7,68 @@ import sys import os.path import operator import psycopg2 -from langdetect import detect -import requests -import pdb +from psycopg2 import sql +from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT -def write_totals(spamcheck_datetime_lst, spamcheck_registers_lst): +class Spamcheck: - insert_sql = 'INSERT INTO totals(datetime, registers) VALUES(%s,%s) ON CONFLICT (datetime) DO UPDATE SET (datetime, registers) = (EXCLUDED.datetime, EXCLUDED.registers)' + name = "Spamcheck for Mastodon social servers" - first_date = spamcheck_datetime_lst[0] + def __init__(self, mastodon_hostname=None, mastodon_db=None, mastodon_db_user=None, spamcheck_db=None, spamcheck_db_user=None): - last_date = spamcheck_datetime_lst[len(spamcheck_datetime_lst)-1] + self.config_file = 'config/config.txt' - i = 0 + is_setup = self.__check_setup(self) - while i < len(spamcheck_datetime_lst): + if is_setup: - conn = None + self.mastodon_hostname = self.__get_parameter("mastodon_hostname", self.config_file) + self.mastodon_db = self.__get_parameter("mastodon_db", self.config_file) + self.mastodon_db_user = self.__get_parameter("mastodon_db_user", self.config_file) + self.spamcheck_db = self.__get_parameter("spamcheck_db", self.config_file) + self.spamcheck_db_user = self.__get_parameter("spamcheck_db_user", self.config_file) + + else: + + self.mastodon_hostname, self.mastodon_db, self.mastodon_db_user, self.spamcheck_db, self.spamcheck_db_user = self.__setup(self) + + db_setup = self.__check_dbsetup(self) + + if not db_setup: + self.__createdb(self) + + def new_registers(self, created_at_lst=[], id_lst=[], email_lst=[], ip_lst=[]): + try: - conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432") + conn = None + + conn = psycopg2.connect(database = self.mastodon_db, user = self.mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432") cur = conn.cursor() - if first_date == spamcheck_datetime_lst[i]: + cur.execute("select users.created_at, users.id, users.email, users.sign_up_ip from users where users.approved and users.created_at > now() - interval '7 days'") - cur.execute(insert_sql, (spamcheck_datetime_lst[i], spamcheck_registers_lst[i])) + rows = cur.fetchall() - i += 1 + for row in rows: - else: + if row != None: - cur.execute(insert_sql, (first_date, '0')) + created_at_lst.append(row[0]) - conn.commit() + id_lst.append(row[1]) + + email_lst.append(row[2]) + + ip_lst.append(row[3]) cur.close() except (Exception, psycopg2.DatabaseError) as error: - print(error) + print (error) finally: @@ -55,333 +76,315 @@ def write_totals(spamcheck_datetime_lst, spamcheck_registers_lst): conn.close() - first_date = first_date + timedelta(days=1) + return (created_at_lst, id_lst, email_lst, ip_lst) - if date.today() == last_date + timedelta(days=1): + def save_registers(self, created_at_lst, id_lst, email_lst, ip_lst): - insert_sql = 'INSERT INTO totals(datetime, registers) VALUES(%s,%s) ON CONFLICT (datetime) DO UPDATE SET (datetime, registers) = (EXCLUDED.datetime, EXCLUDED.registers)' - - conn = None + insert_sql = 'INSERT INTO spamcheck(created_at, id, email, ip, tor_exit_node) VALUES(%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING' + + i = 0 - try: + while i < len(id_lst): - conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432") + is_tor_exit_node = self.__check_ip(self, ip_lst[i]) - cur = conn.cursor() - - cur.execute(insert_sql, (date.today(), '0')) - - conn.commit() - - cur.close() - - except (Exception, psycopg2.DatabaseError) as error: + tor_exit_node = 't' if is_tor_exit_node == 't' else 'f' - print(error) + conn = None - finally: + try: - if conn is not None: + conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432") - conn.close() + cur = conn.cursor() -def get_totals(): + cur.execute(insert_sql, (created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i], tor_exit_node)) - spamcheck_datetime_lst = [] + conn.commit() - spamcheck_registers_lst = [] + cur.close() - select_sql = 'select date(created_at), count(ip) as registers from spamcheck group by date(created_at) order by date(created_at)' + except (Exception, psycopg2.DatabaseError) as error: - conn = None + print(error) - try: + finally: - conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432") + if conn is not None: - cur = conn.cursor() + conn.close() + + print(created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i], tor_exit_node) - cur.execute(select_sql) + i += 1 - rows = cur.fetchall() + def get_totals(self): - for row in rows: + spamcheck_datetime_lst = [] - spamcheck_datetime_lst.append(row[0]) + spamcheck_registers_lst = [] - spamcheck_registers_lst.append(row[1]) + select_sql = 'select date(created_at), count(ip) as registers from spamcheck group by date(created_at) order by date(created_at)' - cur.close() + conn = None - except (Exception, psycopg2.DatabaseError) as error: + try: - print(error) + conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432") - finally: + cur = conn.cursor() - if conn is not None: + cur.execute(select_sql) - conn.close() + rows = cur.fetchall() - return (spamcheck_datetime_lst, spamcheck_registers_lst) + for row in rows: -def check_ip(ip): + spamcheck_datetime_lst.append(row[0]) - is_tor_exit_node = 'f' + spamcheck_registers_lst.append(row[1]) - conn = None + cur.close() - try: + except (Exception, psycopg2.DatabaseError) as error: - conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432") + print(error) - cur = conn.cursor() + finally: - cur.execute('select ip from torexit_ips where ip=(%s)', (ip,)) + if conn is not None: - row = cur.fetchone() + conn.close() - if row != None: + return (spamcheck_datetime_lst, spamcheck_registers_lst) - is_tor_exit_node = 't' + def write_totals(self, spamcheck_datetime_lst, spamcheck_registers_lst): - cur.close() + insert_sql = 'INSERT INTO totals(datetime, registers) VALUES(%s,%s) ON CONFLICT (datetime) DO UPDATE SET (datetime, registers) = (EXCLUDED.datetime, EXCLUDED.registers)' - except (Exception, psycopg2.DatabaseError) as error: + first_date = spamcheck_datetime_lst[0] - print(error) + last_date = spamcheck_datetime_lst[len(spamcheck_datetime_lst)-1] - finally: + i = 0 - if conn is not None: + while i < len(spamcheck_datetime_lst): - conn.close() + conn = None - return is_tor_exit_node + try: -def check_approval(user_id): + conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432") - approved = False + cur = conn.cursor() - try: + if first_date == spamcheck_datetime_lst[i]: - conn = None + cur.execute(insert_sql, (spamcheck_datetime_lst[i], spamcheck_registers_lst[i])) - conn = psycopg2.connect(database = mastodon_db, user = mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432") + i += 1 - cur = conn.cursor() + else: - cur.execute("select approved from users where id = (%s)", (user_id,)) + cur.execute(insert_sql, (first_date, '0')) - row = cur.fetchone() + conn.commit() - if row != None: + cur.close() - approved = row[0] + except (Exception, psycopg2.DatabaseError) as error: - cur.close() + print(error) - return approved + finally: - except (Exception, psycopg2.DatabaseError) as error: + if conn is not None: - print (error) + conn.close() - finally: + first_date = first_date + timedelta(days=1) - if conn is not None: + if date.today() == last_date + timedelta(days=1): - conn.close() + insert_sql = 'INSERT INTO totals(datetime, registers) VALUES(%s,%s) ON CONFLICT (datetime) DO UPDATE SET (datetime, registers) = (EXCLUDED.datetime, EXCLUDED.registers)' -def mastodon(): + conn = None - # Load secrets from secrets file - secrets_filepath = "secrets/secrets.txt" - uc_client_id = get_parameter("uc_client_id", secrets_filepath) - uc_client_secret = get_parameter("uc_client_secret", secrets_filepath) - uc_access_token = get_parameter("uc_access_token", secrets_filepath) + try: - # Load configuration from config file - config_filepath = "config/config.txt" - mastodon_hostname = get_parameter("mastodon_hostname", config_filepath) + conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432") - # Initialise Mastodon API - mastodon = Mastodon( - client_id=uc_client_id, - client_secret=uc_client_secret, - access_token=uc_access_token, - api_base_url='https://' + mastodon_hostname, - ) + cur = conn.cursor() - # Initialise access headers - headers = {'Authorization': 'Bearer %s'%uc_access_token} + cur.execute(insert_sql, (date.today(), '0')) - return (mastodon, mastodon_hostname) + conn.commit() -def db_config(): + cur.close() - # Load db configuration from config file - config_filepath = "config/db_config.txt" - mastodon_db = get_parameter("mastodon_db", config_filepath) - mastodon_db_user = get_parameter("mastodon_db_user", config_filepath) - spamcheck_db = get_parameter("spamcheck_db", config_filepath) - spamcheck_db_user = get_parameter("spamcheck_db_user", config_filepath) + except (Exception, psycopg2.DatabaseError) as error: - return (mastodon_db, mastodon_db_user, spamcheck_db, spamcheck_db_user) + print(error) -# Returns the parameter from the specified file -def get_parameter( parameter, file_path ): - # Check if secrets file exists - if not os.path.isfile(file_path): - print("File %s not found, exiting."%file_path) - sys.exit(0) + finally: - # Find parameter in file - with open( file_path ) as f: - for line in f: - if line.startswith( parameter ): - return line.replace(parameter + ":", "").strip() + if conn is not None: - # Cannot find parameter, exit - print(file_path + " Missing parameter %s "%parameter) - sys.exit(0) + conn.close() -############################################################################### -# main + def check_approval(self, user_id): -if __name__ == '__main__': + approved = False - mastodon_db, mastodon_db_user, spamcheck_db, spamcheck_db_user = db_config() + try: - mastodon, mastodon_hostname = mastodon() + conn = None - ############################################################################### - # check new registering - ############################################################################### - - created_at_lst = [] + conn = psycopg2.connect(database = self.mastodon_db, user = self.mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432") - id_lst = [] + cur = conn.cursor() - email_lst = [] + cur.execute("select approved from users where id = (%s)", (user_id,)) - ip_lst = [] + row = cur.fetchone() - text_lst = [] - - try: + if row != None: - conn = None + approved = row[0] - conn = psycopg2.connect(database = mastodon_db, user = mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432") + cur.close() - cur = conn.cursor() + return approved - cur.execute("select users.created_at, users.id, users.email, users.sign_up_ip, user_invite_requests.text from users, user_invite_requests where not users.approved and users.id = user_invite_requests.user_id") + except (Exception, psycopg2.DatabaseError) as error: - rows = cur.fetchall() + print (error) - for row in rows: + finally: - if row != None: + if conn is not None: - created_at_lst.append(row[0]) + conn.close() - id_lst.append(row[1]) + @staticmethod + def __check_ip(self, ip): - email_lst.append(row[2]) + is_tor_exit_node = 'f' - ip_lst.append(row[3]) + if ip == None: - text_lst.append(row[4]) + return - cur.close() + conn = None - except (Exception, psycopg2.DatabaseError) as error: + try: - print (error) + conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432") - finally: + cur = conn.cursor() - if conn is not None: + cur.execute('select ip from torexit_ips where ip=(%s)', (ip,)) - conn.close() + row = cur.fetchone() - ############################################################################### + if row != None: - insert_sql = 'INSERT INTO spamcheck(created_at, id, email, ip, text, tor_exit_node) VALUES(%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING' - - i = 0 + is_tor_exit_node = 't' - while i < len(id_lst): + cur.close() - if len(text_lst[i]) > 1: + except (Exception, psycopg2.DatabaseError) as error: - text_lang = detect(text_lst[i]) + print(error) - else: + finally: - text_lang = 'en' + if conn is not None: - if text_lang != 'ca' or len(text_lst) == 1: + conn.close() - is_tor_exit_node = check_ip(ip_lst[i]) + return is_tor_exit_node - tor_exit_node = 't' if is_tor_exit_node == 't' else 'f' + @staticmethod + def __check_setup(self): - text_length = 200 if len(text_lst[i]) > 200 else len(text_lst[i]) + is_setup = False - reason_text = '{0}...'.format(text_lst[i][:text_length].rsplit(' ', 1)[0]) + if not os.path.isfile(self.config_file): + print(f"File {self.config_file} not found, running setup.\n") + else: + is_setup = True - conn = None + return is_setup + + @staticmethod + def __setup(self): - try: + if not os.path.exists('config'): + os.makedirs('config') - conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432") + self.mastodon_hostname = input("Mastodon hostname, in ex. 'mastodon.social': ") + self.mastodon_db = input("Mastodon's database name: ") + self.mastodon_db_user = input("Mastodon's database user: ") + self.spamcheck_db = input("Spamcheck's database name: ") + self.spamcheck_db_user = input("Spamcheck's database user: ") - cur = conn.cursor() + if not os.path.exists(self.config_file): + with open(self.config_file, 'w'): pass + print(f"\n{self.config_file} created!\n") - cur.execute(insert_sql, (created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i], reason_text, tor_exit_node)) + with open(self.config_file, 'a') as the_file: + print(f"Writing Mastodon hostname parameter to {self.config_file}") + the_file.write(f'mastodon_hostname: {self.mastodon_hostname}\n') + the_file.write(f'mastodon_db: {self.mastodon_db}\n') + the_file.write(f'mastodon_db_user: {self.mastodon_db_user}\n') + the_file.write(f'spamcheck_db: {self.spamcheck_db}\n') + the_file.write(f'spamcheck_db_user: {self.spamcheck_db_user}\n') - conn.commit() + return (self.mastodon_hostname, self.mastodon_db, self.mastodon_db_user, self.spamcheck_db, self.spamcheck_db_user) - cur.close() + @staticmethod + def __check_dbsetup(self): - except (Exception, psycopg2.DatabaseError) as error: + dbsetup = False - print(error) + try: - finally: + conn = None - if conn is not None: + conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432") - conn.close() + dbsetup = True - print(created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i], text_lst[i], tor_exit_node) + except (Exception, psycopg2.DatabaseError) as error: - i = i + 1 + print(error) - ########################################################################## + return dbsetup - approved_users_id_lst = [] + @staticmethod + def __createdb(self): conn = None try: - conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432") + conn = psycopg2.connect(dbname='postgres', + user=self.spamcheck_db_user, host='', + password='') + + conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) cur = conn.cursor() - cur.execute('select id from spamcheck') + print(f"Creating database {self.spamcheck_db}. Please wait...") - rows = cur.fetchall() + cur.execute(sql.SQL("CREATE DATABASE {}").format( + sql.Identifier(self.spamcheck_db)) + ) + print(f"Database {self.spamcheck_db} created!\n") - for row in rows: - - approved_users_id_lst.append(row) - - cur.close() + self.__dbtables_schemes(self) except (Exception, psycopg2.DatabaseError) as error: @@ -393,36 +396,69 @@ if __name__ == '__main__': conn.close() - for user_id in approved_users_id_lst: + @staticmethod + def __dbtables_schemes(self): - approved = check_approval(user_id) + table = "spamcheck" + sql = "create table "+table+" (created_at timestamptz, id bigint PRIMARY KEY, email varchar(200), ip inet, tor_exit_node boolean)" + self.__create_table(self, table, sql) - if approved: + table = "torexit_ips" + sql = "create table "+table+" (created_at timestamptz, ip inet PRIMARY KEY)" + self.__create_table(self, table, sql) - conn = None + table = "totals" + sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, registers int)" + self.__create_table(self, table, sql) - try: + @staticmethod + def __create_table(self, table, sql): - conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432") + conn = None - cur = conn.cursor() + try: - cur.execute('delete from spamcheck where id=(%s)', (user_id,)) + conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432") + cur = conn.cursor() - conn.commit() + print(f"Creating table {table}") + cur.execute(sql) - cur.close() + conn.commit() + print(f"Table {table} created!\n") + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: - except (Exception, psycopg2.DatabaseError) as error: + conn.close() + + @staticmethod + def __get_parameter(parameter, file_path ): - print(error) + with open( file_path ) as f: + for line in f: + if line.startswith( parameter ): + return line.replace(parameter + ":", "").strip() - finally: + print(f'{file_path} Missing parameter {parameter}') + sys.exit(0) - if conn is not None: +############################################################################### +# main + +if __name__ == '__main__': + + spamcheck = Spamcheck() + + created_at_lst, id_lst, email_lst, ip_lst = spamcheck.new_registers() - conn.close() + spamcheck.save_registers(created_at_lst, id_lst, email_lst, ip_lst) - spamcheck_datetime_lst, spamcheck_registers_lst = get_totals() + spamcheck_datetime_lst, spamcheck_registers_lst = spamcheck.get_totals() - write_totals(spamcheck_datetime_lst, spamcheck_registers_lst) + spamcheck.write_totals(spamcheck_datetime_lst, spamcheck_registers_lst) diff --git a/torips.py b/torips.py index 0e3f6c5..d2e5299 100644 --- a/torips.py +++ b/torips.py @@ -2,8 +2,8 @@ import os import datetime import psycopg2 import requests +import re import sys -import pdb def insert_tor_ip(tor_ip): @@ -43,7 +43,7 @@ def get_torbulk_list(): user_agent = {'User-agent': 'Mozilla/5.0'} - response = requests.get('https://check.torproject.org/torbulkexitlist', headers = user_agent, timeout=3) + response = requests.get('https://check.torproject.org/exit-addresses', headers = user_agent, timeout=3) if response.status_code == 200: @@ -68,7 +68,7 @@ def get_torbulk_list(): def db_config(): # Load db configuration from config file - config_filepath = "config/db_config.txt" + config_filepath = "config/config.txt" spamcheck_db = get_parameter("spamcheck_db", config_filepath) spamcheck_db_user = get_parameter("spamcheck_db_user", config_filepath) @@ -111,9 +111,12 @@ if __name__ == '__main__': line = fp.readline().rstrip('\n') - if line != '': + if 'ExitAddress' in line: - insert_tor_ip(line) + pattern = re.compile(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})') + + ip = pattern.search(line)[0] + insert_tor_ip(ip) cnt += 1