Refactored to Class

README.md

@@ -1,10 +1,6 @@
 # Mastodon's spamcheck
 
-Check and store data from spam bots trying to register to your Mastodon server.
-You need to configure your Mastodon server like this:  
-  
-- Registration mode -> Approval required for sign up  
-- Require new users to enter a reason to join -> activated  
+Check and store data from sign-ups registered on your Mastodon server.
 
 ### Dependencies
 
@@ -18,11 +14,11 @@ Within Python Virtual Environment:
 
 1. Run `pip install -r requirements.txt` to install needed Python libraries.
 
-2. Run `python db-setup.py` to setup and create new Postgresql database and needed tables in it.
+2. Run `python torips.py` to write Tor exit nodes IPs to new created database. You need to get the torbulkexitlist from [here](https://check.torproject.org/exit-addresses)  
 
-3. Run `python torips.py` to write Tor exit nodes IPs to database. You need to get the torbulkexitlist from [here](https://blog.torproject.org/changes-tor-exit-list-service/) 
+3. Run `python spamcheck.py` for the first time to configure it. 
 
-4. Use your favourite scheduling method to set `python spamcheck.py` to run regularly.  
+4. Use your favourite scheduling method to set `python spamcheck.py` and `python torips.py` to run regularly.  
 
 
 

db-setup.py

@@ -1,170 +0,0 @@
-#import getpass
-import os
-import sys
-import psycopg2
-from psycopg2 import sql
-from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
-
-# Returns the parameter from the specified file
-def get_parameter( parameter, file_path ):
-    # Check if secrets file exists
-    if not os.path.isfile(file_path):
-        print("File %s not found, asking."%file_path)
-        write_parameter( parameter, file_path )
-        #sys.exit(0)
-
-    # Find parameter in file
-    with open( file_path ) as f:
-        for line in f:
-            if line.startswith( parameter ):
-                return line.replace(parameter + ":", "").strip()
-
-    # Cannot find parameter, exit
-    print(file_path + "  Missing parameter %s "%parameter)
-    sys.exit(0)
-
-def write_parameter( parameter, file_path ):
-    if not os.path.exists('config'):
-        os.makedirs('config')
-    print("Setting up spamcheck parameters...")
-    print("\n")
-    spamcheck_db = input("spamcheck db name: ")
-    spamcheck_db_user = input("spamcheck db user: ")
-    mastodon_db = input("Mastodon db name: ")
-    mastodon_db_user = input("Mastodon db user: ")
-
-
-    with open(file_path, "w") as text_file:
-        print("spamcheck_db: {}".format(spamcheck_db), file=text_file)
-        print("spamcheck_db_user: {}".format(spamcheck_db_user), file=text_file)
-        print("mastodon_db: {}".format(mastodon_db), file=text_file)
-        print("mastodon_db_user: {}".format(mastodon_db_user), file=text_file)
-
-def create_table(db, db_user, table, sql):
-
-    conn = None
-
-    try:
-
-        conn = psycopg2.connect(database = db, user = db_user, password = "", host = "/var/run/postgresql", port = "5432")
-        cur = conn.cursor()
-
-        print("Creating table.. "+table)
-        # Create the table in PostgreSQL database
-        cur.execute(sql)
-
-        conn.commit()
-        print("Table "+table+" created!")
-        print("\n")
-
-    except (Exception, psycopg2.DatabaseError) as error:
-
-        print(error)
-
-    finally:
-
-        if conn is not None:
-
-            conn.close()
-
-###############################################################################
-# main
-
-if __name__ == '__main__':
-
-    # Load configuration from config file
-    config_filepath = "config/db_config.txt"
-    spamcheck_db = get_parameter("spamcheck_db", config_filepath)
-    spamcheck_db_user = get_parameter("spamcheck_db_user", config_filepath)
-    mastodon_db = get_parameter("mastodon_db", config_filepath)
-    mastodon_db_user = get_parameter("mastodon_db_user", config_filepath)
-
-    ############################################################
-    # create database
-    ############################################################
-
-    conn = None
-
-    try:
-
-        conn = psycopg2.connect(dbname='postgres',
-            user=spamcheck_db_user, host='',
-            password='')
-
-        conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
-
-        cur = conn.cursor()
-
-        print("Creating database " + spamcheck_db + ". Please wait...")
-
-        cur.execute(sql.SQL("CREATE DATABASE {}").format(
-            sql.Identifier(spamcheck_db))
-        )
-        print("Database " + spamcheck_db + " created!")
-
-    except (Exception, psycopg2.DatabaseError) as error:
-
-        print(error)
-
-    finally:
-
-        if conn is not None:
-
-            conn.close()
-
-    ############################################################
-
-    try:
-
-        conn = None
-
-        conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
-
-    except (Exception, psycopg2.DatabaseError) as error:
-
-        print(error)
-
-        # Load configuration from config file
-        os.remove("config/db_config.txt")
-
-        print("Exiting. Run db-setup again with right parameters")
-        sys.exit(0)
-
-    finally:
-
-        if conn is not None:
-
-            conn.close()
-
-            print("\n")
-            print("spamcheck parameters saved to db-config.txt!")
-            print("\n")
-
-    ############################################################
-    # Create needed tables
-    ############################################################
-
-    print("Creating table...")
-
-    db = spamcheck_db
-    db_user = spamcheck_db_user
-
-    table = "spamcheck"
-    sql = "create table "+table+" (created_at timestamptz, id bigint PRIMARY KEY, email varchar(200), ip inet, text varchar(200), tor_exit_node boolean)"
-    create_table(db, db_user, table, sql)
-
-    table = "torexit_ips"
-    sql = "create table "+table+" (created_at timestamptz, ip inet PRIMARY KEY)"
-    create_table(db, db_user, table, sql)
-
-    table = "totals"
-    sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, registers int)"
-    create_table(db, db_user, table, sql)
-
-
-
-    ############################################################
-
-    print("Done!")
-    print("Now you can run setup.py!")
-    print("\n")

requirements.txt

@@ -1,3 +1,2 @@
-Mastodon.py>=1.5.1
-wheel>=0.36.2
-psycopg2>=2.8.6
+psycopg2-binary
+requests

spamcheck.py

@@ -1,5 +1,5 @@
-import datetime
-from mastodon import Mastodon
+from datetime import date, datetime, timedelta
+#from mastodon import Mastodon
 import time
 import os
 import json
@@ -7,29 +7,142 @@ import sys
 import os.path
 import operator
 import psycopg2
-from langdetect import detect
-import requests
-import pdb
+from psycopg2 import sql
+from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
 
-def write_totals(spamcheck_datetime_lst, spamcheck_registers_lst):
+class Spamcheck:
 
-    insert_sql = 'INSERT INTO totals(datetime, registers) VALUES(%s,%s) ON CONFLICT (datetime) DO UPDATE SET (datetime, registers) = (EXCLUDED.datetime, EXCLUDED.registers)'
+    name = "Spamcheck for Mastodon social servers"
 
-    i = 0
+    def __init__(self, mastodon_hostname=None, mastodon_db=None, mastodon_db_user=None, spamcheck_db=None, spamcheck_db_user=None):
 
-    while i < len(spamcheck_datetime_lst):
+        self.config_file = 'config/config.txt'
+
+        is_setup = self.__check_setup(self)
+
+        if is_setup:
+
+            self.mastodon_hostname = self.__get_parameter("mastodon_hostname", self.config_file)
+            self.mastodon_db = self.__get_parameter("mastodon_db", self.config_file)
+            self.mastodon_db_user = self.__get_parameter("mastodon_db_user", self.config_file)
+            self.spamcheck_db = self.__get_parameter("spamcheck_db", self.config_file)
+            self.spamcheck_db_user = self.__get_parameter("spamcheck_db_user", self.config_file)
+        
+        else:
+
+            self.mastodon_hostname, self.mastodon_db, self.mastodon_db_user, self.spamcheck_db, self.spamcheck_db_user = self.__setup(self)
+
+        db_setup = self.__check_dbsetup(self)
+
+        if not db_setup:
+
+            self.__createdb(self)
+
+    def new_registers(self, created_at_lst=[], id_lst=[], email_lst=[], ip_lst=[]):
+    
+        try:
+
+            conn = None
+
+            conn = psycopg2.connect(database = self.mastodon_db, user = self.mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432")
+
+            cur = conn.cursor()
+
+            cur.execute("select users.created_at, users.id, users.email, users.sign_up_ip from users where users.approved and users.created_at > now() - interval '7 days'")
+
+            rows = cur.fetchall()
+
+            for row in rows:
+
+                if row != None:
+
+                    created_at_lst.append(row[0])
+
+                    id_lst.append(row[1])
+
+                    email_lst.append(row[2])
+
+                    ip_lst.append(row[3])
+
+            cur.close()
+
+        except (Exception, psycopg2.DatabaseError) as error:
+
+            print (error)
+
+        finally:
+
+            if conn is not None:
+
+                conn.close()
+
+        return (created_at_lst, id_lst, email_lst, ip_lst)
+
+    def save_registers(self, created_at_lst, id_lst, email_lst, ip_lst):
+
+        insert_sql = 'INSERT INTO spamcheck(created_at, id, email, ip, tor_exit_node) VALUES(%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING'
+        
+        i = 0
+
+        while i < len(id_lst):
+
+            is_tor_exit_node = self.__check_ip(self, ip_lst[i])
+
+            tor_exit_node = 't' if is_tor_exit_node == 't' else 'f'
+
+            conn = None
+
+            try:
+
+                conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
+
+                cur = conn.cursor()
+
+                cur.execute(insert_sql, (created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i], tor_exit_node))
+
+                conn.commit()
+
+                cur.close()
+
+            except (Exception, psycopg2.DatabaseError) as error:
+
+                print(error)
+
+            finally:
+
+                if conn is not None:
+
+                    conn.close()
+            
+            print(created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i], tor_exit_node)
+
+            i += 1
+
+    def get_totals(self):
+
+        spamcheck_datetime_lst = []
+
+        spamcheck_registers_lst = []
+
+        select_sql = 'select date(created_at), count(ip) as registers from spamcheck group by date(created_at) order by date(created_at)'
 
         conn = None
 
         try:
 
-            conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
+            conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
 
             cur = conn.cursor()
 
-            cur.execute(insert_sql, (spamcheck_datetime_lst[i], spamcheck_registers_lst[i]))
+            cur.execute(select_sql)
 
-            conn.commit()
+            rows = cur.fetchall()
+
+            for row in rows:
+
+                spamcheck_datetime_lst.append(row[0])
+
+                spamcheck_registers_lst.append(row[1])
 
             cur.close()
 
@@ -43,253 +156,37 @@ def write_totals(spamcheck_datetime_lst, spamcheck_registers_lst):
 
                 conn.close()
 
-        i += 1
+        return (spamcheck_datetime_lst, spamcheck_registers_lst)
 
-def get_totals():
+    def write_totals(self, spamcheck_datetime_lst, spamcheck_registers_lst):
 
-    spamcheck_datetime_lst = []
+        insert_sql = 'INSERT INTO totals(datetime, registers) VALUES(%s,%s) ON CONFLICT (datetime) DO UPDATE SET (datetime, registers) = (EXCLUDED.datetime, EXCLUDED.registers)'
 
-    spamcheck_registers_lst = []
+        first_date = spamcheck_datetime_lst[0]
 
-    select_sql = 'select date(created_at), count(ip) as registers from spamcheck group by date(created_at) order by date(created_at)'
+        last_date = spamcheck_datetime_lst[len(spamcheck_datetime_lst)-1]
 
-    conn = None
+        i = 0
 
-    try:
-
-        conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
-
-        cur = conn.cursor()
-
-        cur.execute(select_sql)
-
-        rows = cur.fetchall()
-
-        for row in rows:
-
-            spamcheck_datetime_lst.append(row[0])
-
-            spamcheck_registers_lst.append(row[1])
-
-        cur.close()
-
-    except (Exception, psycopg2.DatabaseError) as error:
-
-        print(error)
-
-    finally:
-
-        if conn is not None:
-
-            conn.close()
-
-    return (spamcheck_datetime_lst, spamcheck_registers_lst)
-
-def check_ip(ip):
-
-    is_tor_exit_node = 'f'
-
-    conn = None
-
-    try:
-
-        conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
-
-        cur = conn.cursor()
-
-        cur.execute('select ip from torexit_ips where ip=(%s)', (ip,))
-
-        row = cur.fetchone()
-
-        if row != None:
-
-            is_tor_exit_node = 't'
-
-        cur.close()
-
-    except (Exception, psycopg2.DatabaseError) as error:
-
-        print(error)
-
-    finally:
-
-        if conn is not None:
-
-            conn.close()
-
-    return is_tor_exit_node
-
-def check_approval(user_id):
-
-    approved = False
-
-    try:
-
-        conn = None
-
-        conn = psycopg2.connect(database = mastodon_db, user = mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432")
-
-        cur = conn.cursor()
-
-        cur.execute("select approved from users where id = (%s)", (user_id,))
-
-        row = cur.fetchone()
-
-        if row != None:
-
-            approved = row[0]
-
-        cur.close()
-
-        return approved
-
-    except (Exception, psycopg2.DatabaseError) as error:
-
-        print (error)
-
-    finally:
-
-        if conn is not None:
-
-            conn.close()
-
-def mastodon():
-
-    # Load secrets from secrets file
-    secrets_filepath = "secrets/secrets.txt"
-    uc_client_id = get_parameter("uc_client_id",     secrets_filepath)
-    uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
-    uc_access_token = get_parameter("uc_access_token",  secrets_filepath)
-
-    # Load configuration from config file
-    config_filepath = "config/config.txt"
-    mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
-
-    # Initialise Mastodon API
-    mastodon = Mastodon(
-        client_id=uc_client_id,
-        client_secret=uc_client_secret,
-        access_token=uc_access_token,
-        api_base_url='https://' + mastodon_hostname,
-    )
-
-    # Initialise access headers
-    headers = {'Authorization': 'Bearer %s'%uc_access_token}
-
-    return (mastodon, mastodon_hostname)
-
-def db_config():
-
-    # Load db configuration from config file
-    config_filepath = "config/db_config.txt"
-    mastodon_db = get_parameter("mastodon_db", config_filepath)
-    mastodon_db_user = get_parameter("mastodon_db_user", config_filepath)
-    spamcheck_db = get_parameter("spamcheck_db", config_filepath)
-    spamcheck_db_user = get_parameter("spamcheck_db_user", config_filepath)
-
-    return (mastodon_db, mastodon_db_user, spamcheck_db, spamcheck_db_user)
-
-# Returns the parameter from the specified file
-def get_parameter( parameter, file_path ):
-    # Check if secrets file exists
-    if not os.path.isfile(file_path):
-        print("File %s not found, exiting."%file_path)
-        sys.exit(0)
-
-    # Find parameter in file
-    with open( file_path ) as f:
-        for line in f:
-            if line.startswith( parameter ):
-                return line.replace(parameter + ":", "").strip()
-
-    # Cannot find parameter, exit
-    print(file_path + "  Missing parameter %s "%parameter)
-    sys.exit(0)
-
-###############################################################################
-# main
-
-if __name__ == '__main__':
-
-    mastodon_db, mastodon_db_user, spamcheck_db, spamcheck_db_user = db_config()
-
-    mastodon, mastodon_hostname = mastodon()
-
-    ###############################################################################
-    # check new registering
-    ###############################################################################
-    
-    created_at_lst = []
-
-    id_lst = []
-
-    email_lst = []
-
-    ip_lst = []
-
-    text_lst = []
-    
-    try:
-
-        conn = None
-
-        conn = psycopg2.connect(database = mastodon_db, user = mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432")
-
-        cur = conn.cursor()
-
-        cur.execute("select users.created_at, users.id, users.email, users.sign_up_ip, user_invite_requests.text from users, user_invite_requests where not users.approved and users.id = user_invite_requests.user_id")
-
-        rows = cur.fetchall()
-
-        for row in rows:
-
-            if row != None:
-
-                created_at_lst.append(row[0])
-
-                id_lst.append(row[1])
-
-                email_lst.append(row[2])
-
-                ip_lst.append(row[3])
-
-                text_lst.append(row[4])
-
-        cur.close()
-
-    except (Exception, psycopg2.DatabaseError) as error:
-
-        print (error)
-
-    finally:
-
-        if conn is not None:
-
-            conn.close()
-
-    ###############################################################################
-
-    insert_sql = 'INSERT INTO spamcheck(created_at, id, email, ip, text, tor_exit_node) VALUES(%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING'
-    
-    i = 0
-
-    while i < len(id_lst):
-
-        if detect(text_lst[i]) != 'ca':
-
-            is_tor_exit_node = check_ip(ip_lst[i])
-
-            tor_exit_node = 't' if is_tor_exit_node else 'f'
+        while i < len(spamcheck_datetime_lst):
 
             conn = None
 
             try:
 
-                conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
+                conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
 
                 cur = conn.cursor()
 
-                cur.execute(insert_sql, (created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i], text_lst[i], tor_exit_node))
+                if first_date == spamcheck_datetime_lst[i]:
+
+                    cur.execute(insert_sql, (spamcheck_datetime_lst[i], spamcheck_registers_lst[i]))
+
+                    i += 1
+
+                else:
+
+                    cur.execute(insert_sql, (first_date, '0'))
 
                 conn.commit()
 
@@ -305,29 +202,94 @@ if __name__ == '__main__':
 
                     conn.close()
 
-            print(created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i], text_lst[i], tor_exit_node)
+            first_date = first_date + timedelta(days=1)
 
-        i = i + 1
+        if date.today() == last_date + timedelta(days=1):
 
-        ##########################################################################
+            insert_sql = 'INSERT INTO totals(datetime, registers) VALUES(%s,%s) ON CONFLICT (datetime) DO UPDATE SET (datetime, registers) = (EXCLUDED.datetime, EXCLUDED.registers)'
 
-        approved_users_id_lst = []
+            conn = None
+
+            try:
+
+                conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
+
+                cur = conn.cursor()
+
+                cur.execute(insert_sql, (date.today(), '0'))
+
+                conn.commit()
+
+                cur.close()
+
+            except (Exception, psycopg2.DatabaseError) as error:
+
+                print(error)
+
+            finally:
+
+                if conn is not None:
+
+                    conn.close()
+
+    def check_approval(self, user_id):
+
+        approved = False
+
+        try:
+
+            conn = None
+
+            conn = psycopg2.connect(database = self.mastodon_db, user = self.mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432")
+
+            cur = conn.cursor()
+
+            cur.execute("select approved from users where id = (%s)", (user_id,))
+
+            row = cur.fetchone()
+
+            if row != None:
+
+                approved = row[0]
+
+            cur.close()
+
+            return approved
+
+        except (Exception, psycopg2.DatabaseError) as error:
+
+            print (error)
+
+        finally:
+
+            if conn is not None:
+
+                conn.close()
+
+    @staticmethod
+    def __check_ip(self, ip):
+
+        is_tor_exit_node = 'f'
+
+        if ip == None:
+
+            return
 
         conn = None
 
         try:
 
-            conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
+            conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
 
             cur = conn.cursor()
 
-            cur.execute('select id from spamcheck')
+            cur.execute('select ip from torexit_ips where ip=(%s)', (ip,))
 
-            rows = cur.fetchall()
+            row = cur.fetchone()
 
-            for row in rows:
+            if row != None:
 
-                approved_users_id_lst.append(row)
+                is_tor_exit_node = 't'
 
             cur.close()
 
@@ -341,36 +303,162 @@ if __name__ == '__main__':
 
                 conn.close()
 
-        for user_id in approved_users_id_lst:
+        return is_tor_exit_node
 
-            approved = check_approval(user_id)
+    @staticmethod
+    def __check_setup(self):
 
-            if approved:
+        is_setup = False
 
-                conn = None
+        if not os.path.isfile(self.config_file):
+            print(f"File {self.config_file} not found, running setup.\n")
+        else:
+            is_setup = True
 
-                try:
+        return is_setup
+    
+    @staticmethod
+    def __setup(self):
 
-                    conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
+        if not os.path.exists('config'):
+            os.makedirs('config')
 
-                    cur = conn.cursor()
+        self.mastodon_hostname = input("Mastodon hostname, in ex. 'mastodon.social': ")
+        self.mastodon_db = input("Mastodon's database name: ")
+        self.mastodon_db_user = input("Mastodon's database user: ")
+        self.spamcheck_db = input("Spamcheck's database name: ")
+        self.spamcheck_db_user = input("Spamcheck's database user: ")
 
-                    cur.execute('delete from spamcheck where id=(%s)', (user_id,))
+        if not os.path.exists(self.config_file):
+            with open(self.config_file, 'w'): pass
+            print(f"\n{self.config_file} created!\n")
 
-                    conn.commit()
+        with open(self.config_file, 'a') as the_file:
+            print(f"Writing Mastodon hostname parameter to {self.config_file}")
+            the_file.write(f'mastodon_hostname: {self.mastodon_hostname}\n')
+            the_file.write(f'mastodon_db: {self.mastodon_db}\n')
+            the_file.write(f'mastodon_db_user: {self.mastodon_db_user}\n')
+            the_file.write(f'spamcheck_db: {self.spamcheck_db}\n')
+            the_file.write(f'spamcheck_db_user: {self.spamcheck_db_user}\n')
 
-                    cur.close()
+        return (self.mastodon_hostname, self.mastodon_db, self.mastodon_db_user, self.spamcheck_db, self.spamcheck_db_user)
 
-                except (Exception, psycopg2.DatabaseError) as error:
+    @staticmethod
+    def __check_dbsetup(self):
 
-                    print(error)
+        dbsetup = False
 
-                finally:
+        try:
 
-                    if conn is not None:
+            conn = None
 
-                        conn.close()
+            conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
 
-    spamcheck_datetime_lst, spamcheck_registers_lst = get_totals()
+            dbsetup = True
 
-    write_totals(spamcheck_datetime_lst, spamcheck_registers_lst)
+        except (Exception, psycopg2.DatabaseError) as error:
+
+            print(error)
+
+        return dbsetup
+
+    @staticmethod
+    def __createdb(self):
+
+        conn = None
+
+        try:
+
+            conn = psycopg2.connect(dbname='postgres',
+                user=self.spamcheck_db_user, host='',
+                password='')
+
+            conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
+
+            cur = conn.cursor()
+
+            print(f"Creating database {self.spamcheck_db}. Please wait...")
+
+            cur.execute(sql.SQL("CREATE DATABASE {}").format(
+                sql.Identifier(self.spamcheck_db))
+            )
+            print(f"Database {self.spamcheck_db} created!\n")
+
+            self.__dbtables_schemes(self)
+
+        except (Exception, psycopg2.DatabaseError) as error:
+
+            print(error)
+
+        finally:
+
+            if conn is not None:
+
+                conn.close()
+
+    @staticmethod
+    def __dbtables_schemes(self):
+
+        table = "spamcheck"
+        sql = "create table "+table+" (created_at timestamptz, id bigint PRIMARY KEY, email varchar(200), ip inet, tor_exit_node boolean)"
+        self.__create_table(self, table, sql)
+
+        table = "torexit_ips"
+        sql = "create table "+table+" (created_at timestamptz, ip inet PRIMARY KEY)"
+        self.__create_table(self, table, sql)
+
+        table = "totals"
+        sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, registers int)"
+        self.__create_table(self, table, sql)
+
+    @staticmethod
+    def __create_table(self, table, sql):
+
+        conn = None
+
+        try:
+
+            conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
+            cur = conn.cursor()
+
+            print(f"Creating table {table}")
+            cur.execute(sql)
+
+            conn.commit()
+            print(f"Table {table} created!\n")
+
+        except (Exception, psycopg2.DatabaseError) as error:
+
+            print(error)
+
+        finally:
+
+            if conn is not None:
+
+                conn.close()
+
+    @staticmethod
+    def __get_parameter(parameter, file_path ):
+
+        with open( file_path ) as f:
+            for line in f:
+                if line.startswith( parameter ):
+                    return line.replace(parameter + ":", "").strip()
+
+        print(f'{file_path} Missing parameter {parameter}')
+        sys.exit(0)
+
+###############################################################################
+# main
+
+if __name__ == '__main__':
+
+    spamcheck = Spamcheck()
+    
+    created_at_lst, id_lst, email_lst, ip_lst = spamcheck.new_registers()
+
+    spamcheck.save_registers(created_at_lst, id_lst, email_lst, ip_lst)
+
+    spamcheck_datetime_lst, spamcheck_registers_lst = spamcheck.get_totals()
+
+    spamcheck.write_totals(spamcheck_datetime_lst, spamcheck_registers_lst)

torips.py

@@ -1,7 +1,9 @@
 import os
 import datetime
 import psycopg2
-import pdb
+import requests
+import re
+import sys
 
 def insert_tor_ip(tor_ip):
 
@@ -33,17 +35,47 @@ def insert_tor_ip(tor_ip):
 
             conn.close()
 
+def get_torbulk_list():
+
+    is_saved = False
+
+    try:
+
+        user_agent = {'User-agent': 'Mozilla/5.0'}
+
+        response = requests.get('https://check.torproject.org/exit-addresses', headers = user_agent, timeout=3)
+
+        if response.status_code == 200:
+
+            with open('torbulkexitlist', 'wb') as f:
+
+                f.write(response.content)
+
+            is_saved = True
+
+        else:
+
+            sys.exit(f'download error: {response.status_code}')
+
+    except requests.exceptions.ConnectionError as conn_error:
+
+        print(f'{conn_error}')
+
+        pass
+
+    return is_saved
+
 def db_config():
 
     # Load db configuration from config file
-    config_filepath = "config/db_config.txt"
+    config_filepath = "config/config.txt"
     spamcheck_db = get_parameter("spamcheck_db", config_filepath)
     spamcheck_db_user = get_parameter("spamcheck_db_user", config_filepath)
 
     return (spamcheck_db, spamcheck_db_user)
 
 def get_parameter( parameter, file_path ):
-    # Check if secrets file exists
+
     if not os.path.isfile(file_path):
         print("File %s not found, exiting."%file_path)
         sys.exit(0)
@@ -63,16 +95,35 @@ if __name__ == '__main__':
     spamcheck_db, spamcheck_db_user = db_config()
 
     now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+    is_saved = get_torbulk_list()
+
+    if is_saved:
     
-    filepath = 'torbulkexitlist'
-    with open(filepath) as fp:
-       line = fp.readline()
-       cnt = 1
-       while line:
-           #print("Line {}: {}".format(cnt, line.strip()))
-           line = fp.readline().rstrip('\n')
-           if line != '':
-               insert_tor_ip(line)
-           cnt += 1
+        filepath = 'torbulkexitlist'
+
+        with open(filepath) as fp:
+
+            line = fp.readline()
+            cnt = 1
+
+            while line:
+
+                line = fp.readline().rstrip('\n')
+
+                if 'ExitAddress' in line:
+
+                    pattern = re.compile(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})')
+
+                    ip = pattern.search(line)[0]
+                    insert_tor_ip(ip)
+
+                cnt += 1
+
+            print(f'Tor exit nodes: {cnt-1}')
+
+    else:
+
+        sys.exit('error downloading torbulkexitlist file')