Added Tor exit node IP checking

README.md

@@ -18,7 +18,11 @@ Within Python Virtual Environment:
 
 1. Run `pip install -r requirements.txt` to install needed Python libraries.
 
-2. Run `python db-setup.py` to setup and create new Postgresql database and needed tables in it.  
+2. Run `python db-setup.py` to setup and create new Postgresql database and needed tables in it.
+
+3. Run `python torips.py` to write Tor exit nodes IPs to database. You need to get the torbulkexitlist from [here](https://check.torproject.org/torbulkexitlist) 
+
+4. Use your favourite scheduling method to set `python spamcheck.py` to run regularly.  
+
 
-3. Use your favourite scheduling method to set `python spamcheck.py` to run regularly.
 

checktornodes.py

@@ -0,0 +1,148 @@
+import os
+import sys
+import psycopg2
+import pdb
+
+def check_ip(ip):
+
+    is_tor_exit_node = 'f'
+
+    conn = None
+
+    try:
+
+        conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
+
+        cur = conn.cursor()
+
+        cur.execute('select ip from torexit_ips where ip=(%s)', (ip,))
+
+        row = cur.fetchone()
+
+        if row != None:
+
+            print(f'{ip} is a Tor exit node')
+
+            is_tor_exit_node = 't'
+
+        cur.close()
+
+    except (Exception, psycopg2.DatabaseError) as error:
+
+        print(error)
+
+    finally:
+
+        if conn is not None:
+
+            conn.close()
+
+    return is_tor_exit_node
+
+def get_spam_ips():
+
+    spam_ip_lst = []
+
+    conn = None
+
+    try:
+
+        conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
+
+        cur = conn.cursor()
+
+        cur.execute('select ip from spamcheck')
+
+        rows = cur.fetchall()
+
+        for row in rows:
+
+            spam_ip_lst.append(row[0])
+
+        cur.close()
+
+    except (Exception, psycopg2.DatabaseError) as error:
+
+        print(error)
+
+    finally:
+
+        if conn is not None:
+
+            conn.close()
+
+    return spam_ip_lst
+
+def db_config():
+
+    # Load db configuration from config file
+    config_filepath = "config/db_config.txt"
+    spamcheck_db = get_parameter("spamcheck_db", config_filepath)
+    spamcheck_db_user = get_parameter("spamcheck_db_user", config_filepath)
+
+    return (spamcheck_db, spamcheck_db_user)
+
+# Returns the parameter from the specified file
+def get_parameter( parameter, file_path ):
+    # Check if secrets file exists
+    if not os.path.isfile(file_path):
+        print("File %s not found, exiting."%file_path)
+        sys.exit(0)
+
+    # Find parameter in file
+    with open( file_path ) as f:
+        for line in f:
+            if line.startswith( parameter ):
+                return line.replace(parameter + ":", "").strip()
+
+    # Cannot find parameter, exit
+    print(file_path + "  Missing parameter %s "%parameter)
+    sys.exit(0)
+
+###############################################################################
+# main
+
+if __name__ == '__main__':
+
+    spamcheck_db, spamcheck_db_user = db_config()
+
+    spam_ip_lst = get_spam_ips()
+
+    print(f'{len(spam_ip_lst)} IPs found.')
+
+    i = 0
+
+    while i < len(spam_ip_lst):
+
+        is_tor_exit_node = check_ip(spam_ip_lst[i])
+
+        tor_exit_node = 't' if is_tor_exit_node == 't' else 'f'
+
+        update_sql = 'UPDATE spamcheck set tor_exit_node=(%s) where ip=(%s)'
+
+        conn = None
+
+        try:
+
+            conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
+
+            cur = conn.cursor()
+
+            cur.execute(update_sql, (tor_exit_node, spam_ip_lst[i]))
+
+            conn.commit()
+
+            cur.close()
+
+        except (Exception, psycopg2.DatabaseError) as error:
+
+            print(error)
+
+        finally:
+
+            if conn is not None:
+
+                conn.close()
+
+        i += 1
+

db-setup.py

@@ -150,9 +150,14 @@ if __name__ == '__main__':
     db_user = spamcheck_db_user
 
     table = "spamcheck"
-    sql = "create table "+table+" (created_at timestamptz, id bigint PRIMARY KEY, email varchar(200), ip inet, text varchar(200))"
+    sql = "create table "+table+" (created_at timestamptz, id bigint PRIMARY KEY, email varchar(200), ip inet, text varchar(200), tor_exit_node boolean)"
     create_table(db, db_user, table, sql)
 
+    table = "torexit_ips"
+    sql = "create table "+table+" (created_at timestamptz, ip inet PRIMARY KEY)"
+    create_table(db, db_user, table, sql)
+
+
     ############################################################
 
     print("Done!")

spamcheck.py

@@ -7,8 +7,103 @@ import sys
 import os.path
 import operator
 import psycopg2
+from langdetect import detect
+import requests
 import pdb
 
+def check_ip(ip):
+
+    is_tor_exit_node = 'f'
+
+    conn = None
+
+    try:
+
+        conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
+
+        cur = conn.cursor()
+
+        cur.execute('select ip from torexit_ips where ip=(%s)', (ip,))
+
+        row = cur.fetchone()
+
+        if row != None:
+
+            is_tor_exit_node = 't'
+
+        cur.close()
+
+    except (Exception, psycopg2.DatabaseError) as error:
+
+        print(error)
+
+    finally:
+
+        if conn is not None:
+
+            conn.close()
+
+    return is_tor_exit_node
+
+def check_approval(user_id):
+
+    approved = False
+
+    try:
+
+        conn = None
+
+        conn = psycopg2.connect(database = mastodon_db, user = mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432")
+
+        cur = conn.cursor()
+
+        cur.execute("select approved from users where id = (%s)", (user_id,))
+
+        row = cur.fetchone()
+
+        if row != None:
+
+            approved = row[0]
+
+        cur.close()
+
+        return approved
+
+    except (Exception, psycopg2.DatabaseError) as error:
+
+        print (error)
+
+    finally:
+
+        if conn is not None:
+
+            conn.close()
+
+def mastodon():
+
+    # Load secrets from secrets file
+    secrets_filepath = "secrets/secrets.txt"
+    uc_client_id = get_parameter("uc_client_id",     secrets_filepath)
+    uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
+    uc_access_token = get_parameter("uc_access_token",  secrets_filepath)
+
+    # Load configuration from config file
+    config_filepath = "config/config.txt"
+    mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
+
+    # Initialise Mastodon API
+    mastodon = Mastodon(
+        client_id=uc_client_id,
+        client_secret=uc_client_secret,
+        access_token=uc_access_token,
+        api_base_url='https://' + mastodon_hostname,
+    )
+
+    # Initialise access headers
+    headers = {'Authorization': 'Bearer %s'%uc_access_token}
+
+    return (mastodon, mastodon_hostname)
+
 def db_config():
 
     # Load db configuration from config file
@@ -44,6 +139,8 @@ if __name__ == '__main__':
 
     mastodon_db, mastodon_db_user, spamcheck_db, spamcheck_db_user = db_config()
 
+    mastodon, mastodon_hostname = mastodon()
+
     ###############################################################################
     # check new registering
     ###############################################################################
@@ -98,12 +195,50 @@ if __name__ == '__main__':
 
     ###############################################################################
 
-    insert_sql = 'INSERT INTO spamcheck(created_at, id, email, ip, text) VALUES(%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING'
+    insert_sql = 'INSERT INTO spamcheck(created_at, id, email, ip, text: tor_exit_node) VALUES(%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING'
     
     i = 0
 
     while i < len(id_lst):
 
+        if detect(text_lst[i]) != 'ca':
+
+            is_tor_exit_node = check_ip(ip_lst[i])
+
+            tor_exit_node = 't' if is_tor_exit_node else 'f'
+
+            conn = None
+
+            try:
+
+                conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
+
+                cur = conn.cursor()
+
+                cur.execute(insert_sql, (created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i], text_lst[i]), tor_exit_node)
+
+                conn.commit()
+
+                cur.close()
+
+            except (Exception, psycopg2.DatabaseError) as error:
+
+                print(error)
+
+            finally:
+
+                if conn is not None:
+
+                    conn.close()
+
+            print(created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i], text_lst[i])
+
+        i = i + 1
+
+        ##########################################################################
+
+        approved_users_id_lst = []
+
         conn = None
 
         try:
@@ -112,9 +247,13 @@ if __name__ == '__main__':
 
             cur = conn.cursor()
 
-            cur.execute(insert_sql, (created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i], text_lst[i]))
+            cur.execute('select id from spamcheck')
 
-            conn.commit()
+            rows = cur.fetchall()
+
+            for row in rows:
+
+                approved_users_id_lst.append(row)
 
             cur.close()
 
@@ -128,7 +267,32 @@ if __name__ == '__main__':
 
                 conn.close()
 
-        print(created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i], text_lst[i])
+        for user_id in approved_users_id_lst:
 
-        i = i + 1
+            approved = check_approval(user_id)
 
+            if approved:
+
+                conn = None
+
+                try:
+
+                    conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
+
+                    cur = conn.cursor()
+
+                    cur.execute('delete from spamcheck where id=(%s)', (user_id,))
+
+                    conn.commit()
+
+                    cur.close()
+
+                except (Exception, psycopg2.DatabaseError) as error:
+
+                    print(error)
+
+                finally:
+
+                    if conn is not None:
+
+                        conn.close()

torips.py

@@ -0,0 +1,78 @@
+import os
+import datetime
+import psycopg2
+import pdb
+
+def insert_tor_ip(tor_ip):
+
+    insert_sql = 'INSERT INTO torexit_ips(created_at, ip) VALUES(%s,%s) ON CONFLICT DO NOTHING'
+
+    conn = None
+
+    try:
+
+        conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
+
+        cur = conn.cursor()
+
+        cur.execute(insert_sql, (now, tor_ip))
+
+        conn.commit()
+
+        print(f'Tor IP {tor_ip} saved to database')
+
+        cur.close()
+
+    except (Exception, psycopg2.DatabaseError) as error:
+
+        print(error)
+
+    finally:
+
+        if conn is not None:
+
+            conn.close()
+
+def db_config():
+
+    # Load db configuration from config file
+    config_filepath = "config/db_config.txt"
+    spamcheck_db = get_parameter("spamcheck_db", config_filepath)
+    spamcheck_db_user = get_parameter("spamcheck_db_user", config_filepath)
+
+    return (spamcheck_db, spamcheck_db_user)
+
+def get_parameter( parameter, file_path ):
+    # Check if secrets file exists
+    if not os.path.isfile(file_path):
+        print("File %s not found, exiting."%file_path)
+        sys.exit(0)
+
+    # Find parameter in file
+    with open( file_path ) as f:
+        for line in f:
+            if line.startswith( parameter ):
+                return line.replace(parameter + ":", "").strip()
+
+    # Cannot find parameter, exit
+    print(file_path + "  Missing parameter %s "%parameter)
+    sys.exit(0)
+
+if __name__ == '__main__':
+    
+    spamcheck_db, spamcheck_db_user = db_config()
+
+    now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    
+    filepath = 'torbulkexitlist'
+    with open(filepath) as fp:
+       line = fp.readline()
+       cnt = 1
+       while line:
+           #print("Line {}: {}".format(cnt, line.strip()))
+           line = fp.readline().rstrip('\n')
+           if line != '':
+               insert_tor_ip(line)
+           cnt += 1
+
+