Added Tor exit node IP checking

This commit is contained in:
spla 2022-01-16 14:33:44 +01:00
pare 84db650610
commit 8251a6ebe3
S'han modificat 6 arxius amb 2166 adicions i 8 eliminacions

Veure arxiu

@ -18,7 +18,11 @@ Within Python Virtual Environment:
1. Run `pip install -r requirements.txt` to install needed Python libraries.
2. Run `python db-setup.py` to setup and create new Postgresql database and needed tables in it.
2. Run `python db-setup.py` to setup and create new Postgresql database and needed tables in it.
3. Run `python torips.py` to write Tor exit nodes IPs to database. You need to get the torbulkexitlist from [here](https://check.torproject.org/torbulkexitlist)
4. Use your favourite scheduling method to set `python spamcheck.py` to run regularly.
3. Use your favourite scheduling method to set `python spamcheck.py` to run regularly.

148
checktornodes.py Normal file
Veure arxiu

@ -0,0 +1,148 @@
import os
import sys
import psycopg2
import pdb
def check_ip(ip):
is_tor_exit_node = 'f'
conn = None
try:
conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute('select ip from torexit_ips where ip=(%s)', (ip,))
row = cur.fetchone()
if row != None:
print(f'{ip} is a Tor exit node')
is_tor_exit_node = 't'
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
return is_tor_exit_node
def get_spam_ips():
spam_ip_lst = []
conn = None
try:
conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute('select ip from spamcheck')
rows = cur.fetchall()
for row in rows:
spam_ip_lst.append(row[0])
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
return spam_ip_lst
def db_config():
# Load db configuration from config file
config_filepath = "config/db_config.txt"
spamcheck_db = get_parameter("spamcheck_db", config_filepath)
spamcheck_db_user = get_parameter("spamcheck_db_user", config_filepath)
return (spamcheck_db, spamcheck_db_user)
# Returns the parameter from the specified file
def get_parameter( parameter, file_path ):
# Check if secrets file exists
if not os.path.isfile(file_path):
print("File %s not found, exiting."%file_path)
sys.exit(0)
# Find parameter in file
with open( file_path ) as f:
for line in f:
if line.startswith( parameter ):
return line.replace(parameter + ":", "").strip()
# Cannot find parameter, exit
print(file_path + " Missing parameter %s "%parameter)
sys.exit(0)
###############################################################################
# main
if __name__ == '__main__':
spamcheck_db, spamcheck_db_user = db_config()
spam_ip_lst = get_spam_ips()
print(f'{len(spam_ip_lst)} IPs found.')
i = 0
while i < len(spam_ip_lst):
is_tor_exit_node = check_ip(spam_ip_lst[i])
tor_exit_node = 't' if is_tor_exit_node == 't' else 'f'
update_sql = 'UPDATE spamcheck set tor_exit_node=(%s) where ip=(%s)'
conn = None
try:
conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute(update_sql, (tor_exit_node, spam_ip_lst[i]))
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
i += 1

Veure arxiu

@ -150,9 +150,14 @@ if __name__ == '__main__':
db_user = spamcheck_db_user
table = "spamcheck"
sql = "create table "+table+" (created_at timestamptz, id bigint PRIMARY KEY, email varchar(200), ip inet, text varchar(200))"
sql = "create table "+table+" (created_at timestamptz, id bigint PRIMARY KEY, email varchar(200), ip inet, text varchar(200), tor_exit_node boolean)"
create_table(db, db_user, table, sql)
table = "torexit_ips"
sql = "create table "+table+" (created_at timestamptz, ip inet PRIMARY KEY)"
create_table(db, db_user, table, sql)
############################################################
print("Done!")

Veure arxiu

@ -7,8 +7,103 @@ import sys
import os.path
import operator
import psycopg2
from langdetect import detect
import requests
import pdb
def check_ip(ip):
is_tor_exit_node = 'f'
conn = None
try:
conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute('select ip from torexit_ips where ip=(%s)', (ip,))
row = cur.fetchone()
if row != None:
is_tor_exit_node = 't'
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
return is_tor_exit_node
def check_approval(user_id):
approved = False
try:
conn = None
conn = psycopg2.connect(database = mastodon_db, user = mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute("select approved from users where id = (%s)", (user_id,))
row = cur.fetchone()
if row != None:
approved = row[0]
cur.close()
return approved
except (Exception, psycopg2.DatabaseError) as error:
print (error)
finally:
if conn is not None:
conn.close()
def mastodon():
# Load secrets from secrets file
secrets_filepath = "secrets/secrets.txt"
uc_client_id = get_parameter("uc_client_id", secrets_filepath)
uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
uc_access_token = get_parameter("uc_access_token", secrets_filepath)
# Load configuration from config file
config_filepath = "config/config.txt"
mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
# Initialise Mastodon API
mastodon = Mastodon(
client_id=uc_client_id,
client_secret=uc_client_secret,
access_token=uc_access_token,
api_base_url='https://' + mastodon_hostname,
)
# Initialise access headers
headers = {'Authorization': 'Bearer %s'%uc_access_token}
return (mastodon, mastodon_hostname)
def db_config():
# Load db configuration from config file
@ -44,6 +139,8 @@ if __name__ == '__main__':
mastodon_db, mastodon_db_user, spamcheck_db, spamcheck_db_user = db_config()
mastodon, mastodon_hostname = mastodon()
###############################################################################
# check new registering
###############################################################################
@ -98,12 +195,50 @@ if __name__ == '__main__':
###############################################################################
insert_sql = 'INSERT INTO spamcheck(created_at, id, email, ip, text) VALUES(%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING'
insert_sql = 'INSERT INTO spamcheck(created_at, id, email, ip, text: tor_exit_node) VALUES(%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING'
i = 0
while i < len(id_lst):
if detect(text_lst[i]) != 'ca':
is_tor_exit_node = check_ip(ip_lst[i])
tor_exit_node = 't' if is_tor_exit_node else 'f'
conn = None
try:
conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute(insert_sql, (created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i], text_lst[i]), tor_exit_node)
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
print(created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i], text_lst[i])
i = i + 1
##########################################################################
approved_users_id_lst = []
conn = None
try:
@ -112,9 +247,13 @@ if __name__ == '__main__':
cur = conn.cursor()
cur.execute(insert_sql, (created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i], text_lst[i]))
cur.execute('select id from spamcheck')
conn.commit()
rows = cur.fetchall()
for row in rows:
approved_users_id_lst.append(row)
cur.close()
@ -128,7 +267,32 @@ if __name__ == '__main__':
conn.close()
print(created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i], text_lst[i])
for user_id in approved_users_id_lst:
i = i + 1
approved = check_approval(user_id)
if approved:
conn = None
try:
conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute('delete from spamcheck where id=(%s)', (user_id,))
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()

1759
torbulkexitlist Normal file

La diferencia del archivo ha sido suprimido porque es demasiado grande Cargar Diff

78
torips.py Normal file
Veure arxiu

@ -0,0 +1,78 @@
import os
import datetime
import psycopg2
import pdb
def insert_tor_ip(tor_ip):
insert_sql = 'INSERT INTO torexit_ips(created_at, ip) VALUES(%s,%s) ON CONFLICT DO NOTHING'
conn = None
try:
conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute(insert_sql, (now, tor_ip))
conn.commit()
print(f'Tor IP {tor_ip} saved to database')
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
def db_config():
# Load db configuration from config file
config_filepath = "config/db_config.txt"
spamcheck_db = get_parameter("spamcheck_db", config_filepath)
spamcheck_db_user = get_parameter("spamcheck_db_user", config_filepath)
return (spamcheck_db, spamcheck_db_user)
def get_parameter( parameter, file_path ):
# Check if secrets file exists
if not os.path.isfile(file_path):
print("File %s not found, exiting."%file_path)
sys.exit(0)
# Find parameter in file
with open( file_path ) as f:
for line in f:
if line.startswith( parameter ):
return line.replace(parameter + ":", "").strip()
# Cannot find parameter, exit
print(file_path + " Missing parameter %s "%parameter)
sys.exit(0)
if __name__ == '__main__':
spamcheck_db, spamcheck_db_user = db_config()
now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
filepath = 'torbulkexitlist'
with open(filepath) as fp:
line = fp.readline()
cnt = 1
while line:
#print("Line {}: {}".format(cnt, line.strip()))
line = fp.readline().rstrip('\n')
if line != '':
insert_tor_ip(line)
cnt += 1