Refactored to Class
This commit is contained in:
pare
96da1a1118
commit
db24205899
S'han modificat 5 arxius amb 390 adicions i 526 eliminacions
10
README.md
10
README.md
|
|
@ -1,10 +1,6 @@
|
|||
# Mastodon's spamcheck
|
||||
|
||||
Check and store data from spam bots trying to register to your Mastodon server.
|
||||
You need to configure your Mastodon server like this:
|
||||
|
||||
- Registration mode -> Approval required for sign up
|
||||
- Require new users to enter a reason to join -> activated
|
||||
Check and store data from sign-ups registered on your Mastodon server.
|
||||
|
||||
### Dependencies
|
||||
|
||||
|
|
@ -18,9 +14,9 @@ Within Python Virtual Environment:
|
|||
|
||||
1. Run `pip install -r requirements.txt` to install needed Python libraries.
|
||||
|
||||
2. Run `python db-setup.py` to setup and create new Postgresql database and needed tables in it.
|
||||
2. Run `python torips.py` to write Tor exit nodes IPs to new created database. You need to get the torbulkexitlist from [here](https://check.torproject.org/exit-addresses)
|
||||
|
||||
3. Run `python torips.py` to write Tor exit nodes IPs to database. You need to get the torbulkexitlist from [here](https://blog.torproject.org/changes-tor-exit-list-service/)
|
||||
3. Run `python spamcheck.py` for the first time to configure it.
|
||||
|
||||
4. Use your favourite scheduling method to set `python spamcheck.py` and `python torips.py` to run regularly.
|
||||
|
||||
|
|
|
|||
170
db-setup.py
170
db-setup.py
|
|
@ -1,170 +0,0 @@
|
|||
#import getpass
|
||||
import os
|
||||
import sys
|
||||
import psycopg2
|
||||
from psycopg2 import sql
|
||||
from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
|
||||
|
||||
# Returns the parameter from the specified file
|
||||
def get_parameter( parameter, file_path ):
|
||||
# Check if secrets file exists
|
||||
if not os.path.isfile(file_path):
|
||||
print("File %s not found, asking."%file_path)
|
||||
write_parameter( parameter, file_path )
|
||||
#sys.exit(0)
|
||||
|
||||
# Find parameter in file
|
||||
with open( file_path ) as f:
|
||||
for line in f:
|
||||
if line.startswith( parameter ):
|
||||
return line.replace(parameter + ":", "").strip()
|
||||
|
||||
# Cannot find parameter, exit
|
||||
print(file_path + " Missing parameter %s "%parameter)
|
||||
sys.exit(0)
|
||||
|
||||
def write_parameter( parameter, file_path ):
|
||||
if not os.path.exists('config'):
|
||||
os.makedirs('config')
|
||||
print("Setting up spamcheck parameters...")
|
||||
print("\n")
|
||||
spamcheck_db = input("spamcheck db name: ")
|
||||
spamcheck_db_user = input("spamcheck db user: ")
|
||||
mastodon_db = input("Mastodon db name: ")
|
||||
mastodon_db_user = input("Mastodon db user: ")
|
||||
|
||||
|
||||
with open(file_path, "w") as text_file:
|
||||
print("spamcheck_db: {}".format(spamcheck_db), file=text_file)
|
||||
print("spamcheck_db_user: {}".format(spamcheck_db_user), file=text_file)
|
||||
print("mastodon_db: {}".format(mastodon_db), file=text_file)
|
||||
print("mastodon_db_user: {}".format(mastodon_db_user), file=text_file)
|
||||
|
||||
def create_table(db, db_user, table, sql):
|
||||
|
||||
conn = None
|
||||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(database = db, user = db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
cur = conn.cursor()
|
||||
|
||||
print("Creating table.. "+table)
|
||||
# Create the table in PostgreSQL database
|
||||
cur.execute(sql)
|
||||
|
||||
conn.commit()
|
||||
print("Table "+table+" created!")
|
||||
print("\n")
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print(error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
###############################################################################
|
||||
# main
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
# Load configuration from config file
|
||||
config_filepath = "config/db_config.txt"
|
||||
spamcheck_db = get_parameter("spamcheck_db", config_filepath)
|
||||
spamcheck_db_user = get_parameter("spamcheck_db_user", config_filepath)
|
||||
mastodon_db = get_parameter("mastodon_db", config_filepath)
|
||||
mastodon_db_user = get_parameter("mastodon_db_user", config_filepath)
|
||||
|
||||
############################################################
|
||||
# create database
|
||||
############################################################
|
||||
|
||||
conn = None
|
||||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(dbname='postgres',
|
||||
user=spamcheck_db_user, host='',
|
||||
password='')
|
||||
|
||||
conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
print("Creating database " + spamcheck_db + ". Please wait...")
|
||||
|
||||
cur.execute(sql.SQL("CREATE DATABASE {}").format(
|
||||
sql.Identifier(spamcheck_db))
|
||||
)
|
||||
print("Database " + spamcheck_db + " created!")
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print(error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
############################################################
|
||||
|
||||
try:
|
||||
|
||||
conn = None
|
||||
|
||||
conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print(error)
|
||||
|
||||
# Load configuration from config file
|
||||
os.remove("config/db_config.txt")
|
||||
|
||||
print("Exiting. Run db-setup again with right parameters")
|
||||
sys.exit(0)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
print("\n")
|
||||
print("spamcheck parameters saved to db-config.txt!")
|
||||
print("\n")
|
||||
|
||||
############################################################
|
||||
# Create needed tables
|
||||
############################################################
|
||||
|
||||
print("Creating table...")
|
||||
|
||||
db = spamcheck_db
|
||||
db_user = spamcheck_db_user
|
||||
|
||||
table = "spamcheck"
|
||||
sql = "create table "+table+" (created_at timestamptz, id bigint PRIMARY KEY, email varchar(200), ip inet, text varchar(200), tor_exit_node boolean)"
|
||||
create_table(db, db_user, table, sql)
|
||||
|
||||
table = "torexit_ips"
|
||||
sql = "create table "+table+" (created_at timestamptz, ip inet PRIMARY KEY)"
|
||||
create_table(db, db_user, table, sql)
|
||||
|
||||
table = "totals"
|
||||
sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, registers int)"
|
||||
create_table(db, db_user, table, sql)
|
||||
|
||||
|
||||
|
||||
############################################################
|
||||
|
||||
print("Done!")
|
||||
print("Now you can run setup.py!")
|
||||
print("\n")
|
||||
|
|
@ -1,3 +1,2 @@
|
|||
Mastodon.py>=1.5.1
|
||||
wheel>=0.36.2
|
||||
psycopg2>=2.8.6
|
||||
psycopg2-binary
|
||||
requests
|
||||
|
|
|
|||
712
spamcheck.py
712
spamcheck.py
|
|
@ -1,5 +1,5 @@
|
|||
from datetime import date, datetime, timedelta
|
||||
from mastodon import Mastodon
|
||||
#from mastodon import Mastodon
|
||||
import time
|
||||
import os
|
||||
import json
|
||||
|
|
@ -7,341 +7,98 @@ import sys
|
|||
import os.path
|
||||
import operator
|
||||
import psycopg2
|
||||
from langdetect import detect
|
||||
import requests
|
||||
import pdb
|
||||
from psycopg2 import sql
|
||||
from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
|
||||
|
||||
def write_totals(spamcheck_datetime_lst, spamcheck_registers_lst):
|
||||
class Spamcheck:
|
||||
|
||||
insert_sql = 'INSERT INTO totals(datetime, registers) VALUES(%s,%s) ON CONFLICT (datetime) DO UPDATE SET (datetime, registers) = (EXCLUDED.datetime, EXCLUDED.registers)'
|
||||
name = "Spamcheck for Mastodon social servers"
|
||||
|
||||
first_date = spamcheck_datetime_lst[0]
|
||||
def __init__(self, mastodon_hostname=None, mastodon_db=None, mastodon_db_user=None, spamcheck_db=None, spamcheck_db_user=None):
|
||||
|
||||
last_date = spamcheck_datetime_lst[len(spamcheck_datetime_lst)-1]
|
||||
self.config_file = 'config/config.txt'
|
||||
|
||||
i = 0
|
||||
is_setup = self.__check_setup(self)
|
||||
|
||||
while i < len(spamcheck_datetime_lst):
|
||||
if is_setup:
|
||||
|
||||
conn = None
|
||||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
if first_date == spamcheck_datetime_lst[i]:
|
||||
|
||||
cur.execute(insert_sql, (spamcheck_datetime_lst[i], spamcheck_registers_lst[i]))
|
||||
|
||||
i += 1
|
||||
|
||||
else:
|
||||
|
||||
cur.execute(insert_sql, (first_date, '0'))
|
||||
|
||||
conn.commit()
|
||||
|
||||
cur.close()
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print(error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
first_date = first_date + timedelta(days=1)
|
||||
|
||||
if date.today() == last_date + timedelta(days=1):
|
||||
|
||||
insert_sql = 'INSERT INTO totals(datetime, registers) VALUES(%s,%s) ON CONFLICT (datetime) DO UPDATE SET (datetime, registers) = (EXCLUDED.datetime, EXCLUDED.registers)'
|
||||
|
||||
conn = None
|
||||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute(insert_sql, (date.today(), '0'))
|
||||
|
||||
conn.commit()
|
||||
|
||||
cur.close()
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print(error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
def get_totals():
|
||||
|
||||
spamcheck_datetime_lst = []
|
||||
|
||||
spamcheck_registers_lst = []
|
||||
|
||||
select_sql = 'select date(created_at), count(ip) as registers from spamcheck group by date(created_at) order by date(created_at)'
|
||||
|
||||
conn = None
|
||||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute(select_sql)
|
||||
|
||||
rows = cur.fetchall()
|
||||
|
||||
for row in rows:
|
||||
|
||||
spamcheck_datetime_lst.append(row[0])
|
||||
|
||||
spamcheck_registers_lst.append(row[1])
|
||||
|
||||
cur.close()
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print(error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
return (spamcheck_datetime_lst, spamcheck_registers_lst)
|
||||
|
||||
def check_ip(ip):
|
||||
|
||||
is_tor_exit_node = 'f'
|
||||
|
||||
conn = None
|
||||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute('select ip from torexit_ips where ip=(%s)', (ip,))
|
||||
|
||||
row = cur.fetchone()
|
||||
|
||||
if row != None:
|
||||
|
||||
is_tor_exit_node = 't'
|
||||
|
||||
cur.close()
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print(error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
return is_tor_exit_node
|
||||
|
||||
def check_approval(user_id):
|
||||
|
||||
approved = False
|
||||
|
||||
try:
|
||||
|
||||
conn = None
|
||||
|
||||
conn = psycopg2.connect(database = mastodon_db, user = mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute("select approved from users where id = (%s)", (user_id,))
|
||||
|
||||
row = cur.fetchone()
|
||||
|
||||
if row != None:
|
||||
|
||||
approved = row[0]
|
||||
|
||||
cur.close()
|
||||
|
||||
return approved
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print (error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
def mastodon():
|
||||
|
||||
# Load secrets from secrets file
|
||||
secrets_filepath = "secrets/secrets.txt"
|
||||
uc_client_id = get_parameter("uc_client_id", secrets_filepath)
|
||||
uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
|
||||
uc_access_token = get_parameter("uc_access_token", secrets_filepath)
|
||||
|
||||
# Load configuration from config file
|
||||
config_filepath = "config/config.txt"
|
||||
mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
|
||||
|
||||
# Initialise Mastodon API
|
||||
mastodon = Mastodon(
|
||||
client_id=uc_client_id,
|
||||
client_secret=uc_client_secret,
|
||||
access_token=uc_access_token,
|
||||
api_base_url='https://' + mastodon_hostname,
|
||||
)
|
||||
|
||||
# Initialise access headers
|
||||
headers = {'Authorization': 'Bearer %s'%uc_access_token}
|
||||
|
||||
return (mastodon, mastodon_hostname)
|
||||
|
||||
def db_config():
|
||||
|
||||
# Load db configuration from config file
|
||||
config_filepath = "config/db_config.txt"
|
||||
mastodon_db = get_parameter("mastodon_db", config_filepath)
|
||||
mastodon_db_user = get_parameter("mastodon_db_user", config_filepath)
|
||||
spamcheck_db = get_parameter("spamcheck_db", config_filepath)
|
||||
spamcheck_db_user = get_parameter("spamcheck_db_user", config_filepath)
|
||||
|
||||
return (mastodon_db, mastodon_db_user, spamcheck_db, spamcheck_db_user)
|
||||
|
||||
# Returns the parameter from the specified file
|
||||
def get_parameter( parameter, file_path ):
|
||||
# Check if secrets file exists
|
||||
if not os.path.isfile(file_path):
|
||||
print("File %s not found, exiting."%file_path)
|
||||
sys.exit(0)
|
||||
|
||||
# Find parameter in file
|
||||
with open( file_path ) as f:
|
||||
for line in f:
|
||||
if line.startswith( parameter ):
|
||||
return line.replace(parameter + ":", "").strip()
|
||||
|
||||
# Cannot find parameter, exit
|
||||
print(file_path + " Missing parameter %s "%parameter)
|
||||
sys.exit(0)
|
||||
|
||||
###############################################################################
|
||||
# main
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
mastodon_db, mastodon_db_user, spamcheck_db, spamcheck_db_user = db_config()
|
||||
|
||||
mastodon, mastodon_hostname = mastodon()
|
||||
|
||||
###############################################################################
|
||||
# check new registering
|
||||
###############################################################################
|
||||
|
||||
created_at_lst = []
|
||||
|
||||
id_lst = []
|
||||
|
||||
email_lst = []
|
||||
|
||||
ip_lst = []
|
||||
|
||||
text_lst = []
|
||||
|
||||
try:
|
||||
|
||||
conn = None
|
||||
|
||||
conn = psycopg2.connect(database = mastodon_db, user = mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute("select users.created_at, users.id, users.email, users.sign_up_ip, user_invite_requests.text from users, user_invite_requests where not users.approved and users.id = user_invite_requests.user_id")
|
||||
|
||||
rows = cur.fetchall()
|
||||
|
||||
for row in rows:
|
||||
|
||||
if row != None:
|
||||
|
||||
created_at_lst.append(row[0])
|
||||
|
||||
id_lst.append(row[1])
|
||||
|
||||
email_lst.append(row[2])
|
||||
|
||||
ip_lst.append(row[3])
|
||||
|
||||
text_lst.append(row[4])
|
||||
|
||||
cur.close()
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print (error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
###############################################################################
|
||||
|
||||
insert_sql = 'INSERT INTO spamcheck(created_at, id, email, ip, text, tor_exit_node) VALUES(%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING'
|
||||
|
||||
i = 0
|
||||
|
||||
while i < len(id_lst):
|
||||
|
||||
if len(text_lst[i]) > 1:
|
||||
|
||||
text_lang = detect(text_lst[i])
|
||||
self.mastodon_hostname = self.__get_parameter("mastodon_hostname", self.config_file)
|
||||
self.mastodon_db = self.__get_parameter("mastodon_db", self.config_file)
|
||||
self.mastodon_db_user = self.__get_parameter("mastodon_db_user", self.config_file)
|
||||
self.spamcheck_db = self.__get_parameter("spamcheck_db", self.config_file)
|
||||
self.spamcheck_db_user = self.__get_parameter("spamcheck_db_user", self.config_file)
|
||||
|
||||
else:
|
||||
|
||||
text_lang = 'en'
|
||||
self.mastodon_hostname, self.mastodon_db, self.mastodon_db_user, self.spamcheck_db, self.spamcheck_db_user = self.__setup(self)
|
||||
|
||||
if text_lang != 'ca' or len(text_lst) == 1:
|
||||
db_setup = self.__check_dbsetup(self)
|
||||
|
||||
is_tor_exit_node = check_ip(ip_lst[i])
|
||||
if not db_setup:
|
||||
|
||||
self.__createdb(self)
|
||||
|
||||
def new_registers(self, created_at_lst=[], id_lst=[], email_lst=[], ip_lst=[]):
|
||||
|
||||
try:
|
||||
|
||||
conn = None
|
||||
|
||||
conn = psycopg2.connect(database = self.mastodon_db, user = self.mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute("select users.created_at, users.id, users.email, users.sign_up_ip from users where users.approved and users.created_at > now() - interval '7 days'")
|
||||
|
||||
rows = cur.fetchall()
|
||||
|
||||
for row in rows:
|
||||
|
||||
if row != None:
|
||||
|
||||
created_at_lst.append(row[0])
|
||||
|
||||
id_lst.append(row[1])
|
||||
|
||||
email_lst.append(row[2])
|
||||
|
||||
ip_lst.append(row[3])
|
||||
|
||||
cur.close()
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print (error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
return (created_at_lst, id_lst, email_lst, ip_lst)
|
||||
|
||||
def save_registers(self, created_at_lst, id_lst, email_lst, ip_lst):
|
||||
|
||||
insert_sql = 'INSERT INTO spamcheck(created_at, id, email, ip, tor_exit_node) VALUES(%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING'
|
||||
|
||||
i = 0
|
||||
|
||||
while i < len(id_lst):
|
||||
|
||||
is_tor_exit_node = self.__check_ip(self, ip_lst[i])
|
||||
|
||||
tor_exit_node = 't' if is_tor_exit_node == 't' else 'f'
|
||||
|
||||
text_length = 200 if len(text_lst[i]) > 200 else len(text_lst[i])
|
||||
|
||||
reason_text = '{0}...'.format(text_lst[i][:text_length].rsplit(' ', 1)[0])
|
||||
|
||||
conn = None
|
||||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute(insert_sql, (created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i], reason_text, tor_exit_node))
|
||||
cur.execute(insert_sql, (created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i], tor_exit_node))
|
||||
|
||||
conn.commit()
|
||||
|
||||
|
|
@ -357,29 +114,35 @@ if __name__ == '__main__':
|
|||
|
||||
conn.close()
|
||||
|
||||
print(created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i], text_lst[i], tor_exit_node)
|
||||
print(created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i], tor_exit_node)
|
||||
|
||||
i = i + 1
|
||||
i += 1
|
||||
|
||||
##########################################################################
|
||||
def get_totals(self):
|
||||
|
||||
approved_users_id_lst = []
|
||||
spamcheck_datetime_lst = []
|
||||
|
||||
spamcheck_registers_lst = []
|
||||
|
||||
select_sql = 'select date(created_at), count(ip) as registers from spamcheck group by date(created_at) order by date(created_at)'
|
||||
|
||||
conn = None
|
||||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute('select id from spamcheck')
|
||||
cur.execute(select_sql)
|
||||
|
||||
rows = cur.fetchall()
|
||||
|
||||
for row in rows:
|
||||
|
||||
approved_users_id_lst.append(row)
|
||||
spamcheck_datetime_lst.append(row[0])
|
||||
|
||||
spamcheck_registers_lst.append(row[1])
|
||||
|
||||
cur.close()
|
||||
|
||||
|
|
@ -393,36 +156,309 @@ if __name__ == '__main__':
|
|||
|
||||
conn.close()
|
||||
|
||||
for user_id in approved_users_id_lst:
|
||||
return (spamcheck_datetime_lst, spamcheck_registers_lst)
|
||||
|
||||
approved = check_approval(user_id)
|
||||
def write_totals(self, spamcheck_datetime_lst, spamcheck_registers_lst):
|
||||
|
||||
if approved:
|
||||
insert_sql = 'INSERT INTO totals(datetime, registers) VALUES(%s,%s) ON CONFLICT (datetime) DO UPDATE SET (datetime, registers) = (EXCLUDED.datetime, EXCLUDED.registers)'
|
||||
|
||||
conn = None
|
||||
first_date = spamcheck_datetime_lst[0]
|
||||
|
||||
try:
|
||||
last_date = spamcheck_datetime_lst[len(spamcheck_datetime_lst)-1]
|
||||
|
||||
conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
i = 0
|
||||
|
||||
cur = conn.cursor()
|
||||
while i < len(spamcheck_datetime_lst):
|
||||
|
||||
cur.execute('delete from spamcheck where id=(%s)', (user_id,))
|
||||
conn = None
|
||||
|
||||
conn.commit()
|
||||
try:
|
||||
|
||||
cur.close()
|
||||
conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
cur = conn.cursor()
|
||||
|
||||
print(error)
|
||||
if first_date == spamcheck_datetime_lst[i]:
|
||||
|
||||
finally:
|
||||
cur.execute(insert_sql, (spamcheck_datetime_lst[i], spamcheck_registers_lst[i]))
|
||||
|
||||
if conn is not None:
|
||||
i += 1
|
||||
|
||||
conn.close()
|
||||
else:
|
||||
|
||||
spamcheck_datetime_lst, spamcheck_registers_lst = get_totals()
|
||||
cur.execute(insert_sql, (first_date, '0'))
|
||||
|
||||
write_totals(spamcheck_datetime_lst, spamcheck_registers_lst)
|
||||
conn.commit()
|
||||
|
||||
cur.close()
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print(error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
first_date = first_date + timedelta(days=1)
|
||||
|
||||
if date.today() == last_date + timedelta(days=1):
|
||||
|
||||
insert_sql = 'INSERT INTO totals(datetime, registers) VALUES(%s,%s) ON CONFLICT (datetime) DO UPDATE SET (datetime, registers) = (EXCLUDED.datetime, EXCLUDED.registers)'
|
||||
|
||||
conn = None
|
||||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute(insert_sql, (date.today(), '0'))
|
||||
|
||||
conn.commit()
|
||||
|
||||
cur.close()
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print(error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
def check_approval(self, user_id):
|
||||
|
||||
approved = False
|
||||
|
||||
try:
|
||||
|
||||
conn = None
|
||||
|
||||
conn = psycopg2.connect(database = self.mastodon_db, user = self.mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute("select approved from users where id = (%s)", (user_id,))
|
||||
|
||||
row = cur.fetchone()
|
||||
|
||||
if row != None:
|
||||
|
||||
approved = row[0]
|
||||
|
||||
cur.close()
|
||||
|
||||
return approved
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print (error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
@staticmethod
|
||||
def __check_ip(self, ip):
|
||||
|
||||
is_tor_exit_node = 'f'
|
||||
|
||||
if ip == None:
|
||||
|
||||
return
|
||||
|
||||
conn = None
|
||||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute('select ip from torexit_ips where ip=(%s)', (ip,))
|
||||
|
||||
row = cur.fetchone()
|
||||
|
||||
if row != None:
|
||||
|
||||
is_tor_exit_node = 't'
|
||||
|
||||
cur.close()
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print(error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
return is_tor_exit_node
|
||||
|
||||
@staticmethod
|
||||
def __check_setup(self):
|
||||
|
||||
is_setup = False
|
||||
|
||||
if not os.path.isfile(self.config_file):
|
||||
print(f"File {self.config_file} not found, running setup.\n")
|
||||
else:
|
||||
is_setup = True
|
||||
|
||||
return is_setup
|
||||
|
||||
@staticmethod
|
||||
def __setup(self):
|
||||
|
||||
if not os.path.exists('config'):
|
||||
os.makedirs('config')
|
||||
|
||||
self.mastodon_hostname = input("Mastodon hostname, in ex. 'mastodon.social': ")
|
||||
self.mastodon_db = input("Mastodon's database name: ")
|
||||
self.mastodon_db_user = input("Mastodon's database user: ")
|
||||
self.spamcheck_db = input("Spamcheck's database name: ")
|
||||
self.spamcheck_db_user = input("Spamcheck's database user: ")
|
||||
|
||||
if not os.path.exists(self.config_file):
|
||||
with open(self.config_file, 'w'): pass
|
||||
print(f"\n{self.config_file} created!\n")
|
||||
|
||||
with open(self.config_file, 'a') as the_file:
|
||||
print(f"Writing Mastodon hostname parameter to {self.config_file}")
|
||||
the_file.write(f'mastodon_hostname: {self.mastodon_hostname}\n')
|
||||
the_file.write(f'mastodon_db: {self.mastodon_db}\n')
|
||||
the_file.write(f'mastodon_db_user: {self.mastodon_db_user}\n')
|
||||
the_file.write(f'spamcheck_db: {self.spamcheck_db}\n')
|
||||
the_file.write(f'spamcheck_db_user: {self.spamcheck_db_user}\n')
|
||||
|
||||
return (self.mastodon_hostname, self.mastodon_db, self.mastodon_db_user, self.spamcheck_db, self.spamcheck_db_user)
|
||||
|
||||
@staticmethod
|
||||
def __check_dbsetup(self):
|
||||
|
||||
dbsetup = False
|
||||
|
||||
try:
|
||||
|
||||
conn = None
|
||||
|
||||
conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
|
||||
dbsetup = True
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print(error)
|
||||
|
||||
return dbsetup
|
||||
|
||||
@staticmethod
|
||||
def __createdb(self):
|
||||
|
||||
conn = None
|
||||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(dbname='postgres',
|
||||
user=self.spamcheck_db_user, host='',
|
||||
password='')
|
||||
|
||||
conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
print(f"Creating database {self.spamcheck_db}. Please wait...")
|
||||
|
||||
cur.execute(sql.SQL("CREATE DATABASE {}").format(
|
||||
sql.Identifier(self.spamcheck_db))
|
||||
)
|
||||
print(f"Database {self.spamcheck_db} created!\n")
|
||||
|
||||
self.__dbtables_schemes(self)
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print(error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
@staticmethod
|
||||
def __dbtables_schemes(self):
|
||||
|
||||
table = "spamcheck"
|
||||
sql = "create table "+table+" (created_at timestamptz, id bigint PRIMARY KEY, email varchar(200), ip inet, tor_exit_node boolean)"
|
||||
self.__create_table(self, table, sql)
|
||||
|
||||
table = "torexit_ips"
|
||||
sql = "create table "+table+" (created_at timestamptz, ip inet PRIMARY KEY)"
|
||||
self.__create_table(self, table, sql)
|
||||
|
||||
table = "totals"
|
||||
sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, registers int)"
|
||||
self.__create_table(self, table, sql)
|
||||
|
||||
@staticmethod
|
||||
def __create_table(self, table, sql):
|
||||
|
||||
conn = None
|
||||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(database = self.spamcheck_db, user = self.spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
cur = conn.cursor()
|
||||
|
||||
print(f"Creating table {table}")
|
||||
cur.execute(sql)
|
||||
|
||||
conn.commit()
|
||||
print(f"Table {table} created!\n")
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print(error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
@staticmethod
|
||||
def __get_parameter(parameter, file_path ):
|
||||
|
||||
with open( file_path ) as f:
|
||||
for line in f:
|
||||
if line.startswith( parameter ):
|
||||
return line.replace(parameter + ":", "").strip()
|
||||
|
||||
print(f'{file_path} Missing parameter {parameter}')
|
||||
sys.exit(0)
|
||||
|
||||
###############################################################################
|
||||
# main
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
spamcheck = Spamcheck()
|
||||
|
||||
created_at_lst, id_lst, email_lst, ip_lst = spamcheck.new_registers()
|
||||
|
||||
spamcheck.save_registers(created_at_lst, id_lst, email_lst, ip_lst)
|
||||
|
||||
spamcheck_datetime_lst, spamcheck_registers_lst = spamcheck.get_totals()
|
||||
|
||||
spamcheck.write_totals(spamcheck_datetime_lst, spamcheck_registers_lst)
|
||||
|
|
|
|||
13
torips.py
13
torips.py
|
|
@ -2,8 +2,8 @@ import os
|
|||
import datetime
|
||||
import psycopg2
|
||||
import requests
|
||||
import re
|
||||
import sys
|
||||
import pdb
|
||||
|
||||
def insert_tor_ip(tor_ip):
|
||||
|
||||
|
|
@ -43,7 +43,7 @@ def get_torbulk_list():
|
|||
|
||||
user_agent = {'User-agent': 'Mozilla/5.0'}
|
||||
|
||||
response = requests.get('https://check.torproject.org/torbulkexitlist', headers = user_agent, timeout=3)
|
||||
response = requests.get('https://check.torproject.org/exit-addresses', headers = user_agent, timeout=3)
|
||||
|
||||
if response.status_code == 200:
|
||||
|
||||
|
|
@ -68,7 +68,7 @@ def get_torbulk_list():
|
|||
def db_config():
|
||||
|
||||
# Load db configuration from config file
|
||||
config_filepath = "config/db_config.txt"
|
||||
config_filepath = "config/config.txt"
|
||||
spamcheck_db = get_parameter("spamcheck_db", config_filepath)
|
||||
spamcheck_db_user = get_parameter("spamcheck_db_user", config_filepath)
|
||||
|
||||
|
|
@ -111,9 +111,12 @@ if __name__ == '__main__':
|
|||
|
||||
line = fp.readline().rstrip('\n')
|
||||
|
||||
if line != '':
|
||||
if 'ExitAddress' in line:
|
||||
|
||||
insert_tor_ip(line)
|
||||
pattern = re.compile(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})')
|
||||
|
||||
ip = pattern.search(line)[0]
|
||||
insert_tor_ip(ip)
|
||||
|
||||
cnt += 1
|
||||
|
||||
|
|
|
|||
Loading…
Referencia en una nova incidència