spamcheck/spamcheck.py

377 lines
8.9 KiB
Python
Raw Normal View History

2021-07-21 14:46:27 +02:00
import datetime
from mastodon import Mastodon
import time
import os
import json
import sys
import os.path
import operator
import psycopg2
2022-01-16 14:33:44 +01:00
from langdetect import detect
import requests
2021-07-21 14:46:27 +02:00
import pdb
def write_totals(spamcheck_datetime_lst, spamcheck_registers_lst):
insert_sql = 'INSERT INTO totals(datetime, registers) VALUES(%s,%s) ON CONFLICT (datetime) DO UPDATE SET (datetime, registers) = (EXCLUDED.datetime, EXCLUDED.registers)'
i = 0
while i < len(spamcheck_datetime_lst):
conn = None
try:
conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute(insert_sql, (spamcheck_datetime_lst[i], spamcheck_registers_lst[i]))
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
i += 1
def get_totals():
spamcheck_datetime_lst = []
spamcheck_registers_lst = []
select_sql = 'select date(created_at), count(ip) as registers from spamcheck group by date(created_at) order by date(created_at)'
conn = None
try:
conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute(select_sql)
rows = cur.fetchall()
for row in rows:
spamcheck_datetime_lst.append(row[0])
spamcheck_registers_lst.append(row[1])
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
return (spamcheck_datetime_lst, spamcheck_registers_lst)
2022-01-16 14:33:44 +01:00
def check_ip(ip):
is_tor_exit_node = 'f'
conn = None
try:
conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute('select ip from torexit_ips where ip=(%s)', (ip,))
row = cur.fetchone()
if row != None:
is_tor_exit_node = 't'
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
return is_tor_exit_node
def check_approval(user_id):
approved = False
try:
conn = None
conn = psycopg2.connect(database = mastodon_db, user = mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute("select approved from users where id = (%s)", (user_id,))
row = cur.fetchone()
if row != None:
approved = row[0]
cur.close()
return approved
except (Exception, psycopg2.DatabaseError) as error:
print (error)
finally:
if conn is not None:
conn.close()
def mastodon():
# Load secrets from secrets file
secrets_filepath = "secrets/secrets.txt"
uc_client_id = get_parameter("uc_client_id", secrets_filepath)
uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
uc_access_token = get_parameter("uc_access_token", secrets_filepath)
# Load configuration from config file
config_filepath = "config/config.txt"
mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
# Initialise Mastodon API
mastodon = Mastodon(
client_id=uc_client_id,
client_secret=uc_client_secret,
access_token=uc_access_token,
api_base_url='https://' + mastodon_hostname,
)
# Initialise access headers
headers = {'Authorization': 'Bearer %s'%uc_access_token}
return (mastodon, mastodon_hostname)
2021-07-21 14:46:27 +02:00
def db_config():
# Load db configuration from config file
config_filepath = "config/db_config.txt"
mastodon_db = get_parameter("mastodon_db", config_filepath)
mastodon_db_user = get_parameter("mastodon_db_user", config_filepath)
spamcheck_db = get_parameter("spamcheck_db", config_filepath)
spamcheck_db_user = get_parameter("spamcheck_db_user", config_filepath)
return (mastodon_db, mastodon_db_user, spamcheck_db, spamcheck_db_user)
# Returns the parameter from the specified file
def get_parameter( parameter, file_path ):
# Check if secrets file exists
if not os.path.isfile(file_path):
print("File %s not found, exiting."%file_path)
sys.exit(0)
# Find parameter in file
with open( file_path ) as f:
for line in f:
if line.startswith( parameter ):
return line.replace(parameter + ":", "").strip()
# Cannot find parameter, exit
print(file_path + " Missing parameter %s "%parameter)
sys.exit(0)
###############################################################################
# main
if __name__ == '__main__':
mastodon_db, mastodon_db_user, spamcheck_db, spamcheck_db_user = db_config()
2022-01-16 14:33:44 +01:00
mastodon, mastodon_hostname = mastodon()
2021-07-21 14:46:27 +02:00
###############################################################################
# check new registering
###############################################################################
created_at_lst = []
id_lst = []
email_lst = []
ip_lst = []
2021-07-26 15:43:02 +02:00
text_lst = []
2021-07-21 14:46:27 +02:00
try:
conn = None
conn = psycopg2.connect(database = mastodon_db, user = mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
2021-07-26 15:43:02 +02:00
cur.execute("select users.created_at, users.id, users.email, users.sign_up_ip, user_invite_requests.text from users, user_invite_requests where not users.approved and users.id = user_invite_requests.user_id")
2021-07-21 14:46:27 +02:00
rows = cur.fetchall()
for row in rows:
if row != None:
created_at_lst.append(row[0])
id_lst.append(row[1])
email_lst.append(row[2])
ip_lst.append(row[3])
2021-07-26 15:43:02 +02:00
text_lst.append(row[4])
2021-07-21 14:46:27 +02:00
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print (error)
finally:
if conn is not None:
conn.close()
###############################################################################
2022-01-16 17:09:38 +01:00
insert_sql = 'INSERT INTO spamcheck(created_at, id, email, ip, text, tor_exit_node) VALUES(%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING'
2021-07-21 14:46:27 +02:00
i = 0
while i < len(id_lst):
2022-01-16 14:33:44 +01:00
if detect(text_lst[i]) != 'ca':
is_tor_exit_node = check_ip(ip_lst[i])
tor_exit_node = 't' if is_tor_exit_node else 'f'
conn = None
try:
conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
2022-01-16 17:09:38 +01:00
cur.execute(insert_sql, (created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i], text_lst[i], tor_exit_node))
2022-01-16 14:33:44 +01:00
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
2022-01-16 17:09:38 +01:00
print(created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i], text_lst[i], tor_exit_node)
2022-01-16 14:33:44 +01:00
i = i + 1
##########################################################################
approved_users_id_lst = []
2021-07-21 14:46:27 +02:00
conn = None
try:
conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
2022-01-16 14:33:44 +01:00
cur.execute('select id from spamcheck')
rows = cur.fetchall()
2021-07-21 14:46:27 +02:00
2022-01-16 14:33:44 +01:00
for row in rows:
approved_users_id_lst.append(row)
2021-07-21 14:46:27 +02:00
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
2022-01-16 14:33:44 +01:00
for user_id in approved_users_id_lst:
2021-07-21 14:46:27 +02:00
2022-01-16 14:33:44 +01:00
approved = check_approval(user_id)
if approved:
conn = None
try:
conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute('delete from spamcheck where id=(%s)', (user_id,))
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
2021-07-21 14:46:27 +02:00
2022-01-16 14:33:44 +01:00
conn.close()
spamcheck_datetime_lst, spamcheck_registers_lst = get_totals()
write_totals(spamcheck_datetime_lst, spamcheck_registers_lst)