spamcheck/spamcheck.py

416 lines
10 KiB
Python

from datetime import date, datetime, timedelta
from mastodon import Mastodon
import time
import os
import json
import sys
import os.path
import operator
import psycopg2
from langdetect import detect
import requests
import pdb
def write_totals(spamcheck_datetime_lst, spamcheck_registers_lst):
insert_sql = 'INSERT INTO totals(datetime, registers) VALUES(%s,%s) ON CONFLICT (datetime) DO UPDATE SET (datetime, registers) = (EXCLUDED.datetime, EXCLUDED.registers)'
first_date = spamcheck_datetime_lst[0]
last_date = spamcheck_datetime_lst[len(spamcheck_datetime_lst)-1]
i = 0
while i < len(spamcheck_datetime_lst):
conn = None
try:
conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
if first_date == spamcheck_datetime_lst[i]:
cur.execute(insert_sql, (spamcheck_datetime_lst[i], spamcheck_registers_lst[i]))
i += 1
else:
cur.execute(insert_sql, (first_date, '0'))
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
first_date = first_date + timedelta(days=1)
if date.today() == last_date + timedelta(days=1):
insert_sql = 'INSERT INTO totals(datetime, registers) VALUES(%s,%s) ON CONFLICT (datetime) DO UPDATE SET (datetime, registers) = (EXCLUDED.datetime, EXCLUDED.registers)'
conn = None
try:
conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute(insert_sql, (date.today(), '0'))
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
def get_totals():
spamcheck_datetime_lst = []
spamcheck_registers_lst = []
select_sql = 'select date(created_at), count(ip) as registers from spamcheck group by date(created_at) order by date(created_at)'
conn = None
try:
conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute(select_sql)
rows = cur.fetchall()
for row in rows:
spamcheck_datetime_lst.append(row[0])
spamcheck_registers_lst.append(row[1])
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
return (spamcheck_datetime_lst, spamcheck_registers_lst)
def check_ip(ip):
is_tor_exit_node = 'f'
conn = None
try:
conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute('select ip from torexit_ips where ip=(%s)', (ip,))
row = cur.fetchone()
if row != None:
is_tor_exit_node = 't'
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
return is_tor_exit_node
def check_approval(user_id):
approved = False
try:
conn = None
conn = psycopg2.connect(database = mastodon_db, user = mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute("select approved from users where id = (%s)", (user_id,))
row = cur.fetchone()
if row != None:
approved = row[0]
cur.close()
return approved
except (Exception, psycopg2.DatabaseError) as error:
print (error)
finally:
if conn is not None:
conn.close()
def mastodon():
# Load secrets from secrets file
secrets_filepath = "secrets/secrets.txt"
uc_client_id = get_parameter("uc_client_id", secrets_filepath)
uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
uc_access_token = get_parameter("uc_access_token", secrets_filepath)
# Load configuration from config file
config_filepath = "config/config.txt"
mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
# Initialise Mastodon API
mastodon = Mastodon(
client_id=uc_client_id,
client_secret=uc_client_secret,
access_token=uc_access_token,
api_base_url='https://' + mastodon_hostname,
)
# Initialise access headers
headers = {'Authorization': 'Bearer %s'%uc_access_token}
return (mastodon, mastodon_hostname)
def db_config():
# Load db configuration from config file
config_filepath = "config/db_config.txt"
mastodon_db = get_parameter("mastodon_db", config_filepath)
mastodon_db_user = get_parameter("mastodon_db_user", config_filepath)
spamcheck_db = get_parameter("spamcheck_db", config_filepath)
spamcheck_db_user = get_parameter("spamcheck_db_user", config_filepath)
return (mastodon_db, mastodon_db_user, spamcheck_db, spamcheck_db_user)
# Returns the parameter from the specified file
def get_parameter( parameter, file_path ):
# Check if secrets file exists
if not os.path.isfile(file_path):
print("File %s not found, exiting."%file_path)
sys.exit(0)
# Find parameter in file
with open( file_path ) as f:
for line in f:
if line.startswith( parameter ):
return line.replace(parameter + ":", "").strip()
# Cannot find parameter, exit
print(file_path + " Missing parameter %s "%parameter)
sys.exit(0)
###############################################################################
# main
if __name__ == '__main__':
mastodon_db, mastodon_db_user, spamcheck_db, spamcheck_db_user = db_config()
mastodon, mastodon_hostname = mastodon()
###############################################################################
# check new registering
###############################################################################
created_at_lst = []
id_lst = []
email_lst = []
ip_lst = []
text_lst = []
try:
conn = None
conn = psycopg2.connect(database = mastodon_db, user = mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute("select users.created_at, users.id, users.email, users.sign_up_ip, user_invite_requests.text from users, user_invite_requests where not users.approved and users.id = user_invite_requests.user_id")
rows = cur.fetchall()
for row in rows:
if row != None:
created_at_lst.append(row[0])
id_lst.append(row[1])
email_lst.append(row[2])
ip_lst.append(row[3])
text_lst.append(row[4])
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print (error)
finally:
if conn is not None:
conn.close()
###############################################################################
insert_sql = 'INSERT INTO spamcheck(created_at, id, email, ip, text, tor_exit_node) VALUES(%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING'
i = 0
while i < len(id_lst):
if detect(text_lst[i]) != 'ca' or len(text_lst) == 1:
is_tor_exit_node = check_ip(ip_lst[i])
tor_exit_node = 't' if is_tor_exit_node else 'f'
conn = None
try:
conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute(insert_sql, (created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i], text_lst[i], tor_exit_node))
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
print(created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i], text_lst[i], tor_exit_node)
i = i + 1
##########################################################################
approved_users_id_lst = []
conn = None
try:
conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute('select id from spamcheck')
rows = cur.fetchall()
for row in rows:
approved_users_id_lst.append(row)
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
for user_id in approved_users_id_lst:
approved = check_approval(user_id)
if approved:
conn = None
try:
conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute('delete from spamcheck where id=(%s)', (user_id,))
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
spamcheck_datetime_lst, spamcheck_registers_lst = get_totals()
write_totals(spamcheck_datetime_lst, spamcheck_registers_lst)