416 lines
10 KiB
Python
416 lines
10 KiB
Python
from datetime import date, datetime, timedelta
|
|
from mastodon import Mastodon
|
|
import time
|
|
import os
|
|
import json
|
|
import sys
|
|
import os.path
|
|
import operator
|
|
import psycopg2
|
|
from langdetect import detect
|
|
import requests
|
|
import pdb
|
|
|
|
def write_totals(spamcheck_datetime_lst, spamcheck_registers_lst):
|
|
|
|
insert_sql = 'INSERT INTO totals(datetime, registers) VALUES(%s,%s) ON CONFLICT (datetime) DO UPDATE SET (datetime, registers) = (EXCLUDED.datetime, EXCLUDED.registers)'
|
|
|
|
first_date = spamcheck_datetime_lst[0]
|
|
|
|
last_date = spamcheck_datetime_lst[len(spamcheck_datetime_lst)-1]
|
|
|
|
i = 0
|
|
|
|
while i < len(spamcheck_datetime_lst):
|
|
|
|
conn = None
|
|
|
|
try:
|
|
|
|
conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
if first_date == spamcheck_datetime_lst[i]:
|
|
|
|
cur.execute(insert_sql, (spamcheck_datetime_lst[i], spamcheck_registers_lst[i]))
|
|
|
|
i += 1
|
|
|
|
else:
|
|
|
|
cur.execute(insert_sql, (first_date, '0'))
|
|
|
|
conn.commit()
|
|
|
|
cur.close()
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
first_date = first_date + timedelta(days=1)
|
|
|
|
if date.today() == last_date + timedelta(days=1):
|
|
|
|
insert_sql = 'INSERT INTO totals(datetime, registers) VALUES(%s,%s) ON CONFLICT (datetime) DO UPDATE SET (datetime, registers) = (EXCLUDED.datetime, EXCLUDED.registers)'
|
|
|
|
conn = None
|
|
|
|
try:
|
|
|
|
conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
cur.execute(insert_sql, (date.today(), '0'))
|
|
|
|
conn.commit()
|
|
|
|
cur.close()
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
def get_totals():
|
|
|
|
spamcheck_datetime_lst = []
|
|
|
|
spamcheck_registers_lst = []
|
|
|
|
select_sql = 'select date(created_at), count(ip) as registers from spamcheck group by date(created_at) order by date(created_at)'
|
|
|
|
conn = None
|
|
|
|
try:
|
|
|
|
conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
cur.execute(select_sql)
|
|
|
|
rows = cur.fetchall()
|
|
|
|
for row in rows:
|
|
|
|
spamcheck_datetime_lst.append(row[0])
|
|
|
|
spamcheck_registers_lst.append(row[1])
|
|
|
|
cur.close()
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
return (spamcheck_datetime_lst, spamcheck_registers_lst)
|
|
|
|
def check_ip(ip):
|
|
|
|
is_tor_exit_node = 'f'
|
|
|
|
conn = None
|
|
|
|
try:
|
|
|
|
conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
cur.execute('select ip from torexit_ips where ip=(%s)', (ip,))
|
|
|
|
row = cur.fetchone()
|
|
|
|
if row != None:
|
|
|
|
is_tor_exit_node = 't'
|
|
|
|
cur.close()
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
return is_tor_exit_node
|
|
|
|
def check_approval(user_id):
|
|
|
|
approved = False
|
|
|
|
try:
|
|
|
|
conn = None
|
|
|
|
conn = psycopg2.connect(database = mastodon_db, user = mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
cur.execute("select approved from users where id = (%s)", (user_id,))
|
|
|
|
row = cur.fetchone()
|
|
|
|
if row != None:
|
|
|
|
approved = row[0]
|
|
|
|
cur.close()
|
|
|
|
return approved
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print (error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
def mastodon():
|
|
|
|
# Load secrets from secrets file
|
|
secrets_filepath = "secrets/secrets.txt"
|
|
uc_client_id = get_parameter("uc_client_id", secrets_filepath)
|
|
uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
|
|
uc_access_token = get_parameter("uc_access_token", secrets_filepath)
|
|
|
|
# Load configuration from config file
|
|
config_filepath = "config/config.txt"
|
|
mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
|
|
|
|
# Initialise Mastodon API
|
|
mastodon = Mastodon(
|
|
client_id=uc_client_id,
|
|
client_secret=uc_client_secret,
|
|
access_token=uc_access_token,
|
|
api_base_url='https://' + mastodon_hostname,
|
|
)
|
|
|
|
# Initialise access headers
|
|
headers = {'Authorization': 'Bearer %s'%uc_access_token}
|
|
|
|
return (mastodon, mastodon_hostname)
|
|
|
|
def db_config():
|
|
|
|
# Load db configuration from config file
|
|
config_filepath = "config/db_config.txt"
|
|
mastodon_db = get_parameter("mastodon_db", config_filepath)
|
|
mastodon_db_user = get_parameter("mastodon_db_user", config_filepath)
|
|
spamcheck_db = get_parameter("spamcheck_db", config_filepath)
|
|
spamcheck_db_user = get_parameter("spamcheck_db_user", config_filepath)
|
|
|
|
return (mastodon_db, mastodon_db_user, spamcheck_db, spamcheck_db_user)
|
|
|
|
# Returns the parameter from the specified file
|
|
def get_parameter( parameter, file_path ):
|
|
# Check if secrets file exists
|
|
if not os.path.isfile(file_path):
|
|
print("File %s not found, exiting."%file_path)
|
|
sys.exit(0)
|
|
|
|
# Find parameter in file
|
|
with open( file_path ) as f:
|
|
for line in f:
|
|
if line.startswith( parameter ):
|
|
return line.replace(parameter + ":", "").strip()
|
|
|
|
# Cannot find parameter, exit
|
|
print(file_path + " Missing parameter %s "%parameter)
|
|
sys.exit(0)
|
|
|
|
###############################################################################
|
|
# main
|
|
|
|
if __name__ == '__main__':
|
|
|
|
mastodon_db, mastodon_db_user, spamcheck_db, spamcheck_db_user = db_config()
|
|
|
|
mastodon, mastodon_hostname = mastodon()
|
|
|
|
###############################################################################
|
|
# check new registering
|
|
###############################################################################
|
|
|
|
created_at_lst = []
|
|
|
|
id_lst = []
|
|
|
|
email_lst = []
|
|
|
|
ip_lst = []
|
|
|
|
text_lst = []
|
|
|
|
try:
|
|
|
|
conn = None
|
|
|
|
conn = psycopg2.connect(database = mastodon_db, user = mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
cur.execute("select users.created_at, users.id, users.email, users.sign_up_ip, user_invite_requests.text from users, user_invite_requests where not users.approved and users.id = user_invite_requests.user_id")
|
|
|
|
rows = cur.fetchall()
|
|
|
|
for row in rows:
|
|
|
|
if row != None:
|
|
|
|
created_at_lst.append(row[0])
|
|
|
|
id_lst.append(row[1])
|
|
|
|
email_lst.append(row[2])
|
|
|
|
ip_lst.append(row[3])
|
|
|
|
text_lst.append(row[4])
|
|
|
|
cur.close()
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print (error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
###############################################################################
|
|
|
|
insert_sql = 'INSERT INTO spamcheck(created_at, id, email, ip, text, tor_exit_node) VALUES(%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING'
|
|
|
|
i = 0
|
|
|
|
while i < len(id_lst):
|
|
|
|
if detect(text_lst[i]) != 'ca' or len(text_lst) == 1:
|
|
|
|
is_tor_exit_node = check_ip(ip_lst[i])
|
|
|
|
tor_exit_node = 't' if is_tor_exit_node else 'f'
|
|
|
|
conn = None
|
|
|
|
try:
|
|
|
|
conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
cur.execute(insert_sql, (created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i], text_lst[i], tor_exit_node))
|
|
|
|
conn.commit()
|
|
|
|
cur.close()
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
print(created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i], text_lst[i], tor_exit_node)
|
|
|
|
i = i + 1
|
|
|
|
##########################################################################
|
|
|
|
approved_users_id_lst = []
|
|
|
|
conn = None
|
|
|
|
try:
|
|
|
|
conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
cur.execute('select id from spamcheck')
|
|
|
|
rows = cur.fetchall()
|
|
|
|
for row in rows:
|
|
|
|
approved_users_id_lst.append(row)
|
|
|
|
cur.close()
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
for user_id in approved_users_id_lst:
|
|
|
|
approved = check_approval(user_id)
|
|
|
|
if approved:
|
|
|
|
conn = None
|
|
|
|
try:
|
|
|
|
conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
cur.execute('delete from spamcheck where id=(%s)', (user_id,))
|
|
|
|
conn.commit()
|
|
|
|
cur.close()
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
spamcheck_datetime_lst, spamcheck_registers_lst = get_totals()
|
|
|
|
write_totals(spamcheck_datetime_lst, spamcheck_registers_lst)
|