From 7a3f26a263a949755fdb66273f8a9d1b666daf1e Mon Sep 17 00:00:00 2001 From: spla Date: Wed, 21 Jul 2021 14:46:27 +0200 Subject: [PATCH] Mastodon spamcheck first release --- db-setup.py | 160 +++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 3 + spamcheck.py | 130 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 293 insertions(+) create mode 100644 db-setup.py create mode 100644 requirements.txt create mode 100644 spamcheck.py diff --git a/db-setup.py b/db-setup.py new file mode 100644 index 0000000..01cc98c --- /dev/null +++ b/db-setup.py @@ -0,0 +1,160 @@ +#import getpass +import os +import sys +import psycopg2 +from psycopg2 import sql +from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT + +# Returns the parameter from the specified file +def get_parameter( parameter, file_path ): + # Check if secrets file exists + if not os.path.isfile(file_path): + print("File %s not found, asking."%file_path) + write_parameter( parameter, file_path ) + #sys.exit(0) + + # Find parameter in file + with open( file_path ) as f: + for line in f: + if line.startswith( parameter ): + return line.replace(parameter + ":", "").strip() + + # Cannot find parameter, exit + print(file_path + " Missing parameter %s "%parameter) + sys.exit(0) + +def write_parameter( parameter, file_path ): + if not os.path.exists('config'): + os.makedirs('config') + print("Setting up spamcheck parameters...") + print("\n") + spamcheck_db = input("spamcheck db name: ") + spamcheck_db_user = input("spamcheck db user: ") + mastodon_db = input("Mastodon db name: ") + mastodon_db_user = input("Mastodon db user: ") + + + with open(file_path, "w") as text_file: + print("spamcheck_db: {}".format(spamcheck_db), file=text_file) + print("spamcheck_db_user: {}".format(spamcheck_db_user), file=text_file) + print("mastodon_db: {}".format(mastodon_db), file=text_file) + print("mastodon_db_user: {}".format(mastodon_db_user), file=text_file) + +def create_table(db, db_user, table, sql): + + conn = None + + try: + + conn = psycopg2.connect(database = db, user = db_user, password = "", host = "/var/run/postgresql", port = "5432") + cur = conn.cursor() + + print("Creating table.. "+table) + # Create the table in PostgreSQL database + cur.execute(sql) + + conn.commit() + print("Table "+table+" created!") + print("\n") + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + +############################################################################### +# main + +if __name__ == '__main__': + + # Load configuration from config file + config_filepath = "config/db_config.txt" + spamcheck_db = get_parameter("spamcheck_db", config_filepath) + spamcheck_db_user = get_parameter("spamcheck_db_user", config_filepath) + mastodon_db = get_parameter("mastodon_db", config_filepath) + mastodon_db_user = get_parameter("mastodon_db_user", config_filepath) + + ############################################################ + # create database + ############################################################ + + conn = None + + try: + + conn = psycopg2.connect(dbname='postgres', + user=spamcheck_db_user, host='', + password='') + + conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) + + cur = conn.cursor() + + print("Creating database " + spamcheck_db + ". Please wait...") + + cur.execute(sql.SQL("CREATE DATABASE {}").format( + sql.Identifier(spamcheck_db)) + ) + print("Database " + spamcheck_db + " created!") + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + ############################################################ + + try: + + conn = None + + conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432") + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + # Load configuration from config file + os.remove("config/db_config.txt") + + print("Exiting. Run db-setup again with right parameters") + sys.exit(0) + + finally: + + if conn is not None: + + conn.close() + + print("\n") + print("spamcheck parameters saved to db-config.txt!") + print("\n") + + ############################################################ + # Create needed tables + ############################################################ + + print("Creating table...") + + db = spamcheck_db + db_user = spamcheck_db_user + + table = "spamcheck" + sql = "create table "+table+" (created_at timestamptz, id bigint PRIMARY KEY, email varchar(200), ip inet)" + create_table(db, db_user, table, sql) + + ############################################################ + + print("Done!") + print("Now you can run setup.py!") + print("\n") diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..64b2eb5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +Mastodon.py>=1.5.1 +wheel>=0.36.2 +psycopg2>=2.8.6 diff --git a/spamcheck.py b/spamcheck.py new file mode 100644 index 0000000..0c5fc0a --- /dev/null +++ b/spamcheck.py @@ -0,0 +1,130 @@ +import datetime +from mastodon import Mastodon +import time +import os +import json +import sys +import os.path +import operator +import psycopg2 +import pdb + +def db_config(): + + # Load db configuration from config file + config_filepath = "config/db_config.txt" + mastodon_db = get_parameter("mastodon_db", config_filepath) + mastodon_db_user = get_parameter("mastodon_db_user", config_filepath) + spamcheck_db = get_parameter("spamcheck_db", config_filepath) + spamcheck_db_user = get_parameter("spamcheck_db_user", config_filepath) + + return (mastodon_db, mastodon_db_user, spamcheck_db, spamcheck_db_user) + +# Returns the parameter from the specified file +def get_parameter( parameter, file_path ): + # Check if secrets file exists + if not os.path.isfile(file_path): + print("File %s not found, exiting."%file_path) + sys.exit(0) + + # Find parameter in file + with open( file_path ) as f: + for line in f: + if line.startswith( parameter ): + return line.replace(parameter + ":", "").strip() + + # Cannot find parameter, exit + print(file_path + " Missing parameter %s "%parameter) + sys.exit(0) + +############################################################################### +# main + +if __name__ == '__main__': + + mastodon_db, mastodon_db_user, spamcheck_db, spamcheck_db_user = db_config() + + ############################################################################### + # check new registering + ############################################################################### + + created_at_lst = [] + + id_lst = [] + + email_lst = [] + + ip_lst = [] + + try: + + conn = None + + conn = psycopg2.connect(database = mastodon_db, user = mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432") + + cur = conn.cursor() + + cur.execute("select created_at, id, email, sign_up_ip from users where not approved;") + + rows = cur.fetchall() + + for row in rows: + + if row != None: + + created_at_lst.append(row[0]) + + id_lst.append(row[1]) + + email_lst.append(row[2]) + + ip_lst.append(row[3]) + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print (error) + + finally: + + if conn is not None: + + conn.close() + + ############################################################################### + + insert_sql = 'INSERT INTO spamcheck(created_at, id, email, ip) VALUES(%s,%s,%s,%s) ON CONFLICT DO NOTHING' + + i = 0 + + while i < len(id_lst): + + conn = None + + try: + + conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432") + + cur = conn.cursor() + + cur.execute(insert_sql, (created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i])) + + conn.commit() + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + print(created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i]) + + i = i + 1 +