Mastodon spamcheck first release

db-setup.py

@@ -0,0 +1,160 @@
+#import getpass
+import os
+import sys
+import psycopg2
+from psycopg2 import sql
+from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
+
+# Returns the parameter from the specified file
+def get_parameter( parameter, file_path ):
+    # Check if secrets file exists
+    if not os.path.isfile(file_path):
+        print("File %s not found, asking."%file_path)
+        write_parameter( parameter, file_path )
+        #sys.exit(0)
+
+    # Find parameter in file
+    with open( file_path ) as f:
+        for line in f:
+            if line.startswith( parameter ):
+                return line.replace(parameter + ":", "").strip()
+
+    # Cannot find parameter, exit
+    print(file_path + "  Missing parameter %s "%parameter)
+    sys.exit(0)
+
+def write_parameter( parameter, file_path ):
+    if not os.path.exists('config'):
+        os.makedirs('config')
+    print("Setting up spamcheck parameters...")
+    print("\n")
+    spamcheck_db = input("spamcheck db name: ")
+    spamcheck_db_user = input("spamcheck db user: ")
+    mastodon_db = input("Mastodon db name: ")
+    mastodon_db_user = input("Mastodon db user: ")
+
+
+    with open(file_path, "w") as text_file:
+        print("spamcheck_db: {}".format(spamcheck_db), file=text_file)
+        print("spamcheck_db_user: {}".format(spamcheck_db_user), file=text_file)
+        print("mastodon_db: {}".format(mastodon_db), file=text_file)
+        print("mastodon_db_user: {}".format(mastodon_db_user), file=text_file)
+
+def create_table(db, db_user, table, sql):
+
+    conn = None
+
+    try:
+
+        conn = psycopg2.connect(database = db, user = db_user, password = "", host = "/var/run/postgresql", port = "5432")
+        cur = conn.cursor()
+
+        print("Creating table.. "+table)
+        # Create the table in PostgreSQL database
+        cur.execute(sql)
+
+        conn.commit()
+        print("Table "+table+" created!")
+        print("\n")
+
+    except (Exception, psycopg2.DatabaseError) as error:
+
+        print(error)
+
+    finally:
+
+        if conn is not None:
+
+            conn.close()
+
+###############################################################################
+# main
+
+if __name__ == '__main__':
+
+    # Load configuration from config file
+    config_filepath = "config/db_config.txt"
+    spamcheck_db = get_parameter("spamcheck_db", config_filepath)
+    spamcheck_db_user = get_parameter("spamcheck_db_user", config_filepath)
+    mastodon_db = get_parameter("mastodon_db", config_filepath)
+    mastodon_db_user = get_parameter("mastodon_db_user", config_filepath)
+
+    ############################################################
+    # create database
+    ############################################################
+
+    conn = None
+
+    try:
+
+        conn = psycopg2.connect(dbname='postgres',
+            user=spamcheck_db_user, host='',
+            password='')
+
+        conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
+
+        cur = conn.cursor()
+
+        print("Creating database " + spamcheck_db + ". Please wait...")
+
+        cur.execute(sql.SQL("CREATE DATABASE {}").format(
+            sql.Identifier(spamcheck_db))
+        )
+        print("Database " + spamcheck_db + " created!")
+
+    except (Exception, psycopg2.DatabaseError) as error:
+
+        print(error)
+
+    finally:
+
+        if conn is not None:
+
+            conn.close()
+
+    ############################################################
+
+    try:
+
+        conn = None
+
+        conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
+
+    except (Exception, psycopg2.DatabaseError) as error:
+
+        print(error)
+
+        # Load configuration from config file
+        os.remove("config/db_config.txt")
+
+        print("Exiting. Run db-setup again with right parameters")
+        sys.exit(0)
+
+    finally:
+
+        if conn is not None:
+
+            conn.close()
+
+            print("\n")
+            print("spamcheck parameters saved to db-config.txt!")
+            print("\n")
+
+    ############################################################
+    # Create needed tables
+    ############################################################
+
+    print("Creating table...")
+
+    db = spamcheck_db
+    db_user = spamcheck_db_user
+
+    table = "spamcheck"
+    sql = "create table "+table+" (created_at timestamptz, id bigint PRIMARY KEY, email varchar(200), ip inet)"
+    create_table(db, db_user, table, sql)
+
+    ############################################################
+
+    print("Done!")
+    print("Now you can run setup.py!")
+    print("\n")

requirements.txt

@@ -0,0 +1,3 @@
+Mastodon.py>=1.5.1
+wheel>=0.36.2
+psycopg2>=2.8.6

spamcheck.py

@@ -0,0 +1,130 @@
+import datetime
+from mastodon import Mastodon
+import time
+import os
+import json
+import sys
+import os.path
+import operator
+import psycopg2
+import pdb
+
+def db_config():
+
+    # Load db configuration from config file
+    config_filepath = "config/db_config.txt"
+    mastodon_db = get_parameter("mastodon_db", config_filepath)
+    mastodon_db_user = get_parameter("mastodon_db_user", config_filepath)
+    spamcheck_db = get_parameter("spamcheck_db", config_filepath)
+    spamcheck_db_user = get_parameter("spamcheck_db_user", config_filepath)
+
+    return (mastodon_db, mastodon_db_user, spamcheck_db, spamcheck_db_user)
+
+# Returns the parameter from the specified file
+def get_parameter( parameter, file_path ):
+    # Check if secrets file exists
+    if not os.path.isfile(file_path):
+        print("File %s not found, exiting."%file_path)
+        sys.exit(0)
+
+    # Find parameter in file
+    with open( file_path ) as f:
+        for line in f:
+            if line.startswith( parameter ):
+                return line.replace(parameter + ":", "").strip()
+
+    # Cannot find parameter, exit
+    print(file_path + "  Missing parameter %s "%parameter)
+    sys.exit(0)
+
+###############################################################################
+# main
+
+if __name__ == '__main__':
+
+    mastodon_db, mastodon_db_user, spamcheck_db, spamcheck_db_user = db_config()
+
+    ###############################################################################
+    # check new registering
+    ###############################################################################
+    
+    created_at_lst = []
+
+    id_lst = []
+
+    email_lst = []
+
+    ip_lst = []
+    
+    try:
+
+        conn = None
+
+        conn = psycopg2.connect(database = mastodon_db, user = mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432")
+
+        cur = conn.cursor()
+
+        cur.execute("select created_at, id, email, sign_up_ip from users where not approved;")
+
+        rows = cur.fetchall()
+
+        for row in rows:
+
+            if row != None:
+
+                created_at_lst.append(row[0])
+
+                id_lst.append(row[1])
+
+                email_lst.append(row[2])
+
+                ip_lst.append(row[3])
+
+        cur.close()
+
+    except (Exception, psycopg2.DatabaseError) as error:
+
+        print (error)
+
+    finally:
+
+        if conn is not None:
+
+            conn.close()
+
+    ###############################################################################
+
+    insert_sql = 'INSERT INTO spamcheck(created_at, id, email, ip) VALUES(%s,%s,%s,%s) ON CONFLICT DO NOTHING'
+    
+    i = 0
+
+    while i < len(id_lst):
+
+        conn = None
+
+        try:
+
+            conn = psycopg2.connect(database = spamcheck_db, user = spamcheck_db_user, password = "", host = "/var/run/postgresql", port = "5432")
+
+            cur = conn.cursor()
+
+            cur.execute(insert_sql, (created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i]))
+
+            conn.commit()
+
+            cur.close()
+
+        except (Exception, psycopg2.DatabaseError) as error:
+
+            print(error)
+
+        finally:
+
+            if conn is not None:
+
+                conn.close()
+
+        print(created_at_lst[i], id_lst[i], email_lst[i], ip_lst[i])
+
+        i = i + 1
+