Refactored. Added Inactives Class

This commit is contained in:
spla 2022-08-01 11:52:30 +02:00
pare 9fbc5a6d44
commit 17e74f8517
S'han modificat 4 arxius amb 215 adicions i 560 eliminacions

Veure arxiu

@ -8,7 +8,18 @@ import psycopg2
from psycopg2 import sql
from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
# Returns the parameter from the specified file
def get_config():
# Load configuration from config file
config_filepath = "config/db_config.txt"
mastodon_db = get_parameter("mastodon_db", config_filepath)
mastodon_db_user = get_parameter("mastodon_db_user", config_filepath)
mailing_db = get_parameter("mailing_db", config_filepath)
mailing_db_user = get_parameter("mailing_db_user", config_filepath)
mailing_db_table = get_parameter("mailing_db_table", config_filepath)
return (mastodon_db, mastodon_db_user, mailing_db, mailing_db_user, mailing_db_table)
def get_parameter( parameter, file_path ):
# Check if secrets file exists
if not os.path.isfile(file_path):
@ -27,28 +38,81 @@ def get_parameter( parameter, file_path ):
sys.exit(0)
def write_parameter( parameter, file_path ):
print("Setting up mailing DB parameters...")
print("\n")
mastodon_db = input("Mastodon db name: ")
mastodon_db_user = input("Mastodon db user: ")
mailing_db = input("Mailing db name: ")
mailing_db_user = input("Mailing db user: ")
mailing_db_table = input("Mailing db table: ")
with open(file_path, "w") as text_file:
print("mastodon_db: {}".format(mastodon_db), file=text_file)
print("mastodon_db_user: {}".format(mastodon_db_user), file=text_file)
print("mailing_db: {}".format(mailing_db), file=text_file)
print("mailing_db_user: {}".format(mailing_db_user), file=text_file)
print("mailing_db_table: {}".format(mailing_db_table), file=text_file)
if not os.path.exists('config'):
os.makedirs('config')
print("Setting up mailing DB parameters...")
print("\n")
mastodon_db = input("Mastodon db name: ")
mastodon_db_user = input("Mastodon db user: ")
mailing_db = input("Mailing db name: ")
mailing_db_user = input("Mailing db user: ")
mailing_db_table = input("Mailing db table: ")
with open(file_path, "w") as text_file:
print("mastodon_db: {}".format(mastodon_db), file=text_file)
print("mastodon_db_user: {}".format(mastodon_db_user), file=text_file)
print("mailing_db: {}".format(mailing_db), file=text_file)
print("mailing_db_user: {}".format(mailing_db_user), file=text_file)
print("mailing_db_table: {}".format(mailing_db_table), file=text_file)
def create_database():
try:
conn = psycopg2.connect(dbname='postgres',
user=mailing_db_user, host='',
password='')
conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
cur = conn.cursor()
print("Creating database " + mailing_db + ". Please wait...")
cur.execute(sql.SQL("CREATE DATABASE {}").format(
sql.Identifier(mailing_db))
)
print("Database " + mailing_db + " created!")
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
try:
conn = None
conn = psycopg2.connect(database = mailing_db, user = mailing_db_user, password = "", host = "/var/run/postgresql", port = "5432")
except (Exception, psycopg2.DatabaseError) as error:
print(error)
# Load configuration from config file
os.remove("config.txt")
print("Exiting. Run setup again with right parameters")
sys.exit(0)
if conn is not None:
print("\n")
print("Mailing db parameters saved to config.txt!")
print("\n")
def create_table(db, db_user, table, sql):
try:
conn = None
conn = psycopg2.connect(database = db, user = db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
conn = psycopg2.connect(database = db, user = db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
print("Creating table.. "+table)
# Create the table in PostgreSQL database
@ -68,85 +132,25 @@ def create_table(db, db_user, table, sql):
conn.close()
#############################################################################################
# main
# Load configuration from config file
config_filepath = "config.txt"
mastodon_db = get_parameter("mastodon_db", config_filepath) # E.g., mastodon_production
mastodon_db_user = get_parameter("mastodon_db_user", config_filepath) # E.g., mastodon
mailing_db = get_parameter("mailing_db", config_filepath) # E.g., inactive
mailing_db_user = get_parameter("mailing_db_user", config_filepath) # E.g., mastodon
mailing_db_table = get_parameter("mailing_db_table", config_filepath) # E.g., inactive_users
if __name__ == '__main__':
############################################################
# create database
############################################################
mastodon_db, mastodon_db_user, mailing_db, mailing_db_user, mailing_db_table = get_config()
try:
create_database()
conn = psycopg2.connect(dbname='postgres',
user=mailing_db_user, host='',
password='')
########################################
# create mailing DB table
conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
db = mailing_db
db_user = mailing_db_user
table = mailing_db_table
sql = "create table " + table + "(datetime timestamptz, account_id int primary key, username varchar(30), email varchar(50), emailed_at timestamptz,"
sql += "emailed boolean default False, deleted boolean default False, elapsed_days varchar(30), to_be_deleted boolean default False,"
sql += "recipient_error boolean default False, feedback boolean default False, current_sign_in_at timestamptz)"
create_table(db, db_user, table, sql)
cur = conn.cursor()
#####################################
print("Creating database " + mailing_db + ". Please wait...")
cur.execute(sql.SQL("CREATE DATABASE {}").format(
sql.Identifier(mailing_db))
)
print("Database " + mailing_db + " created!")
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
#############################################################################################
try:
conn = None
conn = psycopg2.connect(database = mailing_db, user = mailing_db_user, password = "", host = "/var/run/postgresql", port = "5432")
except (Exception, psycopg2.DatabaseError) as error:
print(error)
# Load configuration from config file
os.remove("config.txt")
print("Exiting. Run setup again with right parameters")
sys.exit(0)
if conn is not None:
print("\n")
print("Mailing db parameters saved to config.txt!")
print("\n")
############################################################
# Create needed tables
############################################################
print("Creating table...")
########################################
db = mailing_db
db_user = mailing_db_user
table = mailing_db_table
sql = "create table " + table + "(datetime timestamptz, account_id int primary key, username varchar(30), email varchar(50), emailed_at timestamptz,"
sql += "emailed boolean default False, deleted boolean default False, elapsed_days varchar(30), to_be_deleted boolean default False,"
sql += "recipient_error boolean default False, feedback boolean default False)"
create_table(db, db_user, table, sql)
#####################################
print("Done!")
print("Now you can run setup.py!")
print("\n")
print("Done!\nNow you can run setup.py!\n")

Veure arxiu

@ -3,150 +3,177 @@
from datetime import datetime, timezone, timedelta
import time
import threading
import os
import sys
import os.path
import psycopg2
def delete_inactives(mailing_db, mailing_db_user, mailing_db_table, deletion_accepted, query, id_array, username_array):
def delete_inactives(deletion_accepted, query):
conn = None
id_array = []
try:
username_array = []
conn = psycopg2.connect(database = mailing_db, user = mailing_db_user, password = "", host = "/var/run/postgresql", port = "5432")
conn = None
cur = conn.cursor()
try:
cur.execute(query)
conn = psycopg2.connect(database = mailing_db, user = mailing_db_user, password = "", host = "/var/run/postgresql", port = "5432")
for row in cur:
cur = conn.cursor()
id_array.append(row[0])
username_array.append(row[1])
cur.execute(query)
i = 0
if len(id_array) == 0:
if deletion_accepted == True:
print("None inactive users who accepted to be deleted found!")
elif deletion_accepted == False:
print("None inactive users to be deleted!")
return
for row in cur:
while i < len(id_array):
id_array.append(row[0])
if deletion_accepted == True:
print("Deleting inactive users who accepted to be deleted...")
elif deletion_accepted == False:
print("Deleting inactive users who do not reply our email...")
username_array.append(row[1])
print("\n")
print("Deleting user " + str(i) + " of " + str(len(id_array)) + " users")
print("Deleting user with id " + str(id_array[i]) + ", username: " + username_array[i])
os.system("RAILS_ENV=production " + rvm_ruby + " " + mastodon_full_path + "/bin/tootctl accounts delete " + username_array[i])
delete_user(id_array[i], username_array[i])
i += 1
i = 0
cur.close()
if len(id_array) == 0:
except (Exception, psycopg2.DatabaseError) as error:
if deletion_accepted == True:
print(error)
print("None inactive users who accepted to be deleted found!")
finally:
elif deletion_accepted == False:
if conn is not None:
print("None inactive users to be deleted!")
conn.close()
return
###################################################################################
# update user as deleted
###################################################################################
while i < len(id_array):
if deletion_accepted == True:
print("Deleting inactive users who accepted to be deleted...")
elif deletion_accepted == False:
print("Deleting inactive users who do not reply our email...")
print(f"\nDeleting user {str(i)} of {str(len(id_array))} users")
print(f"Deleting user with id {str(id_array[i])}, username: {username_array[i]}")
os.system(f"RAILS_ENV=production {rvm_ruby} {mastodon_full_path}/bin/tootctl accounts delete {username_array[i]}")
delete_user(id_array[i], username_array[i])
i += 1
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
def delete_user(id, username):
conn = None
conn = None
try:
try:
conn = psycopg2.connect(database = mailing_db, user = mailing_db_user, password = "", host = "/var/run/postgresql", port = "5432")
conn = psycopg2.connect(database = mailing_db, user = mailing_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur = conn.cursor()
cur.execute("DELETE FROM " + mailing_db_table + " where account_id=(%s)", (str(id),))
print("Deleting user " + str(id) + ", username " + str(username))
cur.execute("DELETE FROM " + mailing_db_table + " where account_id=(%s)", (str(id),))
conn.commit()
print(f"Deleting user {str(id)}, username {str(username)}")
cur.close()
conn.commit()
except (Exception, psycopg2.DatabaseError) as error:
cur.close()
print(error)
except (Exception, psycopg2.DatabaseError) as error:
finally:
print(error)
if conn is not None:
finally:
conn.close()
if conn is not None:
###############################################################################
# INITIALISATION
###############################################################################
conn.close()
# Returns the parameter from the specified file
def get_parameter( parameter, file_path ):
# Check if secrets file exists
if not os.path.isfile(file_path):
if file_path == "secrets/secrets.txt":
print("File %s not found, exiting. Run setup.py."%file_path)
print("File %s not found, exiting. Run setup.py."%file_path)
elif file_path == "config.txt":
print("File %s not found, exiting. Run db-setup.py."%file_path)
print("File %s not found, exiting. Run db-setup.py."%file_path)
sys.exit(0)
# Find parameter in file
with open( file_path ) as f:
for line in f:
if line.startswith( parameter ):
return line.replace(parameter + ":", "").strip()
# Cannot find parameter, exit
print(file_path + " Missing parameter %s "%parameter)
print(f"{file_path} Missing parameter {parameter}")
print("Run setup.py")
sys.exit(0)
# Load secrets from secrets file
secrets_filepath = "secrets/secrets.txt"
mastodon_full_path = get_parameter("mastodon_full_path", secrets_filepath)
def config():
# Load configuration from config file
config_filepath = "config.txt"
mailing_db = get_parameter("mailing_db", config_filepath)
mailing_db_user = get_parameter("mailing_db_user", config_filepath)
mailing_db_table = get_parameter("mailing_db_table", config_filepath)
secrets_filepath = "secrets/secrets.txt"
mastodon_full_path = get_parameter("mastodon_full_path", secrets_filepath)
# Load configuration from config file
config_filepath = "config.txt"
mailing_db = get_parameter("mailing_db", config_filepath)
mailing_db_user = get_parameter("mailing_db_user", config_filepath)
mailing_db_table = get_parameter("mailing_db_table", config_filepath)
return (mastodon_full_path, mailing_db, mailing_db_user, mailing_db_table)
###############################################################################
# main
global rvm_ruby
rvm_ruby = os.environ['HOME'] + "/.rbenv/shims/ruby"
if __name__ == '__main__':
###############################################################################
# select users who replied the warning email saying yes to deletion
###############################################################################
mastodon_full_path, mailing_db, mailing_db_user, mailing_db_table = config()
deletion_accepted = True
query = "select account_id, username, email, to_be_deleted, feedback, recipient_error, elapsed_days from " + mailing_db_table + " where to_be_deleted = 't' and feedback = 't' and recipient_error = 'f' and emailed_at < now() - interval '31 days'"
id_array = []
username_array = []
delete_inactives(mailing_db, mailing_db_user, mailing_db_table, deletion_accepted, query, id_array, username_array)
global rvm_ruby
rvm_ruby = os.environ['HOME'] + "/.rbenv/shims/ruby"
###############################################################################
# select users who don't replied to email after 30 days
###############################################################################
###############################################################################
# select users who replied the warning email saying yes to deletion
###############################################################################
deletion_accepted = False
query = "select account_id, username, email, to_be_deleted, feedback, recipient_error, elapsed_days from " + mailing_db_table + " where to_be_deleted = 'f' and feedback = 'f' and recipient_error = 'f' and emailed_at < now() - interval '31 days'"
id_array = []
username_array = []
delete_inactives(mailing_db, mailing_db_user, mailing_db_table, deletion_accepted, query, id_array, username_array)
deletion_accepted = True
query = "select account_id, username, email, to_be_deleted, feedback, recipient_error, elapsed_days from " + mailing_db_table + " where to_be_deleted = 't' and "
query += "feedback = 't' and recipient_error = 'f' and emailed_at < now() - interval '31 days'"
delete_inactives(deletion_accepted, query)
###############################################################################
# select users who don't replied to email after 30 days
###############################################################################
deletion_accepted = False
query = "select account_id, username, email, to_be_deleted, feedback, recipient_error, elapsed_days from " + mailing_db_table + " where to_be_deleted = 'f' and "
query += "feedback = 'f' and recipient_error = 'f' and emailed_at < now() - interval '31 days'"
delete_inactives(deletion_accepted, query)

Veure arxiu

@ -183,8 +183,7 @@ class Inactives:
cur = conn.cursor()
#cur.execute("select account_id, email from users where current_sign_in_at < now() - interval '180 days' and disabled=False and approved=True order by current_sign_in_at desc;")
cur.execute("select account_id, email, current_sign_in_at from users where current_sign_in_at < now() - interval '180 days' and disabled=False and approved=True and account_id in (select id from accounts where actor_type='Person')")
cur.execute("select account_id, email from users where current_sign_in_at < now() - interval '180 days' and disabled=False and approved=True order by current_sign_in_at desc;")
rows = cur.fetchall()

Veure arxiu

@ -1,396 +1,21 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from datetime import datetime, timezone, timedelta
import time
import threading
import os
import sys
import os.path
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
import smtplib
from smtplib import SMTPException, SMTPAuthenticationError, SMTPConnectError, SMTPRecipientsRefused
import psycopg2
import socket
from socket import gaierror
from inactives import Inactives
###################################################################################
# write to database mailing status of inactive users
###################################################################################
# main
def write_db(now, id, username, email, emailed_at, emailed):
if __name__ == '__main__':
insert_line = "INSERT INTO " + mailing_db_table + "(datetime, account_id, username, email, emailed_at, emailed) VALUES(%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
inactives = Inactives()
conn = None
inactives_ids = inactives.id()
try:
inactives.delete(inactives_ids)
conn = psycopg2.connect(database = mailing_db, user = mailing_db_user, password = "", host = "/var/run/postgresql", port = "5432")
account_ids, account_emails, account_last_sign_in_ats = inactives.get_inactives()
cur = conn.cursor()
account_usernames = inactives.usernames(account_ids)
cur.execute("SELECT account_id FROM " + mailing_db_table + " where account_id=(%s)", (id,))
inactives.mailing(account_ids, account_emails, account_usernames, account_last_sign_in_ats)
row = cur.fetchone()
if row == None:
cur.execute(insert_line, (now, id, username, email, now, emailed))
else:
if emailed == True:
cur.execute("SELECT datetime FROM " + mailing_db_table + " where account_id=(%s)", (id,))
row = cur.fetchone()
delta = now-row[0]
cur.execute("UPDATE " + mailing_db_table + " SET elapsed_days=(%s), email=(%s), emailed=(%s) where account_id=(%s)", (delta, email, emailed, id))
print("Updating user " + str(id))
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
###############################################################################
# check if inactive user had been already emailed
###############################################################################
def email_sent(id):
try:
conn = psycopg2.connect(database = mailing_db, user = mailing_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute("SELECT emailed FROM " + mailing_db_table + " where account_id=(%s)", (id,))
row = cur.fetchone()
if row == None:
been_emailed = False
else:
been_emailed = row[0]
cur.close()
return been_emailed
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
###############################################################################
# Connect to Mastodon's Postgres DB to check if any inactive user is back online
###############################################################################
def check_alive(id):
try:
conn = psycopg2.connect(database = mastodon_db, user = mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute("select last_sign_in_at from users where account_id=(%s)", (id,))
row = cur.fetchone()
if row != None:
seen = row[0]
else:
seen = None
cur.close()
return seen
except (Exception, psycopg2.DatabaseError) as error:
print(error)
###############################################################################
# INITIALISATION
###############################################################################
# Returns the parameter from the specified file
def get_parameter( parameter, file_path ):
# Check if secrets file exists
if not os.path.isfile(file_path):
if file_path == "secrets/secrets.txt":
print("File %s not found, exiting. Run setup.py."%file_path)
elif file_path == "config.txt":
print("File %s not found, exiting. Run db-setup.py."%file_path)
sys.exit(0)
# Find parameter in file
with open( file_path ) as f:
for line in f:
if line.startswith( parameter ):
return line.replace(parameter + ":", "").strip()
# Cannot find parameter, exit
print(file_path + " Missing parameter %s "%parameter)
print("Run setup.py")
sys.exit(0)
# Load secrets from secrets file
secrets_filepath = "secrets/secrets.txt"
smtp_host = get_parameter("smtp_host", secrets_filepath)
smtp_user_login = get_parameter("smtp_user_login", secrets_filepath)
smtp_user_password = get_parameter("smtp_user_password", secrets_filepath)
email_subject = get_parameter("email_subject", secrets_filepath)
# Load configuration from config file
config_filepath = "config.txt"
mastodon_db = get_parameter("mastodon_db", config_filepath)
mastodon_db_user = get_parameter("mastodon_db_user", config_filepath)
mailing_db = get_parameter("mailing_db", config_filepath)
mailing_db_user = get_parameter("mailing_db_user", config_filepath)
mailing_db_table = get_parameter("mailing_db_table", config_filepath)
###############################################################################
# check if inactive user is back online
###############################################################################
try:
conn = psycopg2.connect(database = mailing_db, user = mailing_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute("SELECT account_id FROM " + mailing_db_table)
rows = cur.fetchall()
if rows != []:
inactive_users_id = []
for row in rows:
inactive_users_id.append(row[0])
else:
inactive_users_id = []
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
i = 0
while i < len(inactive_users_id):
seen = check_alive(inactive_users_id[i])
try:
conn = psycopg2.connect(database = mailing_db, user = mailing_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute("SELECT emailed_at FROM " + mailing_db_table + " where account_id=(%s)", (inactive_users_id[i],))
row = cur.fetchone()
email_datetime = row[0]
email_datetime = email_datetime.replace(tzinfo=None)
if seen != None:
reactivated = email_datetime < seen
last_year = datetime.today() - timedelta(days=365)
if reactivated == True or seen == None or seen > last_year: #if inactive user had reactivated its account or had deleted it we must delete related row from 'mailing_db_table'
cur.execute("DELETE FROM " + mailing_db_table + " where account_id=(%s)", (inactive_users_id[i],))
print("Deleting user " + str(inactive_users_id[i]))
conn.commit()
cur.close()
i += 1
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
###############################################################################
now = datetime.now(timezone.utc)
###############################################################################
# Connect to Mastodon's Postgres DB to get last year inactive users
###############################################################################
try:
conn = psycopg2.connect(database = mastodon_db, user = mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute("select account_id, email from users where last_sign_in_at < now() - interval '365 days' and disabled=False and approved=True order by last_sign_in_at desc;")
rows = cur.fetchall()
inactive_account_id = []
inactive_email = []
for row in rows:
inactive_account_id.append(row[0])
inactive_email.append(row[1])
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
########################################################################################################
# get related usernames #
########################################################################################################
inactive_usernames = []
i = 0
while i < len(inactive_account_id):
try:
conn = psycopg2.connect(database = mastodon_db, user = mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
inactive_id = inactive_account_id[i]
cur.execute("select username from accounts where id = '%s';", [inactive_id])
row = cur.fetchone()
new_username = row[0]
inactive_usernames.append(new_username)
i += 1
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
###########################################################################################################
# email inactive users
###########################################################################################################
try:
fp = open('message.txt')
text = fp.read()
message = MIMEText(text)
fp.close()
except:
print("message.txt file not found! Create it and write in the message you want for your inactive users.")
sys.exit(0)
i = 0
while i < len(inactive_email):
been_emailed = email_sent(inactive_account_id[i])
if been_emailed == False:
# Create message object instance
msg = MIMEMultipart()
# Declare message elements
msg['From'] = smtp_user_login
msg['To'] = inactive_email[i]
msg['Subject'] = inactive_usernames[i] + " " + email_subject
# Add the message body to the object instance
msg.attach(message)
try:
# Create the server connection
server = smtplib.SMTP(smtp_host)
# Switch the connection over to TLS encryption
server.starttls()
# Authenticate with the server
server.login(smtp_user_login, smtp_user_password)
# Send the message
server.sendmail(msg['From'], msg['To'], msg.as_string())
# Disconnect
server.quit()
print("Successfully sent email message to %s" % msg['To'])
emailed = True
write_db(now, inactive_account_id[i], inactive_usernames[i], inactive_email[i], now, emailed)
i += 1
time.sleep(5)
except SMTPAuthenticationError as auth_error:
print(auth_error)
sys.exit(":-(")
except socket.gaierror as socket_error:
print(socket_error)
print("Unknown SMTP server")
sys.exit(":-(")
except SMTPRecipientsRefused as recip_error:
print(recip_error)
emailed = False
write_db(now, inactive_account_id[i], inactive_usernames[i], inactive_email[i], now, emailed)
i += 1
else:
emailed = True
write_db(now, inactive_account_id[i], inactive_usernames[i], inactive_email[i], now, emailed)
i += 1