Refactored. Added Inactives Class

This commit is contained in:
spla 2022-08-01 11:52:30 +02:00
pare 9fbc5a6d44
commit 17e74f8517
S'han modificat 4 arxius amb 215 adicions i 560 eliminacions

Veure arxiu

@ -8,7 +8,18 @@ import psycopg2
from psycopg2 import sql from psycopg2 import sql
from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
# Returns the parameter from the specified file def get_config():
# Load configuration from config file
config_filepath = "config/db_config.txt"
mastodon_db = get_parameter("mastodon_db", config_filepath)
mastodon_db_user = get_parameter("mastodon_db_user", config_filepath)
mailing_db = get_parameter("mailing_db", config_filepath)
mailing_db_user = get_parameter("mailing_db_user", config_filepath)
mailing_db_table = get_parameter("mailing_db_table", config_filepath)
return (mastodon_db, mastodon_db_user, mailing_db, mailing_db_user, mailing_db_table)
def get_parameter( parameter, file_path ): def get_parameter( parameter, file_path ):
# Check if secrets file exists # Check if secrets file exists
if not os.path.isfile(file_path): if not os.path.isfile(file_path):
@ -27,6 +38,10 @@ def get_parameter( parameter, file_path ):
sys.exit(0) sys.exit(0)
def write_parameter( parameter, file_path ): def write_parameter( parameter, file_path ):
if not os.path.exists('config'):
os.makedirs('config')
print("Setting up mailing DB parameters...") print("Setting up mailing DB parameters...")
print("\n") print("\n")
mastodon_db = input("Mastodon db name: ") mastodon_db = input("Mastodon db name: ")
@ -41,46 +56,7 @@ def write_parameter( parameter, file_path ):
print("mailing_db_user: {}".format(mailing_db_user), file=text_file) print("mailing_db_user: {}".format(mailing_db_user), file=text_file)
print("mailing_db_table: {}".format(mailing_db_table), file=text_file) print("mailing_db_table: {}".format(mailing_db_table), file=text_file)
def create_table(db, db_user, table, sql): def create_database():
try:
conn = None
conn = psycopg2.connect(database = db, user = db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
print("Creating table.. "+table)
# Create the table in PostgreSQL database
cur.execute(sql)
conn.commit()
print("Table "+table+" created!")
print("\n")
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
#############################################################################################
# Load configuration from config file
config_filepath = "config.txt"
mastodon_db = get_parameter("mastodon_db", config_filepath) # E.g., mastodon_production
mastodon_db_user = get_parameter("mastodon_db_user", config_filepath) # E.g., mastodon
mailing_db = get_parameter("mailing_db", config_filepath) # E.g., inactive
mailing_db_user = get_parameter("mailing_db_user", config_filepath) # E.g., mastodon
mailing_db_table = get_parameter("mailing_db_table", config_filepath) # E.g., inactive_users
############################################################
# create database
############################################################
try: try:
@ -108,11 +84,10 @@ finally:
conn.close() conn.close()
#############################################################################################
try: try:
conn = None conn = None
conn = psycopg2.connect(database = mailing_db, user = mailing_db_user, password = "", host = "/var/run/postgresql", port = "5432") conn = psycopg2.connect(database = mailing_db, user = mailing_db_user, password = "", host = "/var/run/postgresql", port = "5432")
except (Exception, psycopg2.DatabaseError) as error: except (Exception, psycopg2.DatabaseError) as error:
@ -129,24 +104,53 @@ if conn is not None:
print("Mailing db parameters saved to config.txt!") print("Mailing db parameters saved to config.txt!")
print("\n") print("\n")
############################################################ def create_table(db, db_user, table, sql):
# Create needed tables
############################################################
print("Creating table...") try:
conn = None
conn = psycopg2.connect(database = db, user = db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
print("Creating table.. "+table)
# Create the table in PostgreSQL database
cur.execute(sql)
conn.commit()
print("Table "+table+" created!")
print("\n")
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
# main
if __name__ == '__main__':
mastodon_db, mastodon_db_user, mailing_db, mailing_db_user, mailing_db_table = get_config()
create_database()
######################################## ########################################
# create mailing DB table
db = mailing_db db = mailing_db
db_user = mailing_db_user db_user = mailing_db_user
table = mailing_db_table table = mailing_db_table
sql = "create table " + table + "(datetime timestamptz, account_id int primary key, username varchar(30), email varchar(50), emailed_at timestamptz," sql = "create table " + table + "(datetime timestamptz, account_id int primary key, username varchar(30), email varchar(50), emailed_at timestamptz,"
sql += "emailed boolean default False, deleted boolean default False, elapsed_days varchar(30), to_be_deleted boolean default False," sql += "emailed boolean default False, deleted boolean default False, elapsed_days varchar(30), to_be_deleted boolean default False,"
sql += "recipient_error boolean default False, feedback boolean default False)" sql += "recipient_error boolean default False, feedback boolean default False, current_sign_in_at timestamptz)"
create_table(db, db_user, table, sql) create_table(db, db_user, table, sql)
##################################### #####################################
print("Done!") print("Done!\nNow you can run setup.py!\n")
print("Now you can run setup.py!")
print("\n")

Veure arxiu

@ -3,13 +3,16 @@
from datetime import datetime, timezone, timedelta from datetime import datetime, timezone, timedelta
import time import time
import threading
import os import os
import sys import sys
import os.path import os.path
import psycopg2 import psycopg2
def delete_inactives(mailing_db, mailing_db_user, mailing_db_table, deletion_accepted, query, id_array, username_array): def delete_inactives(deletion_accepted, query):
id_array = []
username_array = []
conn = None conn = None
@ -24,28 +27,41 @@ def delete_inactives(mailing_db, mailing_db_user, mailing_db_table, deletion_acc
for row in cur: for row in cur:
id_array.append(row[0]) id_array.append(row[0])
username_array.append(row[1]) username_array.append(row[1])
i = 0 i = 0
if len(id_array) == 0: if len(id_array) == 0:
if deletion_accepted == True: if deletion_accepted == True:
print("None inactive users who accepted to be deleted found!") print("None inactive users who accepted to be deleted found!")
elif deletion_accepted == False: elif deletion_accepted == False:
print("None inactive users to be deleted!") print("None inactive users to be deleted!")
return return
while i < len(id_array): while i < len(id_array):
if deletion_accepted == True: if deletion_accepted == True:
print("Deleting inactive users who accepted to be deleted...") print("Deleting inactive users who accepted to be deleted...")
elif deletion_accepted == False: elif deletion_accepted == False:
print("Deleting inactive users who do not reply our email...") print("Deleting inactive users who do not reply our email...")
print("\n") print(f"\nDeleting user {str(i)} of {str(len(id_array))} users")
print("Deleting user " + str(i) + " of " + str(len(id_array)) + " users")
print("Deleting user with id " + str(id_array[i]) + ", username: " + username_array[i]) print(f"Deleting user with id {str(id_array[i])}, username: {username_array[i]}")
os.system("RAILS_ENV=production " + rvm_ruby + " " + mastodon_full_path + "/bin/tootctl accounts delete " + username_array[i])
os.system(f"RAILS_ENV=production {rvm_ruby} {mastodon_full_path}/bin/tootctl accounts delete {username_array[i]}")
delete_user(id_array[i], username_array[i]) delete_user(id_array[i], username_array[i])
i += 1 i += 1
cur.close() cur.close()
@ -60,10 +76,6 @@ def delete_inactives(mailing_db, mailing_db_user, mailing_db_table, deletion_acc
conn.close() conn.close()
###################################################################################
# update user as deleted
###################################################################################
def delete_user(id, username): def delete_user(id, username):
conn = None conn = None
@ -75,7 +87,8 @@ def delete_user(id, username):
cur = conn.cursor() cur = conn.cursor()
cur.execute("DELETE FROM " + mailing_db_table + " where account_id=(%s)", (str(id),)) cur.execute("DELETE FROM " + mailing_db_table + " where account_id=(%s)", (str(id),))
print("Deleting user " + str(id) + ", username " + str(username))
print(f"Deleting user {str(id)}, username {str(username)}")
conn.commit() conn.commit()
@ -91,32 +104,37 @@ def delete_user(id, username):
conn.close() conn.close()
###############################################################################
# INITIALISATION
###############################################################################
# Returns the parameter from the specified file
def get_parameter( parameter, file_path ): def get_parameter( parameter, file_path ):
# Check if secrets file exists # Check if secrets file exists
if not os.path.isfile(file_path): if not os.path.isfile(file_path):
if file_path == "secrets/secrets.txt": if file_path == "secrets/secrets.txt":
print("File %s not found, exiting. Run setup.py."%file_path) print("File %s not found, exiting. Run setup.py."%file_path)
elif file_path == "config.txt": elif file_path == "config.txt":
print("File %s not found, exiting. Run db-setup.py."%file_path) print("File %s not found, exiting. Run db-setup.py."%file_path)
sys.exit(0) sys.exit(0)
# Find parameter in file # Find parameter in file
with open( file_path ) as f: with open( file_path ) as f:
for line in f: for line in f:
if line.startswith( parameter ): if line.startswith( parameter ):
return line.replace(parameter + ":", "").strip() return line.replace(parameter + ":", "").strip()
# Cannot find parameter, exit # Cannot find parameter, exit
print(file_path + " Missing parameter %s "%parameter) print(f"{file_path} Missing parameter {parameter}")
print("Run setup.py") print("Run setup.py")
sys.exit(0) sys.exit(0)
# Load secrets from secrets file def config():
secrets_filepath = "secrets/secrets.txt" secrets_filepath = "secrets/secrets.txt"
mastodon_full_path = get_parameter("mastodon_full_path", secrets_filepath) mastodon_full_path = get_parameter("mastodon_full_path", secrets_filepath)
@ -126,7 +144,14 @@ mailing_db = get_parameter("mailing_db", config_filepath)
mailing_db_user = get_parameter("mailing_db_user", config_filepath) mailing_db_user = get_parameter("mailing_db_user", config_filepath)
mailing_db_table = get_parameter("mailing_db_table", config_filepath) mailing_db_table = get_parameter("mailing_db_table", config_filepath)
return (mastodon_full_path, mailing_db, mailing_db_user, mailing_db_table)
############################################################################### ###############################################################################
# main
if __name__ == '__main__':
mastodon_full_path, mailing_db, mailing_db_user, mailing_db_table = config()
global rvm_ruby global rvm_ruby
rvm_ruby = os.environ['HOME'] + "/.rbenv/shims/ruby" rvm_ruby = os.environ['HOME'] + "/.rbenv/shims/ruby"
@ -136,17 +161,19 @@ rvm_ruby = os.environ['HOME'] + "/.rbenv/shims/ruby"
############################################################################### ###############################################################################
deletion_accepted = True deletion_accepted = True
query = "select account_id, username, email, to_be_deleted, feedback, recipient_error, elapsed_days from " + mailing_db_table + " where to_be_deleted = 't' and feedback = 't' and recipient_error = 'f' and emailed_at < now() - interval '31 days'"
id_array = [] query = "select account_id, username, email, to_be_deleted, feedback, recipient_error, elapsed_days from " + mailing_db_table + " where to_be_deleted = 't' and "
username_array = [] query += "feedback = 't' and recipient_error = 'f' and emailed_at < now() - interval '31 days'"
delete_inactives(mailing_db, mailing_db_user, mailing_db_table, deletion_accepted, query, id_array, username_array)
delete_inactives(deletion_accepted, query)
############################################################################### ###############################################################################
# select users who don't replied to email after 30 days # select users who don't replied to email after 30 days
############################################################################### ###############################################################################
deletion_accepted = False deletion_accepted = False
query = "select account_id, username, email, to_be_deleted, feedback, recipient_error, elapsed_days from " + mailing_db_table + " where to_be_deleted = 'f' and feedback = 'f' and recipient_error = 'f' and emailed_at < now() - interval '31 days'"
id_array = [] query = "select account_id, username, email, to_be_deleted, feedback, recipient_error, elapsed_days from " + mailing_db_table + " where to_be_deleted = 'f' and "
username_array = [] query += "feedback = 'f' and recipient_error = 'f' and emailed_at < now() - interval '31 days'"
delete_inactives(mailing_db, mailing_db_user, mailing_db_table, deletion_accepted, query, id_array, username_array)
delete_inactives(deletion_accepted, query)

Veure arxiu

@ -183,8 +183,7 @@ class Inactives:
cur = conn.cursor() cur = conn.cursor()
#cur.execute("select account_id, email from users where current_sign_in_at < now() - interval '180 days' and disabled=False and approved=True order by current_sign_in_at desc;") cur.execute("select account_id, email from users where current_sign_in_at < now() - interval '180 days' and disabled=False and approved=True order by current_sign_in_at desc;")
cur.execute("select account_id, email, current_sign_in_at from users where current_sign_in_at < now() - interval '180 days' and disabled=False and approved=True and account_id in (select id from accounts where actor_type='Person')")
rows = cur.fetchall() rows = cur.fetchall()

Veure arxiu

@ -1,396 +1,21 @@
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from datetime import datetime, timezone, timedelta from inactives import Inactives
import time
import threading
import os
import sys
import os.path
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
import smtplib
from smtplib import SMTPException, SMTPAuthenticationError, SMTPConnectError, SMTPRecipientsRefused
import psycopg2
import socket
from socket import gaierror
################################################################################### # main
# write to database mailing status of inactive users
###################################################################################
def write_db(now, id, username, email, emailed_at, emailed): if __name__ == '__main__':
insert_line = "INSERT INTO " + mailing_db_table + "(datetime, account_id, username, email, emailed_at, emailed) VALUES(%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING" inactives = Inactives()
conn = None inactives_ids = inactives.id()
try: inactives.delete(inactives_ids)
conn = psycopg2.connect(database = mailing_db, user = mailing_db_user, password = "", host = "/var/run/postgresql", port = "5432") account_ids, account_emails, account_last_sign_in_ats = inactives.get_inactives()
cur = conn.cursor() account_usernames = inactives.usernames(account_ids)
cur.execute("SELECT account_id FROM " + mailing_db_table + " where account_id=(%s)", (id,)) inactives.mailing(account_ids, account_emails, account_usernames, account_last_sign_in_ats)
row = cur.fetchone()
if row == None:
cur.execute(insert_line, (now, id, username, email, now, emailed))
else:
if emailed == True:
cur.execute("SELECT datetime FROM " + mailing_db_table + " where account_id=(%s)", (id,))
row = cur.fetchone()
delta = now-row[0]
cur.execute("UPDATE " + mailing_db_table + " SET elapsed_days=(%s), email=(%s), emailed=(%s) where account_id=(%s)", (delta, email, emailed, id))
print("Updating user " + str(id))
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
###############################################################################
# check if inactive user had been already emailed
###############################################################################
def email_sent(id):
try:
conn = psycopg2.connect(database = mailing_db, user = mailing_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute("SELECT emailed FROM " + mailing_db_table + " where account_id=(%s)", (id,))
row = cur.fetchone()
if row == None:
been_emailed = False
else:
been_emailed = row[0]
cur.close()
return been_emailed
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
###############################################################################
# Connect to Mastodon's Postgres DB to check if any inactive user is back online
###############################################################################
def check_alive(id):
try:
conn = psycopg2.connect(database = mastodon_db, user = mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute("select last_sign_in_at from users where account_id=(%s)", (id,))
row = cur.fetchone()
if row != None:
seen = row[0]
else:
seen = None
cur.close()
return seen
except (Exception, psycopg2.DatabaseError) as error:
print(error)
###############################################################################
# INITIALISATION
###############################################################################
# Returns the parameter from the specified file
def get_parameter( parameter, file_path ):
# Check if secrets file exists
if not os.path.isfile(file_path):
if file_path == "secrets/secrets.txt":
print("File %s not found, exiting. Run setup.py."%file_path)
elif file_path == "config.txt":
print("File %s not found, exiting. Run db-setup.py."%file_path)
sys.exit(0)
# Find parameter in file
with open( file_path ) as f:
for line in f:
if line.startswith( parameter ):
return line.replace(parameter + ":", "").strip()
# Cannot find parameter, exit
print(file_path + " Missing parameter %s "%parameter)
print("Run setup.py")
sys.exit(0)
# Load secrets from secrets file
secrets_filepath = "secrets/secrets.txt"
smtp_host = get_parameter("smtp_host", secrets_filepath)
smtp_user_login = get_parameter("smtp_user_login", secrets_filepath)
smtp_user_password = get_parameter("smtp_user_password", secrets_filepath)
email_subject = get_parameter("email_subject", secrets_filepath)
# Load configuration from config file
config_filepath = "config.txt"
mastodon_db = get_parameter("mastodon_db", config_filepath)
mastodon_db_user = get_parameter("mastodon_db_user", config_filepath)
mailing_db = get_parameter("mailing_db", config_filepath)
mailing_db_user = get_parameter("mailing_db_user", config_filepath)
mailing_db_table = get_parameter("mailing_db_table", config_filepath)
###############################################################################
# check if inactive user is back online
###############################################################################
try:
conn = psycopg2.connect(database = mailing_db, user = mailing_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute("SELECT account_id FROM " + mailing_db_table)
rows = cur.fetchall()
if rows != []:
inactive_users_id = []
for row in rows:
inactive_users_id.append(row[0])
else:
inactive_users_id = []
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
i = 0
while i < len(inactive_users_id):
seen = check_alive(inactive_users_id[i])
try:
conn = psycopg2.connect(database = mailing_db, user = mailing_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute("SELECT emailed_at FROM " + mailing_db_table + " where account_id=(%s)", (inactive_users_id[i],))
row = cur.fetchone()
email_datetime = row[0]
email_datetime = email_datetime.replace(tzinfo=None)
if seen != None:
reactivated = email_datetime < seen
last_year = datetime.today() - timedelta(days=365)
if reactivated == True or seen == None or seen > last_year: #if inactive user had reactivated its account or had deleted it we must delete related row from 'mailing_db_table'
cur.execute("DELETE FROM " + mailing_db_table + " where account_id=(%s)", (inactive_users_id[i],))
print("Deleting user " + str(inactive_users_id[i]))
conn.commit()
cur.close()
i += 1
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
###############################################################################
now = datetime.now(timezone.utc)
###############################################################################
# Connect to Mastodon's Postgres DB to get last year inactive users
###############################################################################
try:
conn = psycopg2.connect(database = mastodon_db, user = mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute("select account_id, email from users where last_sign_in_at < now() - interval '365 days' and disabled=False and approved=True order by last_sign_in_at desc;")
rows = cur.fetchall()
inactive_account_id = []
inactive_email = []
for row in rows:
inactive_account_id.append(row[0])
inactive_email.append(row[1])
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
########################################################################################################
# get related usernames #
########################################################################################################
inactive_usernames = []
i = 0
while i < len(inactive_account_id):
try:
conn = psycopg2.connect(database = mastodon_db, user = mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
inactive_id = inactive_account_id[i]
cur.execute("select username from accounts where id = '%s';", [inactive_id])
row = cur.fetchone()
new_username = row[0]
inactive_usernames.append(new_username)
i += 1
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
###########################################################################################################
# email inactive users
###########################################################################################################
try:
fp = open('message.txt')
text = fp.read()
message = MIMEText(text)
fp.close()
except:
print("message.txt file not found! Create it and write in the message you want for your inactive users.")
sys.exit(0)
i = 0
while i < len(inactive_email):
been_emailed = email_sent(inactive_account_id[i])
if been_emailed == False:
# Create message object instance
msg = MIMEMultipart()
# Declare message elements
msg['From'] = smtp_user_login
msg['To'] = inactive_email[i]
msg['Subject'] = inactive_usernames[i] + " " + email_subject
# Add the message body to the object instance
msg.attach(message)
try:
# Create the server connection
server = smtplib.SMTP(smtp_host)
# Switch the connection over to TLS encryption
server.starttls()
# Authenticate with the server
server.login(smtp_user_login, smtp_user_password)
# Send the message
server.sendmail(msg['From'], msg['To'], msg.as_string())
# Disconnect
server.quit()
print("Successfully sent email message to %s" % msg['To'])
emailed = True
write_db(now, inactive_account_id[i], inactive_usernames[i], inactive_email[i], now, emailed)
i += 1
time.sleep(5)
except SMTPAuthenticationError as auth_error:
print(auth_error)
sys.exit(":-(")
except socket.gaierror as socket_error:
print(socket_error)
print("Unknown SMTP server")
sys.exit(":-(")
except SMTPRecipientsRefused as recip_error:
print(recip_error)
emailed = False
write_db(now, inactive_account_id[i], inactive_usernames[i], inactive_email[i], now, emailed)
i += 1
else:
emailed = True
write_db(now, inactive_account_id[i], inactive_usernames[i], inactive_email[i], now, emailed)
i += 1