diff --git a/welcome.py b/welcome.py index 7e6000d..626e1da 100644 --- a/welcome.py +++ b/welcome.py @@ -1,29 +1,61 @@ -from six.moves import urllib +#from six.moves import urllib import datetime -from subprocess import call +#from subprocess import call from mastodon import Mastodon -import time -import threading -import csv +#import time +#import threading +#import csv import os import json -import time -import signal +#import time +#import signal import sys import os.path import requests import operator -import redis -import calendar +#import redis +#import calendar import psycopg2 import pdb -from decimal import * -getcontext().prec = 2 +#from decimal import * +#getcontext().prec = 2 -############################################################################### -# INITIALISATION -############################################################################### +def mastodon(): + + # Load secrets from secrets file + secrets_filepath = "secrets/secrets.txt" + uc_client_id = get_parameter("uc_client_id", secrets_filepath) + uc_client_secret = get_parameter("uc_client_secret", secrets_filepath) + uc_access_token = get_parameter("uc_access_token", secrets_filepath) + + # Load configuration from config file + config_filepath = "config/config.txt" + mastodon_hostname = get_parameter("mastodon_hostname", config_filepath) + + # Initialise Mastodon API + mastodon = Mastodon( + client_id=uc_client_id, + client_secret=uc_client_secret, + access_token=uc_access_token, + api_base_url='https://' + mastodon_hostname, + ) + + # Initialise access headers + headers = {'Authorization': 'Bearer %s'%uc_access_token} + + return (mastodon, mastodon_hostname) + +def db_config(): + + # Load db configuration from config file + config_filepath = "config/db_config.txt" + mastodon_db = get_parameter("mastodon_db", config_filepath) + mastodon_db_user = get_parameter("mastodon_db_user", config_filepath) + welcome_db = get_parameter("welcome_db", config_filepath) + welcome_db_user = get_parameter("welcome_db_user", config_filepath) + + return (mastodon_db, mastodon_db_user, welcome_db, welcome_db_user) # Returns the parameter from the specified file def get_parameter( parameter, file_path ): @@ -42,423 +74,326 @@ def get_parameter( parameter, file_path ): print(file_path + " Missing parameter %s "%parameter) sys.exit(0) -# Load secrets from secrets file -secrets_filepath = "secrets/secrets.txt" -uc_client_id = get_parameter("uc_client_id", secrets_filepath) -uc_client_secret = get_parameter("uc_client_secret", secrets_filepath) -uc_access_token = get_parameter("uc_access_token", secrets_filepath) - -# Load configuration from config file -config_filepath = "config.txt" -mastodon_hostname = get_parameter("mastodon_hostname", config_filepath) # E.g., mastodon.social - -# Initialise Mastodon API -mastodon = Mastodon( - client_id = uc_client_id, - client_secret = uc_client_secret, - access_token = uc_access_token, - api_base_url = 'https://' + mastodon_hostname, -) - -# Initialise access headers -headers={ 'Authorization': 'Bearer %s'%uc_access_token } - -############################################################################### -# current registered users ############################################################################### +# main -try: +if __name__ == '__main__': + + mastodon, mastodon_hostname = mastodon() + + mastodon_db, mastodon_db_user, welcome_db, welcome_db_user = db_config() + + ############################################################################### + # current registered users + ############################################################################### + + try: + + conn = None + + conn = psycopg2.connect(database = mastodon_db, user = mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432") + + cur = conn.cursor() + + cur.execute("SELECT count(*) FROM users WHERE disabled='f' and approved='t'") + + row = cur.fetchone() + + if row != None: + + current_id = row[0] + + else: + + current_id = 0 + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print (error) + + finally: + + if conn is not None: + + conn.close() + + print("Current users: %s "% current_id) + + ############################################################################### + + now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + users_before = current_id + users_hour = 0 conn = None - conn = psycopg2.connect(database = "mastodon_production", user = "mastodon", password = "", host = "/var/run/postgresql", port = "5432") + try: - cur = conn.cursor() + conn = psycopg2.connect(database = welcome_db, user = welcome_db_user, password = "", host = "/var/run/postgresql", port = "5432") - cur.execute("SELECT count(*) FROM users WHERE disabled='f' and approved='t'") + cur = conn.cursor() - row = cur.fetchone() + cur.execute("SELECT DISTINCT ON (datetime) users FROM welcome WHERE datetime > current_timestamp - INTERVAL '70 minutes' ORDER BY datetime asc LIMIT 1") - if row != None: + row = cur.fetchone() - current_id = row[0] - else: + users_before = row[0] - current_id = 0 + cur.close() - cur.close() + users_hour = current_id - users_before -except (Exception, psycopg2.DatabaseError) as error: + except (Exception, psycopg2.DatabaseError) as error: - print (error) + print (error) -finally: + finally: - if conn is not None: + if conn is not None: - conn.close() + conn.close() -print("Número d'usuaris: %s "% current_id) + print("-----------------") + print("current users: "+str(current_id)) + print("users before: "+str(users_before)) + print("users / hour: "+str(users_hour)) -############################################################################### + insert_sql = """INSERT INTO welcome(datetime, users, users_hour) + VALUES(%s,%s,%s) RETURNING datetime;""" + conn = None -ara = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") -############################################################################### -# Redis 25/10/18 connectar amb Redis per a obtenir l'activitat setmanal -############################################################################### + try: -redis_host = "localhost" -redis_port = 6379 -redis_password = "" + conn = psycopg2.connect(database = welcome_db, user = welcome_db_user, password = "", host = "/var/run/postgresql", port = "5432") -setmana_actual = str(datetime.datetime.now().isocalendar()[1]) + cur = conn.cursor() -r = redis.StrictRedis(host=redis_host, port=redis_port, password=redis_password, decode_responses=True) + cur.execute(insert_sql, (now, current_id, users_hour)) -# Interaccions setmana actual + datetime = cur.fetchone()[0] -interaccions = r.get("activity:interactions:"+setmana_actual) -if interaccions == None: - interaccions = 0 -print("Interaccions d'aquesta setmana: %s "% interaccions) + conn.commit() -actius = r.pfcount("activity:logins:"+setmana_actual) -print("Usuaris actius aquesta setmana: %s "% actius) + cur.close() -############################################################################### -# Connectar amb la bbdd Postgres per obtenir la darrera fila i els seus usuaris -############################################################################### + except (Exception, psycopg2.DatabaseError) as error: -usuaris_abans = current_id -toots_abans = num_toots -instancies_abans = num_instances -usuarishora = 0 -tootshora = 0 -instancieshora = 0 -toots_inici_setmana = num_toots -inc_disc_space_hour = 0 -toots_actius = 0.00 + print(error) -conn = None + finally: -try: + if conn is not None: - conn = psycopg2.connect(database = "pggrafana", user = "mastodon", password = "", host = "/var/run/postgresql", port = "5432") + conn.close() - cur = conn.cursor() + ############################################################################### + # WORK OUT THE TOOT TEXT + ############################################################################### - cur.execute("SELECT DISTINCT ON (datetime) usuaris,toots,instancies,datetime,used_disk_space FROM grafana WHERE datetime > current_timestamp - INTERVAL '70 minutes' ORDER BY datetime asc LIMIT 1") + # Calculate difference in times - row = cur.fetchone() - usuaris_abans = row[0] - toots_abans = row[1] - instancies_abans = row[2] - disc_space_before = row[4] + hourly_change_string = "" + daily_change_string = "" + weekly_change_string = "" + users_hourly_change_string = "" - # quants toots en l'inici de la setmana - cur.execute("SELECT DISTINCT ON (datetime) toots, datetime FROM grafana WHERE datetime > date_trunc('week', now()::timestamp) ORDER by datetime asc LIMIT 1") - row = cur.fetchone() - if row == None: - toots_inici_setmana = num_toots - else: - toots_inici_setmana = row[0] + try: - cur.close() + conn = None - usuarishora = current_id - usuaris_abans - tootshora = num_toots - toots_abans - instancieshora = num_instances - instancies_abans - inc_disc_space_hour = db_disk_space - disc_space_before + conn = psycopg2.connect(database = welcome_db, user = welcome_db_user, password = "", host = "/var/run/postgresql", port = "5432") -except (Exception, psycopg2.DatabaseError) as error: - print (error) -finally: - if conn is not None: - conn.close() + cur = conn.cursor() -if toots_inici_setmana == num_toots: - toots_actius = 0.00 -elif actius == 0: - toots_actius = 0.00 -else: - toots_actius = round((num_toots-toots_inici_setmana)/actius, 2) + cur.execute("SELECT DISTINCT ON (datetime) users, datetime FROM welcome WHERE datetime > current_timestamp - INTERVAL '62 minutes' ORDER BY datetime asc LIMIT 1") -print("-----------------") -print("current_id: "+str(current_id)) -print("usuaris abans: "+str(usuaris_abans)) -print("usuaris hora: "+str(usuarishora)) -print("-----------------") -print("toots: "+str(num_toots)) -print("toots abans: "+str(toots_abans)) -print("toots x hora: "+str(tootshora)) -print("-----------------") -print("instancies: "+str(num_instances)) -print("instancies abans: "+str(instancies_abans)) -print("instancies x hora: "+str(instancieshora)) -print("-----------------") -print("toots aquesta setmana:"+str(num_toots-toots_inici_setmana)) -print("usuaris actius:"+str(actius)) -print("toots x actius: "+str(toots_actius)) -print("------------------------") -print(" spla @ 2018 - 2020 ") -print("------------------------") + row = cur.fetchone() -###################################################################################################################################### -# Connectar amb la bbdd Postgres pggrafana per guardar les variables per grafana + if row[0] == None: -inserta_linia = """INSERT INTO grafana(datetime, usuaris, usuarishora, toots, tootshora, tootsusuari, interaccions, actius, actius30, instancies, instancieshora, tootsactius, - used_disk_space, disc_space_hour) - VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) RETURNING datetime;""" -conn = None + users_last_hour = 0 -ara = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + else: -try: + users_last_hour = row[0] - conn = psycopg2.connect(database = "pggrafana", user = "mastodon", password = "", host = "/var/run/postgresql", port = "5432") + cur.execute("SELECT DISTINCT ON (datetime) users,datetime FROM welcome WHERE datetime > current_timestamp - INTERVAL '1 day' ORDER BY datetime asc LIMIT 1") - cur = conn.cursor() - # executem INSERT - cur.execute(inserta_linia, (ara, current_id, usuarishora, num_toots, tootshora, toots_per_usuari, interaccions, actius, actius30, num_instances, instancieshora, toots_actius, db_disk_space, inc_disc_space_hour)) - # obté l'id - datetime = cur.fetchone()[0] - # salvar els canvis en la base de dades - conn.commit() - # i tancar la connexió amb la base de dades - cur.close() + row = cur.fetchone() -except (Exception, psycopg2.DatabaseError) as error: + if row[0] == None: - print(error) + users_last_day = 0 -finally: + else: - if conn is not None: + users_last_day = row[0] - conn.close() + cur.execute("SELECT DISTINCT ON (datetime) users,datetime FROM welcome WHERE datetime > current_timestamp - INTERVAL '7 days' ORDER BY datetime asc LIMIT 1") -############################################################################### -# WORK OUT THE TOOT TEXT -############################################################################### + row = cur.fetchone() -# Calculate difference in times + if row[0] == None: -hourly_change_string = "" -daily_change_string = "" -weekly_change_string = "" + users_last_week = 0 + else: -# canvi horari usuaris 19/05/18 -users_hourly_change_string = "" + users_last_week = row[0] -####################################################################################################################### -# Connectar amb la bbdd Postgres per obtenir increment usuaris en la darrera hora, el darrer dia i en la darrera setmana -######################################################################################################################## + cur.close() -try: + except (Exception, psycopg2.DatabaseError) as error: + + print (error) + + finally: + + if conn is not None: + + conn.close() + + ################################################################################# + + welcome_users = [] + + if time.localtime().tm_isdst == 0: + + interval_time = '60 minutes' + + elif time.localtime().tm_isdst == 1: + + interval_time = '120 minutes' conn = None - conn = psycopg2.connect(database = "pggrafana", user = "mastodon", password = "", host = "/var/run/postgresql", port = "5432") + try: - cur = conn.cursor() + conn = psycopg2.connect(database = mastodon_db, user = mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432") - cur.execute("SELECT DISTINCT ON (datetime) usuaris,datetime FROM grafana WHERE datetime > current_timestamp - INTERVAL '62 minutes' ORDER BY datetime asc LIMIT 1") + cur = conn.cursor() - row = cur.fetchone() + cur.execute("select username, domain from accounts where (created_at + interval '" + interval_time + "' > (now() - interval '1 hour')) and domain is null and id in (select account_id from users where approved and not disabled)") - if row[0] == None: + rows = cur.fetchall() - usuaris_fa_una_hora = 0 + for row in rows: - else: + if row[1] == None: - usuaris_fa_una_hora = row[0] + welcome_users.append(row[0]) - cur.execute("SELECT DISTINCT ON (datetime) usuaris,datetime FROM grafana WHERE datetime > current_timestamp - INTERVAL '1 day' ORDER BY datetime asc LIMIT 1") + cur.close() - row = cur.fetchone() + except (Exception, psycopg2.DatabaseError) as error: - if row[0] == None: + print(error) - usuaris_fa_un_dia = 0 + finally: - else: + if conn is not None: - usuaris_fa_un_dia = row[0] + conn.close() - cur.execute("SELECT DISTINCT ON (datetime) usuaris,datetime FROM grafana WHERE datetime > current_timestamp - INTERVAL '7 days' ORDER BY datetime asc LIMIT 1") + i = 0 - row = cur.fetchone() + new_users_string = "" - if row[0] == None: + while i < len(welcome_users): - usuaris_fa_una_setmana = 0 - else: + new_users_string = new_users_string+"@"+welcome_users[i] - usuaris_fa_una_setmana = row[0] + i += 1 - cur.close() + if i < len(welcome_users): -except (Exception, psycopg2.DatabaseError) as error: + new_users_string = new_users_string+", " - print (error) + ######################################################################################################### -finally: + hour_inc = current_id - users_last_hour + day_inc = current_id - users_last_day + week_inc = current_id - users_last_week - if conn is not None: + print("------------------------") + print("new users last hour: "+str(hour_inc)) + print("new users last day: "+str(day_inc)) + print("new users last week: "+str(week_inc)) - conn.close() + ################################################################################### -################################################################################# + # Hourly change + if hour_inc != 0: -saludar_usuaris = [] + users_hourly_change = current_id - users_last_hour + print("Hourly users evolution: %s"%users_hourly_change) + if users_hourly_change > 0: + hourly_change_string = "+" + format(users_hourly_change, ",d") + " en la darrera hora\n" -if time.localtime().tm_isdst == 0: + # Daily change + if day_inc != 0: - interval_time = '60 minutes' + daily_change = current_id - users_last_day + print("Daily users evolution: %s"%daily_change) + if daily_change > 0: + daily_change_string = "+" + format(daily_change, ",d") + " in the last day\n" -elif time.localtime().tm_isdst == 1: + # Weekly change + if week_inc != 0: - interval_time = '120 minutes' + weekly_change = current_id - users_last_week + print("Weekly users evolution: %s"%weekly_change) + if weekly_change > 0: + weekly_change_string = "+" + format(weekly_change, ",d") + " in the last week\n" -conn = None + elif users_hourly_change < 0: -try: - - conn = psycopg2.connect(database = "mastodon_production", user = "mastodon", password = "", host = "/var/run/postgresql", port = "5432") + hourly_change_string = format(users_hourly_change, ",d") + " in the last hour\n" - cur = conn.cursor() - cur.execute("select username, domain from accounts where (created_at + interval '" + interval_time + "' > (now() - interval '1 hour')) and domain is null and id in (select account_id from users where approved and not disabled)") - #cur.execute("select username, domain from accounts where id in (select account_id from follows where (created_at + interval '" + interval_time + "' > (now() - interval '60 minutes')) and target_account_id = '1')") + # Daily change + if day_inc != 0: + daily_change = current_id - users_last_day + print("Daily evolution: %s"%daily_change) + if daily_change < 0: + daily_change_string = format(daily_change, ",d") + " in the last day\n" - rows = cur.fetchall() + # Weekly change + if week_inc != 0: - for row in rows: + weekly_change = current_id - users_last_week + print("Weekly evolution: %s"%weekly_change) + if weekly_change < 0: + weekly_change_string = format(weekly_change, ",d") + " in the last week\n" - if row[1] == None: + ############################################################################### + # TOOT IT! + ############################################################################### - saludar_usuaris.append(row[0]) ## guarda els username dels usuaris en l'array saludar_usuaris + if len(welcome_users) > 0: - cur.close() + toot_text = "\n" + toot_text += mastodon_hostname + " welcomes:" + "\n\n" + toot_text += new_users_string + "\n" + toot_text += "\n" + toot_text += "\n\nWe have " + str(current_id) + " users\n\n" + toot_text += hourly_change_string + toot_text += daily_change_string + toot_text += weekly_change_string + toot_text += "\n" -except (Exception, psycopg2.DatabaseError) as error: + print("Tooting...") + print(toot_text) + if len(toot_text) < 500: + mastodon.status_post(toot_text, in_reply_to_id=None, ) + else: + toot_text1, toot_text2 = toot_text[:int(len(toot_text)/2)], toot_text[int(len(toot_text)/2):] + toot_id = mastodon.status_post(toot_text1, in_reply_to_id=None,) + mastodon.status_post(toot_text2, in_reply_to_id=toot_id,) + print(toot_text1) + print(toot_text2) - print(error) - -finally: - - if conn is not None: - - conn.close() - -i = 0 - -cadena_nous = "" - -while i < len(saludar_usuaris): - - cadena_nous = cadena_nous+"@"+saludar_usuaris[i] - - i += 1 - - if i < len(saludar_usuaris): - - cadena_nous = cadena_nous+", " - -######################################################################################################### - -inc_hora = current_id - usuaris_fa_una_hora -inc_dia = current_id - usuaris_fa_un_dia -inc_setmana = current_id - usuaris_fa_una_setmana - -print("------------------------") -print("increment usuaris hora: "+str(inc_hora)) -print("increment usuaris dia: "+str(inc_dia)) -print("increment usuaris setmana: "+str(inc_setmana)) -print("------------------------") -print(" spla @ 2018 - 2021 ") -print("------------------------") - -################################################################################### - -# Hourly change -if inc_hora != 0: - - users_hourly_change = current_id - usuaris_fa_una_hora - print("Evolució horaria usuaris: %s"%users_hourly_change) - if users_hourly_change > 0: - hourly_change_string = "+" + format(users_hourly_change, ",d") + " en la darrera hora\n" - - # Daily change - if inc_dia != 0: - - daily_change = current_id - usuaris_fa_un_dia - print("Evolució diaria: %s"%daily_change) - if daily_change > 0: - daily_change_string = "+" + format(daily_change, ",d") + " en el darrer dia\n" - - # Weekly change - if inc_setmana != 0: - - weekly_change = current_id - usuaris_fa_una_setmana - print("Evolució setmanal: %s"%weekly_change) - if weekly_change > 0: - weekly_change_string = "+" + format(weekly_change, ",d") + " en la darrera setmana\n" - - elif users_hourly_change < 0: - - hourly_change_string = format(users_hourly_change, ",d") + " en la darrera hora\n" - - # Daily change - if inc_dia != 0: - daily_change = current_id - usuaris_fa_un_dia - print("Evolució diaria: %s"%daily_change) - if daily_change < 0: - daily_change_string = format(daily_change, ",d") + " en el darrer dia\n" - - # Weekly change - if inc_setmana != 0: - - weekly_change = current_id - usuaris_fa_una_setmana - print("Evolució setmanal: %s"%weekly_change) - if weekly_change < 0: - weekly_change_string = format(weekly_change, ",d") + " en la darrera setmana\n" - -############################################################################### -# TOOT IT! -############################################################################### - -if len(saludar_usuaris) > 0: - - toot_text = "\n" - toot_text += "mastodont.cat dona la benvinguda a:" + "\n\n" - toot_text += cadena_nous + "\n" - toot_text += "\n" - toot_text += " :mastodon: :senyera: :brindis: " + "\n" - toot_text += "\n" - #toot_text += "Toots publicats: %s "% num_toots + "\n" - #toot_text += "Toots per usuari: %s "% toots_per_usuari + "\n" - #toot_text += "Activitat setmana actual" + "\n" - #toot_text += "Interaccions: %s "% interaccions + "\n" - ##toot_text += "Usuaris actius: %s "% actius + "\n" - #toot_text += "Instàncies connectades: %s "% num_instances + "\n" - toot_text += "\najuda: https://ajuda.mastodont.cat" - toot_text += "\n\nJa som " + str(current_id) + " usuaris\n\n" - toot_text += hourly_change_string - toot_text += daily_change_string - toot_text += weekly_change_string - toot_text += "\n\nMés #estadístiques gràfiques a: https://grafana.mastodont.cat/d/imEDcXfmk/mastodont-cat?orgId=1&kiosk" + "\n" - - print("Tootejant...") - print(toot_text) - if len(toot_text) < 500: - mastodon.status_post(toot_text, in_reply_to_id=None, ) - else: - toot_text1, toot_text2 = toot_text[:int(len(toot_text)/2)], toot_text[int(len(toot_text)/2):] - toot_id = mastodon.status_post(toot_text1, in_reply_to_id=None,) - mastodon.status_post(toot_text2, in_reply_to_id=toot_id,) - print(toot_text1) - print(toot_text2) - - print("Tootejat amb èxit!") + print("Done!")