wip

welcome.py

@@ -1,29 +1,61 @@
-from six.moves import urllib
+#from six.moves import urllib
 import datetime
-from subprocess import call
+#from subprocess import call
 from mastodon import Mastodon
-import time
+#import time
-import threading
+#import threading
-import csv
+#import csv
 import os
 import json
-import time
+#import time
-import signal
+#import signal
 import sys
 import os.path
 import requests
 import operator
-import redis
+#import redis
-import calendar
+#import calendar
 import psycopg2
 import pdb
 
-from decimal import *
+#from decimal import *
-getcontext().prec = 2
+#getcontext().prec = 2
 
-###############################################################################
+def mastodon():
-# INITIALISATION
+
-###############################################################################
+    # Load secrets from secrets file
+    secrets_filepath = "secrets/secrets.txt"
+    uc_client_id = get_parameter("uc_client_id",     secrets_filepath)
+    uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
+    uc_access_token = get_parameter("uc_access_token",  secrets_filepath)
+
+    # Load configuration from config file
+    config_filepath = "config/config.txt"
+    mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
+
+    # Initialise Mastodon API
+    mastodon = Mastodon(
+        client_id=uc_client_id,
+        client_secret=uc_client_secret,
+        access_token=uc_access_token,
+        api_base_url='https://' + mastodon_hostname,
+    )
+
+    # Initialise access headers
+    headers = {'Authorization': 'Bearer %s'%uc_access_token}
+
+    return (mastodon, mastodon_hostname)
+
+def db_config():
+
+    # Load db configuration from config file
+    config_filepath = "config/db_config.txt"
+    mastodon_db = get_parameter("mastodon_db", config_filepath)
+    mastodon_db_user = get_parameter("mastodon_db_user", config_filepath)
+    welcome_db = get_parameter("welcome_db", config_filepath)
+    welcome_db_user = get_parameter("welcome_db_user", config_filepath)
+
+    return (mastodon_db, mastodon_db_user, welcome_db, welcome_db_user)
 
 # Returns the parameter from the specified file
 def get_parameter( parameter, file_path ):
@@ -42,36 +74,24 @@ def get_parameter( parameter, file_path ):
     print(file_path + "  Missing parameter %s "%parameter)
     sys.exit(0)
 
-# Load secrets from secrets file
-secrets_filepath = "secrets/secrets.txt"
-uc_client_id     = get_parameter("uc_client_id",     secrets_filepath)
-uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
-uc_access_token  = get_parameter("uc_access_token",  secrets_filepath)
-
-# Load configuration from config file
-config_filepath = "config.txt"
-mastodon_hostname = get_parameter("mastodon_hostname", config_filepath) # E.g., mastodon.social
-
-# Initialise Mastodon API
-mastodon = Mastodon(
-    client_id = uc_client_id,
-    client_secret = uc_client_secret,
-    access_token = uc_access_token,
-    api_base_url = 'https://' + mastodon_hostname,
-)
-
-# Initialise access headers
-headers={ 'Authorization': 'Bearer %s'%uc_access_token }
-
-###############################################################################
-# current registered users
 ###############################################################################
+# main
 
-try:
+if __name__ == '__main__':
+
+    mastodon, mastodon_hostname = mastodon()
+
+    mastodon_db, mastodon_db_user, welcome_db, welcome_db_user = db_config()
+
+    ###############################################################################
+    # current registered users
+    ###############################################################################
+
+    try:
 
         conn = None
 
-    conn = psycopg2.connect(database = "mastodon_production", user = "mastodon", password = "", host = "/var/run/postgresql", port = "5432")
+        conn = psycopg2.connect(database = mastodon_db, user = mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432")
 
         cur = conn.cursor()
 
@@ -82,255 +102,182 @@ try:
         if row != None:
 
             current_id = row[0]
+
         else:
 
             current_id = 0
 
         cur.close()
 
-except (Exception, psycopg2.DatabaseError) as error:
+    except (Exception, psycopg2.DatabaseError) as error:
 
         print (error)
 
-finally:
+    finally:
 
         if conn is not None:
 
             conn.close()
 
-print("Número d'usuaris: %s "% current_id)
+    print("Current users: %s "% current_id)
 
-###############################################################################
+    ###############################################################################
 
-ara = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
 
-###############################################################################
+    users_before = current_id
-# Redis 25/10/18 connectar amb Redis per a obtenir l'activitat setmanal
+    users_hour = 0
-###############################################################################
-
-redis_host = "localhost"
-redis_port = 6379
-redis_password = ""
-
-setmana_actual = str(datetime.datetime.now().isocalendar()[1])
-
-r = redis.StrictRedis(host=redis_host, port=redis_port, password=redis_password, decode_responses=True)
-
-# Interaccions setmana actual
-
-interaccions = r.get("activity:interactions:"+setmana_actual)
-if interaccions == None:
-   interaccions = 0
-print("Interaccions d'aquesta setmana: %s "% interaccions)
-
-actius = r.pfcount("activity:logins:"+setmana_actual)
-print("Usuaris actius aquesta setmana: %s "% actius)
-
-###############################################################################
-# Connectar amb la bbdd Postgres per obtenir la darrera fila i els seus usuaris
-###############################################################################
-
-usuaris_abans = current_id
-toots_abans = num_toots
-instancies_abans = num_instances
-usuarishora = 0
-tootshora = 0
-instancieshora = 0
-toots_inici_setmana = num_toots
-inc_disc_space_hour = 0
-toots_actius = 0.00
-
-conn = None
-
-try:
-
-  conn = psycopg2.connect(database = "pggrafana", user = "mastodon", password = "", host = "/var/run/postgresql", port = "5432")
-
-  cur = conn.cursor()
-
-  cur.execute("SELECT DISTINCT ON (datetime) usuaris,toots,instancies,datetime,used_disk_space FROM grafana WHERE datetime > current_timestamp - INTERVAL '70 minutes' ORDER BY datetime asc LIMIT 1")
-
-  row = cur.fetchone()
-  usuaris_abans = row[0]
-  toots_abans = row[1]
-  instancies_abans = row[2]
-  disc_space_before = row[4]
-
-  # quants toots en l'inici de la setmana
-  cur.execute("SELECT DISTINCT ON (datetime) toots, datetime FROM grafana WHERE datetime > date_trunc('week', now()::timestamp) ORDER by datetime asc LIMIT 1")
-  row = cur.fetchone()
-  if row == None:
-    toots_inici_setmana = num_toots
-  else:
-    toots_inici_setmana = row[0]
-
-  cur.close()
-
-  usuarishora = current_id - usuaris_abans
-  tootshora = num_toots - toots_abans
-  instancieshora = num_instances - instancies_abans
-  inc_disc_space_hour = db_disk_space - disc_space_before
-
-except (Exception, psycopg2.DatabaseError) as error:
-  print (error)
-finally:
-  if conn is not None:
-      conn.close()
-
-if toots_inici_setmana == num_toots:
-  toots_actius = 0.00
-elif actius == 0:
-  toots_actius = 0.00
-else:
-  toots_actius = round((num_toots-toots_inici_setmana)/actius, 2)
-
-print("-----------------")
-print("current_id: "+str(current_id))
-print("usuaris abans: "+str(usuaris_abans))
-print("usuaris hora: "+str(usuarishora))
-print("-----------------")
-print("toots: "+str(num_toots))
-print("toots abans: "+str(toots_abans))
-print("toots x hora: "+str(tootshora))
-print("-----------------")
-print("instancies: "+str(num_instances))
-print("instancies abans: "+str(instancies_abans))
-print("instancies x hora: "+str(instancieshora))
-print("-----------------")
-print("toots aquesta setmana:"+str(num_toots-toots_inici_setmana))
-print("usuaris actius:"+str(actius))
-print("toots x actius: "+str(toots_actius))
-print("------------------------")
-print("   spla @ 2018 - 2020   ")
-print("------------------------")
-
-######################################################################################################################################
-# Connectar amb la bbdd Postgres pggrafana per guardar les variables per grafana
-
-inserta_linia = """INSERT INTO grafana(datetime, usuaris, usuarishora, toots, tootshora, tootsusuari, interaccions, actius, actius30, instancies, instancieshora, tootsactius,
-                   used_disk_space, disc_space_hour)
-             VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) RETURNING datetime;"""
-conn = None
-
-ara = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-
-try:
-
-    conn = psycopg2.connect(database = "pggrafana", user = "mastodon", password = "", host = "/var/run/postgresql", port = "5432")
-
-    cur = conn.cursor()
-    # executem INSERT
-    cur.execute(inserta_linia, (ara, current_id, usuarishora, num_toots, tootshora, toots_per_usuari, interaccions, actius, actius30, num_instances, instancieshora, toots_actius, db_disk_space, inc_disc_space_hour))
-    # obté l'id
-    datetime = cur.fetchone()[0]
-    # salvar els canvis en la base de dades
-    conn.commit()
-    # i tancar la connexió amb la base de dades
-    cur.close()
-
-except (Exception, psycopg2.DatabaseError) as error:
-
-    print(error)
-
-finally:
-
-    if conn is not None:
-
-        conn.close()
-
-###############################################################################
-# WORK OUT THE TOOT TEXT
-###############################################################################
-
-# Calculate difference in times
-
-hourly_change_string = ""
-daily_change_string  = ""
-weekly_change_string = ""
-
-# canvi horari usuaris 19/05/18
-users_hourly_change_string = ""
-
-#######################################################################################################################
-# Connectar amb la bbdd Postgres per obtenir increment usuaris en la darrera hora, el darrer dia i en la darrera setmana
-########################################################################################################################
-
-try:
 
     conn = None
 
-    conn = psycopg2.connect(database = "pggrafana", user = "mastodon", password = "", host = "/var/run/postgresql", port = "5432")
+    try:
+
+      conn = psycopg2.connect(database = welcome_db, user = welcome_db_user, password = "", host = "/var/run/postgresql", port = "5432")
 
       cur = conn.cursor()
 
-    cur.execute("SELECT DISTINCT ON (datetime) usuaris,datetime FROM grafana WHERE datetime > current_timestamp - INTERVAL '62 minutes' ORDER BY datetime asc LIMIT 1")
+      cur.execute("SELECT DISTINCT ON (datetime) users FROM welcome WHERE datetime > current_timestamp - INTERVAL '70 minutes' ORDER BY datetime asc LIMIT 1")
 
       row = cur.fetchone()
 
-    if row[0] == None:
+      users_before = row[0]
-
-        usuaris_fa_una_hora = 0
-
-    else:
-
-        usuaris_fa_una_hora = row[0]
-
-    cur.execute("SELECT DISTINCT ON (datetime) usuaris,datetime FROM grafana WHERE datetime > current_timestamp - INTERVAL '1 day' ORDER BY datetime asc LIMIT 1")
-
-    row = cur.fetchone()
-
-    if row[0] == None:
-
-        usuaris_fa_un_dia = 0
-
-    else:
-
-        usuaris_fa_un_dia = row[0]
-
-    cur.execute("SELECT DISTINCT ON (datetime) usuaris,datetime FROM grafana WHERE datetime > current_timestamp - INTERVAL '7 days' ORDER BY datetime asc LIMIT 1")
-
-    row = cur.fetchone()
-
-    if row[0] == None:
-
-        usuaris_fa_una_setmana = 0
-    else:
-
-        usuaris_fa_una_setmana = row[0]
 
       cur.close()
 
-except (Exception, psycopg2.DatabaseError) as error:
+      users_hour = current_id - users_before
+
+    except (Exception, psycopg2.DatabaseError) as error:
 
         print (error)
 
-finally:
+    finally:
 
         if conn is not None:
 
             conn.close()
 
-#################################################################################
+    print("-----------------")
+    print("current users: "+str(current_id))
+    print("users before: "+str(users_before))
+    print("users / hour: "+str(users_hour))
 
-saludar_usuaris = []
+    insert_sql = """INSERT INTO welcome(datetime, users, users_hour)
+                 VALUES(%s,%s,%s) RETURNING datetime;"""
+    conn = None
 
-if time.localtime().tm_isdst == 0:
+    now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+    try:
+
+        conn = psycopg2.connect(database = welcome_db, user = welcome_db_user, password = "", host = "/var/run/postgresql", port = "5432")
+
+        cur = conn.cursor()
+
+        cur.execute(insert_sql, (now, current_id, users_hour))
+
+        datetime = cur.fetchone()[0]
+
+        conn.commit()
+
+        cur.close()
+
+    except (Exception, psycopg2.DatabaseError) as error:
+
+        print(error)
+
+    finally:
+
+        if conn is not None:
+
+            conn.close()
+
+    ###############################################################################
+    # WORK OUT THE TOOT TEXT
+    ###############################################################################
+
+    # Calculate difference in times
+
+    hourly_change_string = ""
+    daily_change_string  = ""
+    weekly_change_string = ""
+    users_hourly_change_string = ""
+
+    try:
+
+        conn = None
+
+        conn = psycopg2.connect(database = welcome_db, user = welcome_db_user, password = "", host = "/var/run/postgresql", port = "5432")
+
+        cur = conn.cursor()
+
+        cur.execute("SELECT DISTINCT ON (datetime) users, datetime FROM welcome WHERE datetime > current_timestamp - INTERVAL '62 minutes' ORDER BY datetime asc LIMIT 1")
+
+        row = cur.fetchone()
+
+        if row[0] == None:
+
+            users_last_hour = 0
+
+        else:
+
+            users_last_hour = row[0]
+
+        cur.execute("SELECT DISTINCT ON (datetime) users,datetime FROM welcome WHERE datetime > current_timestamp - INTERVAL '1 day' ORDER BY datetime asc LIMIT 1")
+
+        row = cur.fetchone()
+
+        if row[0] == None:
+
+            users_last_day = 0
+
+        else:
+
+            users_last_day = row[0]
+
+        cur.execute("SELECT DISTINCT ON (datetime) users,datetime FROM welcome WHERE datetime > current_timestamp - INTERVAL '7 days' ORDER BY datetime asc LIMIT 1")
+
+        row = cur.fetchone()
+
+        if row[0] == None:
+
+            users_last_week = 0
+        else:
+
+            users_last_week = row[0]
+
+        cur.close()
+
+    except (Exception, psycopg2.DatabaseError) as error:
+
+        print (error)
+
+    finally:
+
+        if conn is not None:
+
+            conn.close()
+
+    #################################################################################
+
+    welcome_users = []
+
+    if time.localtime().tm_isdst == 0:
 
         interval_time = '60 minutes'
 
-elif time.localtime().tm_isdst == 1:
+    elif time.localtime().tm_isdst == 1:
 
         interval_time = '120 minutes'
 
-conn = None
+    conn = None
 
-try:
+    try:
 
-    conn = psycopg2.connect(database = "mastodon_production", user = "mastodon", password = "", host = "/var/run/postgresql", port = "5432")
+        conn = psycopg2.connect(database = mastodon_db, user = mastodon_db_user, password = "", host = "/var/run/postgresql", port = "5432")
 
         cur = conn.cursor()
+
         cur.execute("select username, domain from accounts where (created_at + interval '" + interval_time + "' > (now() - interval '1 hour')) and domain is null and id in (select account_id from users where approved and not disabled)")
-    #cur.execute("select username, domain from accounts where id in (select account_id from follows where (created_at + interval '" + interval_time + "' > (now() - interval '60 minutes')) and target_account_id = '1')")
 
         rows = cur.fetchall()
 
@@ -338,119 +285,107 @@ try:
 
             if row[1] == None:
 
-            saludar_usuaris.append(row[0]) ## guarda els username dels usuaris en l'array saludar_usuaris
+                welcome_users.append(row[0])
 
         cur.close()
 
-except (Exception, psycopg2.DatabaseError) as error:
+    except (Exception, psycopg2.DatabaseError) as error:
 
         print(error)
 
-finally:
+    finally:
 
         if conn is not None:
 
             conn.close()
 
-i = 0
+    i = 0
 
-cadena_nous = ""
+    new_users_string = ""
 
-while i < len(saludar_usuaris):
+    while i < len(welcome_users):
 
-    cadena_nous = cadena_nous+"@"+saludar_usuaris[i]
+        new_users_string = new_users_string+"@"+welcome_users[i]
 
         i += 1
 
-    if i < len(saludar_usuaris):
+        if i < len(welcome_users):
 
-        cadena_nous = cadena_nous+", "
+            new_users_string = new_users_string+", "
 
-#########################################################################################################
+    #########################################################################################################
 
-inc_hora = current_id - usuaris_fa_una_hora
+    hour_inc = current_id - users_last_hour
-inc_dia = current_id - usuaris_fa_un_dia
+    day_inc = current_id - users_last_day
-inc_setmana = current_id - usuaris_fa_una_setmana
+    week_inc = current_id - users_last_week
 
-print("------------------------")
+    print("------------------------")
-print("increment usuaris hora: "+str(inc_hora))
+    print("new users last hour: "+str(hour_inc))
-print("increment usuaris dia: "+str(inc_dia))
+    print("new users last day: "+str(day_inc))
-print("increment usuaris setmana: "+str(inc_setmana))
+    print("new users last week: "+str(week_inc))
-print("------------------------")
-print("   spla @ 2018 - 2021   ")
-print("------------------------")
 
-###################################################################################
+    ###################################################################################
 
-# Hourly change
+    # Hourly change
-if inc_hora != 0:
+    if hour_inc != 0:
 
-  users_hourly_change = current_id - usuaris_fa_una_hora
+      users_hourly_change = current_id - users_last_hour
-  print("Evolució horaria usuaris: %s"%users_hourly_change)
+      print("Hourly users evolution: %s"%users_hourly_change)
       if users_hourly_change > 0:
         hourly_change_string = "+" + format(users_hourly_change, ",d") + " en la darrera hora\n"
 
         # Daily change
-    if inc_dia != 0:
+        if day_inc != 0:
 
-      daily_change = current_id - usuaris_fa_un_dia
+          daily_change = current_id - users_last_day
-      print("Evolució diaria: %s"%daily_change)
+          print("Daily users evolution: %s"%daily_change)
           if daily_change > 0:
-        daily_change_string = "+" + format(daily_change, ",d") + " en el darrer dia\n"
+            daily_change_string = "+" + format(daily_change, ",d") + " in the last day\n"
 
         # Weekly change
-    if inc_setmana != 0:
+        if week_inc != 0:
 
-      weekly_change = current_id - usuaris_fa_una_setmana
+          weekly_change = current_id - users_last_week
-      print("Evolució setmanal: %s"%weekly_change)
+          print("Weekly users evolution: %s"%weekly_change)
           if weekly_change > 0:
-        weekly_change_string = "+" + format(weekly_change, ",d") + " en la darrera setmana\n"
+            weekly_change_string = "+" + format(weekly_change, ",d") + " in the last week\n"
 
       elif users_hourly_change < 0:
 
-    hourly_change_string = format(users_hourly_change, ",d") + " en la darrera hora\n"
+        hourly_change_string = format(users_hourly_change, ",d") + " in the last hour\n"
 
         # Daily change
-    if inc_dia != 0:
+        if day_inc != 0:
-      daily_change = current_id - usuaris_fa_un_dia
+          daily_change = current_id - users_last_day
-      print("Evolució diaria: %s"%daily_change)
+          print("Daily evolution: %s"%daily_change)
           if daily_change < 0:
-        daily_change_string = format(daily_change, ",d") + " en el darrer dia\n"
+            daily_change_string = format(daily_change, ",d") + " in the last day\n"
 
         # Weekly change
-    if inc_setmana != 0:
+        if week_inc != 0:
 
-      weekly_change = current_id - usuaris_fa_una_setmana
+          weekly_change = current_id - users_last_week
-      print("Evolució setmanal: %s"%weekly_change)
+          print("Weekly evolution: %s"%weekly_change)
           if weekly_change < 0:
-        weekly_change_string = format(weekly_change, ",d") + " en la darrera setmana\n"
+            weekly_change_string = format(weekly_change, ",d") + " in the last week\n"
 
-###############################################################################
+    ###############################################################################
-# TOOT IT!
+    # TOOT IT!
-###############################################################################
+    ###############################################################################
 
-if len(saludar_usuaris) > 0:
+    if len(welcome_users) > 0:
 
         toot_text = "\n"
-    toot_text += "mastodont.cat dona la benvinguda a:" + "\n\n"
+        toot_text += mastodon_hostname + " welcomes:" + "\n\n"
-    toot_text += cadena_nous + "\n"
+        toot_text += new_users_string + "\n"
         toot_text += "\n"
-    toot_text += " :mastodon: :senyera: :brindis: " + "\n"
+        toot_text += "\n\nWe have " + str(current_id) + " users\n\n"
-    toot_text += "\n"
-    #toot_text += "Toots publicats: %s "% num_toots + "\n"
-    #toot_text += "Toots per usuari: %s "% toots_per_usuari + "\n"
-    #toot_text += "Activitat setmana actual" + "\n"
-    #toot_text += "Interaccions: %s "% interaccions + "\n"
-    ##toot_text += "Usuaris actius: %s "% actius + "\n"
-    #toot_text += "Instàncies connectades: %s "% num_instances + "\n"
-    toot_text += "\najuda: https://ajuda.mastodont.cat"
-    toot_text += "\n\nJa som " + str(current_id) + " usuaris\n\n"
         toot_text += hourly_change_string
         toot_text += daily_change_string
         toot_text += weekly_change_string
-    toot_text += "\n\nMés #estadístiques gràfiques a: https://grafana.mastodont.cat/d/imEDcXfmk/mastodont-cat?orgId=1&kiosk" + "\n"
+        toot_text += "\n"
 
-    print("Tootejant...")
+        print("Tooting...")
         print(toot_text)
         if len(toot_text) < 500:
           mastodon.status_post(toot_text, in_reply_to_id=None, )
@@ -461,4 +396,4 @@ if len(saludar_usuaris) > 0:
           print(toot_text1)
           print(toot_text2)
 
-    print("Tootejat amb èxit!")
+        print("Done!")