Fix #1 and added stats

corpus.py

@@ -1,6 +1,7 @@
 from mastodonbot import Mastodonbot
 from mastodon import MastodonNetworkError 
 from database import Database
+from datetime import datetime, time
 import sys
 import re
 import pdb
@@ -148,7 +149,6 @@ if __name__ == '__main__':
 
                             bot.mastodon.status_post(f'@{mention.acct}, error al esborrar tots els tuts.', in_reply_to_id=mention.status_id, visibility='direct')
 
-
                 print(f"Dismissing notification id {mention.id}")
 
                 bot.mastodon.notifications_dismiss(mention.id)
@@ -159,14 +159,62 @@ if __name__ == '__main__':
 
                 bot.mastodon.notifications_dismiss(mention.id)
 
-        elif notif.type == 'status' and notif.status.visibility == 'public' and notif.status.language == 'ca':
+        elif notif.type == 'status':
 
-            found_it, pending = db.check_user(notif.account.acct)
+            if 'visibility' in notif.status:
 
-            if found_it and not pending and notif.status.in_reply_to_account_id == None:
-                
-                is_saved = db.save_post(notif.account.acct, unescape(cleanhtml(notif.status.content)))
+                if notif.status.visibility == 'public' and notif.status.language == 'ca':
 
-            print(f"Dismissing notification id {notif.id}")
+                    found_it, pending = db.check_user(notif.account.acct)
+
+                    if found_it and not pending and notif.status.in_reply_to_account_id == None:
+
+                        is_saved = db.save_post(notif.account.acct, unescape(cleanhtml(notif.status.content)))
+
+                    print(f"Dismissing notification id {notif.id}")
+
+                    bot.mastodon.notifications_dismiss(notif.id)
+
+                else:
+
+                    bot.mastodon.notifications_dismiss(notif.id)
+        else:
 
             bot.mastodon.notifications_dismiss(notif.id)
+
+    now = datetime.now()
+
+    print(now.hour, now.minute)
+
+    if now.hour == 18 or now.hour == 0 or now.hour == 6 or now.hour == 12:
+
+        if now.minute == 0:
+
+            post = 'Vols ajudar fàcilment a desenvolupar i millorar les tecnologies de llengua per el #català?\n'
+
+            post += "Només cal demanar-me l'alta i començaré a enregistrar tots els teus tuts públics. "
+
+            post += "Els teus tuts públics podran alimentar projectes lingüistics com per exemple el de Common Voice de #Mozilla.\n"
+
+            post += "Com donar-te d'alta?\n\n@corpus alta"
+
+            print(post)
+
+            bot.mastodon.status_post(post, in_reply_to_id=None, visibility='public')
+
+    elif now.hour == 20:
+
+        if now.minute == 0:
+
+            total_users = db.total_users()
+
+            total_posts = db.total_posts()
+
+            post = f"\nUsuaris donats d'alta: {total_users}\n"
+
+            post += f'Missatges públics cedits: {total_posts}\n'
+
+            print(post)
+
+            bot.mastodon.status_post(post, in_reply_to_id=None, visibility='public')
+

database.py

@@ -249,6 +249,221 @@ class Database():
 
         return are_deleted
 
+    def users(self):
+
+        user_list = []
+
+        sql = "select distinct username from corpus order by 1 asc"
+
+        conn = None
+
+        try:
+
+            conn = psycopg2.connect(database = self.corpus_db, user = self.corpus_db_user, password = "", host = "/var/run/postgresql", port = "5432")
+
+            cur = conn.cursor()
+
+            cur.execute(sql)
+
+            rows = cur.fetchall()
+
+            for row in rows:
+
+                user_list.append(row)
+
+            cur.close()
+
+        except (Exception, psycopg2.DatabaseError) as error:
+
+            print(error)
+
+        finally:
+
+            if conn is not None:
+
+                conn.close()
+
+        return user_list
+
+    def user_posts(self, username):
+
+        posts = []
+
+        sql = "select text from corpus where username = (%s)"
+
+        conn = None
+
+        try:
+
+            conn = psycopg2.connect(database = self.corpus_db, user = self.corpus_db_user, password = "", host = "/var/run/postgresql", port = "5432")
+
+            cur = conn.cursor()
+
+            cur.execute(sql, (username,))
+
+            rows = cur.fetchall()
+
+            for row in rows:
+
+                posts.append(row)
+
+            cur.close()
+
+        except (Exception, psycopg2.DatabaseError) as error:
+
+            print(error)
+
+        finally:
+
+            if conn is not None:
+
+                conn.close()
+
+        return posts
+
+    def total_users(self):
+
+        total_posts = 0
+
+        sql = "select count(username) from users"
+
+        conn = None
+
+        try:
+
+            conn = psycopg2.connect(database = self.corpus_db, user = self.corpus_db_user, password = "", host = "/var/run/postgresql", port = "5432")
+
+            cur = conn.cursor()
+
+            cur.execute(sql)
+
+            row = cur.fetchone()
+
+            if row != None:
+
+                total_users = row[0]
+
+            cur.close()
+
+        except (Exception, psycopg2.DatabaseError) as error:
+
+            print(error)
+
+        finally:
+
+            if conn is not None:
+
+                conn.close()
+
+        return total_users
+
+    def total_posts(self):
+
+        total_posts = 0
+
+        sql = "select count(text) from corpus"
+
+        conn = None
+
+        try:
+
+            conn = psycopg2.connect(database = self.corpus_db, user = self.corpus_db_user, password = "", host = "/var/run/postgresql", port = "5432")
+
+            cur = conn.cursor()
+
+            cur.execute(sql)
+
+            row = cur.fetchone()
+
+            if row != None:
+
+                total_posts = row[0]
+
+            cur.close()
+
+        except (Exception, psycopg2.DatabaseError) as error:
+
+            print(error)
+
+        finally:
+
+            if conn is not None:
+
+                conn.close()
+
+        return total_posts
+
+    def export(self, username, post, lt_errors, language):
+
+        is_saved = False
+
+        sql = "INSERT INTO exported(id, username, text, lt_errors, language, exported_at) VALUES(%s, %s, %s, %s, %s, %s)"
+
+        unique_id = str(uuid.uuid4())
+
+        now = datetime.now()
+
+        conn = None
+
+        try:
+
+            conn = psycopg2.connect(database = self.corpus_db, user = self.corpus_db_user, password = "", host = "/var/run/postgresql", port = "5432")
+
+            cur = conn.cursor()
+
+            cur.execute(sql, (unique_id, username, post, lt_errors, language, now))
+
+            conn.commit()
+
+            cur.close()
+
+            is_saved = True
+
+        except (Exception, psycopg2.DatabaseError) as error:
+
+            print(error)
+
+        finally:
+
+            if conn is not None:
+
+                conn.close()
+
+        return is_saved
+
+    def csv_save(self, filename):
+
+        is_saved = False
+
+        sql = "copy (select * from exported) to stdout with csv delimiter ';'"
+
+        conn = None
+
+        try:
+
+            conn = psycopg2.connect(database = self.corpus_db, user = self.corpus_db_user, password = "", host = "/var/run/postgresql", port = "5432")
+
+            cur = conn.cursor()
+
+            with open(filename, "w") as file:
+                  cur.copy_expert(sql, file)
+
+            is_saved = True
+
+            cur.close()
+
+        except (Exception, psycopg2.DatabaseError) as error:
+
+            print(error)
+
+        finally:
+
+            if conn is not None:
+
+                conn.close()
+
+        return is_saved
+
     @staticmethod
     def __check_dbsetup(self):
 
@@ -306,13 +521,17 @@ class Database():
     def __dbtables_schemes(self):
 
         table = "corpus"
-        sql = "create table "+table+" (id uuid, username varchar(50), text varchar(500), created_at timestamptz, PRIMARY KEY (id))"
+        sql = "create table "+table+" (id uuid, username varchar(50), text varchar(500), created_at timestamptz, exported boolean default False, PRIMARY KEY (id))"
         self.__create_table(self, table, sql)
 
         table = "users"
         sql = "create table "+table+" (id uuid, username varchar(50), pending boolean default True, post_id bigint, PRIMARY KEY (id))"
         self.__create_table(self, table, sql)
 
+        table = "exported"
+        sql = "create table "+table+" (id uuid, username varchar(50), text varchar(500), lt_errors int, language varchar(2), PRIMARY KEY (id))"
+        self.__create_table(self, table, sql)
+
     @staticmethod
     def __create_table(self, table, sql):
 

mastodonbot.py

@@ -266,7 +266,14 @@ class Mastodonbot:
 
         visibility = notif.status.visibility
 
-        reply, question = self.__get_question(self, text)
+        if " " in text:
+            
+            reply, question = self.__get_question(self, text)
+        else:
+
+            reply = False
+
+            question = ''
 
         mention_dict = {'reply': reply, 'question': question, 'id': id, 'account_id': account_id, 'acct': acct, 'status_id': status_id, 'text': text, 'visibility': visibility}
 
@@ -318,6 +325,10 @@ class Mastodonbot:
 
         except:
 
+            reply = False
+
+            question = ''
+
             pass
 
         return (reply, question)

requeriments.txt

@@ -2,3 +2,5 @@ Mastodon.py
 unidecode
 psycopg2-binary
 pytz
+language_tool_python
+langdetect