diff --git a/corpus.py b/corpus.py index 037a04e..0d1a5bb 100644 --- a/corpus.py +++ b/corpus.py @@ -1,6 +1,7 @@ from mastodonbot import Mastodonbot from mastodon import MastodonNetworkError from database import Database +from datetime import datetime, time import sys import re import pdb @@ -148,7 +149,6 @@ if __name__ == '__main__': bot.mastodon.status_post(f'@{mention.acct}, error al esborrar tots els tuts.', in_reply_to_id=mention.status_id, visibility='direct') - print(f"Dismissing notification id {mention.id}") bot.mastodon.notifications_dismiss(mention.id) @@ -159,14 +159,62 @@ if __name__ == '__main__': bot.mastodon.notifications_dismiss(mention.id) - elif notif.type == 'status' and notif.status.visibility == 'public' and notif.status.language == 'ca': + elif notif.type == 'status': - found_it, pending = db.check_user(notif.account.acct) + if 'visibility' in notif.status: - if found_it and not pending and notif.status.in_reply_to_account_id == None: - - is_saved = db.save_post(notif.account.acct, unescape(cleanhtml(notif.status.content))) + if notif.status.visibility == 'public' and notif.status.language == 'ca': - print(f"Dismissing notification id {notif.id}") + found_it, pending = db.check_user(notif.account.acct) + + if found_it and not pending and notif.status.in_reply_to_account_id == None: + + is_saved = db.save_post(notif.account.acct, unescape(cleanhtml(notif.status.content))) + + print(f"Dismissing notification id {notif.id}") + + bot.mastodon.notifications_dismiss(notif.id) + + else: + + bot.mastodon.notifications_dismiss(notif.id) + else: bot.mastodon.notifications_dismiss(notif.id) + + now = datetime.now() + + print(now.hour, now.minute) + + if now.hour == 18 or now.hour == 0 or now.hour == 6 or now.hour == 12: + + if now.minute == 0: + + post = 'Vols ajudar fàcilment a desenvolupar i millorar les tecnologies de llengua per el #català?\n' + + post += "Només cal demanar-me l'alta i començaré a enregistrar tots els teus tuts públics. " + + post += "Els teus tuts públics podran alimentar projectes lingüistics com per exemple el de Common Voice de #Mozilla.\n" + + post += "Com donar-te d'alta?\n\n@corpus alta" + + print(post) + + bot.mastodon.status_post(post, in_reply_to_id=None, visibility='public') + + elif now.hour == 20: + + if now.minute == 0: + + total_users = db.total_users() + + total_posts = db.total_posts() + + post = f"\nUsuaris donats d'alta: {total_users}\n" + + post += f'Missatges públics cedits: {total_posts}\n' + + print(post) + + bot.mastodon.status_post(post, in_reply_to_id=None, visibility='public') + diff --git a/database.py b/database.py index 63aeb58..0a94ddd 100644 --- a/database.py +++ b/database.py @@ -249,6 +249,221 @@ class Database(): return are_deleted + def users(self): + + user_list = [] + + sql = "select distinct username from corpus order by 1 asc" + + conn = None + + try: + + conn = psycopg2.connect(database = self.corpus_db, user = self.corpus_db_user, password = "", host = "/var/run/postgresql", port = "5432") + + cur = conn.cursor() + + cur.execute(sql) + + rows = cur.fetchall() + + for row in rows: + + user_list.append(row) + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + return user_list + + def user_posts(self, username): + + posts = [] + + sql = "select text from corpus where username = (%s)" + + conn = None + + try: + + conn = psycopg2.connect(database = self.corpus_db, user = self.corpus_db_user, password = "", host = "/var/run/postgresql", port = "5432") + + cur = conn.cursor() + + cur.execute(sql, (username,)) + + rows = cur.fetchall() + + for row in rows: + + posts.append(row) + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + return posts + + def total_users(self): + + total_posts = 0 + + sql = "select count(username) from users" + + conn = None + + try: + + conn = psycopg2.connect(database = self.corpus_db, user = self.corpus_db_user, password = "", host = "/var/run/postgresql", port = "5432") + + cur = conn.cursor() + + cur.execute(sql) + + row = cur.fetchone() + + if row != None: + + total_users = row[0] + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + return total_users + + def total_posts(self): + + total_posts = 0 + + sql = "select count(text) from corpus" + + conn = None + + try: + + conn = psycopg2.connect(database = self.corpus_db, user = self.corpus_db_user, password = "", host = "/var/run/postgresql", port = "5432") + + cur = conn.cursor() + + cur.execute(sql) + + row = cur.fetchone() + + if row != None: + + total_posts = row[0] + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + return total_posts + + def export(self, username, post, lt_errors, language): + + is_saved = False + + sql = "INSERT INTO exported(id, username, text, lt_errors, language, exported_at) VALUES(%s, %s, %s, %s, %s, %s)" + + unique_id = str(uuid.uuid4()) + + now = datetime.now() + + conn = None + + try: + + conn = psycopg2.connect(database = self.corpus_db, user = self.corpus_db_user, password = "", host = "/var/run/postgresql", port = "5432") + + cur = conn.cursor() + + cur.execute(sql, (unique_id, username, post, lt_errors, language, now)) + + conn.commit() + + cur.close() + + is_saved = True + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + return is_saved + + def csv_save(self, filename): + + is_saved = False + + sql = "copy (select * from exported) to stdout with csv delimiter ';'" + + conn = None + + try: + + conn = psycopg2.connect(database = self.corpus_db, user = self.corpus_db_user, password = "", host = "/var/run/postgresql", port = "5432") + + cur = conn.cursor() + + with open(filename, "w") as file: + cur.copy_expert(sql, file) + + is_saved = True + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + return is_saved + @staticmethod def __check_dbsetup(self): @@ -306,13 +521,17 @@ class Database(): def __dbtables_schemes(self): table = "corpus" - sql = "create table "+table+" (id uuid, username varchar(50), text varchar(500), created_at timestamptz, PRIMARY KEY (id))" + sql = "create table "+table+" (id uuid, username varchar(50), text varchar(500), created_at timestamptz, exported boolean default False, PRIMARY KEY (id))" self.__create_table(self, table, sql) table = "users" sql = "create table "+table+" (id uuid, username varchar(50), pending boolean default True, post_id bigint, PRIMARY KEY (id))" self.__create_table(self, table, sql) + table = "exported" + sql = "create table "+table+" (id uuid, username varchar(50), text varchar(500), lt_errors int, language varchar(2), PRIMARY KEY (id))" + self.__create_table(self, table, sql) + @staticmethod def __create_table(self, table, sql): diff --git a/mastodonbot.py b/mastodonbot.py index d94a08d..161e9fd 100644 --- a/mastodonbot.py +++ b/mastodonbot.py @@ -266,7 +266,14 @@ class Mastodonbot: visibility = notif.status.visibility - reply, question = self.__get_question(self, text) + if " " in text: + + reply, question = self.__get_question(self, text) + else: + + reply = False + + question = '' mention_dict = {'reply': reply, 'question': question, 'id': id, 'account_id': account_id, 'acct': acct, 'status_id': status_id, 'text': text, 'visibility': visibility} @@ -318,6 +325,10 @@ class Mastodonbot: except: + reply = False + + question = '' + pass return (reply, question) diff --git a/requeriments.txt b/requeriments.txt index 07a4851..7fc5ea9 100644 --- a/requeriments.txt +++ b/requeriments.txt @@ -2,3 +2,5 @@ Mastodon.py unidecode psycopg2-binary pytz +language_tool_python +langdetect