From 773860b92f1b7d2da76718724b3dfdacf72d82d6 Mon Sep 17 00:00:00 2001 From: spla Date: Mon, 21 Mar 2022 14:44:23 +0100 Subject: [PATCH] RAY parallel replies! --- README.md | 2 + requirements.txt | 1 + viquicat.py | 168 +++++++++++++++++++++++++---------------------- 3 files changed, 91 insertions(+), 80 deletions(-) diff --git a/README.md b/README.md index d6d41c6..a1c0e43 100644 --- a/README.md +++ b/README.md @@ -23,3 +23,5 @@ Query the bot is this easy: 6. Use your favourite scheduling method to set `python viquicat.py` to run every minute. +21.3.2022 **New feature** Ray Parallel processing! The code replies all queries at the same time! + diff --git a/requirements.txt b/requirements.txt index 80aa9f8..5d43345 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ wheel>=0.36.2a wikipedia>=1.4.0 Mastodon.py>=1.5.1 unidecode>=1.1.1 +ray>=1.11.0 diff --git a/viquicat.py b/viquicat.py index 598bc39..6e45bd0 100644 --- a/viquicat.py +++ b/viquicat.py @@ -1,16 +1,16 @@ -from six.moves import urllib -from datetime import datetime, timedelta from mastodon import Mastodon import unidecode -import time import re import os import time import sys import os.path import wikipedia +import ray import pdb +ray.init(num_cpus = 8) # Specify this system CPUs. + def cleanhtml(raw_html): cleanr = re.compile('<.*?>') cleantext = re.sub(cleanr, '', raw_html) @@ -20,6 +20,87 @@ def unescape(s): s = s.replace("'", "'") return s +class Notifications: + + name = 'Notifications' + + def __init__(self, type=None, id=None, account_id=None, username=None, status_id=None, text='', visibility=None): + + self.type = type + self.id = id + self.account_id = account_id + self.username = username + self.status_id = status_id + self.text = text + self.visibility = visibility + + @ray.remote + def get_data(self): + + notification_id = self.id + + if self.type != 'mention': + + print(f'dismissing notification {notification_id}') + + mastodon.notifications_dismiss(notification_id) + + return + + account_id = self.account.id + + username = self.account.acct + + status_id = self.status.id + + text = self.status.content + + visibility = self.status.visibility + + reply, question = get_question(text) + + if reply: + + try: + + wiki_result = wikipedia.page(question) + + except wikipedia.exceptions.DisambiguationError as error: + + wiki_result = wikipedia.search(question, results=10) + + ambiguous_reply = "\n- ".join(str(x) for x in wiki_result) + + toot_text = f"@{username} '{question}' és una consulta ambigua. Potser et referies a:\n\n{ambiguous_reply}" + + toot_text = (toot_text[:400] + '... ') if len(toot_text) > 400 else toot_text + + except wikipedia.exceptions.PageError as page_error: + + toot_text = f"@{username} l'article '{question}' no existeix. Varia una mica la consulta o si comprobes que l'article no existeix t'animem a crear-lo. Som l'enciclopèdia lliure que tu també pots editar.\n" + + toot_text += ":viqui:" + + else: + + toot_text = f'@{username}\n' + toot_text += wikipedia.summary(question, sentences=2) + "\n" + toot_text = (toot_text[:400] + '... ') if len(toot_text) > 400 else toot_text + toot_text += f'{str(wiki_result.url)}\n' + toot_text += ":viqui:" + + mastodon.status_post(toot_text, in_reply_to_id=status_id,visibility=visibility) + + print(f'Replied notification {notification_id}') + + mastodon.notifications_dismiss(notification_id) + + else: + + print(f'dismissing notification {notification_id}') + + mastodon.notifications_dismiss(notification_id) + def get_question(text): reply = False @@ -124,83 +205,10 @@ if __name__ == '__main__': bot_notifications = mastodon.notifications() - if len(bot_notifications) == 0: + notifications = Notifications() - print('No mentions') + ray_start = time.time() - sys.exit(0) + results = ray.get([notifications.get_data.remote(mention) for mention in bot_notifications]) - i = 0 - - while i < len(bot_notifications): - - notification_id = bot_notifications[i].id - - if bot_notifications[i].type != 'mention': - - i += 1 - - print(f'dismissing notification {notification_id}') - - mastodon.notifications_dismiss(notification_id) - - continue - - account_id = bot_notifications[i] - - username = bot_notifications[i].account.acct - - status_id = bot_notifications[i].status.id - - text = bot_notifications[i].status.content - - visibility = bot_notifications[i].status.visibility - - reply, question = get_question(text) - - if reply: - - try: - - wiki_result = wikipedia.page(question) - - except wikipedia.exceptions.DisambiguationError as error: - - wiki_result = wikipedia.search(question, results=10) - - ambiguous_reply = "\n- ".join(str(x) for x in wiki_result) - - toot_text = f"@{username} '{question}' és una consulta ambigua. Potser et referies a:\n\n{ambiguous_reply}" - - toot_text = (toot_text[:400] + '... ') if len(toot_text) > 400 else toot_text - - except wikipedia.exceptions.PageError as page_error: - - toot_text = f"@{username} l'article '{question}' no existeix. Varia una mica la consulta o si comprobes que l'article no existeix t'animem a crear-lo. Som l'enciclopèdia lliure que tu també pots editar.\n" - - toot_text += ":viqui:" - - else: - - toot_text = f'@{username}\n' - - toot_text += wikipedia.summary(question, sentences=2) + "\n" - toot_text = (toot_text[:400] + '... ') if len(toot_text) > 400 else toot_text - toot_text += f'{str(wiki_result.url)}\n' - toot_text += ":viqui:" - - mastodon.status_post(toot_text, in_reply_to_id=status_id,visibility=visibility) - - print(f'Replied notification {notification_id}') - - mastodon.notifications_dismiss(notification_id) - - else: - - print(f'dismissing notification {notification_id}') - - mastodon.notifications_dismiss(notification_id) - - i += 1 - - + print(f"duration = {time.time() - ray_start}.\nprocessed queries: {len(results)}")