RAY parallel replies!

1 year ago · 773860b92f
parent 20c9dd2a6f
commit 773860b92f
3 changed files with 91 additions and 80 deletions
--- a/README.md
+++ b/README.md
@ -23,3 +23,5 @@ Query the bot is this easy:

 6. Use your favourite scheduling method to set `python viquicat.py` to run every minute.  

+21.3.2022 **New feature** Ray Parallel processing! The code replies all queries at the same time!   
+
--- a/requirements.txt
+++ b/requirements.txt
@ -2,3 +2,4 @@ wheel>=0.36.2a
 wikipedia>=1.4.0
 Mastodon.py>=1.5.1
 unidecode>=1.1.1
+ray>=1.11.0
--- a/viquicat.py
+++ b/viquicat.py
@ -1,16 +1,16 @@
-from six.moves import urllib
-from datetime import datetime, timedelta
 from mastodon import Mastodon
 import unidecode
-import time
 import re
 import os
 import time
 import sys
 import os.path
 import wikipedia
+import ray
 import pdb

+ray.init(num_cpus = 8) # Specify this system CPUs.
+
 def cleanhtml(raw_html):
    cleanr = re.compile('<.*?>')
    cleantext = re.sub(cleanr, '', raw_html)
@ -20,6 +20,87 @@ def unescape(s):
    s = s.replace("&apos;", "'")
    return s

+class Notifications:
+
+    name = 'Notifications'
+
+    def __init__(self, type=None, id=None, account_id=None, username=None, status_id=None, text='', visibility=None):
+
+        self.type = type
+        self.id = id
+        self.account_id = account_id
+        self.username = username
+        self.status_id = status_id
+        self.text = text
+        self.visibility = visibility
+
+    @ray.remote
+    def get_data(self):
+
+        notification_id = self.id
+
+        if self.type != 'mention':
+
+            print(f'dismissing notification {notification_id}')
+
+            mastodon.notifications_dismiss(notification_id)
+
+            return
+
+        account_id = self.account.id
+
+        username = self.account.acct
+
+        status_id = self.status.id
+
+        text  = self.status.content
+
+        visibility = self.status.visibility
+
+        reply, question = get_question(text)
+
+        if reply:
+
+            try:
+
+                wiki_result = wikipedia.page(question)
+
+            except wikipedia.exceptions.DisambiguationError as error:
+
+                wiki_result = wikipedia.search(question, results=10)
+
+                ambiguous_reply = "\n- ".join(str(x) for x in wiki_result)
+
+                toot_text = f"@{username} '{question}' és una consulta ambigua. Potser et referies a:\n\n{ambiguous_reply}"
+
+                toot_text = (toot_text[:400] + '... ') if len(toot_text) > 400 else toot_text
+
+            except wikipedia.exceptions.PageError as page_error:
+
+                toot_text = f"@{username} l'article '{question}' no existeix. Varia una mica la consulta o si comprobes que l'article no existeix t'animem a crear-lo. Som l'enciclopèdia lliure que tu també pots editar.\n"
+
+                toot_text += ":viqui:"
+
+            else:
+
+                toot_text = f'@{username}\n'
+                toot_text += wikipedia.summary(question, sentences=2) + "\n"
+                toot_text = (toot_text[:400] + '... ') if len(toot_text) > 400 else toot_text
+                toot_text += f'{str(wiki_result.url)}\n'
+                toot_text += ":viqui:"
+
+            mastodon.status_post(toot_text, in_reply_to_id=status_id,visibility=visibility)
+
+            print(f'Replied notification {notification_id}')
+
+            mastodon.notifications_dismiss(notification_id)
+
+        else:
+
+            print(f'dismissing notification {notification_id}')
+
+            mastodon.notifications_dismiss(notification_id)
+
 def get_question(text):

    reply = False
@ -124,83 +205,10 @@ if __name__ == '__main__':

    bot_notifications = mastodon.notifications()

-    if len(bot_notifications) == 0:
+    notifications = Notifications()

-        print('No mentions')
-
-        sys.exit(0)
-
-    i = 0
-
-    while i < len(bot_notifications):
-
-        notification_id = bot_notifications[i].id
-
-        if bot_notifications[i].type != 'mention':
-
-            i += 1
-
-            print(f'dismissing notification {notification_id}')
-
-            mastodon.notifications_dismiss(notification_id)
-
-            continue
-
-        account_id = bot_notifications[i]
-
-        username = bot_notifications[i].account.acct
-
-        status_id = bot_notifications[i].status.id
-
-        text  = bot_notifications[i].status.content
-
-        visibility = bot_notifications[i].status.visibility
-
-        reply, question = get_question(text)
-
-        if reply:
-
-            try:
-
-                wiki_result = wikipedia.page(question)
-
-            except wikipedia.exceptions.DisambiguationError as error:
-
-                wiki_result = wikipedia.search(question, results=10)
-
-                ambiguous_reply = "\n- ".join(str(x) for x in wiki_result)
-
-                toot_text = f"@{username} '{question}' és una consulta ambigua. Potser et referies a:\n\n{ambiguous_reply}"
-
-                toot_text = (toot_text[:400] + '... ') if len(toot_text) > 400 else toot_text
-
-            except wikipedia.exceptions.PageError as page_error:
-
-                toot_text = f"@{username} l'article '{question}' no existeix. Varia una mica la consulta o si comprobes que l'article no existeix t'animem a crear-lo. Som l'enciclopèdia lliure que tu també pots editar.\n"
-
-                toot_text += ":viqui:"
-
-            else:
-
-                toot_text = f'@{username}\n'
-
-                toot_text += wikipedia.summary(question, sentences=2) + "\n"
-                toot_text = (toot_text[:400] + '... ') if len(toot_text) > 400 else toot_text
-                toot_text += f'{str(wiki_result.url)}\n'
-                toot_text += ":viqui:"
-
-            mastodon.status_post(toot_text, in_reply_to_id=status_id,visibility=visibility)
-
-            print(f'Replied notification {notification_id}')
-
-            mastodon.notifications_dismiss(notification_id)
-
-        else:
-
-            print(f'dismissing notification {notification_id}')
-
-            mastodon.notifications_dismiss(notification_id)
+    ray_start = time.time()

-        i += 1
+    results = ray.get([notifications.get_data.remote(mention) for mention in bot_notifications])

-                        
+    print(f"duration = {time.time() - ray_start}.\nprocessed queries: {len(results)}")