RAY parallel replies!

This commit is contained in:
spla 2022-03-21 14:44:23 +01:00
pare 20c9dd2a6f
commit 773860b92f
S'han modificat 3 arxius amb 91 adicions i 80 eliminacions

Veure arxiu

@ -23,3 +23,5 @@ Query the bot is this easy:
6. Use your favourite scheduling method to set `python viquicat.py` to run every minute. 6. Use your favourite scheduling method to set `python viquicat.py` to run every minute.
21.3.2022 **New feature** Ray Parallel processing! The code replies all queries at the same time!

Veure arxiu

@ -2,3 +2,4 @@ wheel>=0.36.2a
wikipedia>=1.4.0 wikipedia>=1.4.0
Mastodon.py>=1.5.1 Mastodon.py>=1.5.1
unidecode>=1.1.1 unidecode>=1.1.1
ray>=1.11.0

Veure arxiu

@ -1,16 +1,16 @@
from six.moves import urllib
from datetime import datetime, timedelta
from mastodon import Mastodon from mastodon import Mastodon
import unidecode import unidecode
import time
import re import re
import os import os
import time import time
import sys import sys
import os.path import os.path
import wikipedia import wikipedia
import ray
import pdb import pdb
ray.init(num_cpus = 8) # Specify this system CPUs.
def cleanhtml(raw_html): def cleanhtml(raw_html):
cleanr = re.compile('<.*?>') cleanr = re.compile('<.*?>')
cleantext = re.sub(cleanr, '', raw_html) cleantext = re.sub(cleanr, '', raw_html)
@ -20,6 +20,87 @@ def unescape(s):
s = s.replace("&apos;", "'") s = s.replace("&apos;", "'")
return s return s
class Notifications:
name = 'Notifications'
def __init__(self, type=None, id=None, account_id=None, username=None, status_id=None, text='', visibility=None):
self.type = type
self.id = id
self.account_id = account_id
self.username = username
self.status_id = status_id
self.text = text
self.visibility = visibility
@ray.remote
def get_data(self):
notification_id = self.id
if self.type != 'mention':
print(f'dismissing notification {notification_id}')
mastodon.notifications_dismiss(notification_id)
return
account_id = self.account.id
username = self.account.acct
status_id = self.status.id
text = self.status.content
visibility = self.status.visibility
reply, question = get_question(text)
if reply:
try:
wiki_result = wikipedia.page(question)
except wikipedia.exceptions.DisambiguationError as error:
wiki_result = wikipedia.search(question, results=10)
ambiguous_reply = "\n- ".join(str(x) for x in wiki_result)
toot_text = f"@{username} '{question}' és una consulta ambigua. Potser et referies a:\n\n{ambiguous_reply}"
toot_text = (toot_text[:400] + '... ') if len(toot_text) > 400 else toot_text
except wikipedia.exceptions.PageError as page_error:
toot_text = f"@{username} l'article '{question}' no existeix. Varia una mica la consulta o si comprobes que l'article no existeix t'animem a crear-lo. Som l'enciclopèdia lliure que tu també pots editar.\n"
toot_text += ":viqui:"
else:
toot_text = f'@{username}\n'
toot_text += wikipedia.summary(question, sentences=2) + "\n"
toot_text = (toot_text[:400] + '... ') if len(toot_text) > 400 else toot_text
toot_text += f'{str(wiki_result.url)}\n'
toot_text += ":viqui:"
mastodon.status_post(toot_text, in_reply_to_id=status_id,visibility=visibility)
print(f'Replied notification {notification_id}')
mastodon.notifications_dismiss(notification_id)
else:
print(f'dismissing notification {notification_id}')
mastodon.notifications_dismiss(notification_id)
def get_question(text): def get_question(text):
reply = False reply = False
@ -124,83 +205,10 @@ if __name__ == '__main__':
bot_notifications = mastodon.notifications() bot_notifications = mastodon.notifications()
if len(bot_notifications) == 0: notifications = Notifications()
print('No mentions') ray_start = time.time()
sys.exit(0)
i = 0
while i < len(bot_notifications):
notification_id = bot_notifications[i].id
if bot_notifications[i].type != 'mention':
i += 1
print(f'dismissing notification {notification_id}')
mastodon.notifications_dismiss(notification_id)
continue
account_id = bot_notifications[i]
username = bot_notifications[i].account.acct
status_id = bot_notifications[i].status.id
text = bot_notifications[i].status.content
visibility = bot_notifications[i].status.visibility
reply, question = get_question(text)
if reply:
try:
wiki_result = wikipedia.page(question)
except wikipedia.exceptions.DisambiguationError as error:
wiki_result = wikipedia.search(question, results=10)
ambiguous_reply = "\n- ".join(str(x) for x in wiki_result)
toot_text = f"@{username} '{question}' és una consulta ambigua. Potser et referies a:\n\n{ambiguous_reply}"
toot_text = (toot_text[:400] + '... ') if len(toot_text) > 400 else toot_text
except wikipedia.exceptions.PageError as page_error:
toot_text = f"@{username} l'article '{question}' no existeix. Varia una mica la consulta o si comprobes que l'article no existeix t'animem a crear-lo. Som l'enciclopèdia lliure que tu també pots editar.\n"
toot_text += ":viqui:"
else:
toot_text = f'@{username}\n'
toot_text += wikipedia.summary(question, sentences=2) + "\n"
toot_text = (toot_text[:400] + '... ') if len(toot_text) > 400 else toot_text
toot_text += f'{str(wiki_result.url)}\n'
toot_text += ":viqui:"
mastodon.status_post(toot_text, in_reply_to_id=status_id,visibility=visibility)
print(f'Replied notification {notification_id}')
mastodon.notifications_dismiss(notification_id)
else:
print(f'dismissing notification {notification_id}')
mastodon.notifications_dismiss(notification_id)
i += 1
results = ray.get([notifications.get_data.remote(mention) for mention in bot_notifications])
print(f"duration = {time.time() - ray_start}.\nprocessed queries: {len(results)}")