RAY parallel replies!
This commit is contained in:
spla
pare
20c9dd2a6f
commit
773860b92f
S'han modificat 3 arxius amb 91 adicions i 80 eliminacions
|
|
@ -23,3 +23,5 @@ Query the bot is this easy:
|
|||
|
||||
6. Use your favourite scheduling method to set `python viquicat.py` to run every minute.
|
||||
|
||||
21.3.2022 **New feature** Ray Parallel processing! The code replies all queries at the same time!
|
||||
|
||||
|
|
|
|||
|
|
@ -2,3 +2,4 @@ wheel>=0.36.2a
|
|||
wikipedia>=1.4.0
|
||||
Mastodon.py>=1.5.1
|
||||
unidecode>=1.1.1
|
||||
ray>=1.11.0
|
||||
|
|
|
|||
168
viquicat.py
168
viquicat.py
|
|
@ -1,16 +1,16 @@
|
|||
from six.moves import urllib
|
||||
from datetime import datetime, timedelta
|
||||
from mastodon import Mastodon
|
||||
import unidecode
|
||||
import time
|
||||
import re
|
||||
import os
|
||||
import time
|
||||
import sys
|
||||
import os.path
|
||||
import wikipedia
|
||||
import ray
|
||||
import pdb
|
||||
|
||||
ray.init(num_cpus = 8) # Specify this system CPUs.
|
||||
|
||||
def cleanhtml(raw_html):
|
||||
cleanr = re.compile('<.*?>')
|
||||
cleantext = re.sub(cleanr, '', raw_html)
|
||||
|
|
@ -20,6 +20,87 @@ def unescape(s):
|
|||
s = s.replace("'", "'")
|
||||
return s
|
||||
|
||||
class Notifications:
|
||||
|
||||
name = 'Notifications'
|
||||
|
||||
def __init__(self, type=None, id=None, account_id=None, username=None, status_id=None, text='', visibility=None):
|
||||
|
||||
self.type = type
|
||||
self.id = id
|
||||
self.account_id = account_id
|
||||
self.username = username
|
||||
self.status_id = status_id
|
||||
self.text = text
|
||||
self.visibility = visibility
|
||||
|
||||
@ray.remote
|
||||
def get_data(self):
|
||||
|
||||
notification_id = self.id
|
||||
|
||||
if self.type != 'mention':
|
||||
|
||||
print(f'dismissing notification {notification_id}')
|
||||
|
||||
mastodon.notifications_dismiss(notification_id)
|
||||
|
||||
return
|
||||
|
||||
account_id = self.account.id
|
||||
|
||||
username = self.account.acct
|
||||
|
||||
status_id = self.status.id
|
||||
|
||||
text = self.status.content
|
||||
|
||||
visibility = self.status.visibility
|
||||
|
||||
reply, question = get_question(text)
|
||||
|
||||
if reply:
|
||||
|
||||
try:
|
||||
|
||||
wiki_result = wikipedia.page(question)
|
||||
|
||||
except wikipedia.exceptions.DisambiguationError as error:
|
||||
|
||||
wiki_result = wikipedia.search(question, results=10)
|
||||
|
||||
ambiguous_reply = "\n- ".join(str(x) for x in wiki_result)
|
||||
|
||||
toot_text = f"@{username} '{question}' és una consulta ambigua. Potser et referies a:\n\n{ambiguous_reply}"
|
||||
|
||||
toot_text = (toot_text[:400] + '... ') if len(toot_text) > 400 else toot_text
|
||||
|
||||
except wikipedia.exceptions.PageError as page_error:
|
||||
|
||||
toot_text = f"@{username} l'article '{question}' no existeix. Varia una mica la consulta o si comprobes que l'article no existeix t'animem a crear-lo. Som l'enciclopèdia lliure que tu també pots editar.\n"
|
||||
|
||||
toot_text += ":viqui:"
|
||||
|
||||
else:
|
||||
|
||||
toot_text = f'@{username}\n'
|
||||
toot_text += wikipedia.summary(question, sentences=2) + "\n"
|
||||
toot_text = (toot_text[:400] + '... ') if len(toot_text) > 400 else toot_text
|
||||
toot_text += f'{str(wiki_result.url)}\n'
|
||||
toot_text += ":viqui:"
|
||||
|
||||
mastodon.status_post(toot_text, in_reply_to_id=status_id,visibility=visibility)
|
||||
|
||||
print(f'Replied notification {notification_id}')
|
||||
|
||||
mastodon.notifications_dismiss(notification_id)
|
||||
|
||||
else:
|
||||
|
||||
print(f'dismissing notification {notification_id}')
|
||||
|
||||
mastodon.notifications_dismiss(notification_id)
|
||||
|
||||
def get_question(text):
|
||||
|
||||
reply = False
|
||||
|
|
@ -124,83 +205,10 @@ if __name__ == '__main__':
|
|||
|
||||
bot_notifications = mastodon.notifications()
|
||||
|
||||
if len(bot_notifications) == 0:
|
||||
notifications = Notifications()
|
||||
|
||||
print('No mentions')
|
||||
|
||||
sys.exit(0)
|
||||
|
||||
i = 0
|
||||
|
||||
while i < len(bot_notifications):
|
||||
|
||||
notification_id = bot_notifications[i].id
|
||||
|
||||
if bot_notifications[i].type != 'mention':
|
||||
|
||||
i += 1
|
||||
|
||||
print(f'dismissing notification {notification_id}')
|
||||
|
||||
mastodon.notifications_dismiss(notification_id)
|
||||
|
||||
continue
|
||||
|
||||
account_id = bot_notifications[i]
|
||||
|
||||
username = bot_notifications[i].account.acct
|
||||
|
||||
status_id = bot_notifications[i].status.id
|
||||
|
||||
text = bot_notifications[i].status.content
|
||||
|
||||
visibility = bot_notifications[i].status.visibility
|
||||
|
||||
reply, question = get_question(text)
|
||||
|
||||
if reply:
|
||||
|
||||
try:
|
||||
|
||||
wiki_result = wikipedia.page(question)
|
||||
|
||||
except wikipedia.exceptions.DisambiguationError as error:
|
||||
|
||||
wiki_result = wikipedia.search(question, results=10)
|
||||
|
||||
ambiguous_reply = "\n- ".join(str(x) for x in wiki_result)
|
||||
|
||||
toot_text = f"@{username} '{question}' és una consulta ambigua. Potser et referies a:\n\n{ambiguous_reply}"
|
||||
|
||||
toot_text = (toot_text[:400] + '... ') if len(toot_text) > 400 else toot_text
|
||||
|
||||
except wikipedia.exceptions.PageError as page_error:
|
||||
|
||||
toot_text = f"@{username} l'article '{question}' no existeix. Varia una mica la consulta o si comprobes que l'article no existeix t'animem a crear-lo. Som l'enciclopèdia lliure que tu també pots editar.\n"
|
||||
|
||||
toot_text += ":viqui:"
|
||||
|
||||
else:
|
||||
|
||||
toot_text = f'@{username}\n'
|
||||
|
||||
toot_text += wikipedia.summary(question, sentences=2) + "\n"
|
||||
toot_text = (toot_text[:400] + '... ') if len(toot_text) > 400 else toot_text
|
||||
toot_text += f'{str(wiki_result.url)}\n'
|
||||
toot_text += ":viqui:"
|
||||
|
||||
mastodon.status_post(toot_text, in_reply_to_id=status_id,visibility=visibility)
|
||||
|
||||
print(f'Replied notification {notification_id}')
|
||||
|
||||
mastodon.notifications_dismiss(notification_id)
|
||||
|
||||
else:
|
||||
|
||||
print(f'dismissing notification {notification_id}')
|
||||
|
||||
mastodon.notifications_dismiss(notification_id)
|
||||
|
||||
i += 1
|
||||
ray_start = time.time()
|
||||
|
||||
results = ray.get([notifications.get_data.remote(mention) for mention in bot_notifications])
|
||||
|
||||
print(f"duration = {time.time() - ray_start}.\nprocessed queries: {len(results)}")
|
||||
|
|
|
|||
Loading…
Referencia en una nova incidència