Fix errors and switch to ray paralel processing

This commit is contained in:
spla 2023-04-12 12:47:45 +02:00
pare f21e3f8461
commit 7f51380d03
S'han modificat 2 arxius amb 26 adicions i 5 eliminacions

Veure arxiu

@ -8,8 +8,11 @@ from datetime import datetime
import requests import requests
import json import json
import psycopg2 import psycopg2
import ray
import pdb import pdb
ray.init(num_cpus = 25) # Specify this system CPUs.
def get_totals(): def get_totals():
conn = None conn = None
@ -198,6 +201,7 @@ def write_totals(alive, not_alive, total):
conn.close() conn.close()
@ray.remote
def check_peers(peer): def check_peers(peer):
cleanserver_db, cleanserver_db_user = dbconfig() cleanserver_db, cleanserver_db_user = dbconfig()
@ -218,6 +222,26 @@ def check_peers(peer):
write_server(peer, alive, checked) write_server(peer, alive, checked)
except requests.exceptions.ChunkedEncodingError as chunk_err:
print(f'Server: {peer} is dead')
alive = False
checked = True
write_server(peer, alive, checked)
except requests.exceptions.InvalidSchema as invalid_err:
print(f'Server: {peer} is dead')
alive = False
checked = True
write_server(peer, alive, checked)
except requests.exceptions.ConnectTimeout as err_ct: except requests.exceptions.ConnectTimeout as err_ct:
print(f'Server: {peer} is dead') print(f'Server: {peer} is dead')
@ -341,11 +365,7 @@ if __name__ == '__main__':
hostname_peers = res.json() hostname_peers = res.json()
with get_context("spawn").Pool(processes=32) as pool: ray.get([check_peers.remote(peer) for peer in hostname_peers])
res = pool.starmap(check_peers, product(hostname_peers))
pool.close()
pool.join()
alive_peers, not_alive_peers, total_peers = get_totals() alive_peers, not_alive_peers, total_peers = get_totals()

Veure arxiu

@ -1,2 +1,3 @@
requests>=2.27.1 requests>=2.27.1
psycopg2>=2.9.3a psycopg2>=2.9.3a
ray