diff --git a/cleanserver.py b/cleanserver.py index 5ce05e5..cedad8d 100644 --- a/cleanserver.py +++ b/cleanserver.py @@ -8,8 +8,11 @@ from datetime import datetime import requests import json import psycopg2 +import ray import pdb +ray.init(num_cpus = 25) # Specify this system CPUs. + def get_totals(): conn = None @@ -198,6 +201,7 @@ def write_totals(alive, not_alive, total): conn.close() +@ray.remote def check_peers(peer): cleanserver_db, cleanserver_db_user = dbconfig() @@ -218,6 +222,26 @@ def check_peers(peer): write_server(peer, alive, checked) + except requests.exceptions.ChunkedEncodingError as chunk_err: + + print(f'Server: {peer} is dead') + + alive = False + + checked = True + + write_server(peer, alive, checked) + + except requests.exceptions.InvalidSchema as invalid_err: + + print(f'Server: {peer} is dead') + + alive = False + + checked = True + + write_server(peer, alive, checked) + except requests.exceptions.ConnectTimeout as err_ct: print(f'Server: {peer} is dead') @@ -341,11 +365,7 @@ if __name__ == '__main__': hostname_peers = res.json() - with get_context("spawn").Pool(processes=32) as pool: - - res = pool.starmap(check_peers, product(hostname_peers)) - pool.close() - pool.join() + ray.get([check_peers.remote(peer) for peer in hostname_peers]) alive_peers, not_alive_peers, total_peers = get_totals() diff --git a/requirements.txt b/requirements.txt index 503a66e..1102f7b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ requests>=2.27.1 psycopg2>=2.9.3a +ray