Fix errors and switch to ray paralel processing

cleanserver.py

@@ -8,8 +8,11 @@ from datetime import datetime
 import requests
 import json
 import psycopg2
+import ray
 import pdb
 
+ray.init(num_cpus = 25) # Specify this system CPUs.
+
 def get_totals():
 
     conn = None
@@ -198,6 +201,7 @@ def write_totals(alive, not_alive, total):
 
             conn.close()
 
+@ray.remote
 def check_peers(peer):
 
     cleanserver_db, cleanserver_db_user = dbconfig()
@@ -218,6 +222,26 @@ def check_peers(peer):
 
             write_server(peer, alive, checked)
 
+    except requests.exceptions.ChunkedEncodingError as chunk_err:
+
+        print(f'Server: {peer} is dead')
+
+        alive = False
+
+        checked = True
+
+        write_server(peer, alive, checked)
+
+    except requests.exceptions.InvalidSchema as invalid_err:
+
+        print(f'Server: {peer} is dead')
+
+        alive = False
+
+        checked = True
+
+        write_server(peer, alive, checked)
+
     except requests.exceptions.ConnectTimeout as err_ct:
 
         print(f'Server: {peer} is dead')
@@ -341,11 +365,7 @@ if __name__ == '__main__':
 
     hostname_peers = res.json()
 
-    with get_context("spawn").Pool(processes=32) as pool:
-
-        res = pool.starmap(check_peers, product(hostname_peers))
-        pool.close()
-        pool.join()
+    ray.get([check_peers.remote(peer) for peer in hostname_peers])
 
     alive_peers, not_alive_peers, total_peers = get_totals()
     

requirements.txt

@@ -1,2 +1,3 @@
 requests>=2.27.1
 psycopg2>=2.9.3a
+ray