torips.py now downloads the torbulkexitlist file

README.md

@@ -22,7 +22,7 @@ Within Python Virtual Environment:
 
 3. Run `python torips.py` to write Tor exit nodes IPs to database. You need to get the torbulkexitlist from [here](https://blog.torproject.org/changes-tor-exit-list-service/) 
 
-4. Use your favourite scheduling method to set `python spamcheck.py` to run regularly.  
+4. Use your favourite scheduling method to set `python spamcheck.py` and `python torips.py` to run regularly.  
 
 
 

torips.py

@@ -1,6 +1,8 @@
 import os
 import datetime
 import psycopg2
+import requests
+import sys
 import pdb
 
 def insert_tor_ip(tor_ip):
@@ -33,6 +35,36 @@ def insert_tor_ip(tor_ip):
 
             conn.close()
 
+def get_torbulk_list():
+
+    is_saved = False
+
+    try:
+
+        user_agent = {'User-agent': 'Mozilla/5.0'}
+
+        response = requests.get('https://check.torproject.org/torbulkexitlist', headers = user_agent, timeout=3)
+
+        if response.status_code == 200:
+
+            with open('torbulkexitlist', 'wb') as f:
+
+                f.write(response.content)
+
+            is_saved = True
+
+        else:
+
+            sys.exit(f'download error: {response.status_code}')
+
+    except requests.exceptions.ConnectionError as conn_error:
+
+        print(f'{conn_error}')
+
+        pass
+
+    return is_saved
+
 def db_config():
 
     # Load db configuration from config file
@@ -43,7 +75,7 @@ def db_config():
     return (spamcheck_db, spamcheck_db_user)
 
 def get_parameter( parameter, file_path ):
-    # Check if secrets file exists
+
     if not os.path.isfile(file_path):
         print("File %s not found, exiting."%file_path)
         sys.exit(0)
@@ -63,16 +95,32 @@ if __name__ == '__main__':
     spamcheck_db, spamcheck_db_user = db_config()
 
     now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+    is_saved = get_torbulk_list()
+
+    if is_saved:
     
-    filepath = 'torbulkexitlist'
+        filepath = 'torbulkexitlist'
-    with open(filepath) as fp:
+
-       line = fp.readline()
+        with open(filepath) as fp:
-       cnt = 1
+
-       while line:
+            line = fp.readline()
-           #print("Line {}: {}".format(cnt, line.strip()))
+            cnt = 1
-           line = fp.readline().rstrip('\n')
+
-           if line != '':
+            while line:
-               insert_tor_ip(line)
+
-           cnt += 1
+                line = fp.readline().rstrip('\n')
+
+                if line != '':
+
+                    insert_tor_ip(line)
+
+                cnt += 1
+
+            print(f'Tor exit nodes: {cnt-1}')
+
+    else:
+
+        sys.exit('error downloading torbulkexitlist file')