torips.py now downloads the torbulkexitlist file

This commit is contained in:
spla 2022-01-30 20:16:07 +01:00
pare a7113472a6
commit a36a16d43c
S'han modificat 3 arxius amb 226 adicions i 734 eliminacions

Veure arxiu

@ -22,7 +22,7 @@ Within Python Virtual Environment:
3. Run `python torips.py` to write Tor exit nodes IPs to database. You need to get the torbulkexitlist from [here](https://blog.torproject.org/changes-tor-exit-list-service/)
4. Use your favourite scheduling method to set `python spamcheck.py` to run regularly.
4. Use your favourite scheduling method to set `python spamcheck.py` and `python torips.py` to run regularly.

La diferencia del archivo ha sido suprimido porque es demasiado grande Cargar Diff

Veure arxiu

@ -1,6 +1,8 @@
import os
import datetime
import psycopg2
import requests
import sys
import pdb
def insert_tor_ip(tor_ip):
@ -33,6 +35,36 @@ def insert_tor_ip(tor_ip):
conn.close()
def get_torbulk_list():
is_saved = False
try:
user_agent = {'User-agent': 'Mozilla/5.0'}
response = requests.get('https://check.torproject.org/torbulkexitlist', headers = user_agent, timeout=3)
if response.status_code == 200:
with open('torbulkexitlist', 'wb') as f:
f.write(response.content)
is_saved = True
else:
sys.exit(f'download error: {response.status_code}')
except requests.exceptions.ConnectionError as conn_error:
print(f'{conn_error}')
pass
return is_saved
def db_config():
# Load db configuration from config file
@ -43,7 +75,7 @@ def db_config():
return (spamcheck_db, spamcheck_db_user)
def get_parameter( parameter, file_path ):
# Check if secrets file exists
if not os.path.isfile(file_path):
print("File %s not found, exiting."%file_path)
sys.exit(0)
@ -64,15 +96,31 @@ if __name__ == '__main__':
now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
filepath = 'torbulkexitlist'
with open(filepath) as fp:
line = fp.readline()
cnt = 1
while line:
#print("Line {}: {}".format(cnt, line.strip()))
line = fp.readline().rstrip('\n')
if line != '':
insert_tor_ip(line)
cnt += 1
is_saved = get_torbulk_list()
if is_saved:
filepath = 'torbulkexitlist'
with open(filepath) as fp:
line = fp.readline()
cnt = 1
while line:
line = fp.readline().rstrip('\n')
if line != '':
insert_tor_ip(line)
cnt += 1
print(f'Tor exit nodes: {cnt-1}')
else:
sys.exit('error downloading torbulkexitlist file')