diff --git a/fetchservers.py b/fetchservers.py index d8b2278..850da48 100644 --- a/fetchservers.py +++ b/fetchservers.py @@ -1,38 +1,17 @@ import time -start_time = time.time() -import urllib3 -from urllib3 import exceptions from datetime import datetime -from subprocess import call -from mastodon import Mastodon -import threading import os import json -import signal import sys import os.path -import requests -from requests import exceptions -import operator -import calendar import psycopg2 -from itertools import product - -from multiprocessing import Pool, Lock, Process, Queue, current_process, Manager -import multiprocessing - +from multiprocessing import Pool, Manager import aiohttp -import aiodns import asyncio -from aiohttp import ClientError, ClientSession, ClientConnectionError, ClientConnectorError, ClientSSLError, ClientConnectorSSLError, ServerTimeoutError -from asyncio import TimeoutError import socket -from socket import gaierror, gethostbyname -from decimal import * -getcontext().prec = 2 - -apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?', '/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?','/wp-json/nodeinfo/2.0?'] +apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?', + '/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?', '/wp-json/nodeinfo/2.0?'] client_exceptions = ( aiohttp.ClientResponseError, @@ -43,11 +22,6 @@ client_exceptions = ( socket.gaierror, ) -now = datetime.now() - -############################################################################### -# INITIALISATION -############################################################################### def is_json(myjson): try: @@ -56,20 +30,23 @@ def is_json(myjson): return False return True -def write_api(server, software, users, alive, api, soft_version): +def write_api(server, software, users, alive, api, soft_version): insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING" conn = None try: - conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") + conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", + port="5432") cur = conn.cursor() cur.execute(insert_sql, (server, now, software, users, alive, api, soft_version)) - cur.execute("UPDATE fediverse SET updated_at=(%s), software=(%s), users=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)", (now, software, users, alive, api, soft_version, server)) + cur.execute( + "UPDATE fediverse SET updated_at=(%s), software=(%s), users=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)", + (now, software, users, alive, api, soft_version, server)) cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,)) @@ -84,11 +61,10 @@ def write_api(server, software, users, alive, api, soft_version): finally: if conn is not None: - conn.close() -async def getsoft(server): +async def getsoft(server): try: socket.gethostbyname(server) @@ -106,7 +82,7 @@ async def getsoft(server): async with aiohttp.ClientSession(timeout=timeout) as session: for api in apis: try: - async with session.get(url+api) as response: + async with session.get(url + api) as response: if response.status == 200: try: response_json = await response.json() @@ -115,29 +91,30 @@ async def getsoft(server): except aiohttp.ClientConnectorError as err: pass else: - if response.status == 200 and api != '/api/v1/instance?': - try: - soft = response_json['software']['name'] - soft = soft.lower() - soft_version = response_json['software']['version'] - users = response_json['usage']['users']['total'] - if users > 1000000: - return - alive = True - write_api(server, soft, users, alive, api, soft_version) - print("Server " + server + " (" + soft + " " + soft_version + ") is alive!") - return - except: - pass - if response.status == 200 and soft == '' and api == "/api/v1/instance?": - soft = 'mastodon' - users = response_json['stats']['user_count'] - soft_version = response_json['version'] - if users > 1000000: - return - alive = True - write_api(server, soft, users, alive, api) - print("Server " + server + " (" + soft + ") is alive!") + if response.status == 200 and api != '/api/v1/instance?': + try: + soft = response_json['software']['name'] + soft = soft.lower() + soft_version = response_json['software']['version'] + users = response_json['usage']['users']['total'] + if users > 1000000: + return + alive = True + write_api(server, soft, users, alive, api, soft_version) + print("Server " + server + " (" + soft + " " + soft_version + ") is alive!") + return + except: + pass + if response.status == 200 and soft == '' and api == "/api/v1/instance?": + soft = 'mastodon' + users = response_json['stats']['user_count'] + soft_version = response_json['version'] + if users > 1000000: + return + alive = True + write_api(server, soft, users, alive, api) + print("Server " + server + " (" + soft + ") is alive!") + def getserver(server, x): @@ -146,11 +123,11 @@ def getserver(server, x): if server.find(".") == -1: return if server.find("@") != -1: - return + return if server.find("/") != -1: - return + return if server.find(":") != -1: - return + return try: @@ -162,21 +139,22 @@ def getserver(server, x): pass + # Returns the parameter from the specified file -def get_parameter( parameter, file_path ): +def get_parameter(parameter, file_path): # Check if secrets file exists if not os.path.isfile(file_path): - print("File %s not found, exiting."%file_path) + print("File %s not found, exiting." % file_path) sys.exit(0) # Find parameter in file - with open( file_path ) as f: + with open(file_path) as f: for line in f: - if line.startswith( parameter ): + if line.startswith(parameter): return line.replace(parameter + ":", "").strip() # Cannot find parameter, exit - print(file_path + " Missing parameter %s "%parameter) + print(file_path + " Missing parameter %s " % parameter) sys.exit(0) @@ -194,21 +172,25 @@ fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath) if __name__ == '__main__': + now = datetime.now() + start_time = time.time() + world_servers = [] try: conn = None - conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") + + conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", + port="5432") cur = conn.cursor() - ### get world servers list + # get world servers list cur.execute("select server from world where checked='f'") for row in cur: - world_servers.append(row[0]) cur.close() @@ -222,7 +204,6 @@ if __name__ == '__main__': finally: if conn is not None: - conn.close() ########################################################################### @@ -239,4 +220,3 @@ if __name__ == '__main__': pool.starmap(getserver, pool_tuple) print('Done.') -