import time from datetime import datetime import os import json import sys import os.path import requests import urllib3 import socket from database import Database from setup import Setup import pdb apis = ['/api/v1/instance?', '/api/v1/nodeinfo?', '/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/nodeinfo/2.1.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?', '/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/wp-json/nodeinfo/2.0?', '/api/v1/instance/nodeinfo/2.0?', '/.well-known/x-nodeinfo2?' ] def is_json(myjson): try: json_object = json.loads(myjson) except ValueError as e: return False return True class Query(): name = "Query server data" def __init__(self, server=None, db=None, setup=None, soft=None, soft_version=None, version=None, users=None, mau=None, alive=False): self.server = server self.db = Database() self.setup = Setup() self.soft = '' self.soft_version = '' self.users = 0 self.mau = 0 self.alive = alive def getsoft(self, server): if server.find(".") == -1: return if server.find("@") != -1: return if server.find("/") != -1: return if server.find(":") != -1: return is_nodeinfo = False url = 'https://' + server try: response = requests.get(url + '/.well-known/nodeinfo', headers = self.setup.user_agent, timeout=3) if response.status_code == 200: try: response_json = response.json() if len(response_json['links']) == 1: nodeinfo = response_json['links'][0]['href'].replace(f'https://{server}','') elif len(response_json['links']) == 2: nodeinfo = response_json['links'][1]['href'].replace(f'https://{server}','') try: nodeinfo_data = requests.get(url + nodeinfo, headers = self.setup.user_agent, timeout=3) if nodeinfo_data.status_code == 200: nodeinfo_json = nodeinfo_data.json() is_nodeinfo = True else: print(f"{nodeinfo} not responding: error code {nodeinfo_data.status_code}") except: pass except: print(f'{server} is not responding: error code {response.status_code}') print('*********************************************************************') pass else: for api in apis: try: response = requests.get(url + api, headers = self.setup.user_agent, timeout=3) if response.status_code == 200: if is_json(response.text): nodeinfo_json = response.json() if 'software' in nodeinfo_json: nodeinfo = api is_nodeinfo = True break elif 'title' in nodeinfo_json: if nodeinfo_json['title'] == 'Zap': nodeinfo = api is_nodeinfo = True soft = 'zap' break elif 'version' in nodeinfo_json: nodeinfo = api is_nodeinfo = True break except: pass except requests.exceptions.SSLError as errssl: pass except requests.exceptions.HTTPError as errh: pass except requests.exceptions.ConnectionError as errc: pass except requests.exceptions.ReadTimeout as to_err: pass except requests.exceptions.TooManyRedirects as tmr_err: pass except urllib3.exceptions.LocationParseError as lp_err: pass except requests.exceptions.InvalidURL as iu_err: pass except requests.exceptions.ChunkedEncodingError as chunk_err: print(f'ChunkedEncodingError! {server}') pass except ray.exceptions.RaySystemError as ray_sys_error: print(ray_sys_error) pass else: if is_nodeinfo: if nodeinfo != '/api/v1/instance?': if nodeinfo != '/.well-known/x-nodeinfo2?': try: self.soft = nodeinfo_json['software']['name'] self.soft = self.soft.lower() self.soft_version = nodeinfo_json['software']['version'] self.users = nodeinfo_json.get('usage').get('users').get('total') or '0' if int(self.users) > 1000000: return self.mau = nodeinfo_json.get('usage').get('users').get('activeMonth') or 0 self.alive = True self.db.write_api(server, self.soft, self.users, self.mau, self.alive, nodeinfo, self.soft_version) print(f"Server {server} ({self.soft} {self.soft_version}) is alive!") print('*********************************************************************') return (server, self.soft, self.soft_version, self.users, self.mau, self.alive) except: pass else: try: self.soft = nodeinfo_json['server']['software'] self.soft = soft.lower() self.soft_version = nodeinfo_json['server']['version'] self.users = nodeinfo_json['usage']['users']['total'] if int(self.users) > 1000000: return self.mau = nodeinfo_json.get('usage').get('users').get('activeMonth') or 0 self.alive = True if self.soft == 'socialhome': self.db.write_api(server, soft, users, alive, nodeinfo, soft_version) print('*********************************************************************') print(f"Server {server} ({self.soft} {self.soft_version}) is alive!") print('*********************************************************************') return except: pass if self.soft == '' and nodeinfo == "/api/v1/instance?": self.soft = 'mastodon' try: self.users = nodeinfo_json['stats']['user_count'] if int(self.users) > 1000000: return except: self.users = 0 try: self.soft_version = nodeinfo_json['version'] except: self.soft_version = 'unknown' alive = True self.db.write_api(server, self.soft, self.users, self.alive, nodeinfo, self.soft_version) print('*********************************************************************') print(f"Server {server} ({self.soft}) is alive!") elif self.soft == 'zap' and nodeinfo == "/api/v1/instance?": self.soft = 'zap' self.users = nodeinfo_json['stats']['user_count'] self.soft_version = nodeinfo_json['version'] self.alive = True print(server, self.soft, self.users, self.alive, api) print('*********************************************************************') print(f"Server {server} ({self.soft}) is alive!") else: print(f'Server {server} is dead') print('*********************************************************************') return (server, self.soft, self.soft_version, self.users, self.mau, self.alive)