diff --git a/fediquery.py b/fediquery.py index e25589f..81b03a2 100644 --- a/fediquery.py +++ b/fediquery.py @@ -6,6 +6,7 @@ from datetime import datetime, timedelta from setup import Setup from mastodon import Mastodon from database import Database +from query import Query import pdb def cleanhtml(raw_html): @@ -82,6 +83,8 @@ if __name__ == '__main__': db = Database() + query = Query() + now = datetime.now() bot_id = mastodon.me().id @@ -157,6 +160,12 @@ if __name__ == '__main__': toot_text = f'@{username}, my data for {search_server}:\n\n' + if server == '': + + toot_text += f"\nServer not found but it's alive. Added!\n\n" + + server, software, version, users, mau, alive = query.getsoft(search_server) + if server != '': toot_text += f'server: {server}\nsoftware: :{software}:\nversion: {version}\nMAU: {mau:,}\nusers: {users:,}\nalive: {alive}' diff --git a/fediverse.py b/fediverse.py index 250c9bc..3f22827 100644 --- a/fediverse.py +++ b/fediverse.py @@ -61,6 +61,9 @@ class Server: try: users = nodeinfo_json.get('usage').get('users').get('total') or '0' + if users > 1000000: + + users = 1 mau = nodeinfo_json.get('usage').get('users').get('activeMonth') or '0' @@ -94,6 +97,10 @@ class Server: try: users = nodeinfo_json.get('stats').get('user_count') or '0' + if users > 1000000: + + users = 1 + soft_version = nodeinfo_json['version'] alive = True mau = 0 diff --git a/query.py b/query.py new file mode 100644 index 0000000..aebae18 --- /dev/null +++ b/query.py @@ -0,0 +1,297 @@ +import time +from datetime import datetime +import os +import json +import sys +import os.path +import requests +import urllib3 +import socket +from database import Database +from setup import Setup +import pdb + +apis = ['/api/v1/instance?', + '/api/v1/nodeinfo?', + '/nodeinfo/2.0?', + '/nodeinfo/2.0.json?', + '/nodeinfo/2.1.json?', + '/main/nodeinfo/2.0?', + '/api/statusnet/config?', + '/api/nodeinfo/2.0.json?', + '/api/nodeinfo?', + '/wp-json/nodeinfo/2.0?', + '/api/v1/instance/nodeinfo/2.0?', + '/.well-known/x-nodeinfo2?' + ] + +def is_json(myjson): + + try: + json_object = json.loads(myjson) + except ValueError as e: + return False + return True + +class Query(): + + name = "Query server data" + + def __init__(self, server=None, db=None, setup=None): + + self.server = server + self.db = Database() + self.setup = Setup() + + def getsoft(self, server): + + if server.find(".") == -1: + return + if server.find("@") != -1: + return + if server.find("/") != -1: + return + if server.find(":") != -1: + return + + soft = '' + + is_nodeinfo = False + + url = 'https://' + server + + try: + + response = requests.get(url + '/.well-known/nodeinfo', headers = self.setup.user_agent, timeout=3) + + if response.status_code == 200: + + try: + + response_json = response.json() + + if len(response_json['links']) == 1: + + nodeinfo = response_json['links'][0]['href'].replace(f'https://{server}','') + + elif len(response_json['links']) == 2: + + nodeinfo = response_json['links'][1]['href'].replace(f'https://{server}','') + + try: + + nodeinfo_data = requests.get(url + nodeinfo, headers = self.setup.user_agent, timeout=3) + + if nodeinfo_data.status_code == 200: + + nodeinfo_json = nodeinfo_data.json() + + is_nodeinfo = True + + else: + + print(f"{nodeinfo} not responding: error code {nodeinfo_data.status_code}") + + except: + + pass + + except: + + print(f'{server} is not responding: error code {response.status_code}') + print('*********************************************************************') + + pass + else: + + for api in apis: + + try: + + response = requests.get(url + api, headers = self.setup.user_agent, timeout=3) + + if response.status_code == 200: + + if is_json(response.text): + + nodeinfo_json = response.json() + + if 'software' in nodeinfo_json: + + nodeinfo = api + + is_nodeinfo = True + + break + + elif 'title' in nodeinfo_json: + + if nodeinfo_json['title'] == 'Zap': + + nodeinfo = api + + is_nodeinfo = True + + soft = 'zap' + + break + + elif 'version' in nodeinfo_json: + + nodeinfo = api + + is_nodeinfo = True + + break + + except: + + pass + + except requests.exceptions.SSLError as errssl: + + pass + + except requests.exceptions.HTTPError as errh: + + pass + + except requests.exceptions.ConnectionError as errc: + + pass + + except requests.exceptions.ReadTimeout as to_err: + + pass + + except requests.exceptions.TooManyRedirects as tmr_err: + + pass + + except urllib3.exceptions.LocationParseError as lp_err: + + pass + + except requests.exceptions.InvalidURL as iu_err: + + pass + + except requests.exceptions.ChunkedEncodingError as chunk_err: + + print(f'ChunkedEncodingError! {server}') + pass + + except ray.exceptions.RaySystemError as ray_sys_error: + + print(ray_sys_error) + pass + + else: + + if is_nodeinfo: + + if nodeinfo != '/api/v1/instance?': + + if nodeinfo != '/.well-known/x-nodeinfo2?': + + try: + + soft = nodeinfo_json['software']['name'] + soft = soft.lower() + soft_version = nodeinfo_json['software']['version'] + users = nodeinfo_json.get('usage').get('users').get('total') or '0' + if users > 1000000: + return + mau = nodeinfo_json.get('usage').get('users').get('activeMonth') or 0 + + alive = True + + self.db.write_api(server, soft, users, alive, nodeinfo, soft_version) + + print(f"Server {server} ({soft} {soft_version}) is alive!") + print('*********************************************************************') + + return (server, soft, soft_version, users, mau, alive) + + except: + + pass + + else: + + try: + + soft = nodeinfo_json['server']['software'] + soft = soft.lower() + soft_version = nodeinfo_json['server']['version'] + users = nodeinfo_json['usage']['users']['total'] + if users > 1000000: + return + mau = nodeinfo_json.get('usage').get('users').get('activeMonth') or 0 + + alive = True + + if soft == 'socialhome': + + self.db.write_api(server, soft, users, alive, nodeinfo, soft_version) + + print('*********************************************************************') + print(f"Server {serve}r ({soft} {soft_version}) is alive!") + print('*********************************************************************') + + return + + except: + + pass + + if soft == '' and nodeinfo == "/api/v1/instance?": + + soft = 'mastodon' + + try: + + users = nodeinfo_json['stats']['user_count'] + + if users > 1000000: + + return + + except: + + users = 0 + + try: + + soft_version = nodeinfo_json['version'] + + except: + + soft_version = 'unknown' + + alive = True + + self.db.write_api(server, soft, users, alive, nodeinfo, soft_version) + + print('*********************************************************************') + print(f"Server {server} ({soft}) is alive!") + + elif soft == 'zap' and nodeinfo == "/api/v1/instance?": + + soft = 'zap' + users = nodeinfo_json['stats']['user_count'] + soft_version = nodeinfo_json['version'] + alive = True + + print(server, soft, users, alive, api) + + print('*********************************************************************') + print(f"Server {server} ({soft}) is alive!") + + else: + + print(f'Server {server} is dead') + print('*********************************************************************') + + return (server, software, version, users, mau, alive) +