When query a not found server, we will be added if it's alive

fediquery.py

@@ -6,6 +6,7 @@ from datetime import datetime, timedelta
 from setup import Setup
 from mastodon import Mastodon
 from database import Database
+from query import Query
 import pdb
 
 def cleanhtml(raw_html):
@@ -82,6 +83,8 @@ if __name__ == '__main__':
 
     db = Database()
 
+    query = Query()
+
     now = datetime.now()
 
     bot_id = mastodon.me().id
@@ -157,6 +160,12 @@ if __name__ == '__main__':
 
                     toot_text = f'@{username}, my data for {search_server}:\n\n'
 
+                    if server == '':
+
+                        toot_text += f"\nServer not found but it's alive. Added!\n\n"
+
+                        server, software, version, users, mau, alive = query.getsoft(search_server)
+
                     if server != '':
 
                         toot_text += f'server: {server}\nsoftware: :{software}:\nversion: {version}\nMAU: {mau:,}\nusers: {users:,}\nalive: {alive}'

fediverse.py

@@ -61,6 +61,9 @@ class Server:
             try:
 
                 users = nodeinfo_json.get('usage').get('users').get('total') or '0'
+                if users > 1000000:
+
+                    users = 1
 
                 mau = nodeinfo_json.get('usage').get('users').get('activeMonth') or '0'
 
@@ -94,6 +97,10 @@ class Server:
 
                     try:
                         users = nodeinfo_json.get('stats').get('user_count') or '0'
+                        if users > 1000000:
+
+                            users = 1
+
                         soft_version = nodeinfo_json['version']
                         alive = True
                         mau = 0

query.py

@@ -0,0 +1,297 @@
+import time
+from datetime import datetime
+import os
+import json
+import sys
+import os.path
+import requests
+import urllib3
+import socket
+from database import Database
+from setup import Setup
+import pdb
+
+apis = ['/api/v1/instance?',
+        '/api/v1/nodeinfo?',
+        '/nodeinfo/2.0?',
+        '/nodeinfo/2.0.json?',
+        '/nodeinfo/2.1.json?',
+        '/main/nodeinfo/2.0?',
+        '/api/statusnet/config?',
+        '/api/nodeinfo/2.0.json?',
+        '/api/nodeinfo?',
+        '/wp-json/nodeinfo/2.0?',
+        '/api/v1/instance/nodeinfo/2.0?',
+        '/.well-known/x-nodeinfo2?'
+        ]
+
+def is_json(myjson):
+
+    try:
+        json_object = json.loads(myjson)
+    except ValueError as e:
+        return False
+    return True
+
+class Query():
+
+    name = "Query server data"
+
+    def __init__(self, server=None, db=None, setup=None):
+
+        self.server = server
+        self.db = Database()
+        self.setup = Setup()
+
+    def getsoft(self, server):
+
+        if server.find(".") == -1:
+            return
+        if server.find("@") != -1:
+            return
+        if server.find("/") != -1:
+            return
+        if server.find(":") != -1:
+            return
+
+        soft = ''
+
+        is_nodeinfo = False
+
+        url = 'https://' + server
+
+        try:
+
+            response = requests.get(url + '/.well-known/nodeinfo', headers = self.setup.user_agent, timeout=3)
+
+            if response.status_code == 200:
+
+                try:
+
+                    response_json = response.json()
+
+                    if len(response_json['links']) == 1:
+
+                        nodeinfo = response_json['links'][0]['href'].replace(f'https://{server}','')
+
+                    elif len(response_json['links']) == 2:
+
+                        nodeinfo = response_json['links'][1]['href'].replace(f'https://{server}','')
+
+                    try:
+
+                        nodeinfo_data = requests.get(url + nodeinfo, headers = self.setup.user_agent, timeout=3)
+
+                        if nodeinfo_data.status_code == 200:
+
+                            nodeinfo_json = nodeinfo_data.json()
+
+                            is_nodeinfo = True
+
+                        else:
+
+                            print(f"{nodeinfo} not responding: error code {nodeinfo_data.status_code}")
+
+                    except:
+
+                        pass
+
+                except:
+
+                    print(f'{server} is not responding: error code {response.status_code}')
+                    print('*********************************************************************')
+
+                    pass
+            else:
+
+                for api in apis:
+
+                    try:
+
+                        response = requests.get(url + api, headers = self.setup.user_agent, timeout=3)
+
+                        if response.status_code == 200:
+
+                            if is_json(response.text):
+
+                                nodeinfo_json = response.json()
+
+                                if 'software' in nodeinfo_json:
+
+                                    nodeinfo = api
+
+                                    is_nodeinfo = True
+
+                                    break
+
+                                elif 'title' in nodeinfo_json:
+
+                                    if nodeinfo_json['title'] == 'Zap':
+
+                                        nodeinfo = api
+
+                                        is_nodeinfo = True
+
+                                        soft = 'zap'
+
+                                        break
+
+                                elif 'version' in nodeinfo_json:
+
+                                    nodeinfo = api
+
+                                    is_nodeinfo = True
+
+                                    break
+
+                    except:
+
+                        pass
+
+        except requests.exceptions.SSLError as errssl:
+
+            pass
+
+        except requests.exceptions.HTTPError as errh:
+
+            pass
+
+        except requests.exceptions.ConnectionError as errc:
+
+            pass
+
+        except requests.exceptions.ReadTimeout as to_err:
+
+            pass
+
+        except requests.exceptions.TooManyRedirects as tmr_err:
+
+            pass
+
+        except urllib3.exceptions.LocationParseError as lp_err:
+
+            pass
+
+        except requests.exceptions.InvalidURL as iu_err:
+
+            pass
+
+        except requests.exceptions.ChunkedEncodingError as chunk_err:
+
+            print(f'ChunkedEncodingError! {server}')
+            pass
+
+        except ray.exceptions.RaySystemError as ray_sys_error:
+
+            print(ray_sys_error) 
+            pass
+
+        else:
+
+            if is_nodeinfo:
+
+                if nodeinfo != '/api/v1/instance?':
+
+                    if nodeinfo != '/.well-known/x-nodeinfo2?':
+
+                        try:
+
+                            soft = nodeinfo_json['software']['name']
+                            soft = soft.lower()
+                            soft_version = nodeinfo_json['software']['version']
+                            users = nodeinfo_json.get('usage').get('users').get('total') or '0'
+                            if users > 1000000:
+                                return
+                            mau = nodeinfo_json.get('usage').get('users').get('activeMonth') or 0
+
+                            alive = True
+
+                            self.db.write_api(server, soft, users, alive, nodeinfo, soft_version)
+
+                            print(f"Server {server} ({soft} {soft_version}) is alive!")
+                            print('*********************************************************************')
+
+                            return (server, soft, soft_version, users, mau, alive)
+
+                        except:
+
+                            pass
+
+                    else:
+
+                        try:
+
+                            soft = nodeinfo_json['server']['software']
+                            soft = soft.lower()
+                            soft_version = nodeinfo_json['server']['version']
+                            users = nodeinfo_json['usage']['users']['total']
+                            if users > 1000000:
+                                return
+                            mau = nodeinfo_json.get('usage').get('users').get('activeMonth') or 0
+
+                            alive = True
+
+                            if soft == 'socialhome':
+
+                                self.db.write_api(server, soft, users, alive, nodeinfo, soft_version)
+
+                                print('*********************************************************************')
+                                print(f"Server {serve}r ({soft} {soft_version}) is alive!")
+                                print('*********************************************************************')
+
+                            return
+
+                        except:
+
+                            pass
+
+                if soft == '' and nodeinfo == "/api/v1/instance?":
+
+                    soft = 'mastodon'
+
+                    try:
+
+                        users = nodeinfo_json['stats']['user_count']
+
+                        if users > 1000000:
+
+                            return
+
+                    except:
+
+                        users = 0
+
+                    try:
+
+                        soft_version = nodeinfo_json['version']
+
+                    except:
+
+                        soft_version = 'unknown'
+
+                    alive = True
+
+                    self.db.write_api(server, soft, users, alive, nodeinfo, soft_version)
+
+                    print('*********************************************************************')
+                    print(f"Server {server} ({soft}) is alive!")
+
+                elif soft == 'zap' and nodeinfo == "/api/v1/instance?":
+
+                    soft = 'zap'
+                    users = nodeinfo_json['stats']['user_count']
+                    soft_version = nodeinfo_json['version']
+                    alive = True
+
+                    print(server, soft, users, alive, api)
+
+                    print('*********************************************************************')
+                    print(f"Server {server} ({soft}) is alive!")
+
+            else:
+
+                print(f'Server {server} is dead')
+                print('*********************************************************************')
+
+            return (server, software, version, users, mau, alive)
+