When query a not found server, we will be added if it's alive

This commit is contained in:
spla 2023-01-07 16:05:47 +01:00
pare 7057bbd899
commit d152aa584a
S'han modificat 3 arxius amb 313 adicions i 0 eliminacions

Veure arxiu

@ -6,6 +6,7 @@ from datetime import datetime, timedelta
from setup import Setup from setup import Setup
from mastodon import Mastodon from mastodon import Mastodon
from database import Database from database import Database
from query import Query
import pdb import pdb
def cleanhtml(raw_html): def cleanhtml(raw_html):
@ -82,6 +83,8 @@ if __name__ == '__main__':
db = Database() db = Database()
query = Query()
now = datetime.now() now = datetime.now()
bot_id = mastodon.me().id bot_id = mastodon.me().id
@ -157,6 +160,12 @@ if __name__ == '__main__':
toot_text = f'@{username}, my data for {search_server}:\n\n' toot_text = f'@{username}, my data for {search_server}:\n\n'
if server == '':
toot_text += f"\nServer not found but it's alive. Added!\n\n"
server, software, version, users, mau, alive = query.getsoft(search_server)
if server != '': if server != '':
toot_text += f'server: {server}\nsoftware: :{software}:\nversion: {version}\nMAU: {mau:,}\nusers: {users:,}\nalive: {alive}' toot_text += f'server: {server}\nsoftware: :{software}:\nversion: {version}\nMAU: {mau:,}\nusers: {users:,}\nalive: {alive}'

Veure arxiu

@ -61,6 +61,9 @@ class Server:
try: try:
users = nodeinfo_json.get('usage').get('users').get('total') or '0' users = nodeinfo_json.get('usage').get('users').get('total') or '0'
if users > 1000000:
users = 1
mau = nodeinfo_json.get('usage').get('users').get('activeMonth') or '0' mau = nodeinfo_json.get('usage').get('users').get('activeMonth') or '0'
@ -94,6 +97,10 @@ class Server:
try: try:
users = nodeinfo_json.get('stats').get('user_count') or '0' users = nodeinfo_json.get('stats').get('user_count') or '0'
if users > 1000000:
users = 1
soft_version = nodeinfo_json['version'] soft_version = nodeinfo_json['version']
alive = True alive = True
mau = 0 mau = 0

297
query.py Normal file
Veure arxiu

@ -0,0 +1,297 @@
import time
from datetime import datetime
import os
import json
import sys
import os.path
import requests
import urllib3
import socket
from database import Database
from setup import Setup
import pdb
apis = ['/api/v1/instance?',
'/api/v1/nodeinfo?',
'/nodeinfo/2.0?',
'/nodeinfo/2.0.json?',
'/nodeinfo/2.1.json?',
'/main/nodeinfo/2.0?',
'/api/statusnet/config?',
'/api/nodeinfo/2.0.json?',
'/api/nodeinfo?',
'/wp-json/nodeinfo/2.0?',
'/api/v1/instance/nodeinfo/2.0?',
'/.well-known/x-nodeinfo2?'
]
def is_json(myjson):
try:
json_object = json.loads(myjson)
except ValueError as e:
return False
return True
class Query():
name = "Query server data"
def __init__(self, server=None, db=None, setup=None):
self.server = server
self.db = Database()
self.setup = Setup()
def getsoft(self, server):
if server.find(".") == -1:
return
if server.find("@") != -1:
return
if server.find("/") != -1:
return
if server.find(":") != -1:
return
soft = ''
is_nodeinfo = False
url = 'https://' + server
try:
response = requests.get(url + '/.well-known/nodeinfo', headers = self.setup.user_agent, timeout=3)
if response.status_code == 200:
try:
response_json = response.json()
if len(response_json['links']) == 1:
nodeinfo = response_json['links'][0]['href'].replace(f'https://{server}','')
elif len(response_json['links']) == 2:
nodeinfo = response_json['links'][1]['href'].replace(f'https://{server}','')
try:
nodeinfo_data = requests.get(url + nodeinfo, headers = self.setup.user_agent, timeout=3)
if nodeinfo_data.status_code == 200:
nodeinfo_json = nodeinfo_data.json()
is_nodeinfo = True
else:
print(f"{nodeinfo} not responding: error code {nodeinfo_data.status_code}")
except:
pass
except:
print(f'{server} is not responding: error code {response.status_code}')
print('*********************************************************************')
pass
else:
for api in apis:
try:
response = requests.get(url + api, headers = self.setup.user_agent, timeout=3)
if response.status_code == 200:
if is_json(response.text):
nodeinfo_json = response.json()
if 'software' in nodeinfo_json:
nodeinfo = api
is_nodeinfo = True
break
elif 'title' in nodeinfo_json:
if nodeinfo_json['title'] == 'Zap':
nodeinfo = api
is_nodeinfo = True
soft = 'zap'
break
elif 'version' in nodeinfo_json:
nodeinfo = api
is_nodeinfo = True
break
except:
pass
except requests.exceptions.SSLError as errssl:
pass
except requests.exceptions.HTTPError as errh:
pass
except requests.exceptions.ConnectionError as errc:
pass
except requests.exceptions.ReadTimeout as to_err:
pass
except requests.exceptions.TooManyRedirects as tmr_err:
pass
except urllib3.exceptions.LocationParseError as lp_err:
pass
except requests.exceptions.InvalidURL as iu_err:
pass
except requests.exceptions.ChunkedEncodingError as chunk_err:
print(f'ChunkedEncodingError! {server}')
pass
except ray.exceptions.RaySystemError as ray_sys_error:
print(ray_sys_error)
pass
else:
if is_nodeinfo:
if nodeinfo != '/api/v1/instance?':
if nodeinfo != '/.well-known/x-nodeinfo2?':
try:
soft = nodeinfo_json['software']['name']
soft = soft.lower()
soft_version = nodeinfo_json['software']['version']
users = nodeinfo_json.get('usage').get('users').get('total') or '0'
if users > 1000000:
return
mau = nodeinfo_json.get('usage').get('users').get('activeMonth') or 0
alive = True
self.db.write_api(server, soft, users, alive, nodeinfo, soft_version)
print(f"Server {server} ({soft} {soft_version}) is alive!")
print('*********************************************************************')
return (server, soft, soft_version, users, mau, alive)
except:
pass
else:
try:
soft = nodeinfo_json['server']['software']
soft = soft.lower()
soft_version = nodeinfo_json['server']['version']
users = nodeinfo_json['usage']['users']['total']
if users > 1000000:
return
mau = nodeinfo_json.get('usage').get('users').get('activeMonth') or 0
alive = True
if soft == 'socialhome':
self.db.write_api(server, soft, users, alive, nodeinfo, soft_version)
print('*********************************************************************')
print(f"Server {serve}r ({soft} {soft_version}) is alive!")
print('*********************************************************************')
return
except:
pass
if soft == '' and nodeinfo == "/api/v1/instance?":
soft = 'mastodon'
try:
users = nodeinfo_json['stats']['user_count']
if users > 1000000:
return
except:
users = 0
try:
soft_version = nodeinfo_json['version']
except:
soft_version = 'unknown'
alive = True
self.db.write_api(server, soft, users, alive, nodeinfo, soft_version)
print('*********************************************************************')
print(f"Server {server} ({soft}) is alive!")
elif soft == 'zap' and nodeinfo == "/api/v1/instance?":
soft = 'zap'
users = nodeinfo_json['stats']['user_count']
soft_version = nodeinfo_json['version']
alive = True
print(server, soft, users, alive, api)
print('*********************************************************************')
print(f"Server {server} ({soft}) is alive!")
else:
print(f'Server {server} is dead')
print('*********************************************************************')
return (server, software, version, users, mau, alive)