Fixed some warnings

This commit is contained in:
spla 2021-05-14 13:44:19 +02:00
pare 8185045292
commit 246ccf869c
S'han modificat 1 arxius amb 51 adicions i 71 eliminacions

Veure arxiu

@ -1,38 +1,17 @@
import time import time
start_time = time.time()
import urllib3
from urllib3 import exceptions
from datetime import datetime from datetime import datetime
from subprocess import call
from mastodon import Mastodon
import threading
import os import os
import json import json
import signal
import sys import sys
import os.path import os.path
import requests
from requests import exceptions
import operator
import calendar
import psycopg2 import psycopg2
from itertools import product from multiprocessing import Pool, Manager
from multiprocessing import Pool, Lock, Process, Queue, current_process, Manager
import multiprocessing
import aiohttp import aiohttp
import aiodns
import asyncio import asyncio
from aiohttp import ClientError, ClientSession, ClientConnectionError, ClientConnectorError, ClientSSLError, ClientConnectorSSLError, ServerTimeoutError
from asyncio import TimeoutError
import socket import socket
from socket import gaierror, gethostbyname
from decimal import * apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?',
getcontext().prec = 2 '/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?', '/wp-json/nodeinfo/2.0?']
apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?', '/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?','/wp-json/nodeinfo/2.0?']
client_exceptions = ( client_exceptions = (
aiohttp.ClientResponseError, aiohttp.ClientResponseError,
@ -43,11 +22,6 @@ client_exceptions = (
socket.gaierror, socket.gaierror,
) )
now = datetime.now()
###############################################################################
# INITIALISATION
###############################################################################
def is_json(myjson): def is_json(myjson):
try: try:
@ -56,20 +30,23 @@ def is_json(myjson):
return False return False
return True return True
def write_api(server, software, users, alive, api, soft_version):
def write_api(server, software, users, alive, api, soft_version):
insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING" insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
conn = None conn = None
try: try:
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql",
port="5432")
cur = conn.cursor() cur = conn.cursor()
cur.execute(insert_sql, (server, now, software, users, alive, api, soft_version)) cur.execute(insert_sql, (server, now, software, users, alive, api, soft_version))
cur.execute("UPDATE fediverse SET updated_at=(%s), software=(%s), users=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)", (now, software, users, alive, api, soft_version, server)) cur.execute(
"UPDATE fediverse SET updated_at=(%s), software=(%s), users=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)",
(now, software, users, alive, api, soft_version, server))
cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,)) cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
@ -84,11 +61,10 @@ def write_api(server, software, users, alive, api, soft_version):
finally: finally:
if conn is not None: if conn is not None:
conn.close() conn.close()
async def getsoft(server):
async def getsoft(server):
try: try:
socket.gethostbyname(server) socket.gethostbyname(server)
@ -106,7 +82,7 @@ async def getsoft(server):
async with aiohttp.ClientSession(timeout=timeout) as session: async with aiohttp.ClientSession(timeout=timeout) as session:
for api in apis: for api in apis:
try: try:
async with session.get(url+api) as response: async with session.get(url + api) as response:
if response.status == 200: if response.status == 200:
try: try:
response_json = await response.json() response_json = await response.json()
@ -115,29 +91,30 @@ async def getsoft(server):
except aiohttp.ClientConnectorError as err: except aiohttp.ClientConnectorError as err:
pass pass
else: else:
if response.status == 200 and api != '/api/v1/instance?': if response.status == 200 and api != '/api/v1/instance?':
try: try:
soft = response_json['software']['name'] soft = response_json['software']['name']
soft = soft.lower() soft = soft.lower()
soft_version = response_json['software']['version'] soft_version = response_json['software']['version']
users = response_json['usage']['users']['total'] users = response_json['usage']['users']['total']
if users > 1000000: if users > 1000000:
return return
alive = True alive = True
write_api(server, soft, users, alive, api, soft_version) write_api(server, soft, users, alive, api, soft_version)
print("Server " + server + " (" + soft + " " + soft_version + ") is alive!") print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
return return
except: except:
pass pass
if response.status == 200 and soft == '' and api == "/api/v1/instance?": if response.status == 200 and soft == '' and api == "/api/v1/instance?":
soft = 'mastodon' soft = 'mastodon'
users = response_json['stats']['user_count'] users = response_json['stats']['user_count']
soft_version = response_json['version'] soft_version = response_json['version']
if users > 1000000: if users > 1000000:
return return
alive = True alive = True
write_api(server, soft, users, alive, api) write_api(server, soft, users, alive, api)
print("Server " + server + " (" + soft + ") is alive!") print("Server " + server + " (" + soft + ") is alive!")
def getserver(server, x): def getserver(server, x):
@ -146,11 +123,11 @@ def getserver(server, x):
if server.find(".") == -1: if server.find(".") == -1:
return return
if server.find("@") != -1: if server.find("@") != -1:
return return
if server.find("/") != -1: if server.find("/") != -1:
return return
if server.find(":") != -1: if server.find(":") != -1:
return return
try: try:
@ -162,21 +139,22 @@ def getserver(server, x):
pass pass
# Returns the parameter from the specified file # Returns the parameter from the specified file
def get_parameter( parameter, file_path ): def get_parameter(parameter, file_path):
# Check if secrets file exists # Check if secrets file exists
if not os.path.isfile(file_path): if not os.path.isfile(file_path):
print("File %s not found, exiting."%file_path) print("File %s not found, exiting." % file_path)
sys.exit(0) sys.exit(0)
# Find parameter in file # Find parameter in file
with open( file_path ) as f: with open(file_path) as f:
for line in f: for line in f:
if line.startswith( parameter ): if line.startswith(parameter):
return line.replace(parameter + ":", "").strip() return line.replace(parameter + ":", "").strip()
# Cannot find parameter, exit # Cannot find parameter, exit
print(file_path + " Missing parameter %s "%parameter) print(file_path + " Missing parameter %s " % parameter)
sys.exit(0) sys.exit(0)
@ -194,21 +172,25 @@ fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath)
if __name__ == '__main__': if __name__ == '__main__':
now = datetime.now()
start_time = time.time()
world_servers = [] world_servers = []
try: try:
conn = None conn = None
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql",
port="5432")
cur = conn.cursor() cur = conn.cursor()
### get world servers list # get world servers list
cur.execute("select server from world where checked='f'") cur.execute("select server from world where checked='f'")
for row in cur: for row in cur:
world_servers.append(row[0]) world_servers.append(row[0])
cur.close() cur.close()
@ -222,7 +204,6 @@ if __name__ == '__main__':
finally: finally:
if conn is not None: if conn is not None:
conn.close() conn.close()
########################################################################### ###########################################################################
@ -239,4 +220,3 @@ if __name__ == '__main__':
pool.starmap(getserver, pool_tuple) pool.starmap(getserver, pool_tuple)
print('Done.') print('Done.')