From 81850452922685471071dc82be4071a41f0f03cf Mon Sep 17 00:00:00 2001 From: spla Date: Fri, 14 May 2021 13:14:31 +0200 Subject: [PATCH] Fixed some warnings --- fediverse.py | 157 +++++++++++++++++++++++++-------------------------- 1 file changed, 77 insertions(+), 80 deletions(-) diff --git a/fediverse.py b/fediverse.py index ce75b8a..b257433 100644 --- a/fediverse.py +++ b/fediverse.py @@ -1,34 +1,18 @@ import time import urllib3 -from urllib3 import exceptions from datetime import datetime -from subprocess import call from mastodon import Mastodon -import threading import os import json -import signal import sys import os.path import requests -from requests import exceptions -import operator -import calendar import psycopg2 from itertools import product - -from multiprocessing import Pool, Lock, Process, Queue, current_process, Manager import multiprocessing - import aiohttp -import aiodns import asyncio -from aiohttp import ClientError, ClientSession, ClientConnectionError, ClientConnectorError, ClientSSLError, ClientConnectorSSLError, ServerTimeoutError -from asyncio import TimeoutError import socket -from socket import gaierror, gethostbyname - -import matplotlib import matplotlib.pyplot as plt plt.style.use('seaborn') @@ -38,7 +22,7 @@ start_time = time.time() from decimal import * getcontext().prec = 2 -apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?', '/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?','/wp-json/nodeinfo/2.0?'] +apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?', '/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?', '/wp-json/nodeinfo/2.0?'] client_exceptions = ( aiohttp.ClientResponseError, @@ -50,23 +34,25 @@ client_exceptions = ( ) def is_json(myjson): + try: json_object = json.loads(myjson) except ValueError as e: + print(e) return False return True def get_alive_servers(server): - + serv_api = '' serv_soft = '' soft_version = '' - conn = None - try: - conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") + conn = None + + conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432") cur = conn.cursor() @@ -74,7 +60,7 @@ def get_alive_servers(server): row = cur.fetchone() - if row != None: + if row is not None: was_alive = row[0] serv_soft = row[1] @@ -172,10 +158,10 @@ def get_alive_servers(server): if alive: - if soft_version != "" and soft_version != None: - print("Server " + str(server) + " (" + serv_soft + " " + soft_version + ") is alive!") + if soft_version is not "" and soft_version is not None: + print("Server " + str(server) + " (" + serv_soft + " " + soft_version + ") is alive!") else: - print("Server " + str(server) + " (" + serv_soft + ") is alive!") + print("Server " + str(server) + " (" + serv_soft + ") is alive!") insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING" @@ -183,7 +169,7 @@ def get_alive_servers(server): try: - conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") + conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432") cur = conn.cursor() @@ -207,54 +193,63 @@ def get_alive_servers(server): except urllib3.exceptions.ProtocolError as protoerr: + print(protoerr) print("Server " + server + " is dead :-(") alive = False pass except requests.exceptions.ChunkedEncodingError as chunkerr: + print(chunkerr) print("Server " + server + " is dead :-(") alive = False pass except KeyError as e: + print(e) print("Server " + server + " is dead :-(") alive = False pass except ValueError as verr: + print(verr) print("Server " + server + " is dead :-(") alive = False pass except requests.exceptions.SSLError as errssl: + print(errssl) print("Server " + server + " is dead :-(") alive = False pass except requests.exceptions.HTTPError as errh: + print(errh) print("Server " + server + " is dead :-(") alive = False pass except requests.exceptions.ConnectionError as errc: + print(errc) print("Server " + server + " is dead :-(") alive = False pass except requests.exceptions.Timeout as errt: + print(errt) print("Server " + server + " is dead :-(") alive = False pass except requests.exceptions.RequestException as err: + print(err) print("Server " + server + " is dead :-(") alive = False pass @@ -264,7 +259,7 @@ def get_alive_servers(server): print(gai_error) pass - if alive == False: + if alive is False: conn = None @@ -298,7 +293,7 @@ def write_api(server, software, users, alive, api, soft_version): try: - conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") + conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432") cur = conn.cursor() @@ -348,31 +343,32 @@ async def getsoft(server): except: pass except aiohttp.ClientConnectorError as err: + print(err) pass else: - if response.status == 200 and api != '/api/v1/instance?': - try: - soft = response_json['software']['name'] - soft = soft.lower() - soft_version = response_json['software']['version'] - users = response_json['usage']['users']['total'] - if users > 1000000: - return - alive = True - write_api(server, soft, users, alive, api, soft_version) - print("Server " + server + " (" + soft + " " + soft_version + ") is alive!") - return - except: - pass - if response.status == 200 and soft == '' and api == "/api/v1/instance?": - soft = 'mastodon' - users = response_json['stats']['user_count'] - soft_version = response_json['version'] - if users > 1000000: - return - alive = True - write_api(server, soft, users, alive, api) - print("Server " + server + " (" + soft + ") is alive!") + if response.status == 200 and api != '/api/v1/instance?': + try: + soft = response_json['software']['name'] + soft = soft.lower() + soft_version = response_json['software']['version'] + users = response_json['usage']['users']['total'] + if users > 1000000: + return + alive = True + write_api(server, soft, users, alive, api, soft_version) + print("Server " + server + " (" + soft + " " + soft_version + ") is alive!") + return + except: + pass + if response.status == 200 and soft == '' and api == "/api/v1/instance?": + soft = 'mastodon' + users = response_json['stats']['user_count'] + soft_version = response_json['version'] + if users > 1000000: + return + alive = True + write_api(server, soft, users, alive, api) + print("Server " + server + " (" + soft + ") is alive!") def getserver(server, x): @@ -381,11 +377,11 @@ def getserver(server, x): if server.find(".") == -1: return if server.find("@") != -1: - return + return if server.find("/") != -1: - return + return if server.find(":") != -1: - return + return try: @@ -406,7 +402,7 @@ def set_world_servers_check_to_false(): conn = None - conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") + conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432") cur = conn.cursor() @@ -436,11 +432,11 @@ def get_last_checked_servers(): try: conn = None - conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") + conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432") cur = conn.cursor() - ### get world servers list + # get world servers list cur.execute("select server from world where server in (select server from fediverse where users_api != '')") @@ -468,9 +464,9 @@ def mastodon(): # Load secrets from secrets file secrets_filepath = "secrets/secrets.txt" - uc_client_id = get_parameter("uc_client_id", secrets_filepath) + uc_client_id = get_parameter("uc_client_id", secrets_filepath) uc_client_secret = get_parameter("uc_client_secret", secrets_filepath) - uc_access_token = get_parameter("uc_access_token", secrets_filepath) + uc_access_token = get_parameter("uc_access_token", secrets_filepath) # Load configuration from config file config_filepath = "config/config.txt" @@ -478,14 +474,14 @@ def mastodon(): # Initialise Mastodon API mastodon = Mastodon( - client_id = uc_client_id, - client_secret = uc_client_secret, - access_token = uc_access_token, - api_base_url = 'https://' + mastodon_hostname, + client_id=uc_client_id, + client_secret=uc_client_secret, + access_token=uc_access_token, + api_base_url='https://' + mastodon_hostname, ) # Initialise access headers - headers={ 'Authorization': 'Bearer %s'%uc_access_token } + headers = {'Authorization': 'Bearer %s'%uc_access_token} return (mastodon, mastodon_hostname) @@ -504,16 +500,17 @@ def usage(): print('usage: python ' + sys.argv[0] + ' --mono' + ' (one process, slow)') # Returns the parameter from the specified file -def get_parameter( parameter, file_path ): +def get_parameter(parameter, file_path): + # Check if secrets file exists if not os.path.isfile(file_path): print("File %s not found, exiting."%file_path) sys.exit(0) # Find parameter in file - with open( file_path ) as f: + with open(file_path) as f: for line in f: - if line.startswith( parameter ): + if line.startswith(parameter): return line.replace(parameter + ":", "").strip() # Cannot find parameter, exit @@ -594,7 +591,7 @@ if __name__ == '__main__': try: conn = None - conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") + conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432") cur = conn.cursor() @@ -632,7 +629,7 @@ if __name__ == '__main__': try: conn = None - conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") + conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432") cur = conn.cursor() @@ -640,7 +637,7 @@ if __name__ == '__main__': row = cur.fetchone() - if row != None: + if row is not None: servers_before = row[0] users_before = row[1] @@ -678,7 +675,7 @@ if __name__ == '__main__': try: - conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") + conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432") cur = conn.cursor() @@ -705,7 +702,7 @@ if __name__ == '__main__': try: - conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") + conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432") cur = conn.cursor() @@ -735,7 +732,7 @@ if __name__ == '__main__': try: - conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") + conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432") cur = conn.cursor() @@ -743,7 +740,7 @@ if __name__ == '__main__': row = cur.fetchone() - if row != None: + if row is not None: max_servers = row[0] @@ -755,7 +752,7 @@ if __name__ == '__main__': row = cur.fetchone() - if row != None: + if row is not None: max_users = row[0] @@ -786,7 +783,7 @@ if __name__ == '__main__': try: - conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") + conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432") cur = conn.cursor() @@ -815,11 +812,11 @@ if __name__ == '__main__': ############################################################################### # generate graphs - plt.plot([-6, -5, -4, -3, -2, -1, 0], [servers_plots[6], servers_plots[5],servers_plots[4],servers_plots[3],servers_plots[2],servers_plots[1],servers_plots[0]], marker='o', color='mediumseagreen') + plt.plot([-6, -5, -4, -3, -2, -1, 0], [servers_plots[6], servers_plots[5], servers_plots[4], servers_plots[3], servers_plots[2], servers_plots[1], servers_plots[0]], marker='o', color='mediumseagreen') plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_servers, max_servers, max_servers, max_servers, max_servers, max_servers, max_servers], color='red') - plt.title('fediverse: total alive servers (max: ' + str(f"{max_servers:,}" + ')') , loc='right', color='blue') + plt.title('fediverse: total alive servers (max: ' + str(f"{max_servers:,}" + ')'), loc='right', color='blue') plt.xlabel('Last seven days') @@ -835,7 +832,7 @@ if __name__ == '__main__': plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_users, max_users, max_users, max_users, max_users, max_users, max_users], color='red') - plt.title('fediverse: total registered users (max: ' + str(f"{max_users:,}" + ')') , loc='right', color='royalblue') + plt.title('fediverse: total registered users (max: ' + str(f"{max_users:,}" + ')'), loc='right', color='royalblue') plt.legend(('users', 'max'), shadow=True, loc=(0.01, 0.80), handlelength=1.5, fontsize=10) @@ -866,7 +863,7 @@ if __name__ == '__main__': toot_text += "max: " + str(f"{max_users:,}") + "\n" toot_text += "\n" - toot_text += "top ten (soft users servers):" + " \n" + toot_text += "top ten (soft users servers):" + " \n" toot_text += "\n" i = 0