Browse Source

Fixed some warnings

master
spla 4 months ago
parent
commit
8185045292
  1. 157
      fediverse.py

157
fediverse.py

@ -1,34 +1,18 @@
import time
import urllib3
from urllib3 import exceptions
from datetime import datetime
from subprocess import call
from mastodon import Mastodon
import threading
import os
import json
import signal
import sys
import os.path
import requests
from requests import exceptions
import operator
import calendar
import psycopg2
from itertools import product
from multiprocessing import Pool, Lock, Process, Queue, current_process, Manager
import multiprocessing
import aiohttp
import aiodns
import asyncio
from aiohttp import ClientError, ClientSession, ClientConnectionError, ClientConnectorError, ClientSSLError, ClientConnectorSSLError, ServerTimeoutError
from asyncio import TimeoutError
import socket
from socket import gaierror, gethostbyname
import matplotlib
import matplotlib.pyplot as plt
plt.style.use('seaborn')
@ -38,7 +22,7 @@ start_time = time.time()
from decimal import *
getcontext().prec = 2
apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?', '/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?','/wp-json/nodeinfo/2.0?']
apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?', '/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?', '/wp-json/nodeinfo/2.0?']
client_exceptions = (
aiohttp.ClientResponseError,
@ -50,23 +34,25 @@ client_exceptions = (
)
def is_json(myjson):
try:
json_object = json.loads(myjson)
except ValueError as e:
print(e)
return False
return True
def get_alive_servers(server):
serv_api = ''
serv_soft = ''
soft_version = ''
conn = None
try:
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
conn = None
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor()
@ -74,7 +60,7 @@ def get_alive_servers(server):
row = cur.fetchone()
if row != None:
if row is not None:
was_alive = row[0]
serv_soft = row[1]
@ -172,10 +158,10 @@ def get_alive_servers(server):
if alive:
if soft_version != "" and soft_version != None:
print("Server " + str(server) + " (" + serv_soft + " " + soft_version + ") is alive!")
if soft_version is not "" and soft_version is not None:
print("Server " + str(server) + " (" + serv_soft + " " + soft_version + ") is alive!")
else:
print("Server " + str(server) + " (" + serv_soft + ") is alive!")
print("Server " + str(server) + " (" + serv_soft + ") is alive!")
insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
@ -183,7 +169,7 @@ def get_alive_servers(server):
try:
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor()
@ -207,54 +193,63 @@ def get_alive_servers(server):
except urllib3.exceptions.ProtocolError as protoerr:
print(protoerr)
print("Server " + server + " is dead :-(")
alive = False
pass
except requests.exceptions.ChunkedEncodingError as chunkerr:
print(chunkerr)
print("Server " + server + " is dead :-(")
alive = False
pass
except KeyError as e:
print(e)
print("Server " + server + " is dead :-(")
alive = False
pass
except ValueError as verr:
print(verr)
print("Server " + server + " is dead :-(")
alive = False
pass
except requests.exceptions.SSLError as errssl:
print(errssl)
print("Server " + server + " is dead :-(")
alive = False
pass
except requests.exceptions.HTTPError as errh:
print(errh)
print("Server " + server + " is dead :-(")
alive = False
pass
except requests.exceptions.ConnectionError as errc:
print(errc)
print("Server " + server + " is dead :-(")
alive = False
pass
except requests.exceptions.Timeout as errt:
print(errt)
print("Server " + server + " is dead :-(")
alive = False
pass
except requests.exceptions.RequestException as err:
print(err)
print("Server " + server + " is dead :-(")
alive = False
pass
@ -264,7 +259,7 @@ def get_alive_servers(server):
print(gai_error)
pass
if alive == False:
if alive is False:
conn = None
@ -298,7 +293,7 @@ def write_api(server, software, users, alive, api, soft_version):
try:
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor()
@ -348,31 +343,32 @@ async def getsoft(server):
except:
pass
except aiohttp.ClientConnectorError as err:
print(err)
pass
else:
if response.status == 200 and api != '/api/v1/instance?':
try:
soft = response_json['software']['name']
soft = soft.lower()
soft_version = response_json['software']['version']
users = response_json['usage']['users']['total']
if users > 1000000:
return
alive = True
write_api(server, soft, users, alive, api, soft_version)
print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
return
except:
pass
if response.status == 200 and soft == '' and api == "/api/v1/instance?":
soft = 'mastodon'
users = response_json['stats']['user_count']
soft_version = response_json['version']
if users > 1000000:
return
alive = True
write_api(server, soft, users, alive, api)
print("Server " + server + " (" + soft + ") is alive!")
if response.status == 200 and api != '/api/v1/instance?':
try:
soft = response_json['software']['name']
soft = soft.lower()
soft_version = response_json['software']['version']
users = response_json['usage']['users']['total']
if users > 1000000:
return
alive = True
write_api(server, soft, users, alive, api, soft_version)
print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
return
except:
pass
if response.status == 200 and soft == '' and api == "/api/v1/instance?":
soft = 'mastodon'
users = response_json['stats']['user_count']
soft_version = response_json['version']
if users > 1000000:
return
alive = True
write_api(server, soft, users, alive, api)
print("Server " + server + " (" + soft + ") is alive!")
def getserver(server, x):
@ -381,11 +377,11 @@ def getserver(server, x):
if server.find(".") == -1:
return
if server.find("@") != -1:
return
return
if server.find("/") != -1:
return
return
if server.find(":") != -1:
return
return
try:
@ -406,7 +402,7 @@ def set_world_servers_check_to_false():
conn = None
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor()
@ -436,11 +432,11 @@ def get_last_checked_servers():
try:
conn = None
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor()
### get world servers list
# get world servers list
cur.execute("select server from world where server in (select server from fediverse where users_api != '')")
@ -468,9 +464,9 @@ def mastodon():
# Load secrets from secrets file
secrets_filepath = "secrets/secrets.txt"
uc_client_id = get_parameter("uc_client_id", secrets_filepath)
uc_client_id = get_parameter("uc_client_id", secrets_filepath)
uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
uc_access_token = get_parameter("uc_access_token", secrets_filepath)
uc_access_token = get_parameter("uc_access_token", secrets_filepath)
# Load configuration from config file
config_filepath = "config/config.txt"
@ -478,14 +474,14 @@ def mastodon():
# Initialise Mastodon API
mastodon = Mastodon(
client_id = uc_client_id,
client_secret = uc_client_secret,
access_token = uc_access_token,
api_base_url = 'https://' + mastodon_hostname,
client_id=uc_client_id,
client_secret=uc_client_secret,
access_token=uc_access_token,
api_base_url='https://' + mastodon_hostname,
)
# Initialise access headers
headers={ 'Authorization': 'Bearer %s'%uc_access_token }
headers = {'Authorization': 'Bearer %s'%uc_access_token}
return (mastodon, mastodon_hostname)
@ -504,16 +500,17 @@ def usage():
print('usage: python ' + sys.argv[0] + ' --mono' + ' (one process, slow)')
# Returns the parameter from the specified file
def get_parameter( parameter, file_path ):
def get_parameter(parameter, file_path):
# Check if secrets file exists
if not os.path.isfile(file_path):
print("File %s not found, exiting."%file_path)
sys.exit(0)
# Find parameter in file
with open( file_path ) as f:
with open(file_path) as f:
for line in f:
if line.startswith( parameter ):
if line.startswith(parameter):
return line.replace(parameter + ":", "").strip()
# Cannot find parameter, exit
@ -594,7 +591,7 @@ if __name__ == '__main__':
try:
conn = None
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor()
@ -632,7 +629,7 @@ if __name__ == '__main__':
try:
conn = None
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor()
@ -640,7 +637,7 @@ if __name__ == '__main__':
row = cur.fetchone()
if row != None:
if row is not None:
servers_before = row[0]
users_before = row[1]
@ -678,7 +675,7 @@ if __name__ == '__main__':
try:
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor()
@ -705,7 +702,7 @@ if __name__ == '__main__':
try:
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor()
@ -735,7 +732,7 @@ if __name__ == '__main__':
try:
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor()
@ -743,7 +740,7 @@ if __name__ == '__main__':
row = cur.fetchone()
if row != None:
if row is not None:
max_servers = row[0]
@ -755,7 +752,7 @@ if __name__ == '__main__':
row = cur.fetchone()
if row != None:
if row is not None:
max_users = row[0]
@ -786,7 +783,7 @@ if __name__ == '__main__':
try:
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor()
@ -815,11 +812,11 @@ if __name__ == '__main__':
###############################################################################
# generate graphs
plt.plot([-6, -5, -4, -3, -2, -1, 0], [servers_plots[6], servers_plots[5],servers_plots[4],servers_plots[3],servers_plots[2],servers_plots[1],servers_plots[0]], marker='o', color='mediumseagreen')
plt.plot([-6, -5, -4, -3, -2, -1, 0], [servers_plots[6], servers_plots[5], servers_plots[4], servers_plots[3], servers_plots[2], servers_plots[1], servers_plots[0]], marker='o', color='mediumseagreen')
plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_servers, max_servers, max_servers, max_servers, max_servers, max_servers, max_servers], color='red')
plt.title('fediverse: total alive servers (max: ' + str(f"{max_servers:,}" + ')') , loc='right', color='blue')
plt.title('fediverse: total alive servers (max: ' + str(f"{max_servers:,}" + ')'), loc='right', color='blue')
plt.xlabel('Last seven days')
@ -835,7 +832,7 @@ if __name__ == '__main__':
plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_users, max_users, max_users, max_users, max_users, max_users, max_users], color='red')
plt.title('fediverse: total registered users (max: ' + str(f"{max_users:,}" + ')') , loc='right', color='royalblue')
plt.title('fediverse: total registered users (max: ' + str(f"{max_users:,}" + ')'), loc='right', color='royalblue')
plt.legend(('users', 'max'), shadow=True, loc=(0.01, 0.80), handlelength=1.5, fontsize=10)
@ -866,7 +863,7 @@ if __name__ == '__main__':
toot_text += "max: " + str(f"{max_users:,}") + "\n"
toot_text += "\n"
toot_text += "top ten (soft users servers):" + " \n"
toot_text += "top ten (soft users servers):" + " \n"
toot_text += "\n"
i = 0

Loading…
Cancel
Save