spla
/
fediverse
1
1
Fork 1
Codi Incidències Sol·licituds Pull Projectes Llançaments Wiki Activitat

Fixed some warnings

This commit is contained in:
spla 2021-05-14 13:14:31 +02:00
pare a935ef8029
commit 8185045292
S'han modificat 1 arxius amb 77 adicions i 80 eliminacions
Mostrar estadístiques Descarregar el fitxer de pedaç Descarregar el fitxer de diferències Expand all files Collapse all files

157
fediverse.py
Veure arxiu

@ -1,34 +1,18 @@
import time import time
import urllib3 import urllib3
from urllib3 import exceptions
from datetime import datetime from datetime import datetime
from subprocess import call
from mastodon import Mastodon from mastodon import Mastodon
import threading
import os import os
import json import json
import signal
import sys import sys
import os.path import os.path
import requests import requests
from requests import exceptions
import operator
import calendar
import psycopg2 import psycopg2
from itertools import product from itertools import product
from multiprocessing import Pool, Lock, Process, Queue, current_process, Manager
import multiprocessing import multiprocessing
import aiohttp import aiohttp
import aiodns
import asyncio import asyncio
from aiohttp import ClientError, ClientSession, ClientConnectionError, ClientConnectorError, ClientSSLError, ClientConnectorSSLError, ServerTimeoutError
from asyncio import TimeoutError
import socket import socket
from socket import gaierror, gethostbyname
import matplotlib
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
plt.style.use('seaborn') plt.style.use('seaborn')
@ -38,7 +22,7 @@ start_time = time.time()
from decimal import * from decimal import *
getcontext().prec = 2 getcontext().prec = 2
apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?', '/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?','/wp-json/nodeinfo/2.0?'] apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?', '/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?', '/wp-json/nodeinfo/2.0?']
client_exceptions = ( client_exceptions = (
aiohttp.ClientResponseError, aiohttp.ClientResponseError,
@ -50,23 +34,25 @@ client_exceptions = (
) )
def is_json(myjson): def is_json(myjson):
try: try:
json_object = json.loads(myjson) json_object = json.loads(myjson)
except ValueError as e: except ValueError as e:
print(e)
return False return False
return True return True
def get_alive_servers(server): def get_alive_servers(server):
serv_api = '' serv_api = ''
serv_soft = '' serv_soft = ''
soft_version = '' soft_version = ''
conn = None
try: try:
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") conn = None
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor() cur = conn.cursor()
@ -74,7 +60,7 @@ def get_alive_servers(server):
row = cur.fetchone() row = cur.fetchone()
if row != None: if row is not None:
was_alive = row[0] was_alive = row[0]
serv_soft = row[1] serv_soft = row[1]
@ -172,10 +158,10 @@ def get_alive_servers(server):
if alive: if alive:
if soft_version != "" and soft_version != None: if soft_version is not "" and soft_version is not None:
print("Server " + str(server) + " (" + serv_soft + " " + soft_version + ") is alive!") print("Server " + str(server) + " (" + serv_soft + " " + soft_version + ") is alive!")
else: else:
print("Server " + str(server) + " (" + serv_soft + ") is alive!") print("Server " + str(server) + " (" + serv_soft + ") is alive!")
insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING" insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
@ -183,7 +169,7 @@ def get_alive_servers(server):
try: try:
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor() cur = conn.cursor()
@ -207,54 +193,63 @@ def get_alive_servers(server):
except urllib3.exceptions.ProtocolError as protoerr: except urllib3.exceptions.ProtocolError as protoerr:
print(protoerr)
print("Server " + server + " is dead :-(") print("Server " + server + " is dead :-(")
alive = False alive = False
pass pass
except requests.exceptions.ChunkedEncodingError as chunkerr: except requests.exceptions.ChunkedEncodingError as chunkerr:
print(chunkerr)
print("Server " + server + " is dead :-(") print("Server " + server + " is dead :-(")
alive = False alive = False
pass pass
except KeyError as e: except KeyError as e:
print(e)
print("Server " + server + " is dead :-(") print("Server " + server + " is dead :-(")
alive = False alive = False
pass pass
except ValueError as verr: except ValueError as verr:
print(verr)
print("Server " + server + " is dead :-(") print("Server " + server + " is dead :-(")
alive = False alive = False
pass pass
except requests.exceptions.SSLError as errssl: except requests.exceptions.SSLError as errssl:
print(errssl)
print("Server " + server + " is dead :-(") print("Server " + server + " is dead :-(")
alive = False alive = False
pass pass
except requests.exceptions.HTTPError as errh: except requests.exceptions.HTTPError as errh:
print(errh)
print("Server " + server + " is dead :-(") print("Server " + server + " is dead :-(")
alive = False alive = False
pass pass
except requests.exceptions.ConnectionError as errc: except requests.exceptions.ConnectionError as errc:
print(errc)
print("Server " + server + " is dead :-(") print("Server " + server + " is dead :-(")
alive = False alive = False
pass pass
except requests.exceptions.Timeout as errt: except requests.exceptions.Timeout as errt:
print(errt)
print("Server " + server + " is dead :-(") print("Server " + server + " is dead :-(")
alive = False alive = False
pass pass
except requests.exceptions.RequestException as err: except requests.exceptions.RequestException as err:
print(err)
print("Server " + server + " is dead :-(") print("Server " + server + " is dead :-(")
alive = False alive = False
pass pass
@ -264,7 +259,7 @@ def get_alive_servers(server):
print(gai_error) print(gai_error)
pass pass
if alive == False: if alive is False:
conn = None conn = None
@ -298,7 +293,7 @@ def write_api(server, software, users, alive, api, soft_version):
try: try:
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor() cur = conn.cursor()
@ -348,31 +343,32 @@ async def getsoft(server):
except: except:
pass pass
except aiohttp.ClientConnectorError as err: except aiohttp.ClientConnectorError as err:
print(err)
pass pass
else: else:
if response.status == 200 and api != '/api/v1/instance?': if response.status == 200 and api != '/api/v1/instance?':
try: try:
soft = response_json['software']['name'] soft = response_json['software']['name']
soft = soft.lower() soft = soft.lower()
soft_version = response_json['software']['version'] soft_version = response_json['software']['version']
users = response_json['usage']['users']['total'] users = response_json['usage']['users']['total']
if users > 1000000: if users > 1000000:
return return
alive = True alive = True
write_api(server, soft, users, alive, api, soft_version) write_api(server, soft, users, alive, api, soft_version)
print("Server " + server + " (" + soft + " " + soft_version + ") is alive!") print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
return return
except: except:
pass pass
if response.status == 200 and soft == '' and api == "/api/v1/instance?": if response.status == 200 and soft == '' and api == "/api/v1/instance?":
soft = 'mastodon' soft = 'mastodon'
users = response_json['stats']['user_count'] users = response_json['stats']['user_count']
soft_version = response_json['version'] soft_version = response_json['version']
if users > 1000000: if users > 1000000:
return return
alive = True alive = True
write_api(server, soft, users, alive, api) write_api(server, soft, users, alive, api)
print("Server " + server + " (" + soft + ") is alive!") print("Server " + server + " (" + soft + ") is alive!")
def getserver(server, x): def getserver(server, x):
@ -381,11 +377,11 @@ def getserver(server, x):
if server.find(".") == -1: if server.find(".") == -1:
return return
if server.find("@") != -1: if server.find("@") != -1:
return return
if server.find("/") != -1: if server.find("/") != -1:
return return
if server.find(":") != -1: if server.find(":") != -1:
return return
try: try:
@ -406,7 +402,7 @@ def set_world_servers_check_to_false():
conn = None conn = None
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor() cur = conn.cursor()
@ -436,11 +432,11 @@ def get_last_checked_servers():
try: try:
conn = None conn = None
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor() cur = conn.cursor()
### get world servers list # get world servers list
cur.execute("select server from world where server in (select server from fediverse where users_api != '')") cur.execute("select server from world where server in (select server from fediverse where users_api != '')")
@ -468,9 +464,9 @@ def mastodon():
# Load secrets from secrets file # Load secrets from secrets file
secrets_filepath = "secrets/secrets.txt" secrets_filepath = "secrets/secrets.txt"
uc_client_id = get_parameter("uc_client_id", secrets_filepath) uc_client_id = get_parameter("uc_client_id", secrets_filepath)
uc_client_secret = get_parameter("uc_client_secret", secrets_filepath) uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
uc_access_token = get_parameter("uc_access_token", secrets_filepath) uc_access_token = get_parameter("uc_access_token", secrets_filepath)
# Load configuration from config file # Load configuration from config file
config_filepath = "config/config.txt" config_filepath = "config/config.txt"
@ -478,14 +474,14 @@ def mastodon():
# Initialise Mastodon API # Initialise Mastodon API
mastodon = Mastodon( mastodon = Mastodon(
client_id = uc_client_id, client_id=uc_client_id,
client_secret = uc_client_secret, client_secret=uc_client_secret,
access_token = uc_access_token, access_token=uc_access_token,
api_base_url = 'https://' + mastodon_hostname, api_base_url='https://' + mastodon_hostname,
) )
# Initialise access headers # Initialise access headers
headers={ 'Authorization': 'Bearer %s'%uc_access_token } headers = {'Authorization': 'Bearer %s'%uc_access_token}
return (mastodon, mastodon_hostname) return (mastodon, mastodon_hostname)
@ -504,16 +500,17 @@ def usage():
print('usage: python ' + sys.argv[0] + ' --mono' + ' (one process, slow)') print('usage: python ' + sys.argv[0] + ' --mono' + ' (one process, slow)')
# Returns the parameter from the specified file # Returns the parameter from the specified file
def get_parameter( parameter, file_path ): def get_parameter(parameter, file_path):
# Check if secrets file exists # Check if secrets file exists
if not os.path.isfile(file_path): if not os.path.isfile(file_path):
print("File %s not found, exiting."%file_path) print("File %s not found, exiting."%file_path)
sys.exit(0) sys.exit(0)
# Find parameter in file # Find parameter in file
with open( file_path ) as f: with open(file_path) as f:
for line in f: for line in f:
if line.startswith( parameter ): if line.startswith(parameter):
return line.replace(parameter + ":", "").strip() return line.replace(parameter + ":", "").strip()
# Cannot find parameter, exit # Cannot find parameter, exit
@ -594,7 +591,7 @@ if __name__ == '__main__':
try: try:
conn = None conn = None
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor() cur = conn.cursor()
@ -632,7 +629,7 @@ if __name__ == '__main__':
try: try:
conn = None conn = None
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor() cur = conn.cursor()
@ -640,7 +637,7 @@ if __name__ == '__main__':
row = cur.fetchone() row = cur.fetchone()
if row != None: if row is not None:
servers_before = row[0] servers_before = row[0]
users_before = row[1] users_before = row[1]
@ -678,7 +675,7 @@ if __name__ == '__main__':
try: try:
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor() cur = conn.cursor()
@ -705,7 +702,7 @@ if __name__ == '__main__':
try: try:
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor() cur = conn.cursor()
@ -735,7 +732,7 @@ if __name__ == '__main__':
try: try:
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor() cur = conn.cursor()
@ -743,7 +740,7 @@ if __name__ == '__main__':
row = cur.fetchone() row = cur.fetchone()
if row != None: if row is not None:
max_servers = row[0] max_servers = row[0]
@ -755,7 +752,7 @@ if __name__ == '__main__':
row = cur.fetchone() row = cur.fetchone()
if row != None: if row is not None:
max_users = row[0] max_users = row[0]
@ -786,7 +783,7 @@ if __name__ == '__main__':
try: try:
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
cur = conn.cursor() cur = conn.cursor()
@ -815,11 +812,11 @@ if __name__ == '__main__':
############################################################################### ###############################################################################
# generate graphs # generate graphs
plt.plot([-6, -5, -4, -3, -2, -1, 0], [servers_plots[6], servers_plots[5],servers_plots[4],servers_plots[3],servers_plots[2],servers_plots[1],servers_plots[0]], marker='o', color='mediumseagreen') plt.plot([-6, -5, -4, -3, -2, -1, 0], [servers_plots[6], servers_plots[5], servers_plots[4], servers_plots[3], servers_plots[2], servers_plots[1], servers_plots[0]], marker='o', color='mediumseagreen')
plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_servers, max_servers, max_servers, max_servers, max_servers, max_servers, max_servers], color='red') plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_servers, max_servers, max_servers, max_servers, max_servers, max_servers, max_servers], color='red')
plt.title('fediverse: total alive servers (max: ' + str(f"{max_servers:,}" + ')') , loc='right', color='blue') plt.title('fediverse: total alive servers (max: ' + str(f"{max_servers:,}" + ')'), loc='right', color='blue')
plt.xlabel('Last seven days') plt.xlabel('Last seven days')
@ -835,7 +832,7 @@ if __name__ == '__main__':
plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_users, max_users, max_users, max_users, max_users, max_users, max_users], color='red') plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_users, max_users, max_users, max_users, max_users, max_users, max_users], color='red')
plt.title('fediverse: total registered users (max: ' + str(f"{max_users:,}" + ')') , loc='right', color='royalblue') plt.title('fediverse: total registered users (max: ' + str(f"{max_users:,}" + ')'), loc='right', color='royalblue')
plt.legend(('users', 'max'), shadow=True, loc=(0.01, 0.80), handlelength=1.5, fontsize=10) plt.legend(('users', 'max'), shadow=True, loc=(0.01, 0.80), handlelength=1.5, fontsize=10)
@ -866,7 +863,7 @@ if __name__ == '__main__':
toot_text += "max: " + str(f"{max_users:,}") + "\n" toot_text += "max: " + str(f"{max_users:,}") + "\n"
toot_text += "\n" toot_text += "\n"
toot_text += "top ten (soft users servers):" + " \n" toot_text += "top ten (soft users servers):" + " \n"
toot_text += "\n" toot_text += "\n"
i = 0 i = 0