forkeado de spla/fediverse
Fixed some warnings
This commit is contained in:
spla
pare
a935ef8029
commit
8185045292
S'han modificat 1 arxius amb 77 adicions i 80 eliminacions
155
fediverse.py
155
fediverse.py
|
|
@ -1,34 +1,18 @@
|
|||
import time
|
||||
import urllib3
|
||||
from urllib3 import exceptions
|
||||
from datetime import datetime
|
||||
from subprocess import call
|
||||
from mastodon import Mastodon
|
||||
import threading
|
||||
import os
|
||||
import json
|
||||
import signal
|
||||
import sys
|
||||
import os.path
|
||||
import requests
|
||||
from requests import exceptions
|
||||
import operator
|
||||
import calendar
|
||||
import psycopg2
|
||||
from itertools import product
|
||||
|
||||
from multiprocessing import Pool, Lock, Process, Queue, current_process, Manager
|
||||
import multiprocessing
|
||||
|
||||
import aiohttp
|
||||
import aiodns
|
||||
import asyncio
|
||||
from aiohttp import ClientError, ClientSession, ClientConnectionError, ClientConnectorError, ClientSSLError, ClientConnectorSSLError, ServerTimeoutError
|
||||
from asyncio import TimeoutError
|
||||
import socket
|
||||
from socket import gaierror, gethostbyname
|
||||
|
||||
import matplotlib
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
plt.style.use('seaborn')
|
||||
|
|
@ -38,7 +22,7 @@ start_time = time.time()
|
|||
from decimal import *
|
||||
getcontext().prec = 2
|
||||
|
||||
apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?', '/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?','/wp-json/nodeinfo/2.0?']
|
||||
apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?', '/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?', '/wp-json/nodeinfo/2.0?']
|
||||
|
||||
client_exceptions = (
|
||||
aiohttp.ClientResponseError,
|
||||
|
|
@ -50,9 +34,11 @@ client_exceptions = (
|
|||
)
|
||||
|
||||
def is_json(myjson):
|
||||
|
||||
try:
|
||||
json_object = json.loads(myjson)
|
||||
except ValueError as e:
|
||||
print(e)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
|
@ -62,11 +48,11 @@ def get_alive_servers(server):
|
|||
serv_soft = ''
|
||||
soft_version = ''
|
||||
|
||||
conn = None
|
||||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
conn = None
|
||||
|
||||
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
|
|
@ -74,7 +60,7 @@ def get_alive_servers(server):
|
|||
|
||||
row = cur.fetchone()
|
||||
|
||||
if row != None:
|
||||
if row is not None:
|
||||
|
||||
was_alive = row[0]
|
||||
serv_soft = row[1]
|
||||
|
|
@ -172,10 +158,10 @@ def get_alive_servers(server):
|
|||
|
||||
if alive:
|
||||
|
||||
if soft_version != "" and soft_version != None:
|
||||
print("Server " + str(server) + " (" + serv_soft + " " + soft_version + ") is alive!")
|
||||
if soft_version is not "" and soft_version is not None:
|
||||
print("Server " + str(server) + " (" + serv_soft + " " + soft_version + ") is alive!")
|
||||
else:
|
||||
print("Server " + str(server) + " (" + serv_soft + ") is alive!")
|
||||
print("Server " + str(server) + " (" + serv_soft + ") is alive!")
|
||||
|
||||
insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
|
||||
|
||||
|
|
@ -183,7 +169,7 @@ def get_alive_servers(server):
|
|||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
|
|
@ -207,54 +193,63 @@ def get_alive_servers(server):
|
|||
|
||||
except urllib3.exceptions.ProtocolError as protoerr:
|
||||
|
||||
print(protoerr)
|
||||
print("Server " + server + " is dead :-(")
|
||||
alive = False
|
||||
pass
|
||||
|
||||
except requests.exceptions.ChunkedEncodingError as chunkerr:
|
||||
|
||||
print(chunkerr)
|
||||
print("Server " + server + " is dead :-(")
|
||||
alive = False
|
||||
pass
|
||||
|
||||
except KeyError as e:
|
||||
|
||||
print(e)
|
||||
print("Server " + server + " is dead :-(")
|
||||
alive = False
|
||||
pass
|
||||
|
||||
except ValueError as verr:
|
||||
|
||||
print(verr)
|
||||
print("Server " + server + " is dead :-(")
|
||||
alive = False
|
||||
pass
|
||||
|
||||
except requests.exceptions.SSLError as errssl:
|
||||
|
||||
print(errssl)
|
||||
print("Server " + server + " is dead :-(")
|
||||
alive = False
|
||||
pass
|
||||
|
||||
except requests.exceptions.HTTPError as errh:
|
||||
|
||||
print(errh)
|
||||
print("Server " + server + " is dead :-(")
|
||||
alive = False
|
||||
pass
|
||||
|
||||
except requests.exceptions.ConnectionError as errc:
|
||||
|
||||
print(errc)
|
||||
print("Server " + server + " is dead :-(")
|
||||
alive = False
|
||||
pass
|
||||
|
||||
except requests.exceptions.Timeout as errt:
|
||||
|
||||
print(errt)
|
||||
print("Server " + server + " is dead :-(")
|
||||
alive = False
|
||||
pass
|
||||
|
||||
except requests.exceptions.RequestException as err:
|
||||
|
||||
print(err)
|
||||
print("Server " + server + " is dead :-(")
|
||||
alive = False
|
||||
pass
|
||||
|
|
@ -264,7 +259,7 @@ def get_alive_servers(server):
|
|||
print(gai_error)
|
||||
pass
|
||||
|
||||
if alive == False:
|
||||
if alive is False:
|
||||
|
||||
conn = None
|
||||
|
||||
|
|
@ -298,7 +293,7 @@ def write_api(server, software, users, alive, api, soft_version):
|
|||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
|
|
@ -348,31 +343,32 @@ async def getsoft(server):
|
|||
except:
|
||||
pass
|
||||
except aiohttp.ClientConnectorError as err:
|
||||
print(err)
|
||||
pass
|
||||
else:
|
||||
if response.status == 200 and api != '/api/v1/instance?':
|
||||
try:
|
||||
soft = response_json['software']['name']
|
||||
soft = soft.lower()
|
||||
soft_version = response_json['software']['version']
|
||||
users = response_json['usage']['users']['total']
|
||||
if users > 1000000:
|
||||
return
|
||||
alive = True
|
||||
write_api(server, soft, users, alive, api, soft_version)
|
||||
print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
|
||||
return
|
||||
except:
|
||||
pass
|
||||
if response.status == 200 and soft == '' and api == "/api/v1/instance?":
|
||||
soft = 'mastodon'
|
||||
users = response_json['stats']['user_count']
|
||||
soft_version = response_json['version']
|
||||
if users > 1000000:
|
||||
return
|
||||
alive = True
|
||||
write_api(server, soft, users, alive, api)
|
||||
print("Server " + server + " (" + soft + ") is alive!")
|
||||
if response.status == 200 and api != '/api/v1/instance?':
|
||||
try:
|
||||
soft = response_json['software']['name']
|
||||
soft = soft.lower()
|
||||
soft_version = response_json['software']['version']
|
||||
users = response_json['usage']['users']['total']
|
||||
if users > 1000000:
|
||||
return
|
||||
alive = True
|
||||
write_api(server, soft, users, alive, api, soft_version)
|
||||
print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
|
||||
return
|
||||
except:
|
||||
pass
|
||||
if response.status == 200 and soft == '' and api == "/api/v1/instance?":
|
||||
soft = 'mastodon'
|
||||
users = response_json['stats']['user_count']
|
||||
soft_version = response_json['version']
|
||||
if users > 1000000:
|
||||
return
|
||||
alive = True
|
||||
write_api(server, soft, users, alive, api)
|
||||
print("Server " + server + " (" + soft + ") is alive!")
|
||||
|
||||
def getserver(server, x):
|
||||
|
||||
|
|
@ -381,11 +377,11 @@ def getserver(server, x):
|
|||
if server.find(".") == -1:
|
||||
return
|
||||
if server.find("@") != -1:
|
||||
return
|
||||
return
|
||||
if server.find("/") != -1:
|
||||
return
|
||||
return
|
||||
if server.find(":") != -1:
|
||||
return
|
||||
return
|
||||
|
||||
try:
|
||||
|
||||
|
|
@ -406,7 +402,7 @@ def set_world_servers_check_to_false():
|
|||
|
||||
conn = None
|
||||
|
||||
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
|
|
@ -436,11 +432,11 @@ def get_last_checked_servers():
|
|||
try:
|
||||
|
||||
conn = None
|
||||
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
### get world servers list
|
||||
# get world servers list
|
||||
|
||||
cur.execute("select server from world where server in (select server from fediverse where users_api != '')")
|
||||
|
||||
|
|
@ -468,9 +464,9 @@ def mastodon():
|
|||
|
||||
# Load secrets from secrets file
|
||||
secrets_filepath = "secrets/secrets.txt"
|
||||
uc_client_id = get_parameter("uc_client_id", secrets_filepath)
|
||||
uc_client_id = get_parameter("uc_client_id", secrets_filepath)
|
||||
uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
|
||||
uc_access_token = get_parameter("uc_access_token", secrets_filepath)
|
||||
uc_access_token = get_parameter("uc_access_token", secrets_filepath)
|
||||
|
||||
# Load configuration from config file
|
||||
config_filepath = "config/config.txt"
|
||||
|
|
@ -478,14 +474,14 @@ def mastodon():
|
|||
|
||||
# Initialise Mastodon API
|
||||
mastodon = Mastodon(
|
||||
client_id = uc_client_id,
|
||||
client_secret = uc_client_secret,
|
||||
access_token = uc_access_token,
|
||||
api_base_url = 'https://' + mastodon_hostname,
|
||||
client_id=uc_client_id,
|
||||
client_secret=uc_client_secret,
|
||||
access_token=uc_access_token,
|
||||
api_base_url='https://' + mastodon_hostname,
|
||||
)
|
||||
|
||||
# Initialise access headers
|
||||
headers={ 'Authorization': 'Bearer %s'%uc_access_token }
|
||||
headers = {'Authorization': 'Bearer %s'%uc_access_token}
|
||||
|
||||
return (mastodon, mastodon_hostname)
|
||||
|
||||
|
|
@ -504,16 +500,17 @@ def usage():
|
|||
print('usage: python ' + sys.argv[0] + ' --mono' + ' (one process, slow)')
|
||||
|
||||
# Returns the parameter from the specified file
|
||||
def get_parameter( parameter, file_path ):
|
||||
def get_parameter(parameter, file_path):
|
||||
|
||||
# Check if secrets file exists
|
||||
if not os.path.isfile(file_path):
|
||||
print("File %s not found, exiting."%file_path)
|
||||
sys.exit(0)
|
||||
|
||||
# Find parameter in file
|
||||
with open( file_path ) as f:
|
||||
with open(file_path) as f:
|
||||
for line in f:
|
||||
if line.startswith( parameter ):
|
||||
if line.startswith(parameter):
|
||||
return line.replace(parameter + ":", "").strip()
|
||||
|
||||
# Cannot find parameter, exit
|
||||
|
|
@ -594,7 +591,7 @@ if __name__ == '__main__':
|
|||
try:
|
||||
|
||||
conn = None
|
||||
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
|
|
@ -632,7 +629,7 @@ if __name__ == '__main__':
|
|||
try:
|
||||
|
||||
conn = None
|
||||
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
|
|
@ -640,7 +637,7 @@ if __name__ == '__main__':
|
|||
|
||||
row = cur.fetchone()
|
||||
|
||||
if row != None:
|
||||
if row is not None:
|
||||
|
||||
servers_before = row[0]
|
||||
users_before = row[1]
|
||||
|
|
@ -678,7 +675,7 @@ if __name__ == '__main__':
|
|||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
|
|
@ -705,7 +702,7 @@ if __name__ == '__main__':
|
|||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
|
|
@ -735,7 +732,7 @@ if __name__ == '__main__':
|
|||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
|
|
@ -743,7 +740,7 @@ if __name__ == '__main__':
|
|||
|
||||
row = cur.fetchone()
|
||||
|
||||
if row != None:
|
||||
if row is not None:
|
||||
|
||||
max_servers = row[0]
|
||||
|
||||
|
|
@ -755,7 +752,7 @@ if __name__ == '__main__':
|
|||
|
||||
row = cur.fetchone()
|
||||
|
||||
if row != None:
|
||||
if row is not None:
|
||||
|
||||
max_users = row[0]
|
||||
|
||||
|
|
@ -786,7 +783,7 @@ if __name__ == '__main__':
|
|||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
|
|
@ -815,11 +812,11 @@ if __name__ == '__main__':
|
|||
###############################################################################
|
||||
# generate graphs
|
||||
|
||||
plt.plot([-6, -5, -4, -3, -2, -1, 0], [servers_plots[6], servers_plots[5],servers_plots[4],servers_plots[3],servers_plots[2],servers_plots[1],servers_plots[0]], marker='o', color='mediumseagreen')
|
||||
plt.plot([-6, -5, -4, -3, -2, -1, 0], [servers_plots[6], servers_plots[5], servers_plots[4], servers_plots[3], servers_plots[2], servers_plots[1], servers_plots[0]], marker='o', color='mediumseagreen')
|
||||
|
||||
plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_servers, max_servers, max_servers, max_servers, max_servers, max_servers, max_servers], color='red')
|
||||
|
||||
plt.title('fediverse: total alive servers (max: ' + str(f"{max_servers:,}" + ')') , loc='right', color='blue')
|
||||
plt.title('fediverse: total alive servers (max: ' + str(f"{max_servers:,}" + ')'), loc='right', color='blue')
|
||||
|
||||
plt.xlabel('Last seven days')
|
||||
|
||||
|
|
@ -835,7 +832,7 @@ if __name__ == '__main__':
|
|||
|
||||
plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_users, max_users, max_users, max_users, max_users, max_users, max_users], color='red')
|
||||
|
||||
plt.title('fediverse: total registered users (max: ' + str(f"{max_users:,}" + ')') , loc='right', color='royalblue')
|
||||
plt.title('fediverse: total registered users (max: ' + str(f"{max_users:,}" + ')'), loc='right', color='royalblue')
|
||||
|
||||
plt.legend(('users', 'max'), shadow=True, loc=(0.01, 0.80), handlelength=1.5, fontsize=10)
|
||||
|
||||
|
|
@ -866,7 +863,7 @@ if __name__ == '__main__':
|
|||
toot_text += "max: " + str(f"{max_users:,}") + "\n"
|
||||
|
||||
toot_text += "\n"
|
||||
toot_text += "top ten (soft users servers):" + " \n"
|
||||
toot_text += "top ten (soft users servers):" + " \n"
|
||||
toot_text += "\n"
|
||||
|
||||
i = 0
|
||||
|
|
|
|||
Loading…
Referencia en una nova incidència