forkeado de spla/fediverse
Fixed some warnings
This commit is contained in:
spla
pare
8185045292
commit
246ccf869c
S'han modificat 1 arxius amb 51 adicions i 71 eliminacions
122
fetchservers.py
122
fetchservers.py
|
|
@ -1,38 +1,17 @@
|
|||
import time
|
||||
start_time = time.time()
|
||||
import urllib3
|
||||
from urllib3 import exceptions
|
||||
from datetime import datetime
|
||||
from subprocess import call
|
||||
from mastodon import Mastodon
|
||||
import threading
|
||||
import os
|
||||
import json
|
||||
import signal
|
||||
import sys
|
||||
import os.path
|
||||
import requests
|
||||
from requests import exceptions
|
||||
import operator
|
||||
import calendar
|
||||
import psycopg2
|
||||
from itertools import product
|
||||
|
||||
from multiprocessing import Pool, Lock, Process, Queue, current_process, Manager
|
||||
import multiprocessing
|
||||
|
||||
from multiprocessing import Pool, Manager
|
||||
import aiohttp
|
||||
import aiodns
|
||||
import asyncio
|
||||
from aiohttp import ClientError, ClientSession, ClientConnectionError, ClientConnectorError, ClientSSLError, ClientConnectorSSLError, ServerTimeoutError
|
||||
from asyncio import TimeoutError
|
||||
import socket
|
||||
from socket import gaierror, gethostbyname
|
||||
|
||||
from decimal import *
|
||||
getcontext().prec = 2
|
||||
|
||||
apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?', '/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?','/wp-json/nodeinfo/2.0?']
|
||||
apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?',
|
||||
'/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?', '/wp-json/nodeinfo/2.0?']
|
||||
|
||||
client_exceptions = (
|
||||
aiohttp.ClientResponseError,
|
||||
|
|
@ -43,11 +22,6 @@ client_exceptions = (
|
|||
socket.gaierror,
|
||||
)
|
||||
|
||||
now = datetime.now()
|
||||
|
||||
###############################################################################
|
||||
# INITIALISATION
|
||||
###############################################################################
|
||||
|
||||
def is_json(myjson):
|
||||
try:
|
||||
|
|
@ -56,20 +30,23 @@ def is_json(myjson):
|
|||
return False
|
||||
return True
|
||||
|
||||
def write_api(server, software, users, alive, api, soft_version):
|
||||
|
||||
def write_api(server, software, users, alive, api, soft_version):
|
||||
insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
|
||||
conn = None
|
||||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql",
|
||||
port="5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute(insert_sql, (server, now, software, users, alive, api, soft_version))
|
||||
|
||||
cur.execute("UPDATE fediverse SET updated_at=(%s), software=(%s), users=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)", (now, software, users, alive, api, soft_version, server))
|
||||
cur.execute(
|
||||
"UPDATE fediverse SET updated_at=(%s), software=(%s), users=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)",
|
||||
(now, software, users, alive, api, soft_version, server))
|
||||
|
||||
cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
|
||||
|
||||
|
|
@ -84,11 +61,10 @@ def write_api(server, software, users, alive, api, soft_version):
|
|||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
async def getsoft(server):
|
||||
|
||||
async def getsoft(server):
|
||||
try:
|
||||
|
||||
socket.gethostbyname(server)
|
||||
|
|
@ -106,7 +82,7 @@ async def getsoft(server):
|
|||
async with aiohttp.ClientSession(timeout=timeout) as session:
|
||||
for api in apis:
|
||||
try:
|
||||
async with session.get(url+api) as response:
|
||||
async with session.get(url + api) as response:
|
||||
if response.status == 200:
|
||||
try:
|
||||
response_json = await response.json()
|
||||
|
|
@ -115,29 +91,30 @@ async def getsoft(server):
|
|||
except aiohttp.ClientConnectorError as err:
|
||||
pass
|
||||
else:
|
||||
if response.status == 200 and api != '/api/v1/instance?':
|
||||
try:
|
||||
soft = response_json['software']['name']
|
||||
soft = soft.lower()
|
||||
soft_version = response_json['software']['version']
|
||||
users = response_json['usage']['users']['total']
|
||||
if users > 1000000:
|
||||
return
|
||||
alive = True
|
||||
write_api(server, soft, users, alive, api, soft_version)
|
||||
print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
|
||||
return
|
||||
except:
|
||||
pass
|
||||
if response.status == 200 and soft == '' and api == "/api/v1/instance?":
|
||||
soft = 'mastodon'
|
||||
users = response_json['stats']['user_count']
|
||||
soft_version = response_json['version']
|
||||
if users > 1000000:
|
||||
return
|
||||
alive = True
|
||||
write_api(server, soft, users, alive, api)
|
||||
print("Server " + server + " (" + soft + ") is alive!")
|
||||
if response.status == 200 and api != '/api/v1/instance?':
|
||||
try:
|
||||
soft = response_json['software']['name']
|
||||
soft = soft.lower()
|
||||
soft_version = response_json['software']['version']
|
||||
users = response_json['usage']['users']['total']
|
||||
if users > 1000000:
|
||||
return
|
||||
alive = True
|
||||
write_api(server, soft, users, alive, api, soft_version)
|
||||
print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
|
||||
return
|
||||
except:
|
||||
pass
|
||||
if response.status == 200 and soft == '' and api == "/api/v1/instance?":
|
||||
soft = 'mastodon'
|
||||
users = response_json['stats']['user_count']
|
||||
soft_version = response_json['version']
|
||||
if users > 1000000:
|
||||
return
|
||||
alive = True
|
||||
write_api(server, soft, users, alive, api)
|
||||
print("Server " + server + " (" + soft + ") is alive!")
|
||||
|
||||
|
||||
def getserver(server, x):
|
||||
|
||||
|
|
@ -146,11 +123,11 @@ def getserver(server, x):
|
|||
if server.find(".") == -1:
|
||||
return
|
||||
if server.find("@") != -1:
|
||||
return
|
||||
return
|
||||
if server.find("/") != -1:
|
||||
return
|
||||
return
|
||||
if server.find(":") != -1:
|
||||
return
|
||||
return
|
||||
|
||||
try:
|
||||
|
||||
|
|
@ -162,21 +139,22 @@ def getserver(server, x):
|
|||
|
||||
pass
|
||||
|
||||
|
||||
# Returns the parameter from the specified file
|
||||
def get_parameter( parameter, file_path ):
|
||||
def get_parameter(parameter, file_path):
|
||||
# Check if secrets file exists
|
||||
if not os.path.isfile(file_path):
|
||||
print("File %s not found, exiting."%file_path)
|
||||
print("File %s not found, exiting." % file_path)
|
||||
sys.exit(0)
|
||||
|
||||
# Find parameter in file
|
||||
with open( file_path ) as f:
|
||||
with open(file_path) as f:
|
||||
for line in f:
|
||||
if line.startswith( parameter ):
|
||||
if line.startswith(parameter):
|
||||
return line.replace(parameter + ":", "").strip()
|
||||
|
||||
# Cannot find parameter, exit
|
||||
print(file_path + " Missing parameter %s "%parameter)
|
||||
print(file_path + " Missing parameter %s " % parameter)
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
|
|
@ -194,21 +172,25 @@ fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath)
|
|||
|
||||
if __name__ == '__main__':
|
||||
|
||||
now = datetime.now()
|
||||
start_time = time.time()
|
||||
|
||||
world_servers = []
|
||||
|
||||
try:
|
||||
|
||||
conn = None
|
||||
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
|
||||
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql",
|
||||
port="5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
### get world servers list
|
||||
# get world servers list
|
||||
|
||||
cur.execute("select server from world where checked='f'")
|
||||
|
||||
for row in cur:
|
||||
|
||||
world_servers.append(row[0])
|
||||
|
||||
cur.close()
|
||||
|
|
@ -222,7 +204,6 @@ if __name__ == '__main__':
|
|||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
###########################################################################
|
||||
|
|
@ -239,4 +220,3 @@ if __name__ == '__main__':
|
|||
pool.starmap(getserver, pool_tuple)
|
||||
|
||||
print('Done.')
|
||||
|
||||
|
|
|
|||
Loading…
Referencia en una nova incidència