spla
/
fediverse
1
1
Fork 1
Codi Incidències Sol·licituds Pull Projectes Llançaments Wiki Activitat

Fixed some warnings

This commit is contained in:
spla 2021-05-14 13:44:19 +02:00
pare 8185045292
commit 246ccf869c
S'han modificat 1 arxius amb 51 adicions i 71 eliminacions
Mostrar estadístiques Descarregar el fitxer de pedaç Descarregar el fitxer de diferències Expand all files Collapse all files

122
fetchservers.py
Veure arxiu

@ -1,38 +1,17 @@
import time
start_time = time.time()
import urllib3
from urllib3 import exceptions
from datetime import datetime
from subprocess import call
from mastodon import Mastodon
import threading
import os
import json
import signal
import sys
import os.path
import requests
from requests import exceptions
import operator
import calendar
import psycopg2
from itertools import product
from multiprocessing import Pool, Lock, Process, Queue, current_process, Manager
import multiprocessing
from multiprocessing import Pool, Manager
import aiohttp
import aiodns
import asyncio
from aiohttp import ClientError, ClientSession, ClientConnectionError, ClientConnectorError, ClientSSLError, ClientConnectorSSLError, ServerTimeoutError
from asyncio import TimeoutError
import socket
from socket import gaierror, gethostbyname
from decimal import *
getcontext().prec = 2
apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?', '/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?','/wp-json/nodeinfo/2.0?']
apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?',
'/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?', '/wp-json/nodeinfo/2.0?']
client_exceptions = (
aiohttp.ClientResponseError,
@ -43,11 +22,6 @@ client_exceptions = (
socket.gaierror,
)
now = datetime.now()
###############################################################################
# INITIALISATION
###############################################################################
def is_json(myjson):
try:
@ -56,20 +30,23 @@ def is_json(myjson):
return False
return True
def write_api(server, software, users, alive, api, soft_version):
def write_api(server, software, users, alive, api, soft_version):
insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
conn = None
try:
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql",
port="5432")
cur = conn.cursor()
cur.execute(insert_sql, (server, now, software, users, alive, api, soft_version))
cur.execute("UPDATE fediverse SET updated_at=(%s), software=(%s), users=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)", (now, software, users, alive, api, soft_version, server))
cur.execute(
"UPDATE fediverse SET updated_at=(%s), software=(%s), users=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)",
(now, software, users, alive, api, soft_version, server))
cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
@ -84,11 +61,10 @@ def write_api(server, software, users, alive, api, soft_version):
finally:
if conn is not None:
conn.close()
async def getsoft(server):
async def getsoft(server):
try:
socket.gethostbyname(server)
@ -106,7 +82,7 @@ async def getsoft(server):
async with aiohttp.ClientSession(timeout=timeout) as session:
for api in apis:
try:
async with session.get(url+api) as response:
async with session.get(url + api) as response:
if response.status == 200:
try:
response_json = await response.json()
@ -115,29 +91,30 @@ async def getsoft(server):
except aiohttp.ClientConnectorError as err:
pass
else:
if response.status == 200 and api != '/api/v1/instance?':
try:
soft = response_json['software']['name']
soft = soft.lower()
soft_version = response_json['software']['version']
users = response_json['usage']['users']['total']
if users > 1000000:
return
alive = True
write_api(server, soft, users, alive, api, soft_version)
print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
return
except:
pass
if response.status == 200 and soft == '' and api == "/api/v1/instance?":
soft = 'mastodon'
users = response_json['stats']['user_count']
soft_version = response_json['version']
if users > 1000000:
return
alive = True
write_api(server, soft, users, alive, api)
print("Server " + server + " (" + soft + ") is alive!")
if response.status == 200 and api != '/api/v1/instance?':
try:
soft = response_json['software']['name']
soft = soft.lower()
soft_version = response_json['software']['version']
users = response_json['usage']['users']['total']
if users > 1000000:
return
alive = True
write_api(server, soft, users, alive, api, soft_version)
print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
return
except:
pass
if response.status == 200 and soft == '' and api == "/api/v1/instance?":
soft = 'mastodon'
users = response_json['stats']['user_count']
soft_version = response_json['version']
if users > 1000000:
return
alive = True
write_api(server, soft, users, alive, api)
print("Server " + server + " (" + soft + ") is alive!")
def getserver(server, x):
@ -146,11 +123,11 @@ def getserver(server, x):
if server.find(".") == -1:
return
if server.find("@") != -1:
return
return
if server.find("/") != -1:
return
return
if server.find(":") != -1:
return
return
try:
@ -162,21 +139,22 @@ def getserver(server, x):
pass
# Returns the parameter from the specified file
def get_parameter( parameter, file_path ):
def get_parameter(parameter, file_path):
# Check if secrets file exists
if not os.path.isfile(file_path):
print("File %s not found, exiting."%file_path)
print("File %s not found, exiting." % file_path)
sys.exit(0)
# Find parameter in file
with open( file_path ) as f:
with open(file_path) as f:
for line in f:
if line.startswith( parameter ):
if line.startswith(parameter):
return line.replace(parameter + ":", "").strip()
# Cannot find parameter, exit
print(file_path + " Missing parameter %s "%parameter)
print(file_path + " Missing parameter %s " % parameter)
sys.exit(0)
@ -194,21 +172,25 @@ fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath)
if __name__ == '__main__':
now = datetime.now()
start_time = time.time()
world_servers = []
try:
conn = None
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql",
port="5432")
cur = conn.cursor()
### get world servers list
# get world servers list
cur.execute("select server from world where checked='f'")
for row in cur:
world_servers.append(row[0])
cur.close()
@ -222,7 +204,6 @@ if __name__ == '__main__':
finally:
if conn is not None:
conn.close()
###########################################################################
@ -239,4 +220,3 @@ if __name__ == '__main__':
pool.starmap(getserver, pool_tuple)
print('Done.')