spla/fediverse
1
1
Fork 1
Codi Incidències Sol·licituds Pull repo.project Llançaments Wiki Activitat

Fixed some warnings

Explorar la Font
This commit is contained in:
spla 2021-05-14 13:44:19 +02:00
pare 8185045292
commit 246ccf869c
S'han modificat 1 arxius amb 51 adicions i 71 eliminacions
Mostrar estadístiques Descarregar el fitxer de pedaç Descarregar el fitxer de diferències Expand all files Collapse all files

58
fetchservers.py
Veure arxiu

@ -1,38 +1,17 @@
import time import time
start_time = time.time()
import urllib3
from urllib3 import exceptions
from datetime import datetime from datetime import datetime
from subprocess import call
from mastodon import Mastodon
import threading
import os import os
import json import json
import signal
import sys import sys
import os.path import os.path
import requests
from requests import exceptions
import operator
import calendar
import psycopg2 import psycopg2
from itertools import product from multiprocessing import Pool, Manager
from multiprocessing import Pool, Lock, Process, Queue, current_process, Manager
import multiprocessing
import aiohttp import aiohttp
import aiodns
import asyncio import asyncio
from aiohttp import ClientError, ClientSession, ClientConnectionError, ClientConnectorError, ClientSSLError, ClientConnectorSSLError, ServerTimeoutError
from asyncio import TimeoutError
import socket import socket
from socket import gaierror, gethostbyname
from decimal import * apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?',
getcontext().prec = 2 '/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?', '/wp-json/nodeinfo/2.0?']
apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?', '/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?','/wp-json/nodeinfo/2.0?']
client_exceptions = ( client_exceptions = (
aiohttp.ClientResponseError, aiohttp.ClientResponseError,
@ -43,11 +22,6 @@ client_exceptions = (
socket.gaierror, socket.gaierror,
) )
now = datetime.now()
###############################################################################
# INITIALISATION
###############################################################################
def is_json(myjson): def is_json(myjson):
try: try:
@ -56,20 +30,23 @@ def is_json(myjson):
return False return False
return True return True
def write_api(server, software, users, alive, api, soft_version):
def write_api(server, software, users, alive, api, soft_version):
insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING" insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
conn = None conn = None
try: try:
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql",
port="5432")
cur = conn.cursor() cur = conn.cursor()
cur.execute(insert_sql, (server, now, software, users, alive, api, soft_version)) cur.execute(insert_sql, (server, now, software, users, alive, api, soft_version))
cur.execute("UPDATE fediverse SET updated_at=(%s), software=(%s), users=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)", (now, software, users, alive, api, soft_version, server)) cur.execute(
"UPDATE fediverse SET updated_at=(%s), software=(%s), users=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)",
(now, software, users, alive, api, soft_version, server))
cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,)) cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
@ -84,11 +61,10 @@ def write_api(server, software, users, alive, api, soft_version):
finally: finally:
if conn is not None: if conn is not None:
conn.close() conn.close()
async def getsoft(server):
async def getsoft(server):
try: try:
socket.gethostbyname(server) socket.gethostbyname(server)
@ -139,6 +115,7 @@ async def getsoft(server):
write_api(server, soft, users, alive, api) write_api(server, soft, users, alive, api)
print("Server " + server + " (" + soft + ") is alive!") print("Server " + server + " (" + soft + ") is alive!")
def getserver(server, x): def getserver(server, x):
server = server[0].rstrip('.').lower() server = server[0].rstrip('.').lower()
@ -162,6 +139,7 @@ def getserver(server, x):
pass pass
# Returns the parameter from the specified file # Returns the parameter from the specified file
def get_parameter(parameter, file_path): def get_parameter(parameter, file_path):
# Check if secrets file exists # Check if secrets file exists
@ -194,21 +172,25 @@ fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath)
if __name__ == '__main__': if __name__ == '__main__':
now = datetime.now()
start_time = time.time()
world_servers = [] world_servers = []
try: try:
conn = None conn = None
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql",
port="5432")
cur = conn.cursor() cur = conn.cursor()
### get world servers list # get world servers list
cur.execute("select server from world where checked='f'") cur.execute("select server from world where checked='f'")
for row in cur: for row in cur:
world_servers.append(row[0]) world_servers.append(row[0])
cur.close() cur.close()
@ -222,7 +204,6 @@ if __name__ == '__main__':
finally: finally:
if conn is not None: if conn is not None:
conn.close() conn.close()
########################################################################### ###########################################################################
@ -239,4 +220,3 @@ if __name__ == '__main__':
pool.starmap(getserver, pool_tuple) pool.starmap(getserver, pool_tuple)
print('Done.') print('Done.')