forkeado de spla/fediverse
Added Funkwhale and Socialhome software detections
This commit is contained in:
pare
c9142101eb
commit
ddb7038190
S'han modificat 7 arxius amb 464 adicions i 295 eliminacions
13
README.md
13
README.md
|
@ -73,8 +73,13 @@ Within Python Virtual Environment:
|
||||||
| writefreely | api/nodeinfo | ['software']['name'] |
|
| writefreely | api/nodeinfo | ['software']['name'] |
|
||||||
| zap | nodeinfo/2.0.json | ['software']['name'] |
|
| zap | nodeinfo/2.0.json | ['software']['name'] |
|
||||||
|
|
||||||
5. Use your favourite scheduling method to set `python fediverse.py` to run twice daily, `python fetchservers.py` one time daily and `python getworld.py` to run monthly.
|
7. Run `python uptime_setup.py` to get your Uptime bot's access token of your Mastodon or Pleroma server existing account. It will be saved to 'secrets/uptime_secrets.txt' for further use.
|
||||||
|
|
||||||
18.2.21 - New feature! Added [Lemmy project](https://join.lemmy.ml)
|
8. Use your favourite scheduling method to set `python fediverse.py` to run twice daily, `python fetchservers.py` one time daily, `python getworld.py` to run monthly and `python uptime.py` (choose your desired frequency) if you want to publish best fediverse's servers uptime.
|
||||||
12.5.21 - New feature! Added Wordpress support. The code can now detect Wordpress instances with ActivityPub enabled plugin.
|
|
||||||
12.5.21 - New feature! New shinny creation of servers and users graphs.
|
18.2.2021 - New feature! Added [Lemmy project](https://join.lemmy.ml)
|
||||||
|
12.5.2021 - New feature! Added Wordpress support. The code can now detect Wordpress instances with ActivityPub enabled plugin.
|
||||||
|
12.5.2021 - New feature! New shinny creation of servers and users graphs.
|
||||||
|
21.8.2021 - New feature! Added Best Fediverse's servers Uptime publishing bot.
|
||||||
|
22.10.2021 - New feature! Added [Funkwhale](https://funkwhale.audio) support.
|
||||||
|
26.10.2021 - New feature! Added [Socialhome](https://socialhome.network) support.
|
||||||
|
|
31
db-setup.py
31
db-setup.py
|
@ -1,6 +1,3 @@
|
||||||
#!/usr/bin/env python
|
|
||||||
# -*- coding: utf-8 -*-
|
|
||||||
|
|
||||||
import getpass
|
import getpass
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
|
@ -10,7 +7,6 @@ import psycopg2
|
||||||
from psycopg2 import sql
|
from psycopg2 import sql
|
||||||
from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
|
from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
|
||||||
|
|
||||||
# Returns the parameter from the specified file
|
|
||||||
def get_parameter( parameter, file_path ):
|
def get_parameter( parameter, file_path ):
|
||||||
# Check if secrets file exists
|
# Check if secrets file exists
|
||||||
if not os.path.isfile(file_path):
|
if not os.path.isfile(file_path):
|
||||||
|
@ -29,20 +25,27 @@ def get_parameter( parameter, file_path ):
|
||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
|
|
||||||
def write_parameter( parameter, file_path ):
|
def write_parameter( parameter, file_path ):
|
||||||
|
|
||||||
if not os.path.exists('config'):
|
if not os.path.exists('config'):
|
||||||
os.makedirs('config')
|
os.makedirs('config')
|
||||||
print("Setting up fediverse parameters...")
|
print("Setting up fediverse parameters...")
|
||||||
print("\n")
|
print("\n")
|
||||||
fediverse_db = input("fediverse db name: ")
|
fediverse_db = input("fediverse db name: ")
|
||||||
fediverse_db_user = input("fediverse db user: ")
|
fediverse_db_user = input("fediverse db user: ")
|
||||||
|
mastodon_db = input("Mastodon db name: ")
|
||||||
|
mastodon_db_user = input("Mastodon db user: ")
|
||||||
|
|
||||||
with open(file_path, "w") as text_file:
|
with open(file_path, "w") as text_file:
|
||||||
|
|
||||||
print("fediverse_db: {}".format(fediverse_db), file=text_file)
|
print("fediverse_db: {}".format(fediverse_db), file=text_file)
|
||||||
print("fediverse_db_user: {}".format(fediverse_db_user), file=text_file)
|
print("fediverse_db_user: {}".format(fediverse_db_user), file=text_file)
|
||||||
|
print("mastodon_db: {}".format(mastodon_db), file=text_file)
|
||||||
|
print("mastodon_db_user: {}".format(mastodon_db_user), file=text_file)
|
||||||
|
|
||||||
def create_table(db, db_user, table, sql):
|
def create_table(db, db_user, table, sql):
|
||||||
|
|
||||||
conn = None
|
conn = None
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|
||||||
conn = psycopg2.connect(database = db, user = db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
conn = psycopg2.connect(database = db, user = db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||||
|
@ -53,6 +56,7 @@ def create_table(db, db_user, table, sql):
|
||||||
cur.execute(sql)
|
cur.execute(sql)
|
||||||
|
|
||||||
conn.commit()
|
conn.commit()
|
||||||
|
|
||||||
print("Table "+table+" created!")
|
print("Table "+table+" created!")
|
||||||
print("\n")
|
print("\n")
|
||||||
|
|
||||||
|
@ -66,12 +70,17 @@ def create_table(db, db_user, table, sql):
|
||||||
|
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
#############################################################################################
|
###############################################################################
|
||||||
|
# main
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
|
||||||
# Load configuration from config file
|
# Load configuration from config file
|
||||||
config_filepath = "config/db_config.txt"
|
config_filepath = "config/db_config.txt"
|
||||||
fediverse_db = get_parameter("fediverse_db", config_filepath)
|
fediverse_db = get_parameter("fediverse_db", config_filepath)
|
||||||
fediverse_db_user = get_parameter("fediverse_db_user", config_filepath)
|
fediverse_db_user = get_parameter("fediverse_db_user", config_filepath)
|
||||||
|
mastodon_db = get_parameter("mastodon_db", config_filepath)
|
||||||
|
mastodon_db_user = get_parameter("mastodon_db_user", config_filepath)
|
||||||
|
|
||||||
############################################################
|
############################################################
|
||||||
# create database
|
# create database
|
||||||
|
@ -138,11 +147,11 @@ print("Creating table...")
|
||||||
db = fediverse_db
|
db = fediverse_db
|
||||||
db_user = fediverse_db_user
|
db_user = fediverse_db_user
|
||||||
table = "world"
|
table = "world"
|
||||||
sql = "create table "+table+" (server varchar(200) PRIMARY KEY, federated_with varchar(200), updated_at timestamptz, saved_at timestamptz), checked boolean"
|
sql = "create table "+table+" (server varchar(200) PRIMARY KEY, federated_with varchar(200), updated_at timestamptz, saved_at timestamptz, checked boolean)"
|
||||||
create_table(db, db_user, table, sql)
|
create_table(db, db_user, table, sql)
|
||||||
|
|
||||||
table = "fediverse"
|
table = "fediverse"
|
||||||
sql = "create table "+table+" (server varchar(200) PRIMARY KEY, users INT, updated_at timestamptz, software varchar(50), version varchar(100))"
|
sql = "create table "+table+" (server varchar(200) PRIMARY KEY, users INT, updated_at timestamptz, software varchar(50), version varchar(100), first_checked_at timestamptz, last_checked_at timestamptz, downs int)"
|
||||||
create_table(db, db_user, table, sql)
|
create_table(db, db_user, table, sql)
|
||||||
|
|
||||||
table = "totals"
|
table = "totals"
|
||||||
|
@ -153,6 +162,14 @@ table = "evo"
|
||||||
sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, servers INT, users INT)"
|
sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, servers INT, users INT)"
|
||||||
create_table(db, db_user, table, sql)
|
create_table(db, db_user, table, sql)
|
||||||
|
|
||||||
|
table = "closed"
|
||||||
|
sql = "create table "+table+" (server varchar(200) PRIMARY KEY, software varchar(10), closed boolean)"
|
||||||
|
create_table(db, db_user, table, sql)
|
||||||
|
|
||||||
|
table = "botreplies"
|
||||||
|
sql = "create table "+table+" (status_id bigint PRIMARY KEY, query_user varchar(40), status_created_at timestamptz)"
|
||||||
|
create_table(db, db_user, table, sql)
|
||||||
|
|
||||||
#####################################
|
#####################################
|
||||||
|
|
||||||
print("Done!")
|
print("Done!")
|
||||||
|
|
98
fediverse.py
98
fediverse.py
|
@ -14,12 +14,22 @@ import aiohttp
|
||||||
import asyncio
|
import asyncio
|
||||||
import socket
|
import socket
|
||||||
import matplotlib.pyplot as plt
|
import matplotlib.pyplot as plt
|
||||||
|
import pdb
|
||||||
plt.style.use('seaborn')
|
plt.style.use('seaborn')
|
||||||
|
|
||||||
start_time = time.time()
|
start_time = time.time()
|
||||||
|
|
||||||
apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?', '/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?', '/wp-json/nodeinfo/2.0?']
|
apis = ['/nodeinfo/2.0?',
|
||||||
|
'/nodeinfo/2.0.json?',
|
||||||
|
'/main/nodeinfo/2.0?',
|
||||||
|
'/api/statusnet/config?',
|
||||||
|
'/api/nodeinfo/2.0.json?',
|
||||||
|
'/api/nodeinfo?',
|
||||||
|
'/api/v1/instance?',
|
||||||
|
'/wp-json/nodeinfo/2.0?',
|
||||||
|
'/api/v1/instance/nodeinfo/2.0?',
|
||||||
|
'/.well-known/x-nodeinfo2?'
|
||||||
|
]
|
||||||
|
|
||||||
client_exceptions = (
|
client_exceptions = (
|
||||||
aiohttp.ClientResponseError,
|
aiohttp.ClientResponseError,
|
||||||
|
@ -53,7 +63,7 @@ def get_alive_servers(server):
|
||||||
|
|
||||||
cur = conn.cursor()
|
cur = conn.cursor()
|
||||||
|
|
||||||
cur.execute("select alive, software, users_api, version from fediverse where server=(%s)", (server,))
|
cur.execute("select alive, software, users_api, version, first_checked_at, downs from fediverse where server=(%s)", (server,))
|
||||||
|
|
||||||
row = cur.fetchone()
|
row = cur.fetchone()
|
||||||
|
|
||||||
|
@ -63,6 +73,8 @@ def get_alive_servers(server):
|
||||||
serv_soft = row[1]
|
serv_soft = row[1]
|
||||||
serv_api = row[2]
|
serv_api = row[2]
|
||||||
soft_version = row[3]
|
soft_version = row[3]
|
||||||
|
first_checked_at = row[4]
|
||||||
|
downs_qty = row[5]
|
||||||
|
|
||||||
cur.close()
|
cur.close()
|
||||||
|
|
||||||
|
@ -79,7 +91,9 @@ def get_alive_servers(server):
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|
||||||
data = requests.get('https://' + server + serv_api, timeout=3)
|
user_agent = {'User-agent': 'Mozilla/5.0'}
|
||||||
|
|
||||||
|
data = requests.get('https://' + server + serv_api, headers = user_agent, timeout=3)
|
||||||
|
|
||||||
if serv_soft == "mastodon":
|
if serv_soft == "mastodon":
|
||||||
if serv_api == '/nodeinfo/2.0?':
|
if serv_api == '/nodeinfo/2.0?':
|
||||||
|
@ -153,14 +167,42 @@ def get_alive_servers(server):
|
||||||
users = 0
|
users = 0
|
||||||
soft_version = ""
|
soft_version = ""
|
||||||
|
|
||||||
|
if serv_soft == 'funkwhale':
|
||||||
|
|
||||||
|
try:
|
||||||
|
users = data.json()['usage']['users']['total']
|
||||||
|
soft_version = data.json()['software']['version']
|
||||||
|
alive = True
|
||||||
|
except:
|
||||||
|
users = 0
|
||||||
|
soft_version = ""
|
||||||
|
|
||||||
|
if serv_soft == 'socialhome':
|
||||||
|
|
||||||
|
try:
|
||||||
|
users = data.json()['usage']['users']['total']
|
||||||
|
soft_version = data.json()['server']['version']
|
||||||
|
alive = True
|
||||||
|
except:
|
||||||
|
users = 0
|
||||||
|
soft_version = ""
|
||||||
|
|
||||||
if alive:
|
if alive:
|
||||||
|
|
||||||
|
if downs_qty != None:
|
||||||
|
|
||||||
|
downs = downs_qty
|
||||||
|
|
||||||
|
else:
|
||||||
|
|
||||||
|
downs = 0
|
||||||
|
|
||||||
if soft_version != "" and soft_version is not None:
|
if soft_version != "" and soft_version is not None:
|
||||||
print("Server " + str(server) + " (" + serv_soft + " " + soft_version + ") is alive!")
|
print("Server " + str(server) + " (" + serv_soft + " " + soft_version + ") is alive!")
|
||||||
else:
|
else:
|
||||||
print("Server " + str(server) + " (" + serv_soft + ") is alive!")
|
print("Server " + str(server) + " (" + serv_soft + ") is alive!")
|
||||||
|
|
||||||
insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
|
insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version, first_checked_at, last_checked_at, downs) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
|
||||||
|
|
||||||
conn = None
|
conn = None
|
||||||
|
|
||||||
|
@ -170,9 +212,16 @@ def get_alive_servers(server):
|
||||||
|
|
||||||
cur = conn.cursor()
|
cur = conn.cursor()
|
||||||
|
|
||||||
cur.execute(insert_sql, (server, users, now, serv_soft, alive, serv_api, soft_version))
|
cur.execute(insert_sql, (server, users, now, serv_soft, alive, serv_api, soft_version, now, now, downs))
|
||||||
|
|
||||||
|
if first_checked_at != None:
|
||||||
|
|
||||||
|
cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (users, now, serv_soft, alive, serv_api, soft_version, now, downs, server))
|
||||||
|
|
||||||
|
else:
|
||||||
|
|
||||||
|
cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), first_checked_at=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (users, now, serv_soft, alive, serv_api, soft_version, now, now, downs, server))
|
||||||
|
|
||||||
cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)", (users, now, serv_soft, alive, serv_api, soft_version, server))
|
|
||||||
|
|
||||||
cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
|
cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
|
||||||
|
|
||||||
|
@ -258,6 +307,14 @@ def get_alive_servers(server):
|
||||||
|
|
||||||
if alive is False:
|
if alive is False:
|
||||||
|
|
||||||
|
if downs_qty != None:
|
||||||
|
|
||||||
|
downs = downs_qty + 1
|
||||||
|
|
||||||
|
else:
|
||||||
|
|
||||||
|
downs = 1
|
||||||
|
|
||||||
conn = None
|
conn = None
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
@ -267,7 +324,13 @@ def get_alive_servers(server):
|
||||||
|
|
||||||
cur = conn.cursor()
|
cur = conn.cursor()
|
||||||
|
|
||||||
cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s) where server=(%s)", (now, alive, server))
|
if first_checked_at != None:
|
||||||
|
|
||||||
|
cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), first_checked_at=(%s), downs=(%s) where server=(%s)", (now, alive, now, downs, server))
|
||||||
|
|
||||||
|
else:
|
||||||
|
|
||||||
|
cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), downs=(%s) where server=(%s)", (now, alive, downs, server))
|
||||||
|
|
||||||
cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
|
cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
|
||||||
|
|
||||||
|
@ -329,8 +392,11 @@ async def getsoft(server):
|
||||||
|
|
||||||
url = 'https://' + server
|
url = 'https://' + server
|
||||||
|
|
||||||
|
user_agent = {'User-agent': 'Mozilla/5.0'}
|
||||||
|
|
||||||
timeout = aiohttp.ClientTimeout(total=3)
|
timeout = aiohttp.ClientTimeout(total=3)
|
||||||
async with aiohttp.ClientSession(timeout=timeout) as session:
|
|
||||||
|
async with aiohttp.ClientSession(headers=user_agent, timeout=timeout) as session:
|
||||||
for api in apis:
|
for api in apis:
|
||||||
try:
|
try:
|
||||||
async with session.get(url+api) as response:
|
async with session.get(url+api) as response:
|
||||||
|
@ -871,23 +937,15 @@ if __name__ == '__main__':
|
||||||
project_servers = soft_total_servers[i]
|
project_servers = soft_total_servers[i]
|
||||||
len_pr_soft = len(project_soft)
|
len_pr_soft = len(project_soft)
|
||||||
|
|
||||||
if project_soft == 'writefreely':
|
toot_text += f":{project_soft}: {project_users:,} {project_servers:,}\n"
|
||||||
|
|
||||||
str_len = 11
|
|
||||||
|
|
||||||
else:
|
|
||||||
|
|
||||||
str_len = 13
|
|
||||||
|
|
||||||
toot_text += f"{':'+project_soft+':':<11}" + f"{project_users:>{str_len},}" + " " + f"{project_servers:>5,}" + "\n"
|
|
||||||
|
|
||||||
i += 1
|
i += 1
|
||||||
|
|
||||||
print("Tooting...")
|
print("Tooting...")
|
||||||
print(toot_text)
|
print(toot_text)
|
||||||
|
|
||||||
servers_image_id = mastodon.media_post('servers.png', "image/png").id
|
servers_image_id = mastodon.media_post('servers.png', "image/png", description='servers graph').id
|
||||||
|
|
||||||
users_image_id = mastodon.media_post('users.png', "image/png").id
|
users_image_id = mastodon.media_post('users.png', "image/png", description='users graph').id
|
||||||
|
|
||||||
mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, users_image_id})
|
mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, users_image_id})
|
||||||
|
|
|
@ -10,8 +10,17 @@ import aiohttp
|
||||||
import asyncio
|
import asyncio
|
||||||
import socket
|
import socket
|
||||||
|
|
||||||
apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?',
|
apis = ['/nodeinfo/2.0?',
|
||||||
'/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?', '/wp-json/nodeinfo/2.0?']
|
'/nodeinfo/2.0.json?',
|
||||||
|
'/main/nodeinfo/2.0?',
|
||||||
|
'/api/statusnet/config?',
|
||||||
|
'/api/nodeinfo/2.0.json?',
|
||||||
|
'/api/nodeinfo?',
|
||||||
|
'/api/v1/instance?',
|
||||||
|
'/wp-json/nodeinfo/2.0?',
|
||||||
|
'/api/v1/instance/nodeinfo/2.0?',
|
||||||
|
'/.well-known/x-nodeinfo2?'
|
||||||
|
]
|
||||||
|
|
||||||
client_exceptions = (
|
client_exceptions = (
|
||||||
aiohttp.ClientResponseError,
|
aiohttp.ClientResponseError,
|
||||||
|
@ -24,15 +33,17 @@ client_exceptions = (
|
||||||
|
|
||||||
|
|
||||||
def is_json(myjson):
|
def is_json(myjson):
|
||||||
|
|
||||||
try:
|
try:
|
||||||
json_object = json.loads(myjson)
|
json_object = json.loads(myjson)
|
||||||
except ValueError as e:
|
except ValueError as e:
|
||||||
return False
|
return False
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
def write_api(server, software, users, alive, api, soft_version):
|
def write_api(server, software, users, alive, api, soft_version):
|
||||||
|
|
||||||
insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
|
insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
|
||||||
|
|
||||||
conn = None
|
conn = None
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
@ -46,7 +57,8 @@ def write_api(server, software, users, alive, api, soft_version):
|
||||||
|
|
||||||
cur.execute(
|
cur.execute(
|
||||||
"UPDATE fediverse SET updated_at=(%s), software=(%s), users=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)",
|
"UPDATE fediverse SET updated_at=(%s), software=(%s), users=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)",
|
||||||
(now, software, users, alive, api, soft_version, server))
|
(now, software, users, alive, api, soft_version, server)
|
||||||
|
)
|
||||||
|
|
||||||
cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
|
cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
|
||||||
|
|
||||||
|
@ -63,7 +75,6 @@ def write_api(server, software, users, alive, api, soft_version):
|
||||||
if conn is not None:
|
if conn is not None:
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
async def getsoft(server):
|
async def getsoft(server):
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
@ -79,20 +90,31 @@ async def getsoft(server):
|
||||||
|
|
||||||
url = 'https://' + server
|
url = 'https://' + server
|
||||||
|
|
||||||
|
user_agent = {'User-agent': 'Mozilla/5.0'}
|
||||||
|
|
||||||
timeout = aiohttp.ClientTimeout(total=3)
|
timeout = aiohttp.ClientTimeout(total=3)
|
||||||
async with aiohttp.ClientSession(timeout=timeout) as session:
|
async with aiohttp.ClientSession(timeout=timeout, headers=user_agent) as session:
|
||||||
for api in apis:
|
for api in apis:
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|
||||||
async with session.get(url + api) as response:
|
async with session.get(url + api) as response:
|
||||||
if response.status == 200:
|
if response.status == 200:
|
||||||
try:
|
try:
|
||||||
response_json = await response.json()
|
response_json = await response.json()
|
||||||
except:
|
except:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
except aiohttp.ClientConnectorError as err:
|
except aiohttp.ClientConnectorError as err:
|
||||||
|
|
||||||
pass
|
pass
|
||||||
|
|
||||||
else:
|
else:
|
||||||
|
|
||||||
if response.status == 200 and api != '/api/v1/instance?':
|
if response.status == 200 and api != '/api/v1/instance?':
|
||||||
|
|
||||||
|
if api != '/.well-known/x-nodeinfo2?':
|
||||||
|
|
||||||
try:
|
try:
|
||||||
soft = response_json['software']['name']
|
soft = response_json['software']['name']
|
||||||
soft = soft.lower()
|
soft = soft.lower()
|
||||||
|
@ -106,6 +128,23 @@ async def getsoft(server):
|
||||||
return
|
return
|
||||||
except:
|
except:
|
||||||
pass
|
pass
|
||||||
|
else:
|
||||||
|
|
||||||
|
try:
|
||||||
|
soft = response_json['server']['software']
|
||||||
|
soft = soft.lower()
|
||||||
|
soft_version = response_json['server']['version']
|
||||||
|
users = response_json['usage']['users']['total']
|
||||||
|
if users > 1000000:
|
||||||
|
return
|
||||||
|
alive = True
|
||||||
|
if soft == 'socialhome':
|
||||||
|
write_api(server, soft, users, alive, api, soft_version)
|
||||||
|
print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
|
||||||
|
return
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
if response.status == 200 and soft == '' and api == "/api/v1/instance?":
|
if response.status == 200 and soft == '' and api == "/api/v1/instance?":
|
||||||
soft = 'mastodon'
|
soft = 'mastodon'
|
||||||
users = response_json['stats']['user_count']
|
users = response_json['stats']['user_count']
|
||||||
|
@ -116,7 +155,6 @@ async def getsoft(server):
|
||||||
write_api(server, soft, users, alive, api)
|
write_api(server, soft, users, alive, api)
|
||||||
print("Server " + server + " (" + soft + ") is alive!")
|
print("Server " + server + " (" + soft + ") is alive!")
|
||||||
|
|
||||||
|
|
||||||
def getserver(server, x):
|
def getserver(server, x):
|
||||||
|
|
||||||
server = server[0].rstrip('.').lower()
|
server = server[0].rstrip('.').lower()
|
||||||
|
@ -140,8 +178,6 @@ def getserver(server, x):
|
||||||
|
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
# Returns the parameter from the specified file
|
|
||||||
def get_parameter(parameter, file_path):
|
def get_parameter(parameter, file_path):
|
||||||
# Check if secrets file exists
|
# Check if secrets file exists
|
||||||
if not os.path.isfile(file_path):
|
if not os.path.isfile(file_path):
|
||||||
|
@ -159,14 +195,20 @@ def get_parameter(parameter, file_path):
|
||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
|
|
||||||
|
|
||||||
|
def get_config():
|
||||||
|
|
||||||
# Load configuration from config file
|
# Load configuration from config file
|
||||||
config_filepath = "config/config.txt"
|
config_filepath = "config/config.txt"
|
||||||
mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
|
mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
|
||||||
|
return mastodon_hostname
|
||||||
|
|
||||||
|
def get_db_config():
|
||||||
|
|
||||||
# Load database config from db_config file
|
# Load database config from db_config file
|
||||||
db_config_filepath = "config/db_config.txt"
|
db_config_filepath = "config/db_config.txt"
|
||||||
fediverse_db = get_parameter("fediverse_db", db_config_filepath)
|
fediverse_db = get_parameter("fediverse_db", db_config_filepath)
|
||||||
fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath)
|
fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath)
|
||||||
|
return (fediverse_db, fediverse_db_user)
|
||||||
|
|
||||||
###############################################################################
|
###############################################################################
|
||||||
# main
|
# main
|
||||||
|
@ -176,6 +218,10 @@ if __name__ == '__main__':
|
||||||
now = datetime.now()
|
now = datetime.now()
|
||||||
start_time = time.time()
|
start_time = time.time()
|
||||||
|
|
||||||
|
mastodon_hostname = get_config()
|
||||||
|
|
||||||
|
fediverse_db, fediverse_db_user = get_db_config()
|
||||||
|
|
||||||
world_servers = []
|
world_servers = []
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
@ -207,6 +253,7 @@ if __name__ == '__main__':
|
||||||
finally:
|
finally:
|
||||||
|
|
||||||
if conn is not None:
|
if conn is not None:
|
||||||
|
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
###########################################################################
|
###########################################################################
|
||||||
|
|
|
@ -50,7 +50,9 @@ def get_peers(peer):
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|
||||||
response = requests.get('https://' + peer + peers_api, timeout=2)
|
user_agent = {'User-agent': 'Mozilla/5.0'}
|
||||||
|
|
||||||
|
response = requests.get('https://' + peer + peers_api, headers = user_agent, timeout=3)
|
||||||
|
|
||||||
response_json = response.json()
|
response_json = response.json()
|
||||||
|
|
||||||
|
@ -113,7 +115,6 @@ if __name__ == '__main__':
|
||||||
updated_at = datetime.now()
|
updated_at = datetime.now()
|
||||||
|
|
||||||
peers_api = '/api/v1/instance/peers?'
|
peers_api = '/api/v1/instance/peers?'
|
||||||
lemmy_api = '/api/v2/site?'
|
|
||||||
|
|
||||||
# Load configuration from config file
|
# Load configuration from config file
|
||||||
config_filepath = "config/config.txt"
|
config_filepath = "config/config.txt"
|
||||||
|
@ -126,7 +127,9 @@ if __name__ == '__main__':
|
||||||
|
|
||||||
world_peers = []
|
world_peers = []
|
||||||
|
|
||||||
res = requests.get('https://' + mastodon_hostname + peers_api)
|
user_agent = {'User-agent': 'Mozilla/5.0'}
|
||||||
|
|
||||||
|
res = requests.get('https://' + mastodon_hostname + peers_api, headers = user_agent, timeout=3)
|
||||||
|
|
||||||
hostname_peers = res.json()
|
hostname_peers = res.json()
|
||||||
|
|
||||||
|
|
|
@ -3,3 +3,4 @@ psycopg2-binary>=2.8.4
|
||||||
aiohttp>=3.6.2
|
aiohttp>=3.6.2
|
||||||
aiodns>=2.0.0
|
aiodns>=2.0.0
|
||||||
matplotlib>=3.3.4
|
matplotlib>=3.3.4
|
||||||
|
humanfriendly>=9.2
|
||||||
|
|
52
setup.py
52
setup.py
|
@ -1,6 +1,3 @@
|
||||||
#!/usr/bin/env python
|
|
||||||
# -*- coding: utf-8 -*-
|
|
||||||
|
|
||||||
import getpass
|
import getpass
|
||||||
from mastodon import Mastodon
|
from mastodon import Mastodon
|
||||||
from mastodon.Mastodon import MastodonMalformedEventError, MastodonNetworkError, MastodonReadTimeout, MastodonAPIError, MastodonIllegalArgumentError
|
from mastodon.Mastodon import MastodonMalformedEventError, MastodonNetworkError, MastodonReadTimeout, MastodonAPIError, MastodonIllegalArgumentError
|
||||||
|
@ -9,15 +6,18 @@ import os
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
def create_dir():
|
def create_dir():
|
||||||
|
|
||||||
if not os.path.exists('secrets'):
|
if not os.path.exists('secrets'):
|
||||||
os.makedirs('secrets')
|
os.makedirs('secrets')
|
||||||
|
|
||||||
def create_file():
|
def create_file():
|
||||||
|
|
||||||
if not os.path.exists('secrets/secrets.txt'):
|
if not os.path.exists('secrets/secrets.txt'):
|
||||||
with open('secrets/secrets.txt', 'w'): pass
|
with open('secrets/secrets.txt', 'w'): pass
|
||||||
print(secrets_filepath + " created!")
|
print(secrets_filepath + " created!")
|
||||||
|
|
||||||
def create_config():
|
def create_config():
|
||||||
|
|
||||||
if not os.path.exists('config'):
|
if not os.path.exists('config'):
|
||||||
os.makedirs('config')
|
os.makedirs('config')
|
||||||
if not os.path.exists(config_filepath):
|
if not os.path.exists(config_filepath):
|
||||||
|
@ -25,18 +25,23 @@ def create_config():
|
||||||
with open('config/config.txt', 'w'): pass
|
with open('config/config.txt', 'w'): pass
|
||||||
|
|
||||||
def write_params():
|
def write_params():
|
||||||
|
|
||||||
with open(secrets_filepath, 'a') as the_file:
|
with open(secrets_filepath, 'a') as the_file:
|
||||||
print("Writing secrets parameter names to " + secrets_filepath)
|
print("Writing secrets parameter names to " + secrets_filepath)
|
||||||
the_file.write('uc_client_id: \n'+'uc_client_secret: \n'+'uc_access_token: \n')
|
the_file.write('uc_client_id: \n'+'uc_client_secret: \n'+'uc_access_token: \n')
|
||||||
|
|
||||||
def write_config():
|
def write_config():
|
||||||
|
|
||||||
with open(config_filepath, 'a') as the_file:
|
with open(config_filepath, 'a') as the_file:
|
||||||
the_file.write('mastodon_hostname: \n')
|
the_file.write('mastodon_hostname: \n')
|
||||||
print("adding parameter name 'mastodon_hostname' to "+ config_filepath)
|
print("adding parameter name 'mastodon_hostname' to "+ config_filepath)
|
||||||
|
|
||||||
def read_client_lines(self):
|
def read_client_lines(self):
|
||||||
|
|
||||||
client_path = 'app_clientcred.txt'
|
client_path = 'app_clientcred.txt'
|
||||||
|
|
||||||
with open(client_path) as fp:
|
with open(client_path) as fp:
|
||||||
|
|
||||||
line = fp.readline()
|
line = fp.readline()
|
||||||
cnt = 1
|
cnt = 1
|
||||||
while line:
|
while line:
|
||||||
|
@ -50,27 +55,41 @@ def read_client_lines(self):
|
||||||
cnt += 1
|
cnt += 1
|
||||||
|
|
||||||
def read_token_line(self):
|
def read_token_line(self):
|
||||||
|
|
||||||
token_path = 'app_usercred.txt'
|
token_path = 'app_usercred.txt'
|
||||||
|
|
||||||
with open(token_path) as fp:
|
with open(token_path) as fp:
|
||||||
|
|
||||||
line = fp.readline()
|
line = fp.readline()
|
||||||
print("Writing access token to " + secrets_filepath)
|
print("Writing access token to " + secrets_filepath)
|
||||||
modify_file(secrets_filepath, "uc_access_token: ", value=line.rstrip())
|
modify_file(secrets_filepath, "uc_access_token: ", value=line.rstrip())
|
||||||
|
|
||||||
def read_config_line():
|
def read_config_line():
|
||||||
|
|
||||||
with open(config_filepath) as fp:
|
with open(config_filepath) as fp:
|
||||||
|
|
||||||
line = fp.readline()
|
line = fp.readline()
|
||||||
modify_file(config_filepath, "mastodon_hostname: ", value=hostname)
|
modify_file(config_filepath, "mastodon_hostname: ", value=hostname)
|
||||||
|
modify_file(config_filepath, "bot_username: ", value=bot_username)
|
||||||
|
|
||||||
def log_in():
|
def log_in():
|
||||||
|
|
||||||
error = 0
|
error = 0
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|
||||||
global hostname
|
global hostname
|
||||||
hostname = input("Enter Mastodon hostname: ")
|
hostname = input("Enter Mastodon hostname: ")
|
||||||
user_name = input("User name, ex. user@" + hostname +"? ")
|
user_name = input("User name, ex. user@" + hostname +"? ")
|
||||||
user_password = getpass.getpass("User password? ")
|
user_password = getpass.getpass("User password? ")
|
||||||
|
bot_username = input("Bot's username, ex. fediverse: ")
|
||||||
app_name = input("This app name? ")
|
app_name = input("This app name? ")
|
||||||
Mastodon.create_app(app_name, scopes=["read","write"],
|
Mastodon.create_app(
|
||||||
to_file="app_clientcred.txt", api_base_url=hostname)
|
app_name,
|
||||||
|
scopes=["read","write"],
|
||||||
|
to_file="app_clientcred.txt",
|
||||||
|
api_base_url=hostname
|
||||||
|
)
|
||||||
mastodon = Mastodon(client_id = "app_clientcred.txt", api_base_url = hostname)
|
mastodon = Mastodon(client_id = "app_clientcred.txt", api_base_url = hostname)
|
||||||
mastodon.log_in(
|
mastodon.log_in(
|
||||||
user_name,
|
user_name,
|
||||||
|
@ -78,8 +97,11 @@ def log_in():
|
||||||
scopes = ["read", "write"],
|
scopes = ["read", "write"],
|
||||||
to_file = "app_usercred.txt"
|
to_file = "app_usercred.txt"
|
||||||
)
|
)
|
||||||
|
|
||||||
except MastodonIllegalArgumentError as i_error:
|
except MastodonIllegalArgumentError as i_error:
|
||||||
|
|
||||||
error = 1
|
error = 1
|
||||||
|
|
||||||
if os.path.exists("secrets/secrets.txt"):
|
if os.path.exists("secrets/secrets.txt"):
|
||||||
print("Removing secrets/secrets.txt file..")
|
print("Removing secrets/secrets.txt file..")
|
||||||
os.remove("secrets/secrets.txt")
|
os.remove("secrets/secrets.txt")
|
||||||
|
@ -87,8 +109,11 @@ def log_in():
|
||||||
print("Removing app_clientcred.txt file..")
|
print("Removing app_clientcred.txt file..")
|
||||||
os.remove("app_clientcred.txt")
|
os.remove("app_clientcred.txt")
|
||||||
sys.exit(i_error)
|
sys.exit(i_error)
|
||||||
|
|
||||||
except MastodonNetworkError as n_error:
|
except MastodonNetworkError as n_error:
|
||||||
|
|
||||||
error = 1
|
error = 1
|
||||||
|
|
||||||
if os.path.exists("secrets/secrets.txt"):
|
if os.path.exists("secrets/secrets.txt"):
|
||||||
print("Removing secrets/secrets.txt file..")
|
print("Removing secrets/secrets.txt file..")
|
||||||
os.remove("secrets/secrets.txt")
|
os.remove("secrets/secrets.txt")
|
||||||
|
@ -96,8 +121,11 @@ def log_in():
|
||||||
print("Removing app_clientcred.txt file..")
|
print("Removing app_clientcred.txt file..")
|
||||||
os.remove("app_clientcred.txt")
|
os.remove("app_clientcred.txt")
|
||||||
sys.exit(n_error)
|
sys.exit(n_error)
|
||||||
|
|
||||||
except MastodonReadTimeout as r_error:
|
except MastodonReadTimeout as r_error:
|
||||||
|
|
||||||
error = 1
|
error = 1
|
||||||
|
|
||||||
if os.path.exists("secrets/secrets.txt"):
|
if os.path.exists("secrets/secrets.txt"):
|
||||||
print("Removing secrets/secrets.txt file..")
|
print("Removing secrets/secrets.txt file..")
|
||||||
os.remove("secrets/secrets.txt")
|
os.remove("secrets/secrets.txt")
|
||||||
|
@ -105,8 +133,11 @@ def log_in():
|
||||||
print("Removing app_clientcred.txt file..")
|
print("Removing app_clientcred.txt file..")
|
||||||
os.remove("app_clientcred.txt")
|
os.remove("app_clientcred.txt")
|
||||||
sys.exit(r_error)
|
sys.exit(r_error)
|
||||||
|
|
||||||
except MastodonAPIError as a_error:
|
except MastodonAPIError as a_error:
|
||||||
|
|
||||||
error = 1
|
error = 1
|
||||||
|
|
||||||
if os.path.exists("secrets/secrets.txt"):
|
if os.path.exists("secrets/secrets.txt"):
|
||||||
print("Removing secrets/secrets.txt file..")
|
print("Removing secrets/secrets.txt file..")
|
||||||
os.remove("secrets/secrets.txt")
|
os.remove("secrets/secrets.txt")
|
||||||
|
@ -114,7 +145,9 @@ def log_in():
|
||||||
print("Removing app_clientcred.txt file..")
|
print("Removing app_clientcred.txt file..")
|
||||||
os.remove("app_clientcred.txt")
|
os.remove("app_clientcred.txt")
|
||||||
sys.exit(a_error)
|
sys.exit(a_error)
|
||||||
|
|
||||||
finally:
|
finally:
|
||||||
|
|
||||||
if error == 0:
|
if error == 0:
|
||||||
|
|
||||||
create_dir()
|
create_dir()
|
||||||
|
@ -133,6 +166,7 @@ def log_in():
|
||||||
print("Secrets setup done!\n")
|
print("Secrets setup done!\n")
|
||||||
|
|
||||||
def modify_file(file_name,pattern,value=""):
|
def modify_file(file_name,pattern,value=""):
|
||||||
|
|
||||||
fh=fileinput.input(file_name,inplace=True)
|
fh=fileinput.input(file_name,inplace=True)
|
||||||
for line in fh:
|
for line in fh:
|
||||||
replacement=pattern + value
|
replacement=pattern + value
|
||||||
|
@ -140,7 +174,6 @@ def modify_file(file_name,pattern,value=""):
|
||||||
sys.stdout.write(line)
|
sys.stdout.write(line)
|
||||||
fh.close()
|
fh.close()
|
||||||
|
|
||||||
# Returns the parameter from the specified file
|
|
||||||
def get_parameter( parameter, file_path ):
|
def get_parameter( parameter, file_path ):
|
||||||
# Check if secrets file exists
|
# Check if secrets file exists
|
||||||
if not os.path.isfile(file_path):
|
if not os.path.isfile(file_path):
|
||||||
|
@ -157,7 +190,6 @@ def get_parameter( parameter, file_path ):
|
||||||
print(file_path + " Missing parameter %s "%parameter)
|
print(file_path + " Missing parameter %s "%parameter)
|
||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
|
|
||||||
# Returns the parameter from the specified file
|
|
||||||
def get_hostname( parameter, config_filepath ):
|
def get_hostname( parameter, config_filepath ):
|
||||||
# Check if secrets file exists
|
# Check if secrets file exists
|
||||||
if not os.path.isfile(config_filepath):
|
if not os.path.isfile(config_filepath):
|
||||||
|
@ -177,6 +209,11 @@ def get_hostname( parameter, config_filepath ):
|
||||||
print("setup done!")
|
print("setup done!")
|
||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
|
|
||||||
|
###############################################################################
|
||||||
|
# main
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
|
||||||
# Load secrets from secrets file
|
# Load secrets from secrets file
|
||||||
secrets_filepath = "secrets/secrets.txt"
|
secrets_filepath = "secrets/secrets.txt"
|
||||||
uc_client_id = get_parameter("uc_client_id", secrets_filepath)
|
uc_client_id = get_parameter("uc_client_id", secrets_filepath)
|
||||||
|
@ -186,3 +223,4 @@ uc_access_token = get_parameter("uc_access_token", secrets_filepath)
|
||||||
# Load configuration from config file
|
# Load configuration from config file
|
||||||
config_filepath = "config/config.txt"
|
config_filepath = "config/config.txt"
|
||||||
mastodon_hostname = get_hostname("mastodon_hostname", config_filepath)
|
mastodon_hostname = get_hostname("mastodon_hostname", config_filepath)
|
||||||
|
bot_username = get_parameter("bot_username", config_filepath)
|
||||||
|
|
Loading…
Referencia en una nova incidència