Added Funkwhale and Socialhome software detections

This commit is contained in:
spla 2021-10-26 13:38:12 +02:00
pare c9142101eb
commit ddb7038190
S'han modificat 7 arxius amb 464 adicions i 295 eliminacions

Veure arxiu

@ -73,8 +73,13 @@ Within Python Virtual Environment:
| writefreely | api/nodeinfo | ['software']['name'] | | writefreely | api/nodeinfo | ['software']['name'] |
| zap | nodeinfo/2.0.json | ['software']['name'] | | zap | nodeinfo/2.0.json | ['software']['name'] |
5. Use your favourite scheduling method to set `python fediverse.py` to run twice daily, `python fetchservers.py` one time daily and `python getworld.py` to run monthly. 7. Run `python uptime_setup.py` to get your Uptime bot's access token of your Mastodon or Pleroma server existing account. It will be saved to 'secrets/uptime_secrets.txt' for further use.
18.2.21 - New feature! Added [Lemmy project](https://join.lemmy.ml) 8. Use your favourite scheduling method to set `python fediverse.py` to run twice daily, `python fetchservers.py` one time daily, `python getworld.py` to run monthly and `python uptime.py` (choose your desired frequency) if you want to publish best fediverse's servers uptime.
12.5.21 - New feature! Added Wordpress support. The code can now detect Wordpress instances with ActivityPub enabled plugin.
12.5.21 - New feature! New shinny creation of servers and users graphs. 18.2.2021 - New feature! Added [Lemmy project](https://join.lemmy.ml)
12.5.2021 - New feature! Added Wordpress support. The code can now detect Wordpress instances with ActivityPub enabled plugin.
12.5.2021 - New feature! New shinny creation of servers and users graphs.
21.8.2021 - New feature! Added Best Fediverse's servers Uptime publishing bot.
22.10.2021 - New feature! Added [Funkwhale](https://funkwhale.audio) support.
26.10.2021 - New feature! Added [Socialhome](https://socialhome.network) support.

Veure arxiu

@ -1,6 +1,3 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import getpass import getpass
import os import os
import sys import sys
@ -10,7 +7,6 @@ import psycopg2
from psycopg2 import sql from psycopg2 import sql
from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
# Returns the parameter from the specified file
def get_parameter( parameter, file_path ): def get_parameter( parameter, file_path ):
# Check if secrets file exists # Check if secrets file exists
if not os.path.isfile(file_path): if not os.path.isfile(file_path):
@ -29,132 +25,153 @@ def get_parameter( parameter, file_path ):
sys.exit(0) sys.exit(0)
def write_parameter( parameter, file_path ): def write_parameter( parameter, file_path ):
if not os.path.exists('config'):
os.makedirs('config') if not os.path.exists('config'):
print("Setting up fediverse parameters...") os.makedirs('config')
print("\n") print("Setting up fediverse parameters...")
fediverse_db = input("fediverse db name: ") print("\n")
fediverse_db_user = input("fediverse db user: ") fediverse_db = input("fediverse db name: ")
fediverse_db_user = input("fediverse db user: ")
mastodon_db = input("Mastodon db name: ")
mastodon_db_user = input("Mastodon db user: ")
with open(file_path, "w") as text_file:
with open(file_path, "w") as text_file: print("fediverse_db: {}".format(fediverse_db), file=text_file)
print("fediverse_db: {}".format(fediverse_db), file=text_file) print("fediverse_db_user: {}".format(fediverse_db_user), file=text_file)
print("fediverse_db_user: {}".format(fediverse_db_user), file=text_file) print("mastodon_db: {}".format(mastodon_db), file=text_file)
print("mastodon_db_user: {}".format(mastodon_db_user), file=text_file)
def create_table(db, db_user, table, sql): def create_table(db, db_user, table, sql):
conn = None conn = None
try:
conn = psycopg2.connect(database = db, user = db_user, password = "", host = "/var/run/postgresql", port = "5432") try:
cur = conn.cursor()
print("Creating table.. "+table) conn = psycopg2.connect(database = db, user = db_user, password = "", host = "/var/run/postgresql", port = "5432")
# Create the table in PostgreSQL database cur = conn.cursor()
cur.execute(sql)
conn.commit() print("Creating table.. "+table)
print("Table "+table+" created!") # Create the table in PostgreSQL database
print("\n") cur.execute(sql)
except (Exception, psycopg2.DatabaseError) as error: conn.commit()
print(error) print("Table "+table+" created!")
print("\n")
finally: except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
###############################################################################
# main
if __name__ == '__main__':
# Load configuration from config file
config_filepath = "config/db_config.txt"
fediverse_db = get_parameter("fediverse_db", config_filepath)
fediverse_db_user = get_parameter("fediverse_db_user", config_filepath)
mastodon_db = get_parameter("mastodon_db", config_filepath)
mastodon_db_user = get_parameter("mastodon_db_user", config_filepath)
############################################################
# create database
############################################################
conn = None
try:
conn = psycopg2.connect(dbname='postgres',
user=fediverse_db_user, host='',
password='')
conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
cur = conn.cursor()
print("Creating database " + fediverse_db + ". Please wait...")
cur.execute(sql.SQL("CREATE DATABASE {}").format(
sql.Identifier(fediverse_db))
)
print("Database " + fediverse_db + " created!")
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
#############################################################################################
try:
conn = None
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
except (Exception, psycopg2.DatabaseError) as error:
print(error)
# Load configuration from config file
os.remove("db_config.txt")
print("Exiting. Run db-setup again with right parameters")
sys.exit(0)
if conn is not None: if conn is not None:
conn.close() print("\n")
print("fediverse parameters saved to db-config.txt!")
print("\n")
############################################################################################# ############################################################
# Create needed tables
############################################################
# Load configuration from config file print("Creating table...")
config_filepath = "config/db_config.txt"
fediverse_db = get_parameter("fediverse_db", config_filepath)
fediverse_db_user = get_parameter("fediverse_db_user", config_filepath)
############################################################ ########################################
# create database
############################################################
conn = None db = fediverse_db
db_user = fediverse_db_user
table = "world"
sql = "create table "+table+" (server varchar(200) PRIMARY KEY, federated_with varchar(200), updated_at timestamptz, saved_at timestamptz, checked boolean)"
create_table(db, db_user, table, sql)
try: table = "fediverse"
sql = "create table "+table+" (server varchar(200) PRIMARY KEY, users INT, updated_at timestamptz, software varchar(50), version varchar(100), first_checked_at timestamptz, last_checked_at timestamptz, downs int)"
create_table(db, db_user, table, sql)
conn = psycopg2.connect(dbname='postgres', table = "totals"
user=fediverse_db_user, host='', sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, total_servers INT, total_users INT)"
password='') create_table(db, db_user, table, sql)
conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) table = "evo"
sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, servers INT, users INT)"
create_table(db, db_user, table, sql)
cur = conn.cursor() table = "closed"
sql = "create table "+table+" (server varchar(200) PRIMARY KEY, software varchar(10), closed boolean)"
create_table(db, db_user, table, sql)
print("Creating database " + fediverse_db + ". Please wait...") table = "botreplies"
sql = "create table "+table+" (status_id bigint PRIMARY KEY, query_user varchar(40), status_created_at timestamptz)"
create_table(db, db_user, table, sql)
cur.execute(sql.SQL("CREATE DATABASE {}").format( #####################################
sql.Identifier(fediverse_db))
)
print("Database " + fediverse_db + " created!")
except (Exception, psycopg2.DatabaseError) as error: print("Done!")
print("Now you can run setup.py!")
print(error) print("\n")
finally:
if conn is not None:
conn.close()
#############################################################################################
try:
conn = None
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
except (Exception, psycopg2.DatabaseError) as error:
print(error)
# Load configuration from config file
os.remove("db_config.txt")
print("Exiting. Run db-setup again with right parameters")
sys.exit(0)
if conn is not None:
print("\n")
print("fediverse parameters saved to db-config.txt!")
print("\n")
############################################################
# Create needed tables
############################################################
print("Creating table...")
########################################
db = fediverse_db
db_user = fediverse_db_user
table = "world"
sql = "create table "+table+" (server varchar(200) PRIMARY KEY, federated_with varchar(200), updated_at timestamptz, saved_at timestamptz), checked boolean"
create_table(db, db_user, table, sql)
table = "fediverse"
sql = "create table "+table+" (server varchar(200) PRIMARY KEY, users INT, updated_at timestamptz, software varchar(50), version varchar(100))"
create_table(db, db_user, table, sql)
table = "totals"
sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, total_servers INT, total_users INT)"
create_table(db, db_user, table, sql)
table = "evo"
sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, servers INT, users INT)"
create_table(db, db_user, table, sql)
#####################################
print("Done!")
print("Now you can run setup.py!")
print("\n")

Veure arxiu

@ -14,12 +14,22 @@ import aiohttp
import asyncio import asyncio
import socket import socket
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
import pdb
plt.style.use('seaborn') plt.style.use('seaborn')
start_time = time.time() start_time = time.time()
apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?', '/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?', '/wp-json/nodeinfo/2.0?'] apis = ['/nodeinfo/2.0?',
'/nodeinfo/2.0.json?',
'/main/nodeinfo/2.0?',
'/api/statusnet/config?',
'/api/nodeinfo/2.0.json?',
'/api/nodeinfo?',
'/api/v1/instance?',
'/wp-json/nodeinfo/2.0?',
'/api/v1/instance/nodeinfo/2.0?',
'/.well-known/x-nodeinfo2?'
]
client_exceptions = ( client_exceptions = (
aiohttp.ClientResponseError, aiohttp.ClientResponseError,
@ -53,7 +63,7 @@ def get_alive_servers(server):
cur = conn.cursor() cur = conn.cursor()
cur.execute("select alive, software, users_api, version from fediverse where server=(%s)", (server,)) cur.execute("select alive, software, users_api, version, first_checked_at, downs from fediverse where server=(%s)", (server,))
row = cur.fetchone() row = cur.fetchone()
@ -63,6 +73,8 @@ def get_alive_servers(server):
serv_soft = row[1] serv_soft = row[1]
serv_api = row[2] serv_api = row[2]
soft_version = row[3] soft_version = row[3]
first_checked_at = row[4]
downs_qty = row[5]
cur.close() cur.close()
@ -79,7 +91,9 @@ def get_alive_servers(server):
try: try:
data = requests.get('https://' + server + serv_api, timeout=3) user_agent = {'User-agent': 'Mozilla/5.0'}
data = requests.get('https://' + server + serv_api, headers = user_agent, timeout=3)
if serv_soft == "mastodon": if serv_soft == "mastodon":
if serv_api == '/nodeinfo/2.0?': if serv_api == '/nodeinfo/2.0?':
@ -149,18 +163,46 @@ def get_alive_servers(server):
alive = True alive = True
except: except:
users = 0
soft_version = ""
if serv_soft == 'funkwhale':
try:
users = data.json()['usage']['users']['total']
soft_version = data.json()['software']['version']
alive = True
except:
users = 0
soft_version = ""
if serv_soft == 'socialhome':
try:
users = data.json()['usage']['users']['total']
soft_version = data.json()['server']['version']
alive = True
except:
users = 0 users = 0
soft_version = "" soft_version = ""
if alive: if alive:
if downs_qty != None:
downs = downs_qty
else:
downs = 0
if soft_version != "" and soft_version is not None: if soft_version != "" and soft_version is not None:
print("Server " + str(server) + " (" + serv_soft + " " + soft_version + ") is alive!") print("Server " + str(server) + " (" + serv_soft + " " + soft_version + ") is alive!")
else: else:
print("Server " + str(server) + " (" + serv_soft + ") is alive!") print("Server " + str(server) + " (" + serv_soft + ") is alive!")
insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING" insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version, first_checked_at, last_checked_at, downs) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
conn = None conn = None
@ -170,9 +212,16 @@ def get_alive_servers(server):
cur = conn.cursor() cur = conn.cursor()
cur.execute(insert_sql, (server, users, now, serv_soft, alive, serv_api, soft_version)) cur.execute(insert_sql, (server, users, now, serv_soft, alive, serv_api, soft_version, now, now, downs))
if first_checked_at != None:
cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (users, now, serv_soft, alive, serv_api, soft_version, now, downs, server))
else:
cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), first_checked_at=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (users, now, serv_soft, alive, serv_api, soft_version, now, now, downs, server))
cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)", (users, now, serv_soft, alive, serv_api, soft_version, server))
cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,)) cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
@ -258,6 +307,14 @@ def get_alive_servers(server):
if alive is False: if alive is False:
if downs_qty != None:
downs = downs_qty + 1
else:
downs = 1
conn = None conn = None
try: try:
@ -267,7 +324,13 @@ def get_alive_servers(server):
cur = conn.cursor() cur = conn.cursor()
cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s) where server=(%s)", (now, alive, server)) if first_checked_at != None:
cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), first_checked_at=(%s), downs=(%s) where server=(%s)", (now, alive, now, downs, server))
else:
cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), downs=(%s) where server=(%s)", (now, alive, downs, server))
cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,)) cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
@ -329,8 +392,11 @@ async def getsoft(server):
url = 'https://' + server url = 'https://' + server
user_agent = {'User-agent': 'Mozilla/5.0'}
timeout = aiohttp.ClientTimeout(total=3) timeout = aiohttp.ClientTimeout(total=3)
async with aiohttp.ClientSession(timeout=timeout) as session:
async with aiohttp.ClientSession(headers=user_agent, timeout=timeout) as session:
for api in apis: for api in apis:
try: try:
async with session.get(url+api) as response: async with session.get(url+api) as response:
@ -871,23 +937,15 @@ if __name__ == '__main__':
project_servers = soft_total_servers[i] project_servers = soft_total_servers[i]
len_pr_soft = len(project_soft) len_pr_soft = len(project_soft)
if project_soft == 'writefreely': toot_text += f":{project_soft}: {project_users:,} {project_servers:,}\n"
str_len = 11
else:
str_len = 13
toot_text += f"{':'+project_soft+':':<11}" + f"{project_users:>{str_len},}" + " " + f"{project_servers:>5,}" + "\n"
i += 1 i += 1
print("Tooting...") print("Tooting...")
print(toot_text) print(toot_text)
servers_image_id = mastodon.media_post('servers.png', "image/png").id servers_image_id = mastodon.media_post('servers.png', "image/png", description='servers graph').id
users_image_id = mastodon.media_post('users.png', "image/png").id users_image_id = mastodon.media_post('users.png', "image/png", description='users graph').id
mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, users_image_id}) mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, users_image_id})

Veure arxiu

@ -10,8 +10,17 @@ import aiohttp
import asyncio import asyncio
import socket import socket
apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?', apis = ['/nodeinfo/2.0?',
'/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?', '/wp-json/nodeinfo/2.0?'] '/nodeinfo/2.0.json?',
'/main/nodeinfo/2.0?',
'/api/statusnet/config?',
'/api/nodeinfo/2.0.json?',
'/api/nodeinfo?',
'/api/v1/instance?',
'/wp-json/nodeinfo/2.0?',
'/api/v1/instance/nodeinfo/2.0?',
'/.well-known/x-nodeinfo2?'
]
client_exceptions = ( client_exceptions = (
aiohttp.ClientResponseError, aiohttp.ClientResponseError,
@ -24,15 +33,17 @@ client_exceptions = (
def is_json(myjson): def is_json(myjson):
try: try:
json_object = json.loads(myjson) json_object = json.loads(myjson)
except ValueError as e: except ValueError as e:
return False return False
return True return True
def write_api(server, software, users, alive, api, soft_version): def write_api(server, software, users, alive, api, soft_version):
insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING" insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
conn = None conn = None
try: try:
@ -46,7 +57,8 @@ def write_api(server, software, users, alive, api, soft_version):
cur.execute( cur.execute(
"UPDATE fediverse SET updated_at=(%s), software=(%s), users=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)", "UPDATE fediverse SET updated_at=(%s), software=(%s), users=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)",
(now, software, users, alive, api, soft_version, server)) (now, software, users, alive, api, soft_version, server)
)
cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,)) cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
@ -63,9 +75,8 @@ def write_api(server, software, users, alive, api, soft_version):
if conn is not None: if conn is not None:
conn.close() conn.close()
async def getsoft(server): async def getsoft(server):
try: try:
socket.gethostbyname(server) socket.gethostbyname(server)
@ -79,33 +90,61 @@ async def getsoft(server):
url = 'https://' + server url = 'https://' + server
user_agent = {'User-agent': 'Mozilla/5.0'}
timeout = aiohttp.ClientTimeout(total=3) timeout = aiohttp.ClientTimeout(total=3)
async with aiohttp.ClientSession(timeout=timeout) as session: async with aiohttp.ClientSession(timeout=timeout, headers=user_agent) as session:
for api in apis: for api in apis:
try: try:
async with session.get(url + api) as response: async with session.get(url + api) as response:
if response.status == 200: if response.status == 200:
try: try:
response_json = await response.json() response_json = await response.json()
except: except:
pass pass
except aiohttp.ClientConnectorError as err: except aiohttp.ClientConnectorError as err:
pass pass
else: else:
if response.status == 200 and api != '/api/v1/instance?': if response.status == 200 and api != '/api/v1/instance?':
try:
soft = response_json['software']['name'] if api != '/.well-known/x-nodeinfo2?':
soft = soft.lower()
soft_version = response_json['software']['version'] try:
users = response_json['usage']['users']['total'] soft = response_json['software']['name']
if users > 1000000: soft = soft.lower()
soft_version = response_json['software']['version']
users = response_json['usage']['users']['total']
if users > 1000000:
return
alive = True
write_api(server, soft, users, alive, api, soft_version)
print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
return return
alive = True except:
write_api(server, soft, users, alive, api, soft_version) pass
print("Server " + server + " (" + soft + " " + soft_version + ") is alive!") else:
return
except: try:
pass soft = response_json['server']['software']
soft = soft.lower()
soft_version = response_json['server']['version']
users = response_json['usage']['users']['total']
if users > 1000000:
return
alive = True
if soft == 'socialhome':
write_api(server, soft, users, alive, api, soft_version)
print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
return
except:
pass
if response.status == 200 and soft == '' and api == "/api/v1/instance?": if response.status == 200 and soft == '' and api == "/api/v1/instance?":
soft = 'mastodon' soft = 'mastodon'
users = response_json['stats']['user_count'] users = response_json['stats']['user_count']
@ -116,11 +155,10 @@ async def getsoft(server):
write_api(server, soft, users, alive, api) write_api(server, soft, users, alive, api)
print("Server " + server + " (" + soft + ") is alive!") print("Server " + server + " (" + soft + ") is alive!")
def getserver(server, x): def getserver(server, x):
server = server[0].rstrip('.').lower() server = server[0].rstrip('.').lower()
if server.find(".") == -1: if server.find(".") == -1:
return return
if server.find("@") != -1: if server.find("@") != -1:
@ -140,8 +178,6 @@ def getserver(server, x):
pass pass
# Returns the parameter from the specified file
def get_parameter(parameter, file_path): def get_parameter(parameter, file_path):
# Check if secrets file exists # Check if secrets file exists
if not os.path.isfile(file_path): if not os.path.isfile(file_path):
@ -159,14 +195,20 @@ def get_parameter(parameter, file_path):
sys.exit(0) sys.exit(0)
# Load configuration from config file def get_config():
config_filepath = "config/config.txt"
mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
# Load database config from db_config file # Load configuration from config file
db_config_filepath = "config/db_config.txt" config_filepath = "config/config.txt"
fediverse_db = get_parameter("fediverse_db", db_config_filepath) mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath) return mastodon_hostname
def get_db_config():
# Load database config from db_config file
db_config_filepath = "config/db_config.txt"
fediverse_db = get_parameter("fediverse_db", db_config_filepath)
fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath)
return (fediverse_db, fediverse_db_user)
############################################################################### ###############################################################################
# main # main
@ -176,6 +218,10 @@ if __name__ == '__main__':
now = datetime.now() now = datetime.now()
start_time = time.time() start_time = time.time()
mastodon_hostname = get_config()
fediverse_db, fediverse_db_user = get_db_config()
world_servers = [] world_servers = []
try: try:
@ -207,6 +253,7 @@ if __name__ == '__main__':
finally: finally:
if conn is not None: if conn is not None:
conn.close() conn.close()
########################################################################### ###########################################################################

Veure arxiu

@ -47,11 +47,13 @@ def write_servers(world_servers):
i += 1 i += 1
def get_peers(peer): def get_peers(peer):
try: try:
response = requests.get('https://' + peer + peers_api, timeout=2) user_agent = {'User-agent': 'Mozilla/5.0'}
response = requests.get('https://' + peer + peers_api, headers = user_agent, timeout=3)
response_json = response.json() response_json = response.json()
if response.status_code == 200: if response.status_code == 200:
@ -59,7 +61,7 @@ def get_peers(peer):
try: try:
print("Server: " + peer + ", " + "federated with " + str(len(response_json)) + " servers") print("Server: " + peer + ", " + "federated with " + str(len(response_json)) + " servers")
for peer_peer in response_json: for peer_peer in response_json:
exist_count = world_peers.count(peer_peer) exist_count = world_peers.count(peer_peer)
@ -111,10 +113,9 @@ if __name__ == '__main__':
start_time = time.time() start_time = time.time()
updated_at = datetime.now() updated_at = datetime.now()
peers_api = '/api/v1/instance/peers?' peers_api = '/api/v1/instance/peers?'
lemmy_api = '/api/v2/site?'
# Load configuration from config file # Load configuration from config file
config_filepath = "config/config.txt" config_filepath = "config/config.txt"
mastodon_hostname = get_parameter("mastodon_hostname", config_filepath) mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
@ -123,17 +124,19 @@ if __name__ == '__main__':
db_config_filepath = "config/db_config.txt" db_config_filepath = "config/db_config.txt"
fediverse_db = get_parameter("fediverse_db", db_config_filepath) fediverse_db = get_parameter("fediverse_db", db_config_filepath)
fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath) fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath)
world_peers = [] world_peers = []
res = requests.get('https://' + mastodon_hostname + peers_api) user_agent = {'User-agent': 'Mozilla/5.0'}
res = requests.get('https://' + mastodon_hostname + peers_api, headers = user_agent, timeout=3)
hostname_peers = res.json() hostname_peers = res.json()
for peer in hostname_peers: for peer in hostname_peers:
exist_count = world_peers.count(peer) exist_count = world_peers.count(peer)
if exist_count == 0: if exist_count == 0:
world_peers.append(peer) world_peers.append(peer)
@ -148,6 +151,6 @@ if __name__ == '__main__':
p_thread.join() p_thread.join()
write_servers(world_peers) write_servers(world_peers)
exec_time = str(round((time.time() - start_time), 2)) exec_time = str(round((time.time() - start_time), 2))
print(exec_time) print(exec_time)

Veure arxiu

@ -3,3 +3,4 @@ psycopg2-binary>=2.8.4
aiohttp>=3.6.2 aiohttp>=3.6.2
aiodns>=2.0.0 aiodns>=2.0.0
matplotlib>=3.3.4 matplotlib>=3.3.4
humanfriendly>=9.2

274
setup.py
Veure arxiu

@ -1,6 +1,3 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import getpass import getpass
from mastodon import Mastodon from mastodon import Mastodon
from mastodon.Mastodon import MastodonMalformedEventError, MastodonNetworkError, MastodonReadTimeout, MastodonAPIError, MastodonIllegalArgumentError from mastodon.Mastodon import MastodonMalformedEventError, MastodonNetworkError, MastodonReadTimeout, MastodonAPIError, MastodonIllegalArgumentError
@ -9,130 +6,167 @@ import os
import sys import sys
def create_dir(): def create_dir():
if not os.path.exists('secrets'):
os.makedirs('secrets') if not os.path.exists('secrets'):
os.makedirs('secrets')
def create_file(): def create_file():
if not os.path.exists('secrets/secrets.txt'):
with open('secrets/secrets.txt', 'w'): pass if not os.path.exists('secrets/secrets.txt'):
print(secrets_filepath + " created!") with open('secrets/secrets.txt', 'w'): pass
print(secrets_filepath + " created!")
def create_config(): def create_config():
if not os.path.exists('config'):
os.makedirs('config') if not os.path.exists('config'):
if not os.path.exists(config_filepath): os.makedirs('config')
print(config_filepath + " created!") if not os.path.exists(config_filepath):
with open('config/config.txt', 'w'): pass print(config_filepath + " created!")
with open('config/config.txt', 'w'): pass
def write_params(): def write_params():
with open(secrets_filepath, 'a') as the_file:
print("Writing secrets parameter names to " + secrets_filepath) with open(secrets_filepath, 'a') as the_file:
the_file.write('uc_client_id: \n'+'uc_client_secret: \n'+'uc_access_token: \n') print("Writing secrets parameter names to " + secrets_filepath)
the_file.write('uc_client_id: \n'+'uc_client_secret: \n'+'uc_access_token: \n')
def write_config(): def write_config():
with open(config_filepath, 'a') as the_file:
the_file.write('mastodon_hostname: \n') with open(config_filepath, 'a') as the_file:
print("adding parameter name 'mastodon_hostname' to "+ config_filepath) the_file.write('mastodon_hostname: \n')
print("adding parameter name 'mastodon_hostname' to "+ config_filepath)
def read_client_lines(self): def read_client_lines(self):
client_path = 'app_clientcred.txt'
with open(client_path) as fp: client_path = 'app_clientcred.txt'
line = fp.readline()
cnt = 1 with open(client_path) as fp:
while line:
if cnt == 1: line = fp.readline()
print("Writing client id to " + secrets_filepath) cnt = 1
modify_file(secrets_filepath, "uc_client_id: ", value=line.rstrip()) while line:
elif cnt == 2: if cnt == 1:
print("Writing client secret to " + secrets_filepath) print("Writing client id to " + secrets_filepath)
modify_file(secrets_filepath, "uc_client_secret: ", value=line.rstrip()) modify_file(secrets_filepath, "uc_client_id: ", value=line.rstrip())
line = fp.readline() elif cnt == 2:
cnt += 1 print("Writing client secret to " + secrets_filepath)
modify_file(secrets_filepath, "uc_client_secret: ", value=line.rstrip())
line = fp.readline()
cnt += 1
def read_token_line(self): def read_token_line(self):
token_path = 'app_usercred.txt'
with open(token_path) as fp: token_path = 'app_usercred.txt'
line = fp.readline()
print("Writing access token to " + secrets_filepath) with open(token_path) as fp:
modify_file(secrets_filepath, "uc_access_token: ", value=line.rstrip())
line = fp.readline()
print("Writing access token to " + secrets_filepath)
modify_file(secrets_filepath, "uc_access_token: ", value=line.rstrip())
def read_config_line(): def read_config_line():
with open(config_filepath) as fp:
line = fp.readline() with open(config_filepath) as fp:
modify_file(config_filepath, "mastodon_hostname: ", value=hostname)
line = fp.readline()
modify_file(config_filepath, "mastodon_hostname: ", value=hostname)
modify_file(config_filepath, "bot_username: ", value=bot_username)
def log_in(): def log_in():
error = 0
try:
global hostname
hostname = input("Enter Mastodon hostname: ")
user_name = input("User name, ex. user@" + hostname +"? ")
user_password = getpass.getpass("User password? ")
app_name = input("This app name? ")
Mastodon.create_app(app_name, scopes=["read","write"],
to_file="app_clientcred.txt", api_base_url=hostname)
mastodon = Mastodon(client_id = "app_clientcred.txt", api_base_url = hostname)
mastodon.log_in(
user_name,
user_password,
scopes = ["read", "write"],
to_file = "app_usercred.txt"
)
except MastodonIllegalArgumentError as i_error:
error = 1
if os.path.exists("secrets/secrets.txt"):
print("Removing secrets/secrets.txt file..")
os.remove("secrets/secrets.txt")
if os.path.exists("app_clientcred.txt"):
print("Removing app_clientcred.txt file..")
os.remove("app_clientcred.txt")
sys.exit(i_error)
except MastodonNetworkError as n_error:
error = 1
if os.path.exists("secrets/secrets.txt"):
print("Removing secrets/secrets.txt file..")
os.remove("secrets/secrets.txt")
if os.path.exists("app_clientcred.txt"):
print("Removing app_clientcred.txt file..")
os.remove("app_clientcred.txt")
sys.exit(n_error)
except MastodonReadTimeout as r_error:
error = 1
if os.path.exists("secrets/secrets.txt"):
print("Removing secrets/secrets.txt file..")
os.remove("secrets/secrets.txt")
if os.path.exists("app_clientcred.txt"):
print("Removing app_clientcred.txt file..")
os.remove("app_clientcred.txt")
sys.exit(r_error)
except MastodonAPIError as a_error:
error = 1
if os.path.exists("secrets/secrets.txt"):
print("Removing secrets/secrets.txt file..")
os.remove("secrets/secrets.txt")
if os.path.exists("app_clientcred.txt"):
print("Removing app_clientcred.txt file..")
os.remove("app_clientcred.txt")
sys.exit(a_error)
finally:
if error == 0:
create_dir() error = 0
create_file()
write_params() try:
client_path = 'app_clientcred.txt'
read_client_lines(client_path) global hostname
token_path = 'app_usercred.txt' hostname = input("Enter Mastodon hostname: ")
read_token_line(token_path) user_name = input("User name, ex. user@" + hostname +"? ")
if os.path.exists("app_clientcred.txt"): user_password = getpass.getpass("User password? ")
print("Removing app_clientcred.txt temp file..") bot_username = input("Bot's username, ex. fediverse: ")
os.remove("app_clientcred.txt") app_name = input("This app name? ")
if os.path.exists("app_usercred.txt"): Mastodon.create_app(
print("Removing app_usercred.txt temp file..") app_name,
os.remove("app_usercred.txt") scopes=["read","write"],
print("Secrets setup done!\n") to_file="app_clientcred.txt",
api_base_url=hostname
)
mastodon = Mastodon(client_id = "app_clientcred.txt", api_base_url = hostname)
mastodon.log_in(
user_name,
user_password,
scopes = ["read", "write"],
to_file = "app_usercred.txt"
)
except MastodonIllegalArgumentError as i_error:
error = 1
if os.path.exists("secrets/secrets.txt"):
print("Removing secrets/secrets.txt file..")
os.remove("secrets/secrets.txt")
if os.path.exists("app_clientcred.txt"):
print("Removing app_clientcred.txt file..")
os.remove("app_clientcred.txt")
sys.exit(i_error)
except MastodonNetworkError as n_error:
error = 1
if os.path.exists("secrets/secrets.txt"):
print("Removing secrets/secrets.txt file..")
os.remove("secrets/secrets.txt")
if os.path.exists("app_clientcred.txt"):
print("Removing app_clientcred.txt file..")
os.remove("app_clientcred.txt")
sys.exit(n_error)
except MastodonReadTimeout as r_error:
error = 1
if os.path.exists("secrets/secrets.txt"):
print("Removing secrets/secrets.txt file..")
os.remove("secrets/secrets.txt")
if os.path.exists("app_clientcred.txt"):
print("Removing app_clientcred.txt file..")
os.remove("app_clientcred.txt")
sys.exit(r_error)
except MastodonAPIError as a_error:
error = 1
if os.path.exists("secrets/secrets.txt"):
print("Removing secrets/secrets.txt file..")
os.remove("secrets/secrets.txt")
if os.path.exists("app_clientcred.txt"):
print("Removing app_clientcred.txt file..")
os.remove("app_clientcred.txt")
sys.exit(a_error)
finally:
if error == 0:
create_dir()
create_file()
write_params()
client_path = 'app_clientcred.txt'
read_client_lines(client_path)
token_path = 'app_usercred.txt'
read_token_line(token_path)
if os.path.exists("app_clientcred.txt"):
print("Removing app_clientcred.txt temp file..")
os.remove("app_clientcred.txt")
if os.path.exists("app_usercred.txt"):
print("Removing app_usercred.txt temp file..")
os.remove("app_usercred.txt")
print("Secrets setup done!\n")
def modify_file(file_name,pattern,value=""): def modify_file(file_name,pattern,value=""):
fh=fileinput.input(file_name,inplace=True) fh=fileinput.input(file_name,inplace=True)
for line in fh: for line in fh:
replacement=pattern + value replacement=pattern + value
@ -140,7 +174,6 @@ def modify_file(file_name,pattern,value=""):
sys.stdout.write(line) sys.stdout.write(line)
fh.close() fh.close()
# Returns the parameter from the specified file
def get_parameter( parameter, file_path ): def get_parameter( parameter, file_path ):
# Check if secrets file exists # Check if secrets file exists
if not os.path.isfile(file_path): if not os.path.isfile(file_path):
@ -157,7 +190,6 @@ def get_parameter( parameter, file_path ):
print(file_path + " Missing parameter %s "%parameter) print(file_path + " Missing parameter %s "%parameter)
sys.exit(0) sys.exit(0)
# Returns the parameter from the specified file
def get_hostname( parameter, config_filepath ): def get_hostname( parameter, config_filepath ):
# Check if secrets file exists # Check if secrets file exists
if not os.path.isfile(config_filepath): if not os.path.isfile(config_filepath):
@ -177,12 +209,18 @@ def get_hostname( parameter, config_filepath ):
print("setup done!") print("setup done!")
sys.exit(0) sys.exit(0)
# Load secrets from secrets file ###############################################################################
secrets_filepath = "secrets/secrets.txt" # main
uc_client_id = get_parameter("uc_client_id", secrets_filepath)
uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
uc_access_token = get_parameter("uc_access_token", secrets_filepath)
# Load configuration from config file if __name__ == '__main__':
config_filepath = "config/config.txt"
mastodon_hostname = get_hostname("mastodon_hostname", config_filepath) # Load secrets from secrets file
secrets_filepath = "secrets/secrets.txt"
uc_client_id = get_parameter("uc_client_id", secrets_filepath)
uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
uc_access_token = get_parameter("uc_access_token", secrets_filepath)
# Load configuration from config file
config_filepath = "config/config.txt"
mastodon_hostname = get_hostname("mastodon_hostname", config_filepath)
bot_username = get_parameter("bot_username", config_filepath)