From ddb7038190712612df9a6d415c0feb8a2ed76e2a Mon Sep 17 00:00:00 2001 From: spla Date: Tue, 26 Oct 2021 13:38:12 +0200 Subject: [PATCH] Added Funkwhale and Socialhome software detections --- README.md | 13 ++- db-setup.py | 235 +++++++++++++++++++++------------------- fediverse.py | 102 ++++++++++++++---- fetchservers.py | 107 ++++++++++++------ getpeers.py | 27 ++--- requirements.txt | 1 + setup.py | 274 +++++++++++++++++++++++++++-------------------- 7 files changed, 464 insertions(+), 295 deletions(-) diff --git a/README.md b/README.md index 29416a0..56a74cd 100644 --- a/README.md +++ b/README.md @@ -73,8 +73,13 @@ Within Python Virtual Environment: | writefreely | api/nodeinfo | ['software']['name'] | | zap | nodeinfo/2.0.json | ['software']['name'] | -5. Use your favourite scheduling method to set `python fediverse.py` to run twice daily, `python fetchservers.py` one time daily and `python getworld.py` to run monthly. +7. Run `python uptime_setup.py` to get your Uptime bot's access token of your Mastodon or Pleroma server existing account. It will be saved to 'secrets/uptime_secrets.txt' for further use. -18.2.21 - New feature! Added [Lemmy project](https://join.lemmy.ml) -12.5.21 - New feature! Added Wordpress support. The code can now detect Wordpress instances with ActivityPub enabled plugin. -12.5.21 - New feature! New shinny creation of servers and users graphs. +8. Use your favourite scheduling method to set `python fediverse.py` to run twice daily, `python fetchservers.py` one time daily, `python getworld.py` to run monthly and `python uptime.py` (choose your desired frequency) if you want to publish best fediverse's servers uptime. + +18.2.2021 - New feature! Added [Lemmy project](https://join.lemmy.ml) +12.5.2021 - New feature! Added Wordpress support. The code can now detect Wordpress instances with ActivityPub enabled plugin. +12.5.2021 - New feature! New shinny creation of servers and users graphs. +21.8.2021 - New feature! Added Best Fediverse's servers Uptime publishing bot. +22.10.2021 - New feature! Added [Funkwhale](https://funkwhale.audio) support. +26.10.2021 - New feature! Added [Socialhome](https://socialhome.network) support. diff --git a/db-setup.py b/db-setup.py index a17320c..b0938e7 100644 --- a/db-setup.py +++ b/db-setup.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - import getpass import os import sys @@ -10,7 +7,6 @@ import psycopg2 from psycopg2 import sql from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT -# Returns the parameter from the specified file def get_parameter( parameter, file_path ): # Check if secrets file exists if not os.path.isfile(file_path): @@ -29,132 +25,153 @@ def get_parameter( parameter, file_path ): sys.exit(0) def write_parameter( parameter, file_path ): - if not os.path.exists('config'): - os.makedirs('config') - print("Setting up fediverse parameters...") - print("\n") - fediverse_db = input("fediverse db name: ") - fediverse_db_user = input("fediverse db user: ") + + if not os.path.exists('config'): + os.makedirs('config') + print("Setting up fediverse parameters...") + print("\n") + fediverse_db = input("fediverse db name: ") + fediverse_db_user = input("fediverse db user: ") + mastodon_db = input("Mastodon db name: ") + mastodon_db_user = input("Mastodon db user: ") + + with open(file_path, "w") as text_file: - with open(file_path, "w") as text_file: - print("fediverse_db: {}".format(fediverse_db), file=text_file) - print("fediverse_db_user: {}".format(fediverse_db_user), file=text_file) + print("fediverse_db: {}".format(fediverse_db), file=text_file) + print("fediverse_db_user: {}".format(fediverse_db_user), file=text_file) + print("mastodon_db: {}".format(mastodon_db), file=text_file) + print("mastodon_db_user: {}".format(mastodon_db_user), file=text_file) def create_table(db, db_user, table, sql): - conn = None - try: + conn = None - conn = psycopg2.connect(database = db, user = db_user, password = "", host = "/var/run/postgresql", port = "5432") - cur = conn.cursor() + try: - print("Creating table.. "+table) - # Create the table in PostgreSQL database - cur.execute(sql) + conn = psycopg2.connect(database = db, user = db_user, password = "", host = "/var/run/postgresql", port = "5432") + cur = conn.cursor() - conn.commit() - print("Table "+table+" created!") - print("\n") + print("Creating table.. "+table) + # Create the table in PostgreSQL database + cur.execute(sql) - except (Exception, psycopg2.DatabaseError) as error: + conn.commit() - print(error) + print("Table "+table+" created!") + print("\n") - finally: + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + +############################################################################### +# main + +if __name__ == '__main__': + + # Load configuration from config file + config_filepath = "config/db_config.txt" + fediverse_db = get_parameter("fediverse_db", config_filepath) + fediverse_db_user = get_parameter("fediverse_db_user", config_filepath) + mastodon_db = get_parameter("mastodon_db", config_filepath) + mastodon_db_user = get_parameter("mastodon_db_user", config_filepath) + + ############################################################ + # create database + ############################################################ + + conn = None + + try: + + conn = psycopg2.connect(dbname='postgres', + user=fediverse_db_user, host='', + password='') + + conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) + + cur = conn.cursor() + + print("Creating database " + fediverse_db + ". Please wait...") + + cur.execute(sql.SQL("CREATE DATABASE {}").format( + sql.Identifier(fediverse_db)) + ) + print("Database " + fediverse_db + " created!") + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + ############################################################################################# + + try: + + conn = None + conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + # Load configuration from config file + os.remove("db_config.txt") + print("Exiting. Run db-setup again with right parameters") + sys.exit(0) if conn is not None: - conn.close() + print("\n") + print("fediverse parameters saved to db-config.txt!") + print("\n") -############################################################################################# + ############################################################ + # Create needed tables + ############################################################ -# Load configuration from config file -config_filepath = "config/db_config.txt" -fediverse_db = get_parameter("fediverse_db", config_filepath) -fediverse_db_user = get_parameter("fediverse_db_user", config_filepath) + print("Creating table...") -############################################################ -# create database -############################################################ + ######################################## -conn = None + db = fediverse_db + db_user = fediverse_db_user + table = "world" + sql = "create table "+table+" (server varchar(200) PRIMARY KEY, federated_with varchar(200), updated_at timestamptz, saved_at timestamptz, checked boolean)" + create_table(db, db_user, table, sql) -try: + table = "fediverse" + sql = "create table "+table+" (server varchar(200) PRIMARY KEY, users INT, updated_at timestamptz, software varchar(50), version varchar(100), first_checked_at timestamptz, last_checked_at timestamptz, downs int)" + create_table(db, db_user, table, sql) - conn = psycopg2.connect(dbname='postgres', - user=fediverse_db_user, host='', - password='') + table = "totals" + sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, total_servers INT, total_users INT)" + create_table(db, db_user, table, sql) - conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) + table = "evo" + sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, servers INT, users INT)" + create_table(db, db_user, table, sql) - cur = conn.cursor() + table = "closed" + sql = "create table "+table+" (server varchar(200) PRIMARY KEY, software varchar(10), closed boolean)" + create_table(db, db_user, table, sql) - print("Creating database " + fediverse_db + ". Please wait...") + table = "botreplies" + sql = "create table "+table+" (status_id bigint PRIMARY KEY, query_user varchar(40), status_created_at timestamptz)" + create_table(db, db_user, table, sql) - cur.execute(sql.SQL("CREATE DATABASE {}").format( - sql.Identifier(fediverse_db)) - ) - print("Database " + fediverse_db + " created!") + ##################################### -except (Exception, psycopg2.DatabaseError) as error: - - print(error) - -finally: - - if conn is not None: - - conn.close() - -############################################################################################# - -try: - - conn = None - conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432") - -except (Exception, psycopg2.DatabaseError) as error: - - print(error) - # Load configuration from config file - os.remove("db_config.txt") - print("Exiting. Run db-setup again with right parameters") - sys.exit(0) - -if conn is not None: - - print("\n") - print("fediverse parameters saved to db-config.txt!") - print("\n") - -############################################################ -# Create needed tables -############################################################ - -print("Creating table...") - -######################################## - -db = fediverse_db -db_user = fediverse_db_user -table = "world" -sql = "create table "+table+" (server varchar(200) PRIMARY KEY, federated_with varchar(200), updated_at timestamptz, saved_at timestamptz), checked boolean" -create_table(db, db_user, table, sql) - -table = "fediverse" -sql = "create table "+table+" (server varchar(200) PRIMARY KEY, users INT, updated_at timestamptz, software varchar(50), version varchar(100))" -create_table(db, db_user, table, sql) - -table = "totals" -sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, total_servers INT, total_users INT)" -create_table(db, db_user, table, sql) - -table = "evo" -sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, servers INT, users INT)" -create_table(db, db_user, table, sql) - -##################################### - -print("Done!") -print("Now you can run setup.py!") -print("\n") + print("Done!") + print("Now you can run setup.py!") + print("\n") diff --git a/fediverse.py b/fediverse.py index 7e1bfd5..5024166 100644 --- a/fediverse.py +++ b/fediverse.py @@ -14,12 +14,22 @@ import aiohttp import asyncio import socket import matplotlib.pyplot as plt - +import pdb plt.style.use('seaborn') start_time = time.time() -apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?', '/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?', '/wp-json/nodeinfo/2.0?'] +apis = ['/nodeinfo/2.0?', + '/nodeinfo/2.0.json?', + '/main/nodeinfo/2.0?', + '/api/statusnet/config?', + '/api/nodeinfo/2.0.json?', + '/api/nodeinfo?', + '/api/v1/instance?', + '/wp-json/nodeinfo/2.0?', + '/api/v1/instance/nodeinfo/2.0?', + '/.well-known/x-nodeinfo2?' + ] client_exceptions = ( aiohttp.ClientResponseError, @@ -53,7 +63,7 @@ def get_alive_servers(server): cur = conn.cursor() - cur.execute("select alive, software, users_api, version from fediverse where server=(%s)", (server,)) + cur.execute("select alive, software, users_api, version, first_checked_at, downs from fediverse where server=(%s)", (server,)) row = cur.fetchone() @@ -63,6 +73,8 @@ def get_alive_servers(server): serv_soft = row[1] serv_api = row[2] soft_version = row[3] + first_checked_at = row[4] + downs_qty = row[5] cur.close() @@ -79,7 +91,9 @@ def get_alive_servers(server): try: - data = requests.get('https://' + server + serv_api, timeout=3) + user_agent = {'User-agent': 'Mozilla/5.0'} + + data = requests.get('https://' + server + serv_api, headers = user_agent, timeout=3) if serv_soft == "mastodon": if serv_api == '/nodeinfo/2.0?': @@ -149,18 +163,46 @@ def get_alive_servers(server): alive = True except: - + + users = 0 + soft_version = "" + + if serv_soft == 'funkwhale': + + try: + users = data.json()['usage']['users']['total'] + soft_version = data.json()['software']['version'] + alive = True + except: + users = 0 + soft_version = "" + + if serv_soft == 'socialhome': + + try: + users = data.json()['usage']['users']['total'] + soft_version = data.json()['server']['version'] + alive = True + except: users = 0 soft_version = "" if alive: - + + if downs_qty != None: + + downs = downs_qty + + else: + + downs = 0 + if soft_version != "" and soft_version is not None: print("Server " + str(server) + " (" + serv_soft + " " + soft_version + ") is alive!") else: print("Server " + str(server) + " (" + serv_soft + ") is alive!") - insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING" + insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version, first_checked_at, last_checked_at, downs) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING" conn = None @@ -170,9 +212,16 @@ def get_alive_servers(server): cur = conn.cursor() - cur.execute(insert_sql, (server, users, now, serv_soft, alive, serv_api, soft_version)) + cur.execute(insert_sql, (server, users, now, serv_soft, alive, serv_api, soft_version, now, now, downs)) + + if first_checked_at != None: + + cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (users, now, serv_soft, alive, serv_api, soft_version, now, downs, server)) + + else: + + cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s), first_checked_at=(%s), last_checked_at=(%s), downs=(%s) where server=(%s)", (users, now, serv_soft, alive, serv_api, soft_version, now, now, downs, server)) - cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)", (users, now, serv_soft, alive, serv_api, soft_version, server)) cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,)) @@ -258,6 +307,14 @@ def get_alive_servers(server): if alive is False: + if downs_qty != None: + + downs = downs_qty + 1 + + else: + + downs = 1 + conn = None try: @@ -267,7 +324,13 @@ def get_alive_servers(server): cur = conn.cursor() - cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s) where server=(%s)", (now, alive, server)) + if first_checked_at != None: + + cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), first_checked_at=(%s), downs=(%s) where server=(%s)", (now, alive, now, downs, server)) + + else: + + cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s), downs=(%s) where server=(%s)", (now, alive, downs, server)) cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,)) @@ -329,8 +392,11 @@ async def getsoft(server): url = 'https://' + server + user_agent = {'User-agent': 'Mozilla/5.0'} + timeout = aiohttp.ClientTimeout(total=3) - async with aiohttp.ClientSession(timeout=timeout) as session: + + async with aiohttp.ClientSession(headers=user_agent, timeout=timeout) as session: for api in apis: try: async with session.get(url+api) as response: @@ -871,23 +937,15 @@ if __name__ == '__main__': project_servers = soft_total_servers[i] len_pr_soft = len(project_soft) - if project_soft == 'writefreely': - - str_len = 11 - - else: - - str_len = 13 - - toot_text += f"{':'+project_soft+':':<11}" + f"{project_users:>{str_len},}" + " " + f"{project_servers:>5,}" + "\n" + toot_text += f":{project_soft}: {project_users:,} {project_servers:,}\n" i += 1 print("Tooting...") print(toot_text) - servers_image_id = mastodon.media_post('servers.png', "image/png").id + servers_image_id = mastodon.media_post('servers.png', "image/png", description='servers graph').id - users_image_id = mastodon.media_post('users.png', "image/png").id + users_image_id = mastodon.media_post('users.png', "image/png", description='users graph').id mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, users_image_id}) diff --git a/fetchservers.py b/fetchservers.py index 1a89af6..8e41d35 100644 --- a/fetchservers.py +++ b/fetchservers.py @@ -10,8 +10,17 @@ import aiohttp import asyncio import socket -apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?', - '/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?', '/wp-json/nodeinfo/2.0?'] +apis = ['/nodeinfo/2.0?', + '/nodeinfo/2.0.json?', + '/main/nodeinfo/2.0?', + '/api/statusnet/config?', + '/api/nodeinfo/2.0.json?', + '/api/nodeinfo?', + '/api/v1/instance?', + '/wp-json/nodeinfo/2.0?', + '/api/v1/instance/nodeinfo/2.0?', + '/.well-known/x-nodeinfo2?' + ] client_exceptions = ( aiohttp.ClientResponseError, @@ -24,15 +33,17 @@ client_exceptions = ( def is_json(myjson): + try: json_object = json.loads(myjson) except ValueError as e: return False return True - def write_api(server, software, users, alive, api, soft_version): + insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING" + conn = None try: @@ -46,7 +57,8 @@ def write_api(server, software, users, alive, api, soft_version): cur.execute( "UPDATE fediverse SET updated_at=(%s), software=(%s), users=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)", - (now, software, users, alive, api, soft_version, server)) + (now, software, users, alive, api, soft_version, server) + ) cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,)) @@ -63,9 +75,8 @@ def write_api(server, software, users, alive, api, soft_version): if conn is not None: conn.close() - async def getsoft(server): - + try: socket.gethostbyname(server) @@ -79,33 +90,61 @@ async def getsoft(server): url = 'https://' + server + user_agent = {'User-agent': 'Mozilla/5.0'} + timeout = aiohttp.ClientTimeout(total=3) - async with aiohttp.ClientSession(timeout=timeout) as session: + async with aiohttp.ClientSession(timeout=timeout, headers=user_agent) as session: for api in apis: + try: + async with session.get(url + api) as response: if response.status == 200: try: response_json = await response.json() except: pass + except aiohttp.ClientConnectorError as err: + pass + else: + if response.status == 200 and api != '/api/v1/instance?': - try: - soft = response_json['software']['name'] - soft = soft.lower() - soft_version = response_json['software']['version'] - users = response_json['usage']['users']['total'] - if users > 1000000: + + if api != '/.well-known/x-nodeinfo2?': + + try: + soft = response_json['software']['name'] + soft = soft.lower() + soft_version = response_json['software']['version'] + users = response_json['usage']['users']['total'] + if users > 1000000: + return + alive = True + write_api(server, soft, users, alive, api, soft_version) + print("Server " + server + " (" + soft + " " + soft_version + ") is alive!") return - alive = True - write_api(server, soft, users, alive, api, soft_version) - print("Server " + server + " (" + soft + " " + soft_version + ") is alive!") - return - except: - pass + except: + pass + else: + + try: + soft = response_json['server']['software'] + soft = soft.lower() + soft_version = response_json['server']['version'] + users = response_json['usage']['users']['total'] + if users > 1000000: + return + alive = True + if soft == 'socialhome': + write_api(server, soft, users, alive, api, soft_version) + print("Server " + server + " (" + soft + " " + soft_version + ") is alive!") + return + except: + pass + if response.status == 200 and soft == '' and api == "/api/v1/instance?": soft = 'mastodon' users = response_json['stats']['user_count'] @@ -116,11 +155,10 @@ async def getsoft(server): write_api(server, soft, users, alive, api) print("Server " + server + " (" + soft + ") is alive!") - def getserver(server, x): server = server[0].rstrip('.').lower() - + if server.find(".") == -1: return if server.find("@") != -1: @@ -140,8 +178,6 @@ def getserver(server, x): pass - -# Returns the parameter from the specified file def get_parameter(parameter, file_path): # Check if secrets file exists if not os.path.isfile(file_path): @@ -159,14 +195,20 @@ def get_parameter(parameter, file_path): sys.exit(0) -# Load configuration from config file -config_filepath = "config/config.txt" -mastodon_hostname = get_parameter("mastodon_hostname", config_filepath) +def get_config(): -# Load database config from db_config file -db_config_filepath = "config/db_config.txt" -fediverse_db = get_parameter("fediverse_db", db_config_filepath) -fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath) + # Load configuration from config file + config_filepath = "config/config.txt" + mastodon_hostname = get_parameter("mastodon_hostname", config_filepath) + return mastodon_hostname + +def get_db_config(): + + # Load database config from db_config file + db_config_filepath = "config/db_config.txt" + fediverse_db = get_parameter("fediverse_db", db_config_filepath) + fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath) + return (fediverse_db, fediverse_db_user) ############################################################################### # main @@ -176,6 +218,10 @@ if __name__ == '__main__': now = datetime.now() start_time = time.time() + mastodon_hostname = get_config() + + fediverse_db, fediverse_db_user = get_db_config() + world_servers = [] try: @@ -207,6 +253,7 @@ if __name__ == '__main__': finally: if conn is not None: + conn.close() ########################################################################### diff --git a/getpeers.py b/getpeers.py index cd60fe8..8e8a352 100644 --- a/getpeers.py +++ b/getpeers.py @@ -47,11 +47,13 @@ def write_servers(world_servers): i += 1 def get_peers(peer): - + try: - response = requests.get('https://' + peer + peers_api, timeout=2) - + user_agent = {'User-agent': 'Mozilla/5.0'} + + response = requests.get('https://' + peer + peers_api, headers = user_agent, timeout=3) + response_json = response.json() if response.status_code == 200: @@ -59,7 +61,7 @@ def get_peers(peer): try: print("Server: " + peer + ", " + "federated with " + str(len(response_json)) + " servers") - + for peer_peer in response_json: exist_count = world_peers.count(peer_peer) @@ -111,10 +113,9 @@ if __name__ == '__main__': start_time = time.time() updated_at = datetime.now() - + peers_api = '/api/v1/instance/peers?' - lemmy_api = '/api/v2/site?' - + # Load configuration from config file config_filepath = "config/config.txt" mastodon_hostname = get_parameter("mastodon_hostname", config_filepath) @@ -123,17 +124,19 @@ if __name__ == '__main__': db_config_filepath = "config/db_config.txt" fediverse_db = get_parameter("fediverse_db", db_config_filepath) fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath) - + world_peers = [] - - res = requests.get('https://' + mastodon_hostname + peers_api) + + user_agent = {'User-agent': 'Mozilla/5.0'} + + res = requests.get('https://' + mastodon_hostname + peers_api, headers = user_agent, timeout=3) hostname_peers = res.json() for peer in hostname_peers: exist_count = world_peers.count(peer) - + if exist_count == 0: world_peers.append(peer) @@ -148,6 +151,6 @@ if __name__ == '__main__': p_thread.join() write_servers(world_peers) - + exec_time = str(round((time.time() - start_time), 2)) print(exec_time) diff --git a/requirements.txt b/requirements.txt index d1dfc34..9bfc392 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ psycopg2-binary>=2.8.4 aiohttp>=3.6.2 aiodns>=2.0.0 matplotlib>=3.3.4 +humanfriendly>=9.2 diff --git a/setup.py b/setup.py index ef279ab..639f313 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - import getpass from mastodon import Mastodon from mastodon.Mastodon import MastodonMalformedEventError, MastodonNetworkError, MastodonReadTimeout, MastodonAPIError, MastodonIllegalArgumentError @@ -9,130 +6,167 @@ import os import sys def create_dir(): - if not os.path.exists('secrets'): - os.makedirs('secrets') + + if not os.path.exists('secrets'): + os.makedirs('secrets') def create_file(): - if not os.path.exists('secrets/secrets.txt'): - with open('secrets/secrets.txt', 'w'): pass - print(secrets_filepath + " created!") + + if not os.path.exists('secrets/secrets.txt'): + with open('secrets/secrets.txt', 'w'): pass + print(secrets_filepath + " created!") def create_config(): - if not os.path.exists('config'): - os.makedirs('config') - if not os.path.exists(config_filepath): - print(config_filepath + " created!") - with open('config/config.txt', 'w'): pass + + if not os.path.exists('config'): + os.makedirs('config') + if not os.path.exists(config_filepath): + print(config_filepath + " created!") + with open('config/config.txt', 'w'): pass def write_params(): - with open(secrets_filepath, 'a') as the_file: - print("Writing secrets parameter names to " + secrets_filepath) - the_file.write('uc_client_id: \n'+'uc_client_secret: \n'+'uc_access_token: \n') + + with open(secrets_filepath, 'a') as the_file: + print("Writing secrets parameter names to " + secrets_filepath) + the_file.write('uc_client_id: \n'+'uc_client_secret: \n'+'uc_access_token: \n') def write_config(): - with open(config_filepath, 'a') as the_file: - the_file.write('mastodon_hostname: \n') - print("adding parameter name 'mastodon_hostname' to "+ config_filepath) + + with open(config_filepath, 'a') as the_file: + the_file.write('mastodon_hostname: \n') + print("adding parameter name 'mastodon_hostname' to "+ config_filepath) def read_client_lines(self): - client_path = 'app_clientcred.txt' - with open(client_path) as fp: - line = fp.readline() - cnt = 1 - while line: - if cnt == 1: - print("Writing client id to " + secrets_filepath) - modify_file(secrets_filepath, "uc_client_id: ", value=line.rstrip()) - elif cnt == 2: - print("Writing client secret to " + secrets_filepath) - modify_file(secrets_filepath, "uc_client_secret: ", value=line.rstrip()) - line = fp.readline() - cnt += 1 + + client_path = 'app_clientcred.txt' + + with open(client_path) as fp: + + line = fp.readline() + cnt = 1 + while line: + if cnt == 1: + print("Writing client id to " + secrets_filepath) + modify_file(secrets_filepath, "uc_client_id: ", value=line.rstrip()) + elif cnt == 2: + print("Writing client secret to " + secrets_filepath) + modify_file(secrets_filepath, "uc_client_secret: ", value=line.rstrip()) + line = fp.readline() + cnt += 1 def read_token_line(self): - token_path = 'app_usercred.txt' - with open(token_path) as fp: - line = fp.readline() - print("Writing access token to " + secrets_filepath) - modify_file(secrets_filepath, "uc_access_token: ", value=line.rstrip()) + + token_path = 'app_usercred.txt' + + with open(token_path) as fp: + + line = fp.readline() + print("Writing access token to " + secrets_filepath) + modify_file(secrets_filepath, "uc_access_token: ", value=line.rstrip()) def read_config_line(): - with open(config_filepath) as fp: - line = fp.readline() - modify_file(config_filepath, "mastodon_hostname: ", value=hostname) + + with open(config_filepath) as fp: + + line = fp.readline() + modify_file(config_filepath, "mastodon_hostname: ", value=hostname) + modify_file(config_filepath, "bot_username: ", value=bot_username) def log_in(): - error = 0 - try: - global hostname - hostname = input("Enter Mastodon hostname: ") - user_name = input("User name, ex. user@" + hostname +"? ") - user_password = getpass.getpass("User password? ") - app_name = input("This app name? ") - Mastodon.create_app(app_name, scopes=["read","write"], - to_file="app_clientcred.txt", api_base_url=hostname) - mastodon = Mastodon(client_id = "app_clientcred.txt", api_base_url = hostname) - mastodon.log_in( - user_name, - user_password, - scopes = ["read", "write"], - to_file = "app_usercred.txt" - ) - except MastodonIllegalArgumentError as i_error: - error = 1 - if os.path.exists("secrets/secrets.txt"): - print("Removing secrets/secrets.txt file..") - os.remove("secrets/secrets.txt") - if os.path.exists("app_clientcred.txt"): - print("Removing app_clientcred.txt file..") - os.remove("app_clientcred.txt") - sys.exit(i_error) - except MastodonNetworkError as n_error: - error = 1 - if os.path.exists("secrets/secrets.txt"): - print("Removing secrets/secrets.txt file..") - os.remove("secrets/secrets.txt") - if os.path.exists("app_clientcred.txt"): - print("Removing app_clientcred.txt file..") - os.remove("app_clientcred.txt") - sys.exit(n_error) - except MastodonReadTimeout as r_error: - error = 1 - if os.path.exists("secrets/secrets.txt"): - print("Removing secrets/secrets.txt file..") - os.remove("secrets/secrets.txt") - if os.path.exists("app_clientcred.txt"): - print("Removing app_clientcred.txt file..") - os.remove("app_clientcred.txt") - sys.exit(r_error) - except MastodonAPIError as a_error: - error = 1 - if os.path.exists("secrets/secrets.txt"): - print("Removing secrets/secrets.txt file..") - os.remove("secrets/secrets.txt") - if os.path.exists("app_clientcred.txt"): - print("Removing app_clientcred.txt file..") - os.remove("app_clientcred.txt") - sys.exit(a_error) - finally: - if error == 0: - create_dir() - create_file() - write_params() - client_path = 'app_clientcred.txt' - read_client_lines(client_path) - token_path = 'app_usercred.txt' - read_token_line(token_path) - if os.path.exists("app_clientcred.txt"): - print("Removing app_clientcred.txt temp file..") - os.remove("app_clientcred.txt") - if os.path.exists("app_usercred.txt"): - print("Removing app_usercred.txt temp file..") - os.remove("app_usercred.txt") - print("Secrets setup done!\n") + error = 0 + + try: + + global hostname + hostname = input("Enter Mastodon hostname: ") + user_name = input("User name, ex. user@" + hostname +"? ") + user_password = getpass.getpass("User password? ") + bot_username = input("Bot's username, ex. fediverse: ") + app_name = input("This app name? ") + Mastodon.create_app( + app_name, + scopes=["read","write"], + to_file="app_clientcred.txt", + api_base_url=hostname + ) + mastodon = Mastodon(client_id = "app_clientcred.txt", api_base_url = hostname) + mastodon.log_in( + user_name, + user_password, + scopes = ["read", "write"], + to_file = "app_usercred.txt" + ) + + except MastodonIllegalArgumentError as i_error: + + error = 1 + + if os.path.exists("secrets/secrets.txt"): + print("Removing secrets/secrets.txt file..") + os.remove("secrets/secrets.txt") + if os.path.exists("app_clientcred.txt"): + print("Removing app_clientcred.txt file..") + os.remove("app_clientcred.txt") + sys.exit(i_error) + + except MastodonNetworkError as n_error: + + error = 1 + + if os.path.exists("secrets/secrets.txt"): + print("Removing secrets/secrets.txt file..") + os.remove("secrets/secrets.txt") + if os.path.exists("app_clientcred.txt"): + print("Removing app_clientcred.txt file..") + os.remove("app_clientcred.txt") + sys.exit(n_error) + + except MastodonReadTimeout as r_error: + + error = 1 + + if os.path.exists("secrets/secrets.txt"): + print("Removing secrets/secrets.txt file..") + os.remove("secrets/secrets.txt") + if os.path.exists("app_clientcred.txt"): + print("Removing app_clientcred.txt file..") + os.remove("app_clientcred.txt") + sys.exit(r_error) + + except MastodonAPIError as a_error: + + error = 1 + + if os.path.exists("secrets/secrets.txt"): + print("Removing secrets/secrets.txt file..") + os.remove("secrets/secrets.txt") + if os.path.exists("app_clientcred.txt"): + print("Removing app_clientcred.txt file..") + os.remove("app_clientcred.txt") + sys.exit(a_error) + + finally: + + if error == 0: + + create_dir() + create_file() + write_params() + client_path = 'app_clientcred.txt' + read_client_lines(client_path) + token_path = 'app_usercred.txt' + read_token_line(token_path) + if os.path.exists("app_clientcred.txt"): + print("Removing app_clientcred.txt temp file..") + os.remove("app_clientcred.txt") + if os.path.exists("app_usercred.txt"): + print("Removing app_usercred.txt temp file..") + os.remove("app_usercred.txt") + print("Secrets setup done!\n") def modify_file(file_name,pattern,value=""): + fh=fileinput.input(file_name,inplace=True) for line in fh: replacement=pattern + value @@ -140,7 +174,6 @@ def modify_file(file_name,pattern,value=""): sys.stdout.write(line) fh.close() -# Returns the parameter from the specified file def get_parameter( parameter, file_path ): # Check if secrets file exists if not os.path.isfile(file_path): @@ -157,7 +190,6 @@ def get_parameter( parameter, file_path ): print(file_path + " Missing parameter %s "%parameter) sys.exit(0) -# Returns the parameter from the specified file def get_hostname( parameter, config_filepath ): # Check if secrets file exists if not os.path.isfile(config_filepath): @@ -177,12 +209,18 @@ def get_hostname( parameter, config_filepath ): print("setup done!") sys.exit(0) -# Load secrets from secrets file -secrets_filepath = "secrets/secrets.txt" -uc_client_id = get_parameter("uc_client_id", secrets_filepath) -uc_client_secret = get_parameter("uc_client_secret", secrets_filepath) -uc_access_token = get_parameter("uc_access_token", secrets_filepath) +############################################################################### +# main -# Load configuration from config file -config_filepath = "config/config.txt" -mastodon_hostname = get_hostname("mastodon_hostname", config_filepath) +if __name__ == '__main__': + + # Load secrets from secrets file + secrets_filepath = "secrets/secrets.txt" + uc_client_id = get_parameter("uc_client_id", secrets_filepath) + uc_client_secret = get_parameter("uc_client_secret", secrets_filepath) + uc_access_token = get_parameter("uc_access_token", secrets_filepath) + + # Load configuration from config file + config_filepath = "config/config.txt" + mastodon_hostname = get_hostname("mastodon_hostname", config_filepath) + bot_username = get_parameter("bot_username", config_filepath)