parametrize db connection, with tests

.gitignore

@@ -0,0 +1,75 @@
+secrets/secrets.txt
+config/config.txt
+config/db_config.txt
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+.noseids
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Ipython Notebook
+.ipynb_checkpoints
+
+# vim swap files
+.*sw?
+
+# IDEs
+.vscode/
+*.code-workspace
+

config/test_db_config.txt

@@ -0,0 +1,3 @@
+fediverse_db: fediverse_test
+fediverse_db_user: tester
+fediverse_db_user_password: tester

federation/__init__.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python3
+
+from .database import Database

federation/database.py

@@ -6,7 +6,6 @@ from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
 import uuid
 from datetime import datetime
 import pytz
-import pdb
 
 tz = pytz.timezone('Europe/Madrid')
 
@@ -14,12 +13,14 @@ class Database():
 
     name = 'fediverse database library'
 
-    def __init__(self, config_file=None, fediverse_db=None, fediverse_db_user=None, fediverse_db_user_password=None):
+    def __init__(self, config_file=None, fediverse_db=None, fediverse_db_user=None, fediverse_db_user_password=None, fediverse_db_host='/var/run/postgresql', fediverse_db_port='6432'):
 
-        self.config_file = "config/db_config.txt"
-        self.fediverse_db = self.__get_parameter("fediverse_db", self.config_file)
-        self.fediverse_db_user = self.__get_parameter("fediverse_db_user", self.config_file)
-        self.fediverse_db_user_password = self.__get_parameter("fediverse_db_user_password", self.config_file)
+        self.config_file = config_file or "config/db_config.txt"
+        self.fediverse_db = fediverse_db or self.__get_parameter("fediverse_db", self.config_file)
+        self.fediverse_db_user = fediverse_db_user or self.__get_parameter("fediverse_db_user", self.config_file)
+        self.fediverse_db_user_password = fediverse_db_user_password or self.__get_parameter("fediverse_db_user_password", self.config_file)
+        self.fediverse_db_host = fediverse_db_host
+        self.fediverse_db_port = fediverse_db_port
 
         db_setup = self.__check_dbsetup(self)
 
@@ -43,7 +44,7 @@ class Database():
 
         try:
 
-            conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = "/var/run/postgresql", port = "6432")
+            conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = self.fediverse_db_host, port = self.fediverse_db_port)
 
             cur = conn.cursor()
 
@@ -73,7 +74,7 @@ class Database():
 
         try:
 
-            conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = "/var/run/postgresql", port = "6432")
+            conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = self.fediverse_db_host, port = self.fediverse_db_port)
 
             cur = conn.cursor()
 
@@ -743,7 +744,7 @@ class Database():
 
             conn = None
 
-            conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = "/var/run/postgresql", port = "6432")
+            conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = self.fediverse_db_host, port = self.fediverse_db_port)
 
             cur = conn.cursor()
 
@@ -787,7 +788,7 @@ class Database():
 
             conn = None
 
-            conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = "/var/run/postgresql", port = "6432")
+            conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = self.fediverse_db_host, port = self.fediverse_db_port)
 
             cur = conn.cursor()
 
@@ -935,6 +936,13 @@ class Database():
 
                 conn.close()
 
+    def check_connection(self):
+
+        try:
+            conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = self.fediverse_db_host, port = self.fediverse_db_port)
+        finally:
+            conn.close()
+
     @staticmethod
     def __check_dbsetup(self):
 
@@ -944,7 +952,7 @@ class Database():
 
             conn = None
 
-            conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = "/var/run/postgresql", port = "6432")
+            conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = self.fediverse_db_host, port = self.fediverse_db_port)
 
             db_setup = True
 
@@ -1023,7 +1031,7 @@ class Database():
 
         try:
 
-            conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = "/var/run/postgresql", port = "6432")
+            conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = self.fediverse_db_host, port = self.fediverse_db_port)
 
             cur = conn.cursor()
 

federation/setup.py

@@ -4,7 +4,6 @@ from datetime import datetime
 import pytz
 from mastodon import Mastodon
 from mastodon.Mastodon import MastodonMalformedEventError, MastodonNetworkError, MastodonReadTimeout, MastodonAPIError, MastodonIllegalArgumentError
-import pdb
 
 class Setup():
 

requirements-dev.txt

@@ -0,0 +1,10 @@
+requests
+psycopg2-binary
+pytz
+ray
+Mastodon.py
+matplotlib
+pandas
+humanfriendly
+pytest
+ipdb

scripts/fediverse.py

@@ -6,7 +6,7 @@ import urllib3
 import requests
 import socket
 from setup import Setup
-from database import Database
+from federation import Database
 from mastodon import Mastodon
 from matplotlib import pyplot as plt
 import matplotlib.dates as mdates
@@ -31,7 +31,8 @@ plt.rc('figure', titlesize=BIGGER_SIZE)  # fontsize of the figure title
 mdates.set_epoch('2000-01-01T00:00:00')
 y_formatter = ScalarFormatter(useOffset=False)
 
-ray.init(num_cpus = 25) # Specify this system CPUs.
+ray.init(num_cpus=25)  # Specify this system CPUs.
+
 
 class Server:
 
@@ -54,7 +55,8 @@ class Server:
 
         try:
 
-            data = requests.get('https://' + self + api, headers = setup.user_agent, timeout=3)
+            data = requests.get('https://' + self + api,
+                                headers=setup.user_agent, timeout=3)
 
             nodeinfo_json = data.json()
 
@@ -65,7 +67,8 @@ class Server:
 
                     users = 1
 
-                mau = nodeinfo_json.get('usage').get('users').get('activeMonth') or '0'
+                mau = nodeinfo_json.get('usage').get(
+                    'users').get('activeMonth') or '0'
 
                 if software == 'socialhome':
 
@@ -115,7 +118,8 @@ class Server:
 
                 if soft_version != "" and soft_version is not None:
 
-                    print(f'\n** Server {self} ({software} {soft_version}) is alive! **')
+                    print(
+                        f'\n** Server {self} ({software} {soft_version}) is alive! **')
 
                 else:
 
@@ -123,11 +127,13 @@ class Server:
 
                 if software != 'birdsitelive':
 
-                    db.write_alive_server(self, software, soft_version, alive, api, users, downs, first_checked_at, mau)
+                    db.write_alive_server(
+                        self, software, soft_version, alive, api, users, downs, first_checked_at, mau)
 
                 else:
 
-                    db.write_blocked_software(self, software, soft_version, alive, api, users, downs, first_checked_at)
+                    db.write_blocked_software(
+                        self, software, soft_version, alive, api, users, downs, first_checked_at)
 
         except urllib3.exceptions.ProtocolError as protoerr:
 
@@ -205,10 +211,12 @@ class Server:
 
         return (self, software, soft_version, alive, api, users, downs, first_checked_at, mau)
 
+
 def print_dead(server):
 
     print(f'\nServer {server} is dead :-(')
 
+
 if __name__ == '__main__':
 
     db = Database()
@@ -226,8 +234,8 @@ if __name__ == '__main__':
     now = start
 
     mastodon = Mastodon(
-            access_token = setup.mastodon_app_token,
-            api_base_url= setup.mastodon_hostname
+        access_token=setup.mastodon_app_token,
+        api_base_url=setup.mastodon_hostname
     )
 
     total_servers = 0
@@ -242,9 +250,11 @@ if __name__ == '__main__':
 
     ray_start = time.time()
 
-    results = ray.get([getservers.get_alive_servers.remote(server) for server in alive_servers])
+    results = ray.get([getservers.get_alive_servers.remote(server)
+                      for server in alive_servers])
 
-    print(f"duration = {time.time() - ray_start}.\nprocessed servers: {len(results)}")
+    print(
+        f"duration = {time.time() - ray_start}.\nprocessed servers: {len(results)}")
 
     # get current total servers and users, get users from every software
 
@@ -252,7 +262,8 @@ if __name__ == '__main__':
 
     # get last check values and write current total ones
 
-    evo_servers, evo_users, evo_mau = db.last_values(total_servers, total_users, total_mau)
+    evo_servers, evo_users, evo_mau = db.last_values(
+        total_servers, total_users, total_mau)
 
     # write evo values
 
@@ -273,11 +284,14 @@ if __name__ == '__main__':
     ###############################################################################
     # generate graphs
 
-    plt.plot([-6, -5, -4, -3, -2, -1, 0], [servers_plots[6], servers_plots[5], servers_plots[4], servers_plots[3], servers_plots[2], servers_plots[1], servers_plots[0]], marker='o', color='mediumseagreen')
+    plt.plot([-6, -5, -4, -3, -2, -1, 0], [servers_plots[6], servers_plots[5], servers_plots[4], servers_plots[3],
+             servers_plots[2], servers_plots[1], servers_plots[0]], marker='o', color='mediumseagreen')
 
-    plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_servers, max_servers, max_servers, max_servers, max_servers, max_servers, max_servers], color='red')
+    plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_servers, max_servers, max_servers,
+             max_servers, max_servers, max_servers, max_servers], color='red')
 
-    plt.title('fediverse: total alive servers (max: ' + str(f"{max_servers:,}" + ')'), loc='right', color='blue')
+    plt.title('fediverse: total alive servers (max: ' +
+              str(f"{max_servers:,}" + ')'), loc='right', color='blue')
 
     plt.xlabel('Last seven days')
 
@@ -291,13 +305,17 @@ if __name__ == '__main__':
 
     plt.close()
 
-    plt.plot([-6, -5, -4, -3, -2, -1, 0], [mau_plots[6], mau_plots[5], mau_plots[4], mau_plots[3], mau_plots[2], mau_plots[1], mau_plots[0]], marker='o', color='royalblue')
+    plt.plot([-6, -5, -4, -3, -2, -1, 0], [mau_plots[6], mau_plots[5], mau_plots[4],
+             mau_plots[3], mau_plots[2], mau_plots[1], mau_plots[0]], marker='o', color='royalblue')
 
-    plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_mau, max_mau, max_mau, max_mau, max_mau, max_mau, max_mau], color='red')
+    plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_mau, max_mau, max_mau,
+             max_mau, max_mau, max_mau, max_mau], color='red')
 
-    plt.title('fediverse: total MAU (max: ' + str(f"{max_mau:,}" + ')'), loc='right', color='royalblue')
+    plt.title('fediverse: total MAU (max: ' +
+              str(f"{max_mau:,}" + ')'), loc='right', color='royalblue')
 
-    plt.legend(('mau', 'max'), shadow=True, loc=(0.01, 0.80), handlelength=1.5, fontsize=10)
+    plt.legend(('mau', 'max'), shadow=True, loc=(
+        0.01, 0.80), handlelength=1.5, fontsize=10)
 
     plt.xlabel('Last seven days')
 
@@ -349,25 +367,30 @@ if __name__ == '__main__':
 
     if evo_servers >= 0:
 
-        toot_text += "alive servers: " + str(f"{total_servers:,}") + " (+"+ str(f"{evo_servers:,}") + ") \n"
+        toot_text += "alive servers: " + \
+            str(f"{total_servers:,}") + \
+            " (+" + str(f"{evo_servers:,}") + ") \n"
 
         toot_text += "max: " + str(f"{max_servers:,}") + "\n"
 
     elif evo_servers < 0:
 
-        toot_text += "alive servers: " + str(f"{total_servers:,}") + " ("+ str(f"{evo_servers:,}") + ") \n"
+        toot_text += "alive servers: " + \
+            str(f"{total_servers:,}") + " (" + str(f"{evo_servers:,}") + ") \n"
 
         toot_text += "max: " + str(f"{max_servers:,}") + "\n"
 
     if evo_mau >= 0:
 
-        toot_text += "total MAU: " + str(f"{total_mau:,}") + " (+"+ str(f"{evo_mau:,}") + ") \n"
+        toot_text += "total MAU: " + \
+            str(f"{total_mau:,}") + " (+" + str(f"{evo_mau:,}") + ") \n"
 
         toot_text += "max: " + str(f"{max_mau:,}") + "\n"
 
     elif evo_mau < 0:
 
-        toot_text += "total MAU: " + str(f"{total_mau:,}") + " ("+ str(f"{evo_mau:,}") + ") \n"
+        toot_text += "total MAU: " + \
+            str(f"{total_mau:,}") + " (" + str(f"{evo_mau:,}") + ") \n"
 
         toot_text += "max: " + str(f"{max_mau:,}") + "\n"
 
@@ -397,13 +420,17 @@ if __name__ == '__main__':
 
     print(toot_text)
 
-    servers_image_id = mastodon.media_post('servers.png', "image/png", description='servers graph').id
+    servers_image_id = mastodon.media_post(
+        'servers.png', "image/png", description='servers graph').id
 
-    mau_image_id = mastodon.media_post('mau.png', "image/png", description='MAU graph').id
+    mau_image_id = mastodon.media_post(
+        'mau.png', "image/png", description='MAU graph').id
 
-    global_image_id = mastodon.media_post('global.png', "image/png", description='global graph').id
+    global_image_id = mastodon.media_post(
+        'global.png', "image/png", description='global graph').id
 
-    mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, mau_image_id, global_image_id})
+    mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={
+                         servers_image_id, mau_image_id, global_image_id})
 
     db.delete_dead_servers()
 

tests/__init__.py

@@ -0,0 +1 @@
+#!/usr/bin/env python3

tests/test_database.py

@@ -0,0 +1,15 @@
+from federation import Database
+
+def test__database_connection():
+    db = Database(config_file='config/test_db_config.txt', fediverse_db_host='localhost', fediverse_db_port='5432')
+
+    # noexcept
+    db.check_connection()
+
+
+def test__database_connection__noservers():
+    db = Database(config_file='config/test_db_config.txt', fediverse_db_host='localhost', fediverse_db_port='5432')
+
+    checked_server = db.get_last_checked_servers()
+
+    assert checked_server == []