Added db-setup.py and setup.py

db-setup.py

@@ -0,0 +1,160 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import getpass
+import os
+import sys
+from mastodon import Mastodon
+from mastodon.Mastodon import MastodonMalformedEventError, MastodonNetworkError, MastodonReadTimeout, MastodonAPIError
+import psycopg2
+from psycopg2 import sql
+from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
+
+# Returns the parameter from the specified file
+def get_parameter( parameter, file_path ):
+    # Check if secrets file exists
+    if not os.path.isfile(file_path):
+        print("File %s not found, asking."%file_path)
+        write_parameter( parameter, file_path )
+        #sys.exit(0)
+
+    # Find parameter in file
+    with open( file_path ) as f:
+        for line in f:
+            if line.startswith( parameter ):
+                return line.replace(parameter + ":", "").strip()
+
+    # Cannot find parameter, exit
+    print(file_path + "  Missing parameter %s "%parameter)
+    sys.exit(0)
+
+def write_parameter( parameter, file_path ):
+  if not os.path.exists('config'):
+    os.makedirs('config')
+  print("Setting up fediverse parameters...")
+  print("\n")
+  fediverse_db = input("fediverse db name: ")
+  fediverse_db_user = input("fediverse db user: ")
+
+  with open(file_path, "w") as text_file:
+    print("fediverse_db: {}".format(fediverse_db), file=text_file)
+    print("fediverse_db_user: {}".format(fediverse_db_user), file=text_file)
+
+def create_table(db, db_user, table, sql):
+
+  conn = None
+  try:
+
+    conn = psycopg2.connect(database = db, user = db_user, password = "", host = "/var/run/postgresql", port = "5432")
+    cur = conn.cursor()
+
+    print("Creating table.. "+table)
+    # Create the table in PostgreSQL database
+    cur.execute(sql)
+
+    conn.commit()
+    print("Table "+table+" created!")
+    print("\n")
+
+  except (Exception, psycopg2.DatabaseError) as error:
+
+    print(error)
+
+  finally:
+
+    if conn is not None:
+
+      conn.close()
+
+#############################################################################################
+
+# Load configuration from config file
+config_filepath = "config/db_config.txt"
+fediverse_db = get_parameter("fediverse_db", config_filepath)
+fediverse_db_user = get_parameter("fediverse_db_user", config_filepath)
+
+############################################################
+# create database
+############################################################
+
+conn = None
+
+try:
+
+  conn = psycopg2.connect(dbname='postgres',
+      user=fediverse_db_user, host='',
+      password='')
+
+  conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
+
+  cur = conn.cursor()
+
+  print("Creating database " + fediverse_db + ". Please wait...")
+
+  cur.execute(sql.SQL("CREATE DATABASE {}").format(
+          sql.Identifier(fediverse_db))
+      )
+  print("Database " + fediverse_db + " created!")
+
+except (Exception, psycopg2.DatabaseError) as error:
+
+  print(error)
+
+finally:
+
+  if conn is not None:
+
+    conn.close()
+
+#############################################################################################
+
+try:
+
+  conn = None
+  conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
+
+except (Exception, psycopg2.DatabaseError) as error:
+
+  print(error)
+  # Load configuration from config file
+  os.remove("db_config.txt")
+  print("Exiting. Run db-setup again with right parameters")
+  sys.exit(0)
+
+if conn is not None:
+
+  print("\n")
+  print("fediverse parameters saved to db-config.txt!")
+  print("\n")
+
+############################################################
+# Create needed tables
+############################################################
+
+print("Creating table...")
+
+########################################
+
+db = fediverse_db
+db_user = fediverse_db_user
+table = "world"
+sql = "create table "+table+" (server varchar(200) PRIMARY KEY, federated_with varchar(200), updated_at timestamptz), saved_at timestamptz"
+create_table(db, db_user, table, sql)
+
+table = "fediverse"
+sql = "create table "+table+" (server varchar(200) PRIMARY KEY, users INT, updated_at timestamptz)"
+create_table(db, db_user, table, sql)
+
+table = "totals"
+sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, total_servers INT, total_users INT)"
+create_table(db, db_user, table, sql)
+
+table = "evo"
+sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, servers INT, users INT)"
+create_table(db, db_user, table, sql)
+
+#####################################
+
+print("Done!")
+print("Now you can run setup.py!")
+print("\n")

setup.py

@@ -0,0 +1,188 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import getpass
+from mastodon import Mastodon
+from mastodon.Mastodon import MastodonMalformedEventError, MastodonNetworkError, MastodonReadTimeout, MastodonAPIError, MastodonIllegalArgumentError
+import fileinput,re
+import os
+import sys
+
+def create_dir():
+  if not os.path.exists('secrets'):
+    os.makedirs('secrets')
+
+def create_file():
+  if not os.path.exists('secrets/secrets.txt'):
+    with open('secrets/secrets.txt', 'w'): pass
+    print(secrets_filepath + " created!")
+
+def create_config():
+  if not os.path.exists('config'):
+    os.makedirs('config')
+  if not os.path.exists(config_filepath):
+    print(config_filepath + " created!")
+    with open('config/config.txt', 'w'): pass
+
+def write_params():
+  with open(secrets_filepath, 'a') as the_file:
+    print("Writing secrets parameter names to " + secrets_filepath)
+    the_file.write('uc_client_id: \n'+'uc_client_secret: \n'+'uc_access_token: \n')
+
+def write_config():
+  with open(config_filepath, 'a') as the_file:
+    the_file.write('mastodon_hostname: \n')
+    print("adding parameter name 'mastodon_hostname' to "+ config_filepath)
+
+def read_client_lines(self):
+  client_path = 'app_clientcred.txt'
+  with open(client_path) as fp:
+    line = fp.readline()
+    cnt = 1
+    while line:
+       if cnt == 1:
+         print("Writing client id to " + secrets_filepath)
+         modify_file(secrets_filepath, "uc_client_id: ", value=line.rstrip())
+       elif cnt == 2:
+         print("Writing client secret to " + secrets_filepath)
+         modify_file(secrets_filepath, "uc_client_secret: ", value=line.rstrip())
+       line = fp.readline()
+       cnt += 1
+
+def read_token_line(self):
+  token_path = 'app_usercred.txt'
+  with open(token_path) as fp:
+    line = fp.readline()
+    print("Writing access token to " + secrets_filepath)
+    modify_file(secrets_filepath, "uc_access_token: ", value=line.rstrip())
+
+def read_config_line():
+  with open(config_filepath) as fp:
+    line = fp.readline()
+    modify_file(config_filepath, "mastodon_hostname: ", value=hostname)
+
+def log_in():
+  error = 0
+  try:
+    global hostname
+    hostname = input("Enter Mastodon hostname: ")
+    user_name = input("User name, ex. user@" + hostname +"? ")
+    user_password = getpass.getpass("User password? ")
+    app_name = input("This app name? ")
+    Mastodon.create_app(app_name, scopes=["read","write"],
+      to_file="app_clientcred.txt", api_base_url=hostname)
+    mastodon = Mastodon(client_id = "app_clientcred.txt", api_base_url = hostname)
+    mastodon.log_in(
+       user_name,
+       user_password,
+       scopes = ["read", "write"],
+       to_file = "app_usercred.txt"
+    )
+  except MastodonIllegalArgumentError as i_error:
+    error = 1
+    if os.path.exists("secrets/secrets.txt"):
+      print("Removing secrets/secrets.txt file..")
+      os.remove("secrets/secrets.txt")
+    if os.path.exists("app_clientcred.txt"):
+      print("Removing app_clientcred.txt file..")
+      os.remove("app_clientcred.txt")
+    sys.exit(i_error)
+  except MastodonNetworkError as n_error:
+    error = 1
+    if os.path.exists("secrets/secrets.txt"):
+      print("Removing secrets/secrets.txt file..")
+      os.remove("secrets/secrets.txt")
+    if os.path.exists("app_clientcred.txt"):
+      print("Removing app_clientcred.txt file..")
+      os.remove("app_clientcred.txt")
+    sys.exit(n_error)
+  except MastodonReadTimeout as r_error:
+    error = 1
+    if os.path.exists("secrets/secrets.txt"):
+      print("Removing secrets/secrets.txt file..")
+      os.remove("secrets/secrets.txt")
+    if os.path.exists("app_clientcred.txt"):
+      print("Removing app_clientcred.txt file..")
+      os.remove("app_clientcred.txt")
+    sys.exit(r_error)
+  except MastodonAPIError as a_error:
+    error = 1
+    if os.path.exists("secrets/secrets.txt"):
+      print("Removing secrets/secrets.txt file..")
+      os.remove("secrets/secrets.txt")
+    if os.path.exists("app_clientcred.txt"):
+      print("Removing app_clientcred.txt file..")
+      os.remove("app_clientcred.txt")
+    sys.exit(a_error)
+  finally:
+    if error == 0:
+
+      create_dir()
+      create_file()
+      write_params()
+      client_path = 'app_clientcred.txt'
+      read_client_lines(client_path)
+      token_path = 'app_usercred.txt'
+      read_token_line(token_path)
+      if os.path.exists("app_clientcred.txt"):
+        print("Removing app_clientcred.txt temp file..")
+        os.remove("app_clientcred.txt")
+      if os.path.exists("app_usercred.txt"):
+        print("Removing app_usercred.txt temp file..")
+        os.remove("app_usercred.txt")
+      print("Secrets setup done!\n")
+
+def modify_file(file_name,pattern,value=""):
+    fh=fileinput.input(file_name,inplace=True)
+    for line in fh:
+        replacement=pattern + value
+        line=re.sub(pattern,replacement,line)
+        sys.stdout.write(line)
+    fh.close()
+
+# Returns the parameter from the specified file
+def get_parameter( parameter, file_path ):
+    # Check if secrets file exists
+    if not os.path.isfile(file_path):
+        print("File %s not found, creating it."%file_path)
+        log_in()
+
+    # Find parameter in file
+    with open( file_path ) as f:
+        for line in f:
+            if line.startswith( parameter ):
+                return line.replace(parameter + ":", "").strip()
+
+    # Cannot find parameter, exit
+    print(file_path + " Missing parameter %s "%parameter)
+    sys.exit(0)
+
+# Returns the parameter from the specified file
+def get_hostname( parameter, config_filepath ):
+    # Check if secrets file exists
+    if not os.path.isfile(config_filepath):
+        print("File %s not found, creating it."%config_filepath)
+        create_config()
+
+    # Find parameter in file
+    with open( config_filepath ) as f:
+        for line in f:
+            if line.startswith( parameter ):
+                return line.replace(parameter + ":", "").strip()
+
+    # Cannot find parameter, exit
+    print(config_filepath + "  Missing parameter %s "%parameter)
+    write_config()
+    read_config_line()
+    print("setup done!")
+    sys.exit(0)
+
+# Load secrets from secrets file
+secrets_filepath = "secrets/secrets.txt"
+uc_client_id     = get_parameter("uc_client_id",     secrets_filepath)
+uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
+uc_access_token  = get_parameter("uc_access_token",  secrets_filepath)
+
+# Load configuration from config file
+config_filepath = "config/config.txt"
+mastodon_hostname = get_hostname("mastodon_hostname", config_filepath)