README.md

@@ -21,7 +21,5 @@ Within Python Virtual Environment:
 4. Use your favourite scheduling method to set `python fetchpeers.py` to run at least once a day and `python fediverse.py` to run at desired pace to publish the results. Also set `python fediquery.py` to run every minute to accept queries from any fediverse' users about any `server` or `soft`.  
 
 15.4.2023 - Added fediquery.py. It allows queries from any fediverse' user about soft and server (keywords). It replies to the asking user with its data, if any.
-28.5.2023 - Added top table to save top five softwares data.
-28.5.2023 - Added upload.py to upload alive fediverse's dataset to configured Forgejo repository.
 
 

app/libraries/database.py

@@ -6,9 +6,6 @@ from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
 import uuid
 from datetime import datetime
 import pytz
-import sqlalchemy
-from sqlalchemy import URL
-import pandas as pd
 
 tz = pytz.timezone('Europe/Madrid')
 
@@ -41,7 +38,7 @@ class Database():
 
             conn = None
 
-            conn = psycopg2.connect(database = self.mau_db, user = self.mau_db_user, password = self.mau_db_user_password, host = "/var/run/postgresql", port = "5432")
+            conn = psycopg2.connect(database = self.mau_db, user = self.mau_db_user, password = self.mau_db_user_password, host = "/var/run/postgresql", port = "6432")
 
             cur = conn.cursor()
 
@@ -83,7 +80,7 @@ class Database():
 
         try:
 
-            conn = psycopg2.connect(database = self.mau_db, user = self.mau_db_user, password = self.mau_db_user_password, host = "/var/run/postgresql", port = "5432")
+            conn = psycopg2.connect(database = self.mau_db, user = self.mau_db_user, password = self.mau_db_user_password, host = "/var/run/postgresql", port = "6432")
 
             cur = conn.cursor()
 
@@ -115,7 +112,7 @@ class Database():
 
         try:
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
 
             cur = conn.cursor()
 
@@ -156,7 +153,7 @@ class Database():
 
             conn = None
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
 
             cur = conn.cursor()
 
@@ -200,7 +197,7 @@ class Database():
 
             conn = None
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
 
             cur = conn.cursor()
 
@@ -238,7 +235,7 @@ class Database():
 
             conn = None
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
 
             cur = conn.cursor()
 
@@ -276,7 +273,7 @@ class Database():
 
             conn = None
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
 
             cur = conn.cursor()
 
@@ -324,7 +321,7 @@ class Database():
 
             conn = None
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
 
             cur = conn.cursor()
 
@@ -373,7 +370,7 @@ class Database():
 
             conn = None
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
 
             cur = conn.cursor()
 
@@ -411,7 +408,7 @@ class Database():
 
         try:
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
 
             cur = conn.cursor()
 
@@ -443,7 +440,7 @@ class Database():
 
         try:
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
 
             cur = conn.cursor()
 
@@ -474,7 +471,7 @@ class Database():
 
         try:
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
 
             cur = conn.cursor()
 
@@ -504,7 +501,7 @@ class Database():
 
         try:
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
 
             cur = conn.cursor()
 
@@ -535,7 +532,7 @@ class Database():
 
         try:
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
 
             cur = conn.cursor()
 
@@ -565,7 +562,7 @@ class Database():
 
         try:
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
 
             cur = conn.cursor()
 
@@ -603,7 +600,7 @@ class Database():
 
             conn = None
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
 
             cur = conn.cursor()
 
@@ -662,7 +659,7 @@ class Database():
 
             conn = None
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
 
             cur = conn.cursor()
 
@@ -710,36 +707,6 @@ class Database():
 
         return (evo_servers, evo_users, evo_mau)
 
-    def write_top_soft(self, soft_total_project, soft_total_users, soft_total_mau, soft_total_servers):
-
-        now = datetime.now()
-
-        insert_sql = "INSERT INTO top(datetime, software, users, mau, servers) VALUES(%s,%s,%s,%s,%s)"
-
-        conn = None
-
-        try:
-
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
-
-            cur = conn.cursor()
-
-            cur.execute(insert_sql, (now, soft_total_project, soft_total_users, soft_total_mau, soft_total_servers))
-
-            conn.commit()
-
-            cur.close()
-
-        except (Exception, psycopg2.DatabaseError) as error:
-
-            print(error)
-
-        finally:
-
-            if conn is not None:
-
-                conn.close()
-
     def write_evo(self, evo_servers, evo_users, evo_mau):
 
         #  write evo values
@@ -752,7 +719,7 @@ class Database():
 
         try:
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
 
             cur = conn.cursor()
 
@@ -780,7 +747,7 @@ class Database():
 
         try:
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
 
             cur = conn.cursor()
 
@@ -814,7 +781,7 @@ class Database():
 
         try:
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
 
             cur = conn.cursor()
 
@@ -896,7 +863,7 @@ class Database():
 
         try:
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
 
             cur = conn.cursor()
 
@@ -966,7 +933,7 @@ class Database():
 
             conn = None
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
 
             cur = conn.cursor()
 
@@ -1004,7 +971,7 @@ class Database():
 
             conn = None
 
-            conn = psycopg2.connect(database = self.mau_db, user = self.mau_db_user, password = self.mau_db_user_password, host = "/var/run/postgresql", port = "5432")
+            conn = psycopg2.connect(database = self.mau_db, user = self.mau_db_user, password = self.mau_db_user_password, host = "/var/run/postgresql", port = "6432")
 
             cur = conn.cursor()
 
@@ -1048,7 +1015,7 @@ class Database():
 
             conn = None
 
-            conn = psycopg2.connect(database = self.mau_db, user = self.mau_db_user, password = self.mau_db_user_password, host = "/var/run/postgresql", port = "5432")
+            conn = psycopg2.connect(database = self.mau_db, user = self.mau_db_user, password = self.mau_db_user_password, host = "/var/run/postgresql", port = "6432")
 
             cur = conn.cursor()
 
@@ -1110,7 +1077,7 @@ class Database():
 
             conn = None
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password=self.mau_db_user_password, host="/var/run/postgresql", port="5432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password=self.mau_db_user_password, host="/var/run/postgresql", port="6432")
 
             cur = conn.cursor()
 
@@ -1168,7 +1135,7 @@ class Database():
 
             conn = None
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password=self.mau_db_user_password, host="/var/run/postgresql", port="5432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password=self.mau_db_user_password, host="/var/run/postgresql", port="6432")
 
             cur = conn.cursor()
 
@@ -1196,40 +1163,6 @@ class Database():
 
                 conn.close()
 
-    def csv_save(self, filename):
-
-        is_saved = False
-
-        url_object = URL.create(
-            "postgresql+psycopg2",
-            username=self.mau_db_user,
-            password=self.mau_db_user_password,
-            host="localhost",
-            database=self.mau_db,
-        )
-
-        sql = "select * from mau where alive order by mau desc"
-
-        conn = None
-
-        try:
-
-            conn = sqlalchemy.create_engine(url_object)
-
-            with conn.connect() as connection:
-
-                dataframe = pd.read_sql_query(sql, conn)
-
-                dataframe.to_csv(filename, sep='|', header=True, index=False)
-
-                is_saved = True
-
-        except (Exception, psycopg2.DatabaseError) as error:
-
-            print(error)
-
-        return is_saved
-
     @staticmethod
     def __check_dbsetup(self):
 
@@ -1239,7 +1172,7 @@ class Database():
 
             conn = None
 
-            conn = psycopg2.connect(database = self.mau_db, user = self.mau_db_user, password = self.mau_db_user_password, host = "/var/run/postgresql", port = "5432")
+            conn = psycopg2.connect(database = self.mau_db, user = self.mau_db_user, password = self.mau_db_user_password, host = "/var/run/postgresql", port = "6432")
 
             db_setup = True
 
@@ -1292,7 +1225,7 @@ class Database():
         self.__create_table(self, table, sql)
 
         table = "servers"
-        sql = "create table "+table+" (server varchar(200) PRIMARY KEY, api varchar(50), created_at timestamptz, peers boolean, updated_at timestamptz)"
+        sql = "create table "+table+" (server varchar(200) PRIMARY KEY, api varchar(50), created_at timestamptz, peers boolean, update_at timestamptz)"
         self.__create_table(self, table, sql)
 
         table = "deadservers"
@@ -1307,10 +1240,6 @@ class Database():
         sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, servers INT, users INT, mau INT)"
         self.__create_table(self, table, sql)
 
-        table = "top"
-        sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, software varchar(50), users INT, mau INT, servers INT)"
-        self.__create_table(self, table, sql)
-
         table = "execution_time"
         sql = "create table "+table+" (program varchar(20) PRIMARY KEY, start timestamptz, finish timestamptz)"
         self.__create_table(self, table, sql)
@@ -1322,7 +1251,7 @@ class Database():
 
         try:
 
-            conn = psycopg2.connect(database = self.mau_db, user = self.mau_db_user, password = self.mau_db_user_password, host = "/var/run/postgresql", port = "5432")
+            conn = psycopg2.connect(database = self.mau_db, user = self.mau_db_user, password = self.mau_db_user_password, host = "/var/run/postgresql", port = "6432")
 
             cur = conn.cursor()
 

app/libraries/forgejo.py

@@ -1,489 +0,0 @@
-import os
-import sys
-import requests
-from requests.models import urlencode
-
-###
-# Dict helper class.
-# Defined at top level so it can be pickled.
-###
-class AttribAccessDict(dict):
-    def __getattr__(self, attr):
-        if attr in self:
-            return self[attr]
-        else:
-            raise AttributeError("Attribute not found: " + str(attr))
-
-    def __setattr__(self, attr, val):
-        if attr in self:
-            raise AttributeError("Attribute-style access is read only")
-        super(AttribAccessDict, self).__setattr__(attr, val)
-
-class Forgejo:
-
-    name = 'Forgejo API wrapper'
-
-    def __init__(self, api_base_url=None, access_token=None, session=None):
-
-        self.__forgejo_config_path = "config/forgejo.txt"
-
-        is_setup = self.__check_setup(self)
-
-        if is_setup:
-
-            self.api_base_url = self.__get_parameter("api_base_url", self.__forgejo_config_path)
-            self.access_token = self.__get_parameter("access_token", self.__forgejo_config_path)
-            self.stats_repo = self.__get_parameter("stats_repo", self.__forgejo_config_path)
-
-        else:
-
-            self.api_base_url, self.access_token, self.stats_repo = self.__setup(self)
-
-        if session:
-            self.session = session
-        else:
-            self.session = requests.Session()
-
-    def admin_users_create(self, email, username, passwd, full_name=None, must_change_password=True, restricted=False, send_notify=True, source_id='0', visibility='private'):
-
-        data = {'email':email,
-                'full_name':username,
-                'login_name':username,
-                'must_change_password':must_change_password,
-                'password':passwd,
-                'restricted':restricted,
-                'send_notify':send_notify,
-                'source_id':source_id,
-                'username':username,
-                'visibility':visibility
-            }
-
-        endpoint = self.api_base_url + '/api/v1/admin/users?access_token={0}'.format(self.access_token)
-
-        response = self.__api_request('POST', endpoint, data)
-
-        registered = response.ok
-
-        response = self.__json_allow_dict_attrs(response.json())
-
-        return (registered, response)
-
-    def admin_users_list(self, page=None, limit=None):
-
-        params = dict()
-        if page != None:
-            params['page'] = page
-        if limit != None:
-            params['limit'] = limit
-        params['token'] = self.access_token
-        formatted_params = urlencode(params)
-
-        endpoint = self.api_base_url + '/api/v1/admin/users?{0}'.format(formatted_params)
-
-        response = self.__api_request('GET', endpoint)
-
-        response = self.__json_allow_dict_attrs(response.json())
-
-        return response
-
-    def notifications_new(self):
-
-        endpoint = self.api_base_url + '/api/v1/notifications/new?token={0}'.format(self.access_token)
-
-        response = self.__api_request('GET', endpoint)
-
-        response = self.__json_allow_dict_attrs(response.json())
-
-        return response
-
-    ### 
-    ### repository
-    ###
-
-    def repos_get_repo(self, owner, repo):
-
-        params = dict()
-        params['token'] = self.access_token
-        formatted_params = urlencode(params)
-
-        endpoint = self.api_base_url + '/api/v1/repos/{0}/{1}'.format(owner, repo, formatted_params)
-
-        response = self.__api_request('GET', endpoint)
-
-        response = self.__json_allow_dict_attrs(response.json())
-
-        return response
-
-    def repos_issues_search(self, owner, state=None, labels=None, q=None, milestones=None, priority_repo_id=None, issue_type=None, since=None, before=None, assigned=None, 
-            created=None, mentioned=None, review_requested=None, team=None, page=None, limit=None):
-
-        params = dict()
-
-        if state == None:
-            params['state'] = 'open'
-        else:
-            params['state'] = state
-        if labels != None:
-            params['labels'] = labels
-        if milestones != None:
-            params['milestones'] = milestones
-        if q != None:
-            params['q'] = q
-        if priority_repo_id != None:
-            params['priority_repo_id'] = priority_repo_id
-        if issue_type != None:
-            params['type'] = issue_type
-        if since != None:
-            params['since'] = since
-        if before != None:
-            params['before'] = before
-        if assigned != None:
-            params['assigned'] = assigned
-        if created != None:
-            params['created'] = created
-        if mentioned != None:
-            params['mentioned'] = mentioned
-        if review_requested != None:
-            params['review_requested'] = review_requested
-        params['owner'] = owner
-        if team != None:
-            params['team'] = team
-        if page != None:
-            params['page'] = page
-        if limit != None:
-            params['limit'] = limit
-        params['token'] = self.access_token
-        formatted_params = urlencode(params)
-
-        endpoint = self.api_base_url + '/api/v1/repos/issues/search?{0}'.format(formatted_params)
-
-        response = self.__api_request('GET', endpoint)
-
-        response = self.__json_allow_dict_attrs(response.json())
-
-        return response
-
-    def repos_owner_repo_issues(self, owner, repo, state=None, labels=None, q=None, issue_type=None, milestones=None, since=None, before=None, created_by=None, assigned_by=None,
-                                    mentioned_by=None, page=None, limit=None):
-        """
-        if since or before are specified, they must have following format: in. ex. 2022-08-13T08:09:07+02:00
-        """
-        params = dict()
-
-        if state == None:
-            params['state'] = 'open'
-        else:
-            params['state'] = state
-        params['labels'] = labels
-        if q != None:
-            params['q'] = q
-        params['issue_type'] = issue_type
-        params['milestones'] = milestones
-        if since != None:
-            params['since'] = since
-        if before != None:
-            params['before'] = before
-        if created_by != None:
-            params['created_by'] = created_by
-        if assigned_by != None:
-            params['assigned_by'] = assigned_by
-        if mentioned_by != None:
-            params['mentioned_by'] = mentioned_by
-        params['page'] = page
-        params['limit'] = limit
-        formatted_params = urlencode(params)
-
-        endpoint = self.api_base_url + '/api/v1/repos/{0}/{1}/issues?{2}'.format(owner, repo, formatted_params)
-
-        response = self.__api_request('GET', endpoint)
-
-        response = self.__json_allow_dict_attrs(response.json())
-
-        return response
-
-    def repos_owner_repo_issues_comments(self, owner, repo, since=None, before=None, page=None, limit=None):
-        """
-        if since or before are specified, they must have following format: in. ex. 2022-08-13T08:09:07+02:00
-        """
-        params = dict()
-        if since != None:
-            params['since'] = since
-        if before != None:
-            params['before'] = before
-        params['page'] = page
-        params['limit'] = limit
-        formatted_params = urlencode(params)
-
-        endpoint = self.api_base_url + '/api/v1/repos/{0}/{1}/issues/comments?{2}'.format(owner, repo, formatted_params)
-
-        response = self.__api_request('GET', endpoint)#, data)
-
-        response = self.__json_allow_dict_attrs(response.json())
-
-        return response
-
-    def repo_owner_get_metada(self, owner, repo, filepath):
-        """
-        Gets the metadata and contents (if a file) of an entry in a repository, or  list of entries if a dir
-        """
-
-        params = dict()
-        params['token'] = self.access_token
-        formatted_params = urlencode(params)
-
-        endpoint = self.api_base_url + '/api/v1/repos/{0}/{1}/contents/{2}?{3}'.format(owner, repo, filepath, formatted_params)
-
-        response = self.__api_request('GET', endpoint)
-
-        response = self.__json_allow_dict_attrs(response.json())
-
-        return response
-
-    def repo_owner_create_file(self, owner, repo, filepath, author_email, author_name, branch, content, message):
-        """
-        Create a file in a repository
-        """
-
-        data = {"author":[{"email":author_email},{"name":author_name}],
-                "branch":branch,
-                "content":content,
-                "message":message
-        }
-
-        params = dict()
-        params['token'] = self.access_token
-        formatted_params = urlencode(params)
-
-        endpoint = self.api_base_url + '/api/v1/repos/{0}/{1}/contents/{2}?{3}'.format(owner, repo, filepath, formatted_params)
-
-        response = self.__api_request('POST', endpoint, data=data)
-
-        response = self.__json_allow_dict_attrs(response.json())
-
-        return response
-
-    def repo_owner_update_file(self, owner, repo, filepath, author_email, author_name, branch, message, sha):
-        """
-        Update a file in a repository
-        """
-        data = {"author":[{"email":author_email},{"name":author_name}],
-                "branch":branch,
-                "message":message,
-                "sha":sha
-        }
-
-        params = dict()
-        params['token'] = self.access_token
-        formatted_params = urlencode(params)
-
-        endpoint = self.api_base_url + '/api/v1/repos/{0}/{1}/contents/{2}?{3}'.format(owner, repo, filepath, formatted_params)
-
-        response = self.__api_request('PUT', endpoint, data=data)
-
-        response = self.__json_allow_dict_attrs(response.json())
-
-        return response
-
-    def repo_owner_delete_file(self, owner, repo, filepath, author_email, author_name, branch, message, sha):
-        """
-        Delete a file in a repository
-        """
-        data = {"author":[{"email":author_email},{"name":author_name}],
-                "branch":branch,
-                "commiter":[{"email":author_email},{"name":author_name}],
-                "message":message,
-                "new_branch":branch,
-                'sha':sha
-        }
-
-        params = dict()
-        params['token'] = self.access_token
-        formatted_params = urlencode(params)
-
-        endpoint = self.api_base_url + '/api/v1/repos/{0}/{1}/contents/{2}?{3}'.format(owner, repo, filepath, formatted_params)
-
-        response = self.__api_request('DELETE', endpoint, data=data)
-
-        response = self.__json_allow_dict_attrs(response.json())
-
-        return response
-
-    def user(self):
-        """
-        Get the authenticated user
-        """
-
-        params = dict()
-        params['token'] = self.access_token
-        formatted_params = urlencode(params)
-
-        endpoint = self.api_base_url + '/api/v1/user?{0}'.format(formatted_params)
-
-        response = self.__api_request('GET', endpoint)
-
-        response = self.__json_allow_dict_attrs(response.json())
-
-        return response
-
-    @staticmethod
-    def __check_setup(self):
-
-        is_setup = False
-
-        if not os.path.isfile(self.__forgejo_config_path):
-            print(f"File {self.__forgejo_config_path} not found, running setup.")
-        else:
-            is_setup = True
-
-        return is_setup
-
-    @staticmethod
-    def __setup(self):
-
-        if not os.path.exists('config'):
-            os.makedirs('config')
-
-        self.api_base_url = input("Forgejo API base url, in ex. 'https://yourforgejo.instance': ")
-        self.access_token = input("Forgejo access token: ")
-        self.stats_repo = input("Stats repo: ")
-
-
-        if not os.path.exists(self.__forgejo_config_path):
-            with open(self.__forgejo_config_path, 'w'): pass
-            print(f"{self.__forgejo_config_path} created!")
-
-        with open(self.__forgejo_config_path, 'a') as the_file:
-            print("Writing forgejo parameters to " + self.__forgejo_config_path)
-            the_file.write(f'api_base_url: {self.api_base_url}\n'+f'access_token: {self.access_token}\nstats_repo: {self.stats_repo}\n')
-
-        return (self.api_base_url, self.access_token, self.stats_repo)
-
-    @staticmethod
-    def __get_parameter(parameter, file_path ):
-
-        with open( file_path ) as f:
-            for line in f:
-                if line.startswith( parameter ):
-                    return line.replace(parameter + ":", "").strip()
-
-        print(f'{file_path} Missing parameter {parameter}')
-        sys.exit(0)
-
-    def __api_request(self, method, endpoint, data={}):
-
-        response = None
-
-        try:
-
-            kwargs = dict(data=data)
-
-            response = self.session.request(method, url = endpoint, **kwargs)
-
-        except Exception as e:
-
-            raise ForgejoNetworkError(f"Could not complete request: {e}")
-
-        if response is None:
-
-            raise ForgejoIllegalArgumentError("Illegal request.")
-
-        if not response.ok:
-
-                try:
-                    if isinstance(response, dict) and 'error' in response:
-                        error_msg = response['error']
-                    elif isinstance(response, str):
-                        error_msg = response
-                    else:
-                        error_msg = None
-                except ValueError:
-                    error_msg = None
-
-                if response.status_code == 404:
-                    ex_type = ForgejoNotFoundError
-                    if not error_msg:
-                        error_msg = 'Endpoint not found.'
-                        # this is for compatibility with older versions
-                        # which raised ForgejoAPIError('Endpoint not found.')
-                        # on any 404
-                elif response.status_code == 401:
-                    ex_type = ForgejoUnauthorizedError
-                elif response.status_code == 422:
-                    return response
-                elif response.status_code == 500:
-                    ex_type = ForgejoInternalServerError
-                elif response.status_code == 502:
-                    ex_type = ForgejoBadGatewayError
-                elif response.status_code == 503:
-                    ex_type = ForgejoServiceUnavailableError
-                elif response.status_code == 504:
-                    ex_type = ForgejoGatewayTimeoutError
-                elif response.status_code >= 500 and \
-                    response.status_code <= 511:
-                    ex_type = ForgejoServerError
-                else:
-                    ex_type = ForgejoAPIError
-
-                raise ex_type(
-                    'Forgejo API returned error',
-                    response.status_code,
-                    response.reason,
-                    error_msg)
-
-        else:
-
-            return response
-
-    @staticmethod
-    def __json_allow_dict_attrs(json_object):
-        """
-        Makes it possible to use attribute notation to access a dicts
-        elements, while still allowing the dict to act as a dict.
-        """
-        if isinstance(json_object, dict):
-            return AttribAccessDict(json_object)
-        return json_objecte
-
-##
-# Exceptions
-##
-class ForgejoError(Exception):
-    """Base class for Forgejo.py exceptions"""
-
-class ForgejoIOError(IOError, ForgejoError):
-    """Base class for Forgejo.py I/O errors"""
-
-class ForgejoNetworkError(ForgejoIOError):
-    """Raised when network communication with the server fails"""
-    pass
-class ForgejoAPIError(ForgejoError):
-    """Raised when the forgejo API generates a response that cannot be handled"""
-    pass
-class ForgejoServerError(ForgejoAPIError):
-    """Raised if the Server is malconfigured and returns a 5xx error code"""
-    pass
-class ForgejoInternalServerError(ForgejoServerError):
-    """Raised if the Server returns a 500 error"""
-    pass
-
-class ForgejoBadGatewayError(ForgejoServerError):
-    """Raised if the Server returns a 502 error"""
-    pass
-
-class ForgejoServiceUnavailableError(ForgejoServerError):
-    """Raised if the Server returns a 503 error"""
-    pass
-class ForgejoGatewayTimeoutError(ForgejoServerError):
-    """Raised if the Server returns a 504 error"""
-    pass
-class ForgejoNotFoundError(ForgejoAPIError):
-    """Raised when the forgejo API returns a 404 Not Found error"""
-    pass
-
-class ForgejoUnauthorizedError(ForgejoAPIError):
-    """Raised when the forgejo API returns a 401 Unauthorized error
-
-       This happens when an OAuth token is invalid or has been revoked,
-       or when trying to access an endpoint that can't be used without
-       authentication without providing credentials."""
-    pass

app/libraries/setup.py

@@ -15,7 +15,7 @@ class Setup():
         self.config_file = "config/config.txt"
         self.mastodon_hostname = self.__get_parameter("mastodon_hostname", self.config_file)
         self.peers_api = '/api/v1/instance/peers?'
-        self.user_agent = {'User-agent': "fediverse's stats"}
+        self.user_agent = {'User-agent': "fediverse's stats (fediverse@mastodont.cat)"}
 
         self.secrets_filepath = 'secrets/secrets.txt'
 

fediverse.py

@@ -57,15 +57,6 @@ if __name__ == '__main__':
 
     soft_total_project, soft_total_users, soft_total_mau, soft_total_servers, total_servers, total_users, total_mau = db.soft_totals()
 
-    i = 0
-
-    while i < 5:
-
-
-        db.write_top_soft(soft_total_project[i], soft_total_users[i], soft_total_mau[i], soft_total_servers[i])
-
-        i += 1
-
     # get last check values and write current total ones
 
     evo_servers, evo_users, evo_mau = db.last_values(total_servers, total_users, total_mau)

requirements.txt

@@ -5,5 +5,4 @@ fastapi
 uvicorn[standard]
 matplotlib
 pandas
-sqlalchemy
 

upload.py

@@ -1,58 +0,0 @@
-import os
-import base64
-from datetime import datetime
-from app.libraries.database import Database
-from app.libraries.forgejo import Forgejo, ForgejoNotFoundError
-import pdb
-
-def date_string():
-
-    now = datetime.now()
-
-    day = '{:02d}'.format(now.day)
-
-    month = '{:02d}'.format(now.month)
-
-    year = str(now.year)[2:]
-
-    return f'{day}{month}{year}'
-
-if __name__ == '__main__':
-
-    db = Database()
-
-    if not os.path.exists('dataset'):
-
-        os.makedirs('dataset')
-
-    date = date_string()
-
-    if not os.path.exists(f'dataset/{date}'):
-
-        os.makedirs(f'dataset/{date}')
-
-    filename = f'dataset_{date}.csv'
-
-    db.csv_save(f'dataset/{date}/{filename}')
-
-    fgj = Forgejo()
-
-    gituser = fgj.user()
-
-    try:
-
-        with open(f'dataset/{date}/{filename}', 'rb') as input_file:
-            data = input_file.read()
-            file = base64.b64encode(data)
-
-        response = fgj.repo_owner_create_file(gituser.login, fgj.stats_repo, f'dataset/{date}/{filename}', gituser.email, gituser.login, "main", file, f"{date} fediverse's dataset")
-
-        if 'content' in response:
-
-            print(f'{filename} uploaded to {fgj.api_base_url}/{gituser.login}/{fgj.stats_repo}')
-
-    except:
-
-        print(response)
-
-        pass