Merge branch 'main' of https://git.mastodont.cat/spla/stats

README.md

@@ -21,5 +21,7 @@ Within Python Virtual Environment:
 4. Use your favourite scheduling method to set `python fetchpeers.py` to run at least once a day and `python fediverse.py` to run at desired pace to publish the results. Also set `python fediquery.py` to run every minute to accept queries from any fediverse' users about any `server` or `soft`.  
 
 15.4.2023 - Added fediquery.py. It allows queries from any fediverse' user about soft and server (keywords). It replies to the asking user with its data, if any.
+28.5.2023 - Added top table to save top five softwares data.
+28.5.2023 - Added upload.py to upload alive fediverse's dataset to configured Forgejo repository.
 
 

app/libraries/database.py

@@ -6,6 +6,9 @@ from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
 import uuid
 from datetime import datetime
 import pytz
+import sqlalchemy
+from sqlalchemy import URL
+import pandas as pd
 
 tz = pytz.timezone('Europe/Madrid')
 
@@ -38,7 +41,7 @@ class Database():
 
             conn = None
 
-            conn = psycopg2.connect(database = self.mau_db, user = self.mau_db_user, password = self.mau_db_user_password, host = "/var/run/postgresql", port = "6432")
+            conn = psycopg2.connect(database = self.mau_db, user = self.mau_db_user, password = self.mau_db_user_password, host = "/var/run/postgresql", port = "5432")
 
             cur = conn.cursor()
 
@@ -80,7 +83,7 @@ class Database():
 
         try:
 
-            conn = psycopg2.connect(database = self.mau_db, user = self.mau_db_user, password = self.mau_db_user_password, host = "/var/run/postgresql", port = "6432")
+            conn = psycopg2.connect(database = self.mau_db, user = self.mau_db_user, password = self.mau_db_user_password, host = "/var/run/postgresql", port = "5432")
 
             cur = conn.cursor()
 
@@ -112,7 +115,7 @@ class Database():
 
         try:
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
 
             cur = conn.cursor()
 
@@ -153,7 +156,7 @@ class Database():
 
             conn = None
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
 
             cur = conn.cursor()
 
@@ -197,7 +200,7 @@ class Database():
 
             conn = None
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
 
             cur = conn.cursor()
 
@@ -235,7 +238,7 @@ class Database():
 
             conn = None
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
 
             cur = conn.cursor()
 
@@ -273,7 +276,7 @@ class Database():
 
             conn = None
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
 
             cur = conn.cursor()
 
@@ -321,7 +324,7 @@ class Database():
 
             conn = None
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
 
             cur = conn.cursor()
 
@@ -370,7 +373,7 @@ class Database():
 
             conn = None
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
 
             cur = conn.cursor()
 
@@ -408,7 +411,7 @@ class Database():
 
         try:
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
 
             cur = conn.cursor()
 
@@ -440,7 +443,7 @@ class Database():
 
         try:
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
 
             cur = conn.cursor()
 
@@ -471,7 +474,7 @@ class Database():
 
         try:
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
 
             cur = conn.cursor()
 
@@ -501,7 +504,7 @@ class Database():
 
         try:
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
 
             cur = conn.cursor()
 
@@ -532,7 +535,7 @@ class Database():
 
         try:
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
 
             cur = conn.cursor()
 
@@ -562,7 +565,7 @@ class Database():
 
         try:
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
 
             cur = conn.cursor()
 
@@ -600,7 +603,7 @@ class Database():
 
             conn = None
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
 
             cur = conn.cursor()
 
@@ -659,7 +662,7 @@ class Database():
 
             conn = None
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
 
             cur = conn.cursor()
 
@@ -707,6 +710,36 @@ class Database():
 
         return (evo_servers, evo_users, evo_mau)
 
+    def write_top_soft(self, soft_total_project, soft_total_users, soft_total_mau, soft_total_servers):
+
+        now = datetime.now()
+
+        insert_sql = "INSERT INTO top(datetime, software, users, mau, servers) VALUES(%s,%s,%s,%s,%s)"
+
+        conn = None
+
+        try:
+
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
+
+            cur = conn.cursor()
+
+            cur.execute(insert_sql, (now, soft_total_project, soft_total_users, soft_total_mau, soft_total_servers))
+
+            conn.commit()
+
+            cur.close()
+
+        except (Exception, psycopg2.DatabaseError) as error:
+
+            print(error)
+
+        finally:
+
+            if conn is not None:
+
+                conn.close()
+
     def write_evo(self, evo_servers, evo_users, evo_mau):
 
         #  write evo values
@@ -719,7 +752,7 @@ class Database():
 
         try:
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
 
             cur = conn.cursor()
 
@@ -747,7 +780,7 @@ class Database():
 
         try:
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
 
             cur = conn.cursor()
 
@@ -781,7 +814,7 @@ class Database():
 
         try:
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
 
             cur = conn.cursor()
 
@@ -863,7 +896,7 @@ class Database():
 
         try:
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
 
             cur = conn.cursor()
 
@@ -933,7 +966,7 @@ class Database():
 
             conn = None
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
 
             cur = conn.cursor()
 
@@ -971,7 +1004,7 @@ class Database():
 
             conn = None
 
-            conn = psycopg2.connect(database = self.mau_db, user = self.mau_db_user, password = self.mau_db_user_password, host = "/var/run/postgresql", port = "6432")
+            conn = psycopg2.connect(database = self.mau_db, user = self.mau_db_user, password = self.mau_db_user_password, host = "/var/run/postgresql", port = "5432")
 
             cur = conn.cursor()
 
@@ -1015,7 +1048,7 @@ class Database():
 
             conn = None
 
-            conn = psycopg2.connect(database = self.mau_db, user = self.mau_db_user, password = self.mau_db_user_password, host = "/var/run/postgresql", port = "6432")
+            conn = psycopg2.connect(database = self.mau_db, user = self.mau_db_user, password = self.mau_db_user_password, host = "/var/run/postgresql", port = "5432")
 
             cur = conn.cursor()
 
@@ -1077,7 +1110,7 @@ class Database():
 
             conn = None
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password=self.mau_db_user_password, host="/var/run/postgresql", port="6432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password=self.mau_db_user_password, host="/var/run/postgresql", port="5432")
 
             cur = conn.cursor()
 
@@ -1135,7 +1168,7 @@ class Database():
 
             conn = None
 
-            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password=self.mau_db_user_password, host="/var/run/postgresql", port="6432")
+            conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password=self.mau_db_user_password, host="/var/run/postgresql", port="5432")
 
             cur = conn.cursor()
 
@@ -1163,6 +1196,40 @@ class Database():
 
                 conn.close()
 
+    def csv_save(self, filename):
+
+        is_saved = False
+
+        url_object = URL.create(
+            "postgresql+psycopg2",
+            username=self.mau_db_user,
+            password=self.mau_db_user_password,
+            host="localhost",
+            database=self.mau_db,
+        )
+
+        sql = "select * from mau where alive order by mau desc"
+
+        conn = None
+
+        try:
+
+            conn = sqlalchemy.create_engine(url_object)
+
+            with conn.connect() as connection:
+
+                dataframe = pd.read_sql_query(sql, conn)
+
+                dataframe.to_csv(filename, sep='|', header=True, index=False)
+
+                is_saved = True
+
+        except (Exception, psycopg2.DatabaseError) as error:
+
+            print(error)
+
+        return is_saved
+
     @staticmethod
     def __check_dbsetup(self):
 
@@ -1172,7 +1239,7 @@ class Database():
 
             conn = None
 
-            conn = psycopg2.connect(database = self.mau_db, user = self.mau_db_user, password = self.mau_db_user_password, host = "/var/run/postgresql", port = "6432")
+            conn = psycopg2.connect(database = self.mau_db, user = self.mau_db_user, password = self.mau_db_user_password, host = "/var/run/postgresql", port = "5432")
 
             db_setup = True
 
@@ -1225,7 +1292,7 @@ class Database():
         self.__create_table(self, table, sql)
 
         table = "servers"
-        sql = "create table "+table+" (server varchar(200) PRIMARY KEY, api varchar(50), created_at timestamptz, peers boolean, update_at timestamptz)"
+        sql = "create table "+table+" (server varchar(200) PRIMARY KEY, api varchar(50), created_at timestamptz, peers boolean, updated_at timestamptz)"
         self.__create_table(self, table, sql)
 
         table = "deadservers"
@@ -1240,6 +1307,10 @@ class Database():
         sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, servers INT, users INT, mau INT)"
         self.__create_table(self, table, sql)
 
+        table = "top"
+        sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, software varchar(50), users INT, mau INT, servers INT)"
+        self.__create_table(self, table, sql)
+
         table = "execution_time"
         sql = "create table "+table+" (program varchar(20) PRIMARY KEY, start timestamptz, finish timestamptz)"
         self.__create_table(self, table, sql)
@@ -1251,7 +1322,7 @@ class Database():
 
         try:
 
-            conn = psycopg2.connect(database = self.mau_db, user = self.mau_db_user, password = self.mau_db_user_password, host = "/var/run/postgresql", port = "6432")
+            conn = psycopg2.connect(database = self.mau_db, user = self.mau_db_user, password = self.mau_db_user_password, host = "/var/run/postgresql", port = "5432")
 
             cur = conn.cursor()
 

app/libraries/forgejo.py

@@ -0,0 +1,489 @@
+import os
+import sys
+import requests
+from requests.models import urlencode
+
+###
+# Dict helper class.
+# Defined at top level so it can be pickled.
+###
+class AttribAccessDict(dict):
+    def __getattr__(self, attr):
+        if attr in self:
+            return self[attr]
+        else:
+            raise AttributeError("Attribute not found: " + str(attr))
+
+    def __setattr__(self, attr, val):
+        if attr in self:
+            raise AttributeError("Attribute-style access is read only")
+        super(AttribAccessDict, self).__setattr__(attr, val)
+
+class Forgejo:
+
+    name = 'Forgejo API wrapper'
+
+    def __init__(self, api_base_url=None, access_token=None, session=None):
+
+        self.__forgejo_config_path = "config/forgejo.txt"
+
+        is_setup = self.__check_setup(self)
+
+        if is_setup:
+
+            self.api_base_url = self.__get_parameter("api_base_url", self.__forgejo_config_path)
+            self.access_token = self.__get_parameter("access_token", self.__forgejo_config_path)
+            self.stats_repo = self.__get_parameter("stats_repo", self.__forgejo_config_path)
+
+        else:
+
+            self.api_base_url, self.access_token, self.stats_repo = self.__setup(self)
+
+        if session:
+            self.session = session
+        else:
+            self.session = requests.Session()
+
+    def admin_users_create(self, email, username, passwd, full_name=None, must_change_password=True, restricted=False, send_notify=True, source_id='0', visibility='private'):
+
+        data = {'email':email,
+                'full_name':username,
+                'login_name':username,
+                'must_change_password':must_change_password,
+                'password':passwd,
+                'restricted':restricted,
+                'send_notify':send_notify,
+                'source_id':source_id,
+                'username':username,
+                'visibility':visibility
+            }
+
+        endpoint = self.api_base_url + '/api/v1/admin/users?access_token={0}'.format(self.access_token)
+
+        response = self.__api_request('POST', endpoint, data)
+
+        registered = response.ok
+
+        response = self.__json_allow_dict_attrs(response.json())
+
+        return (registered, response)
+
+    def admin_users_list(self, page=None, limit=None):
+
+        params = dict()
+        if page != None:
+            params['page'] = page
+        if limit != None:
+            params['limit'] = limit
+        params['token'] = self.access_token
+        formatted_params = urlencode(params)
+
+        endpoint = self.api_base_url + '/api/v1/admin/users?{0}'.format(formatted_params)
+
+        response = self.__api_request('GET', endpoint)
+
+        response = self.__json_allow_dict_attrs(response.json())
+
+        return response
+
+    def notifications_new(self):
+
+        endpoint = self.api_base_url + '/api/v1/notifications/new?token={0}'.format(self.access_token)
+
+        response = self.__api_request('GET', endpoint)
+
+        response = self.__json_allow_dict_attrs(response.json())
+
+        return response
+
+    ### 
+    ### repository
+    ###
+
+    def repos_get_repo(self, owner, repo):
+
+        params = dict()
+        params['token'] = self.access_token
+        formatted_params = urlencode(params)
+
+        endpoint = self.api_base_url + '/api/v1/repos/{0}/{1}'.format(owner, repo, formatted_params)
+
+        response = self.__api_request('GET', endpoint)
+
+        response = self.__json_allow_dict_attrs(response.json())
+
+        return response
+
+    def repos_issues_search(self, owner, state=None, labels=None, q=None, milestones=None, priority_repo_id=None, issue_type=None, since=None, before=None, assigned=None, 
+            created=None, mentioned=None, review_requested=None, team=None, page=None, limit=None):
+
+        params = dict()
+
+        if state == None:
+            params['state'] = 'open'
+        else:
+            params['state'] = state
+        if labels != None:
+            params['labels'] = labels
+        if milestones != None:
+            params['milestones'] = milestones
+        if q != None:
+            params['q'] = q
+        if priority_repo_id != None:
+            params['priority_repo_id'] = priority_repo_id
+        if issue_type != None:
+            params['type'] = issue_type
+        if since != None:
+            params['since'] = since
+        if before != None:
+            params['before'] = before
+        if assigned != None:
+            params['assigned'] = assigned
+        if created != None:
+            params['created'] = created
+        if mentioned != None:
+            params['mentioned'] = mentioned
+        if review_requested != None:
+            params['review_requested'] = review_requested
+        params['owner'] = owner
+        if team != None:
+            params['team'] = team
+        if page != None:
+            params['page'] = page
+        if limit != None:
+            params['limit'] = limit
+        params['token'] = self.access_token
+        formatted_params = urlencode(params)
+
+        endpoint = self.api_base_url + '/api/v1/repos/issues/search?{0}'.format(formatted_params)
+
+        response = self.__api_request('GET', endpoint)
+
+        response = self.__json_allow_dict_attrs(response.json())
+
+        return response
+
+    def repos_owner_repo_issues(self, owner, repo, state=None, labels=None, q=None, issue_type=None, milestones=None, since=None, before=None, created_by=None, assigned_by=None,
+                                    mentioned_by=None, page=None, limit=None):
+        """
+        if since or before are specified, they must have following format: in. ex. 2022-08-13T08:09:07+02:00
+        """
+        params = dict()
+
+        if state == None:
+            params['state'] = 'open'
+        else:
+            params['state'] = state
+        params['labels'] = labels
+        if q != None:
+            params['q'] = q
+        params['issue_type'] = issue_type
+        params['milestones'] = milestones
+        if since != None:
+            params['since'] = since
+        if before != None:
+            params['before'] = before
+        if created_by != None:
+            params['created_by'] = created_by
+        if assigned_by != None:
+            params['assigned_by'] = assigned_by
+        if mentioned_by != None:
+            params['mentioned_by'] = mentioned_by
+        params['page'] = page
+        params['limit'] = limit
+        formatted_params = urlencode(params)
+
+        endpoint = self.api_base_url + '/api/v1/repos/{0}/{1}/issues?{2}'.format(owner, repo, formatted_params)
+
+        response = self.__api_request('GET', endpoint)
+
+        response = self.__json_allow_dict_attrs(response.json())
+
+        return response
+
+    def repos_owner_repo_issues_comments(self, owner, repo, since=None, before=None, page=None, limit=None):
+        """
+        if since or before are specified, they must have following format: in. ex. 2022-08-13T08:09:07+02:00
+        """
+        params = dict()
+        if since != None:
+            params['since'] = since
+        if before != None:
+            params['before'] = before
+        params['page'] = page
+        params['limit'] = limit
+        formatted_params = urlencode(params)
+
+        endpoint = self.api_base_url + '/api/v1/repos/{0}/{1}/issues/comments?{2}'.format(owner, repo, formatted_params)
+
+        response = self.__api_request('GET', endpoint)#, data)
+
+        response = self.__json_allow_dict_attrs(response.json())
+
+        return response
+
+    def repo_owner_get_metada(self, owner, repo, filepath):
+        """
+        Gets the metadata and contents (if a file) of an entry in a repository, or  list of entries if a dir
+        """
+
+        params = dict()
+        params['token'] = self.access_token
+        formatted_params = urlencode(params)
+
+        endpoint = self.api_base_url + '/api/v1/repos/{0}/{1}/contents/{2}?{3}'.format(owner, repo, filepath, formatted_params)
+
+        response = self.__api_request('GET', endpoint)
+
+        response = self.__json_allow_dict_attrs(response.json())
+
+        return response
+
+    def repo_owner_create_file(self, owner, repo, filepath, author_email, author_name, branch, content, message):
+        """
+        Create a file in a repository
+        """
+
+        data = {"author":[{"email":author_email},{"name":author_name}],
+                "branch":branch,
+                "content":content,
+                "message":message
+        }
+
+        params = dict()
+        params['token'] = self.access_token
+        formatted_params = urlencode(params)
+
+        endpoint = self.api_base_url + '/api/v1/repos/{0}/{1}/contents/{2}?{3}'.format(owner, repo, filepath, formatted_params)
+
+        response = self.__api_request('POST', endpoint, data=data)
+
+        response = self.__json_allow_dict_attrs(response.json())
+
+        return response
+
+    def repo_owner_update_file(self, owner, repo, filepath, author_email, author_name, branch, message, sha):
+        """
+        Update a file in a repository
+        """
+        data = {"author":[{"email":author_email},{"name":author_name}],
+                "branch":branch,
+                "message":message,
+                "sha":sha
+        }
+
+        params = dict()
+        params['token'] = self.access_token
+        formatted_params = urlencode(params)
+
+        endpoint = self.api_base_url + '/api/v1/repos/{0}/{1}/contents/{2}?{3}'.format(owner, repo, filepath, formatted_params)
+
+        response = self.__api_request('PUT', endpoint, data=data)
+
+        response = self.__json_allow_dict_attrs(response.json())
+
+        return response
+
+    def repo_owner_delete_file(self, owner, repo, filepath, author_email, author_name, branch, message, sha):
+        """
+        Delete a file in a repository
+        """
+        data = {"author":[{"email":author_email},{"name":author_name}],
+                "branch":branch,
+                "commiter":[{"email":author_email},{"name":author_name}],
+                "message":message,
+                "new_branch":branch,
+                'sha':sha
+        }
+
+        params = dict()
+        params['token'] = self.access_token
+        formatted_params = urlencode(params)
+
+        endpoint = self.api_base_url + '/api/v1/repos/{0}/{1}/contents/{2}?{3}'.format(owner, repo, filepath, formatted_params)
+
+        response = self.__api_request('DELETE', endpoint, data=data)
+
+        response = self.__json_allow_dict_attrs(response.json())
+
+        return response
+
+    def user(self):
+        """
+        Get the authenticated user
+        """
+
+        params = dict()
+        params['token'] = self.access_token
+        formatted_params = urlencode(params)
+
+        endpoint = self.api_base_url + '/api/v1/user?{0}'.format(formatted_params)
+
+        response = self.__api_request('GET', endpoint)
+
+        response = self.__json_allow_dict_attrs(response.json())
+
+        return response
+
+    @staticmethod
+    def __check_setup(self):
+
+        is_setup = False
+
+        if not os.path.isfile(self.__forgejo_config_path):
+            print(f"File {self.__forgejo_config_path} not found, running setup.")
+        else:
+            is_setup = True
+
+        return is_setup
+
+    @staticmethod
+    def __setup(self):
+
+        if not os.path.exists('config'):
+            os.makedirs('config')
+
+        self.api_base_url = input("Forgejo API base url, in ex. 'https://yourforgejo.instance': ")
+        self.access_token = input("Forgejo access token: ")
+        self.stats_repo = input("Stats repo: ")
+
+
+        if not os.path.exists(self.__forgejo_config_path):
+            with open(self.__forgejo_config_path, 'w'): pass
+            print(f"{self.__forgejo_config_path} created!")
+
+        with open(self.__forgejo_config_path, 'a') as the_file:
+            print("Writing forgejo parameters to " + self.__forgejo_config_path)
+            the_file.write(f'api_base_url: {self.api_base_url}\n'+f'access_token: {self.access_token}\nstats_repo: {self.stats_repo}\n')
+
+        return (self.api_base_url, self.access_token, self.stats_repo)
+
+    @staticmethod
+    def __get_parameter(parameter, file_path ):
+
+        with open( file_path ) as f:
+            for line in f:
+                if line.startswith( parameter ):
+                    return line.replace(parameter + ":", "").strip()
+
+        print(f'{file_path} Missing parameter {parameter}')
+        sys.exit(0)
+
+    def __api_request(self, method, endpoint, data={}):
+
+        response = None
+
+        try:
+
+            kwargs = dict(data=data)
+
+            response = self.session.request(method, url = endpoint, **kwargs)
+
+        except Exception as e:
+
+            raise ForgejoNetworkError(f"Could not complete request: {e}")
+
+        if response is None:
+
+            raise ForgejoIllegalArgumentError("Illegal request.")
+
+        if not response.ok:
+
+                try:
+                    if isinstance(response, dict) and 'error' in response:
+                        error_msg = response['error']
+                    elif isinstance(response, str):
+                        error_msg = response
+                    else:
+                        error_msg = None
+                except ValueError:
+                    error_msg = None
+
+                if response.status_code == 404:
+                    ex_type = ForgejoNotFoundError
+                    if not error_msg:
+                        error_msg = 'Endpoint not found.'
+                        # this is for compatibility with older versions
+                        # which raised ForgejoAPIError('Endpoint not found.')
+                        # on any 404
+                elif response.status_code == 401:
+                    ex_type = ForgejoUnauthorizedError
+                elif response.status_code == 422:
+                    return response
+                elif response.status_code == 500:
+                    ex_type = ForgejoInternalServerError
+                elif response.status_code == 502:
+                    ex_type = ForgejoBadGatewayError
+                elif response.status_code == 503:
+                    ex_type = ForgejoServiceUnavailableError
+                elif response.status_code == 504:
+                    ex_type = ForgejoGatewayTimeoutError
+                elif response.status_code >= 500 and \
+                    response.status_code <= 511:
+                    ex_type = ForgejoServerError
+                else:
+                    ex_type = ForgejoAPIError
+
+                raise ex_type(
+                    'Forgejo API returned error',
+                    response.status_code,
+                    response.reason,
+                    error_msg)
+
+        else:
+
+            return response
+
+    @staticmethod
+    def __json_allow_dict_attrs(json_object):
+        """
+        Makes it possible to use attribute notation to access a dicts
+        elements, while still allowing the dict to act as a dict.
+        """
+        if isinstance(json_object, dict):
+            return AttribAccessDict(json_object)
+        return json_objecte
+
+##
+# Exceptions
+##
+class ForgejoError(Exception):
+    """Base class for Forgejo.py exceptions"""
+
+class ForgejoIOError(IOError, ForgejoError):
+    """Base class for Forgejo.py I/O errors"""
+
+class ForgejoNetworkError(ForgejoIOError):
+    """Raised when network communication with the server fails"""
+    pass
+class ForgejoAPIError(ForgejoError):
+    """Raised when the forgejo API generates a response that cannot be handled"""
+    pass
+class ForgejoServerError(ForgejoAPIError):
+    """Raised if the Server is malconfigured and returns a 5xx error code"""
+    pass
+class ForgejoInternalServerError(ForgejoServerError):
+    """Raised if the Server returns a 500 error"""
+    pass
+
+class ForgejoBadGatewayError(ForgejoServerError):
+    """Raised if the Server returns a 502 error"""
+    pass
+
+class ForgejoServiceUnavailableError(ForgejoServerError):
+    """Raised if the Server returns a 503 error"""
+    pass
+class ForgejoGatewayTimeoutError(ForgejoServerError):
+    """Raised if the Server returns a 504 error"""
+    pass
+class ForgejoNotFoundError(ForgejoAPIError):
+    """Raised when the forgejo API returns a 404 Not Found error"""
+    pass
+
+class ForgejoUnauthorizedError(ForgejoAPIError):
+    """Raised when the forgejo API returns a 401 Unauthorized error
+
+       This happens when an OAuth token is invalid or has been revoked,
+       or when trying to access an endpoint that can't be used without
+       authentication without providing credentials."""
+    pass

app/libraries/setup.py

@@ -15,7 +15,7 @@ class Setup():
         self.config_file = "config/config.txt"
         self.mastodon_hostname = self.__get_parameter("mastodon_hostname", self.config_file)
         self.peers_api = '/api/v1/instance/peers?'
-        self.user_agent = {'User-agent': "fediverse's stats (fediverse@mastodont.cat)"}
+        self.user_agent = {'User-agent': "fediverse's stats"}
 
         self.secrets_filepath = 'secrets/secrets.txt'
 

fediverse.py

@@ -57,6 +57,15 @@ if __name__ == '__main__':
 
     soft_total_project, soft_total_users, soft_total_mau, soft_total_servers, total_servers, total_users, total_mau = db.soft_totals()
 
+    i = 0
+
+    while i < 5:
+
+
+        db.write_top_soft(soft_total_project[i], soft_total_users[i], soft_total_mau[i], soft_total_servers[i])
+
+        i += 1
+
     # get last check values and write current total ones
 
     evo_servers, evo_users, evo_mau = db.last_values(total_servers, total_users, total_mau)

requirements.txt

@@ -5,4 +5,5 @@ fastapi
 uvicorn[standard]
 matplotlib
 pandas
+sqlalchemy
 

upload.py

@@ -0,0 +1,58 @@
+import os
+import base64
+from datetime import datetime
+from app.libraries.database import Database
+from app.libraries.forgejo import Forgejo, ForgejoNotFoundError
+import pdb
+
+def date_string():
+
+    now = datetime.now()
+
+    day = '{:02d}'.format(now.day)
+
+    month = '{:02d}'.format(now.month)
+
+    year = str(now.year)[2:]
+
+    return f'{day}{month}{year}'
+
+if __name__ == '__main__':
+
+    db = Database()
+
+    if not os.path.exists('dataset'):
+
+        os.makedirs('dataset')
+
+    date = date_string()
+
+    if not os.path.exists(f'dataset/{date}'):
+
+        os.makedirs(f'dataset/{date}')
+
+    filename = f'dataset_{date}.csv'
+
+    db.csv_save(f'dataset/{date}/{filename}')
+
+    fgj = Forgejo()
+
+    gituser = fgj.user()
+
+    try:
+
+        with open(f'dataset/{date}/{filename}', 'rb') as input_file:
+            data = input_file.read()
+            file = base64.b64encode(data)
+
+        response = fgj.repo_owner_create_file(gituser.login, fgj.stats_repo, f'dataset/{date}/{filename}', gituser.email, gituser.login, "main", file, f"{date} fediverse's dataset")
+
+        if 'content' in response:
+
+            print(f'{filename} uploaded to {fgj.api_base_url}/{gituser.login}/{fgj.stats_repo}')
+
+    except:
+
+        print(response)
+
+        pass