Added top table to save top five softwares and upload.py to upload alive fediverse's dataset to Forgejo repo

This commit is contained in:
spla 2023-05-28 22:18:12 +02:00
pare 1d9a0b4742
commit 512c2df00d
S'han modificat 7 arxius amb 660 adicions i 30 eliminacions

Veure arxiu

@ -21,5 +21,7 @@ Within Python Virtual Environment:
4. Use your favourite scheduling method to set `python fetchpeers.py` to run at least once a day and `python fediverse.py` to run at desired pace to publish the results. Also set `python fediquery.py` to run every minute to accept queries from any fediverse' users about any `server` or `soft`. 4. Use your favourite scheduling method to set `python fetchpeers.py` to run at least once a day and `python fediverse.py` to run at desired pace to publish the results. Also set `python fediquery.py` to run every minute to accept queries from any fediverse' users about any `server` or `soft`.
15.4.2023 - Added fediquery.py. It allows queries from any fediverse' user about soft and server (keywords). It replies to the asking user with its data, if any. 15.4.2023 - Added fediquery.py. It allows queries from any fediverse' user about soft and server (keywords). It replies to the asking user with its data, if any.
28.5.2023 - Added top table to save top five softwares data.
28.5.2023 - Added upload.py to upload alive fediverse's dataset to configured Forgejo repository.

Veure arxiu

@ -6,6 +6,9 @@ from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
import uuid import uuid
from datetime import datetime from datetime import datetime
import pytz import pytz
import sqlalchemy
from sqlalchemy import URL
import pandas as pd
tz = pytz.timezone('Europe/Madrid') tz = pytz.timezone('Europe/Madrid')
@ -38,7 +41,7 @@ class Database():
conn = None conn = None
conn = psycopg2.connect(database = self.mau_db, user = self.mau_db_user, password = self.mau_db_user_password, host = "/var/run/postgresql", port = "6432") conn = psycopg2.connect(database = self.mau_db, user = self.mau_db_user, password = self.mau_db_user_password, host = "/var/run/postgresql", port = "5432")
cur = conn.cursor() cur = conn.cursor()
@ -80,7 +83,7 @@ class Database():
try: try:
conn = psycopg2.connect(database = self.mau_db, user = self.mau_db_user, password = self.mau_db_user_password, host = "/var/run/postgresql", port = "6432") conn = psycopg2.connect(database = self.mau_db, user = self.mau_db_user, password = self.mau_db_user_password, host = "/var/run/postgresql", port = "5432")
cur = conn.cursor() cur = conn.cursor()
@ -112,7 +115,7 @@ class Database():
try: try:
conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432") conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
cur = conn.cursor() cur = conn.cursor()
@ -153,7 +156,7 @@ class Database():
conn = None conn = None
conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432") conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
cur = conn.cursor() cur = conn.cursor()
@ -197,7 +200,7 @@ class Database():
conn = None conn = None
conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432") conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
cur = conn.cursor() cur = conn.cursor()
@ -235,7 +238,7 @@ class Database():
conn = None conn = None
conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432") conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
cur = conn.cursor() cur = conn.cursor()
@ -273,7 +276,7 @@ class Database():
conn = None conn = None
conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432") conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
cur = conn.cursor() cur = conn.cursor()
@ -321,7 +324,7 @@ class Database():
conn = None conn = None
conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432") conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
cur = conn.cursor() cur = conn.cursor()
@ -370,7 +373,7 @@ class Database():
conn = None conn = None
conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432") conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
cur = conn.cursor() cur = conn.cursor()
@ -408,7 +411,7 @@ class Database():
try: try:
conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432") conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
cur = conn.cursor() cur = conn.cursor()
@ -440,7 +443,7 @@ class Database():
try: try:
conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432") conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
cur = conn.cursor() cur = conn.cursor()
@ -471,7 +474,7 @@ class Database():
try: try:
conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432") conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
cur = conn.cursor() cur = conn.cursor()
@ -501,7 +504,7 @@ class Database():
try: try:
conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432") conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
cur = conn.cursor() cur = conn.cursor()
@ -532,7 +535,7 @@ class Database():
try: try:
conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432") conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
cur = conn.cursor() cur = conn.cursor()
@ -562,7 +565,7 @@ class Database():
try: try:
conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432") conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
cur = conn.cursor() cur = conn.cursor()
@ -600,7 +603,7 @@ class Database():
conn = None conn = None
conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432") conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
cur = conn.cursor() cur = conn.cursor()
@ -659,7 +662,7 @@ class Database():
conn = None conn = None
conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432") conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
cur = conn.cursor() cur = conn.cursor()
@ -707,6 +710,36 @@ class Database():
return (evo_servers, evo_users, evo_mau) return (evo_servers, evo_users, evo_mau)
def write_top_soft(self, soft_total_project, soft_total_users, soft_total_mau, soft_total_servers):
now = datetime.now()
insert_sql = "INSERT INTO top(datetime, software, users, mau, servers) VALUES(%s,%s,%s,%s,%s)"
conn = None
try:
conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
cur = conn.cursor()
cur.execute(insert_sql, (now, soft_total_project, soft_total_users, soft_total_mau, soft_total_servers))
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
def write_evo(self, evo_servers, evo_users, evo_mau): def write_evo(self, evo_servers, evo_users, evo_mau):
# write evo values # write evo values
@ -719,7 +752,7 @@ class Database():
try: try:
conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432") conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
cur = conn.cursor() cur = conn.cursor()
@ -747,7 +780,7 @@ class Database():
try: try:
conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432") conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
cur = conn.cursor() cur = conn.cursor()
@ -781,7 +814,7 @@ class Database():
try: try:
conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432") conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
cur = conn.cursor() cur = conn.cursor()
@ -863,7 +896,7 @@ class Database():
try: try:
conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432") conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
cur = conn.cursor() cur = conn.cursor()
@ -933,7 +966,7 @@ class Database():
conn = None conn = None
conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="6432") conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password = self.mau_db_user_password, host="/var/run/postgresql", port="5432")
cur = conn.cursor() cur = conn.cursor()
@ -971,7 +1004,7 @@ class Database():
conn = None conn = None
conn = psycopg2.connect(database = self.mau_db, user = self.mau_db_user, password = self.mau_db_user_password, host = "/var/run/postgresql", port = "6432") conn = psycopg2.connect(database = self.mau_db, user = self.mau_db_user, password = self.mau_db_user_password, host = "/var/run/postgresql", port = "5432")
cur = conn.cursor() cur = conn.cursor()
@ -1015,7 +1048,7 @@ class Database():
conn = None conn = None
conn = psycopg2.connect(database = self.mau_db, user = self.mau_db_user, password = self.mau_db_user_password, host = "/var/run/postgresql", port = "6432") conn = psycopg2.connect(database = self.mau_db, user = self.mau_db_user, password = self.mau_db_user_password, host = "/var/run/postgresql", port = "5432")
cur = conn.cursor() cur = conn.cursor()
@ -1077,7 +1110,7 @@ class Database():
conn = None conn = None
conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password=self.mau_db_user_password, host="/var/run/postgresql", port="6432") conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password=self.mau_db_user_password, host="/var/run/postgresql", port="5432")
cur = conn.cursor() cur = conn.cursor()
@ -1135,7 +1168,7 @@ class Database():
conn = None conn = None
conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password=self.mau_db_user_password, host="/var/run/postgresql", port="6432") conn = psycopg2.connect(database=self.mau_db, user=self.mau_db_user, password=self.mau_db_user_password, host="/var/run/postgresql", port="5432")
cur = conn.cursor() cur = conn.cursor()
@ -1163,6 +1196,40 @@ class Database():
conn.close() conn.close()
def csv_save(self, filename):
is_saved = False
url_object = URL.create(
"postgresql+psycopg2",
username=self.mau_db_user,
password=self.mau_db_user_password,
host="localhost",
database=self.mau_db,
)
sql = "select * from mau where alive order by mau desc"
conn = None
try:
conn = sqlalchemy.create_engine(url_object)
with conn.connect() as connection:
dataframe = pd.read_sql_query(sql, conn)
dataframe.to_csv(filename, sep='|', header=True, index=False)
is_saved = True
except (Exception, psycopg2.DatabaseError) as error:
print(error)
return is_saved
@staticmethod @staticmethod
def __check_dbsetup(self): def __check_dbsetup(self):
@ -1172,7 +1239,7 @@ class Database():
conn = None conn = None
conn = psycopg2.connect(database = self.mau_db, user = self.mau_db_user, password = self.mau_db_user_password, host = "/var/run/postgresql", port = "6432") conn = psycopg2.connect(database = self.mau_db, user = self.mau_db_user, password = self.mau_db_user_password, host = "/var/run/postgresql", port = "5432")
db_setup = True db_setup = True
@ -1225,7 +1292,7 @@ class Database():
self.__create_table(self, table, sql) self.__create_table(self, table, sql)
table = "servers" table = "servers"
sql = "create table "+table+" (server varchar(200) PRIMARY KEY, api varchar(50), created_at timestamptz, peers boolean, update_at timestamptz)" sql = "create table "+table+" (server varchar(200) PRIMARY KEY, api varchar(50), created_at timestamptz, peers boolean, updated_at timestamptz)"
self.__create_table(self, table, sql) self.__create_table(self, table, sql)
table = "deadservers" table = "deadservers"
@ -1240,6 +1307,10 @@ class Database():
sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, servers INT, users INT, mau INT)" sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, servers INT, users INT, mau INT)"
self.__create_table(self, table, sql) self.__create_table(self, table, sql)
table = "top"
sql = "create table "+table+" (datetime timestamptz PRIMARY KEY, software varchar(50), users INT, mau INT, servers INT)"
self.__create_table(self, table, sql)
table = "execution_time" table = "execution_time"
sql = "create table "+table+" (program varchar(20) PRIMARY KEY, start timestamptz, finish timestamptz)" sql = "create table "+table+" (program varchar(20) PRIMARY KEY, start timestamptz, finish timestamptz)"
self.__create_table(self, table, sql) self.__create_table(self, table, sql)
@ -1251,7 +1322,7 @@ class Database():
try: try:
conn = psycopg2.connect(database = self.mau_db, user = self.mau_db_user, password = self.mau_db_user_password, host = "/var/run/postgresql", port = "6432") conn = psycopg2.connect(database = self.mau_db, user = self.mau_db_user, password = self.mau_db_user_password, host = "/var/run/postgresql", port = "5432")
cur = conn.cursor() cur = conn.cursor()

489
app/libraries/forgejo.py Normal file
Veure arxiu

@ -0,0 +1,489 @@
import os
import sys
import requests
from requests.models import urlencode
###
# Dict helper class.
# Defined at top level so it can be pickled.
###
class AttribAccessDict(dict):
def __getattr__(self, attr):
if attr in self:
return self[attr]
else:
raise AttributeError("Attribute not found: " + str(attr))
def __setattr__(self, attr, val):
if attr in self:
raise AttributeError("Attribute-style access is read only")
super(AttribAccessDict, self).__setattr__(attr, val)
class Forgejo:
name = 'Forgejo API wrapper'
def __init__(self, api_base_url=None, access_token=None, session=None):
self.__forgejo_config_path = "config/forgejo.txt"
is_setup = self.__check_setup(self)
if is_setup:
self.api_base_url = self.__get_parameter("api_base_url", self.__forgejo_config_path)
self.access_token = self.__get_parameter("access_token", self.__forgejo_config_path)
self.stats_repo = self.__get_parameter("stats_repo", self.__forgejo_config_path)
else:
self.api_base_url, self.access_token, self.stats_repo = self.__setup(self)
if session:
self.session = session
else:
self.session = requests.Session()
def admin_users_create(self, email, username, passwd, full_name=None, must_change_password=True, restricted=False, send_notify=True, source_id='0', visibility='private'):
data = {'email':email,
'full_name':username,
'login_name':username,
'must_change_password':must_change_password,
'password':passwd,
'restricted':restricted,
'send_notify':send_notify,
'source_id':source_id,
'username':username,
'visibility':visibility
}
endpoint = self.api_base_url + '/api/v1/admin/users?access_token={0}'.format(self.access_token)
response = self.__api_request('POST', endpoint, data)
registered = response.ok
response = self.__json_allow_dict_attrs(response.json())
return (registered, response)
def admin_users_list(self, page=None, limit=None):
params = dict()
if page != None:
params['page'] = page
if limit != None:
params['limit'] = limit
params['token'] = self.access_token
formatted_params = urlencode(params)
endpoint = self.api_base_url + '/api/v1/admin/users?{0}'.format(formatted_params)
response = self.__api_request('GET', endpoint)
response = self.__json_allow_dict_attrs(response.json())
return response
def notifications_new(self):
endpoint = self.api_base_url + '/api/v1/notifications/new?token={0}'.format(self.access_token)
response = self.__api_request('GET', endpoint)
response = self.__json_allow_dict_attrs(response.json())
return response
###
### repository
###
def repos_get_repo(self, owner, repo):
params = dict()
params['token'] = self.access_token
formatted_params = urlencode(params)
endpoint = self.api_base_url + '/api/v1/repos/{0}/{1}'.format(owner, repo, formatted_params)
response = self.__api_request('GET', endpoint)
response = self.__json_allow_dict_attrs(response.json())
return response
def repos_issues_search(self, owner, state=None, labels=None, q=None, milestones=None, priority_repo_id=None, issue_type=None, since=None, before=None, assigned=None,
created=None, mentioned=None, review_requested=None, team=None, page=None, limit=None):
params = dict()
if state == None:
params['state'] = 'open'
else:
params['state'] = state
if labels != None:
params['labels'] = labels
if milestones != None:
params['milestones'] = milestones
if q != None:
params['q'] = q
if priority_repo_id != None:
params['priority_repo_id'] = priority_repo_id
if issue_type != None:
params['type'] = issue_type
if since != None:
params['since'] = since
if before != None:
params['before'] = before
if assigned != None:
params['assigned'] = assigned
if created != None:
params['created'] = created
if mentioned != None:
params['mentioned'] = mentioned
if review_requested != None:
params['review_requested'] = review_requested
params['owner'] = owner
if team != None:
params['team'] = team
if page != None:
params['page'] = page
if limit != None:
params['limit'] = limit
params['token'] = self.access_token
formatted_params = urlencode(params)
endpoint = self.api_base_url + '/api/v1/repos/issues/search?{0}'.format(formatted_params)
response = self.__api_request('GET', endpoint)
response = self.__json_allow_dict_attrs(response.json())
return response
def repos_owner_repo_issues(self, owner, repo, state=None, labels=None, q=None, issue_type=None, milestones=None, since=None, before=None, created_by=None, assigned_by=None,
mentioned_by=None, page=None, limit=None):
"""
if since or before are specified, they must have following format: in. ex. 2022-08-13T08:09:07+02:00
"""
params = dict()
if state == None:
params['state'] = 'open'
else:
params['state'] = state
params['labels'] = labels
if q != None:
params['q'] = q
params['issue_type'] = issue_type
params['milestones'] = milestones
if since != None:
params['since'] = since
if before != None:
params['before'] = before
if created_by != None:
params['created_by'] = created_by
if assigned_by != None:
params['assigned_by'] = assigned_by
if mentioned_by != None:
params['mentioned_by'] = mentioned_by
params['page'] = page
params['limit'] = limit
formatted_params = urlencode(params)
endpoint = self.api_base_url + '/api/v1/repos/{0}/{1}/issues?{2}'.format(owner, repo, formatted_params)
response = self.__api_request('GET', endpoint)
response = self.__json_allow_dict_attrs(response.json())
return response
def repos_owner_repo_issues_comments(self, owner, repo, since=None, before=None, page=None, limit=None):
"""
if since or before are specified, they must have following format: in. ex. 2022-08-13T08:09:07+02:00
"""
params = dict()
if since != None:
params['since'] = since
if before != None:
params['before'] = before
params['page'] = page
params['limit'] = limit
formatted_params = urlencode(params)
endpoint = self.api_base_url + '/api/v1/repos/{0}/{1}/issues/comments?{2}'.format(owner, repo, formatted_params)
response = self.__api_request('GET', endpoint)#, data)
response = self.__json_allow_dict_attrs(response.json())
return response
def repo_owner_get_metada(self, owner, repo, filepath):
"""
Gets the metadata and contents (if a file) of an entry in a repository, or list of entries if a dir
"""
params = dict()
params['token'] = self.access_token
formatted_params = urlencode(params)
endpoint = self.api_base_url + '/api/v1/repos/{0}/{1}/contents/{2}?{3}'.format(owner, repo, filepath, formatted_params)
response = self.__api_request('GET', endpoint)
response = self.__json_allow_dict_attrs(response.json())
return response
def repo_owner_create_file(self, owner, repo, filepath, author_email, author_name, branch, content, message):
"""
Create a file in a repository
"""
data = {"author":[{"email":author_email},{"name":author_name}],
"branch":branch,
"content":content,
"message":message
}
params = dict()
params['token'] = self.access_token
formatted_params = urlencode(params)
endpoint = self.api_base_url + '/api/v1/repos/{0}/{1}/contents/{2}?{3}'.format(owner, repo, filepath, formatted_params)
response = self.__api_request('POST', endpoint, data=data)
response = self.__json_allow_dict_attrs(response.json())
return response
def repo_owner_update_file(self, owner, repo, filepath, author_email, author_name, branch, message, sha):
"""
Update a file in a repository
"""
data = {"author":[{"email":author_email},{"name":author_name}],
"branch":branch,
"message":message,
"sha":sha
}
params = dict()
params['token'] = self.access_token
formatted_params = urlencode(params)
endpoint = self.api_base_url + '/api/v1/repos/{0}/{1}/contents/{2}?{3}'.format(owner, repo, filepath, formatted_params)
response = self.__api_request('PUT', endpoint, data=data)
response = self.__json_allow_dict_attrs(response.json())
return response
def repo_owner_delete_file(self, owner, repo, filepath, author_email, author_name, branch, message, sha):
"""
Delete a file in a repository
"""
data = {"author":[{"email":author_email},{"name":author_name}],
"branch":branch,
"commiter":[{"email":author_email},{"name":author_name}],
"message":message,
"new_branch":branch,
'sha':sha
}
params = dict()
params['token'] = self.access_token
formatted_params = urlencode(params)
endpoint = self.api_base_url + '/api/v1/repos/{0}/{1}/contents/{2}?{3}'.format(owner, repo, filepath, formatted_params)
response = self.__api_request('DELETE', endpoint, data=data)
response = self.__json_allow_dict_attrs(response.json())
return response
def user(self):
"""
Get the authenticated user
"""
params = dict()
params['token'] = self.access_token
formatted_params = urlencode(params)
endpoint = self.api_base_url + '/api/v1/user?{0}'.format(formatted_params)
response = self.__api_request('GET', endpoint)
response = self.__json_allow_dict_attrs(response.json())
return response
@staticmethod
def __check_setup(self):
is_setup = False
if not os.path.isfile(self.__forgejo_config_path):
print(f"File {self.__forgejo_config_path} not found, running setup.")
else:
is_setup = True
return is_setup
@staticmethod
def __setup(self):
if not os.path.exists('config'):
os.makedirs('config')
self.api_base_url = input("Forgejo API base url, in ex. 'https://yourforgejo.instance': ")
self.access_token = input("Forgejo access token: ")
self.stats_repo = input("Stats repo: ")
if not os.path.exists(self.__forgejo_config_path):
with open(self.__forgejo_config_path, 'w'): pass
print(f"{self.__forgejo_config_path} created!")
with open(self.__forgejo_config_path, 'a') as the_file:
print("Writing forgejo parameters to " + self.__forgejo_config_path)
the_file.write(f'api_base_url: {self.api_base_url}\n'+f'access_token: {self.access_token}\nstats_repo: {self.stats_repo}\n')
return (self.api_base_url, self.access_token, self.stats_repo)
@staticmethod
def __get_parameter(parameter, file_path ):
with open( file_path ) as f:
for line in f:
if line.startswith( parameter ):
return line.replace(parameter + ":", "").strip()
print(f'{file_path} Missing parameter {parameter}')
sys.exit(0)
def __api_request(self, method, endpoint, data={}):
response = None
try:
kwargs = dict(data=data)
response = self.session.request(method, url = endpoint, **kwargs)
except Exception as e:
raise ForgejoNetworkError(f"Could not complete request: {e}")
if response is None:
raise ForgejoIllegalArgumentError("Illegal request.")
if not response.ok:
try:
if isinstance(response, dict) and 'error' in response:
error_msg = response['error']
elif isinstance(response, str):
error_msg = response
else:
error_msg = None
except ValueError:
error_msg = None
if response.status_code == 404:
ex_type = ForgejoNotFoundError
if not error_msg:
error_msg = 'Endpoint not found.'
# this is for compatibility with older versions
# which raised ForgejoAPIError('Endpoint not found.')
# on any 404
elif response.status_code == 401:
ex_type = ForgejoUnauthorizedError
elif response.status_code == 422:
return response
elif response.status_code == 500:
ex_type = ForgejoInternalServerError
elif response.status_code == 502:
ex_type = ForgejoBadGatewayError
elif response.status_code == 503:
ex_type = ForgejoServiceUnavailableError
elif response.status_code == 504:
ex_type = ForgejoGatewayTimeoutError
elif response.status_code >= 500 and \
response.status_code <= 511:
ex_type = ForgejoServerError
else:
ex_type = ForgejoAPIError
raise ex_type(
'Forgejo API returned error',
response.status_code,
response.reason,
error_msg)
else:
return response
@staticmethod
def __json_allow_dict_attrs(json_object):
"""
Makes it possible to use attribute notation to access a dicts
elements, while still allowing the dict to act as a dict.
"""
if isinstance(json_object, dict):
return AttribAccessDict(json_object)
return json_objecte
##
# Exceptions
##
class ForgejoError(Exception):
"""Base class for Forgejo.py exceptions"""
class ForgejoIOError(IOError, ForgejoError):
"""Base class for Forgejo.py I/O errors"""
class ForgejoNetworkError(ForgejoIOError):
"""Raised when network communication with the server fails"""
pass
class ForgejoAPIError(ForgejoError):
"""Raised when the forgejo API generates a response that cannot be handled"""
pass
class ForgejoServerError(ForgejoAPIError):
"""Raised if the Server is malconfigured and returns a 5xx error code"""
pass
class ForgejoInternalServerError(ForgejoServerError):
"""Raised if the Server returns a 500 error"""
pass
class ForgejoBadGatewayError(ForgejoServerError):
"""Raised if the Server returns a 502 error"""
pass
class ForgejoServiceUnavailableError(ForgejoServerError):
"""Raised if the Server returns a 503 error"""
pass
class ForgejoGatewayTimeoutError(ForgejoServerError):
"""Raised if the Server returns a 504 error"""
pass
class ForgejoNotFoundError(ForgejoAPIError):
"""Raised when the forgejo API returns a 404 Not Found error"""
pass
class ForgejoUnauthorizedError(ForgejoAPIError):
"""Raised when the forgejo API returns a 401 Unauthorized error
This happens when an OAuth token is invalid or has been revoked,
or when trying to access an endpoint that can't be used without
authentication without providing credentials."""
pass

Veure arxiu

@ -15,7 +15,7 @@ class Setup():
self.config_file = "config/config.txt" self.config_file = "config/config.txt"
self.mastodon_hostname = self.__get_parameter("mastodon_hostname", self.config_file) self.mastodon_hostname = self.__get_parameter("mastodon_hostname", self.config_file)
self.peers_api = '/api/v1/instance/peers?' self.peers_api = '/api/v1/instance/peers?'
self.user_agent = {'User-agent': "fediverse's stats (fediverse@mastodont.cat)"} self.user_agent = {'User-agent': "fediverse's stats"}
self.secrets_filepath = 'secrets/secrets.txt' self.secrets_filepath = 'secrets/secrets.txt'

Veure arxiu

@ -57,6 +57,15 @@ if __name__ == '__main__':
soft_total_project, soft_total_users, soft_total_mau, soft_total_servers, total_servers, total_users, total_mau = db.soft_totals() soft_total_project, soft_total_users, soft_total_mau, soft_total_servers, total_servers, total_users, total_mau = db.soft_totals()
i = 0
while i < 5:
db.write_top_soft(soft_total_project[i], soft_total_users[i], soft_total_mau[i], soft_total_servers[i])
i += 1
# get last check values and write current total ones # get last check values and write current total ones
evo_servers, evo_users, evo_mau = db.last_values(total_servers, total_users, total_mau) evo_servers, evo_users, evo_mau = db.last_values(total_servers, total_users, total_mau)

Veure arxiu

@ -5,4 +5,5 @@ fastapi
uvicorn[standard] uvicorn[standard]
matplotlib matplotlib
pandas pandas
sqlalchemy

58
upload.py Normal file
Veure arxiu

@ -0,0 +1,58 @@
import os
import base64
from datetime import datetime
from app.libraries.database import Database
from app.libraries.forgejo import Forgejo, ForgejoNotFoundError
import pdb
def date_string():
now = datetime.now()
day = '{:02d}'.format(now.day)
month = '{:02d}'.format(now.month)
year = str(now.year)[2:]
return f'{day}{month}{year}'
if __name__ == '__main__':
db = Database()
if not os.path.exists('dataset'):
os.makedirs('dataset')
date = date_string()
if not os.path.exists(f'dataset/{date}'):
os.makedirs(f'dataset/{date}')
filename = f'dataset_{date}.csv'
db.csv_save(f'dataset/{date}/{filename}')
fgj = Forgejo()
gituser = fgj.user()
try:
with open(f'dataset/{date}/{filename}', 'rb') as input_file:
data = input_file.read()
file = base64.b64encode(data)
response = fgj.repo_owner_create_file(gituser.login, fgj.stats_repo, f'dataset/{date}/{filename}', gituser.email, gituser.login, "main", file, f"{date} fediverse's dataset")
if 'content' in response:
print(f'{filename} uploaded to {fgj.api_base_url}/{gituser.login}/{fgj.stats_repo}')
except:
print(response)
pass