forkeado de spla/fediverse
Comparar commits
No hay commits en común. 'master' y 'master' tienen historias completamente diferentes.
S'han modificat 13 arxius amb 48 adicions i 189 eliminacions
75
.gitignore
vendido
75
.gitignore
vendido
|
|
@ -1,75 +0,0 @@
|
|||
secrets/secrets.txt
|
||||
config/config.txt
|
||||
config/db_config.txt
|
||||
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
env/
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*,cover
|
||||
.hypothesis/
|
||||
.noseids
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
target/
|
||||
|
||||
# Ipython Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# vim swap files
|
||||
.*sw?
|
||||
|
||||
# IDEs
|
||||
.vscode/
|
||||
*.code-workspace
|
||||
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
fediverse_db: fediverse_test
|
||||
fediverse_db_user: tester
|
||||
fediverse_db_user_password: tester
|
||||
|
|
@ -6,6 +6,7 @@ from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
|
|||
import uuid
|
||||
from datetime import datetime
|
||||
import pytz
|
||||
import pdb
|
||||
|
||||
tz = pytz.timezone('Europe/Madrid')
|
||||
|
||||
|
|
@ -13,14 +14,12 @@ class Database():
|
|||
|
||||
name = 'fediverse database library'
|
||||
|
||||
def __init__(self, config_file=None, fediverse_db=None, fediverse_db_user=None, fediverse_db_user_password=None, fediverse_db_host='/var/run/postgresql', fediverse_db_port='6432'):
|
||||
def __init__(self, config_file=None, fediverse_db=None, fediverse_db_user=None, fediverse_db_user_password=None):
|
||||
|
||||
self.config_file = config_file or "config/db_config.txt"
|
||||
self.fediverse_db = fediverse_db or self.__get_parameter("fediverse_db", self.config_file)
|
||||
self.fediverse_db_user = fediverse_db_user or self.__get_parameter("fediverse_db_user", self.config_file)
|
||||
self.fediverse_db_user_password = fediverse_db_user_password or self.__get_parameter("fediverse_db_user_password", self.config_file)
|
||||
self.fediverse_db_host = fediverse_db_host
|
||||
self.fediverse_db_port = fediverse_db_port
|
||||
self.config_file = "config/db_config.txt"
|
||||
self.fediverse_db = self.__get_parameter("fediverse_db", self.config_file)
|
||||
self.fediverse_db_user = self.__get_parameter("fediverse_db_user", self.config_file)
|
||||
self.fediverse_db_user_password = self.__get_parameter("fediverse_db_user_password", self.config_file)
|
||||
|
||||
db_setup = self.__check_dbsetup(self)
|
||||
|
||||
|
|
@ -30,7 +29,7 @@ class Database():
|
|||
self.fediverse_db_user = input("\nFediverse database user: ")
|
||||
self.fediverse_db_user_password = input("\nFediverse database user password: ")
|
||||
|
||||
self.__createdb(self)
|
||||
self.__createdb(self)
|
||||
self.__create_config(self)
|
||||
self.__write_config(self)
|
||||
|
||||
|
|
@ -44,7 +43,7 @@ class Database():
|
|||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = self.fediverse_db_host, port = self.fediverse_db_port)
|
||||
conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = "/var/run/postgresql", port = "6432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
|
|
@ -74,7 +73,7 @@ class Database():
|
|||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = self.fediverse_db_host, port = self.fediverse_db_port)
|
||||
conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = "/var/run/postgresql", port = "6432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
|
|
@ -744,7 +743,7 @@ class Database():
|
|||
|
||||
conn = None
|
||||
|
||||
conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = self.fediverse_db_host, port = self.fediverse_db_port)
|
||||
conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = "/var/run/postgresql", port = "6432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
|
|
@ -788,7 +787,7 @@ class Database():
|
|||
|
||||
conn = None
|
||||
|
||||
conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = self.fediverse_db_host, port = self.fediverse_db_port)
|
||||
conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = "/var/run/postgresql", port = "6432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
|
|
@ -845,7 +844,7 @@ class Database():
|
|||
conn.close()
|
||||
|
||||
def get_uptime(self):
|
||||
|
||||
|
||||
try:
|
||||
|
||||
conn = None
|
||||
|
|
@ -936,13 +935,6 @@ class Database():
|
|||
|
||||
conn.close()
|
||||
|
||||
def check_connection(self):
|
||||
|
||||
try:
|
||||
conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = self.fediverse_db_host, port = self.fediverse_db_port)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
@staticmethod
|
||||
def __check_dbsetup(self):
|
||||
|
||||
|
|
@ -952,7 +944,7 @@ class Database():
|
|||
|
||||
conn = None
|
||||
|
||||
conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = self.fediverse_db_host, port = self.fediverse_db_port)
|
||||
conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = "/var/run/postgresql", port = "6432")
|
||||
|
||||
db_setup = True
|
||||
|
||||
|
|
@ -1031,7 +1023,7 @@ class Database():
|
|||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = self.fediverse_db_host, port = self.fediverse_db_port)
|
||||
conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = "/var/run/postgresql", port = "6432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
from .database import Database
|
||||
|
|
@ -6,7 +6,7 @@ import urllib3
|
|||
import requests
|
||||
import socket
|
||||
from setup import Setup
|
||||
from federation import Database
|
||||
from database import Database
|
||||
from mastodon import Mastodon
|
||||
from matplotlib import pyplot as plt
|
||||
import matplotlib.dates as mdates
|
||||
|
|
@ -31,8 +31,7 @@ plt.rc('figure', titlesize=BIGGER_SIZE) # fontsize of the figure title
|
|||
mdates.set_epoch('2000-01-01T00:00:00')
|
||||
y_formatter = ScalarFormatter(useOffset=False)
|
||||
|
||||
ray.init(num_cpus=25) # Specify this system CPUs.
|
||||
|
||||
ray.init(num_cpus = 25) # Specify this system CPUs.
|
||||
|
||||
class Server:
|
||||
|
||||
|
|
@ -55,8 +54,7 @@ class Server:
|
|||
|
||||
try:
|
||||
|
||||
data = requests.get('https://' + self + api,
|
||||
headers=setup.user_agent, timeout=3)
|
||||
data = requests.get('https://' + self + api, headers = setup.user_agent, timeout=3)
|
||||
|
||||
nodeinfo_json = data.json()
|
||||
|
||||
|
|
@ -67,8 +65,7 @@ class Server:
|
|||
|
||||
users = 1
|
||||
|
||||
mau = nodeinfo_json.get('usage').get(
|
||||
'users').get('activeMonth') or '0'
|
||||
mau = nodeinfo_json.get('usage').get('users').get('activeMonth') or '0'
|
||||
|
||||
if software == 'socialhome':
|
||||
|
||||
|
|
@ -94,7 +91,7 @@ class Server:
|
|||
|
||||
soft_version = ""
|
||||
|
||||
else:
|
||||
else:
|
||||
|
||||
if api == '/api/v1/instance':
|
||||
|
||||
|
|
@ -118,8 +115,7 @@ class Server:
|
|||
|
||||
if soft_version != "" and soft_version is not None:
|
||||
|
||||
print(
|
||||
f'\n** Server {self} ({software} {soft_version}) is alive! **')
|
||||
print(f'\n** Server {self} ({software} {soft_version}) is alive! **')
|
||||
|
||||
else:
|
||||
|
||||
|
|
@ -127,13 +123,11 @@ class Server:
|
|||
|
||||
if software != 'birdsitelive':
|
||||
|
||||
db.write_alive_server(
|
||||
self, software, soft_version, alive, api, users, downs, first_checked_at, mau)
|
||||
db.write_alive_server(self, software, soft_version, alive, api, users, downs, first_checked_at, mau)
|
||||
|
||||
else:
|
||||
|
||||
db.write_blocked_software(
|
||||
self, software, soft_version, alive, api, users, downs, first_checked_at)
|
||||
db.write_blocked_software(self, software, soft_version, alive, api, users, downs, first_checked_at)
|
||||
|
||||
except urllib3.exceptions.ProtocolError as protoerr:
|
||||
|
||||
|
|
@ -211,12 +205,10 @@ class Server:
|
|||
|
||||
return (self, software, soft_version, alive, api, users, downs, first_checked_at, mau)
|
||||
|
||||
|
||||
def print_dead(server):
|
||||
|
||||
print(f'\nServer {server} is dead :-(')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
db = Database()
|
||||
|
|
@ -234,9 +226,9 @@ if __name__ == '__main__':
|
|||
now = start
|
||||
|
||||
mastodon = Mastodon(
|
||||
access_token=setup.mastodon_app_token,
|
||||
api_base_url=setup.mastodon_hostname
|
||||
)
|
||||
access_token = setup.mastodon_app_token,
|
||||
api_base_url= setup.mastodon_hostname
|
||||
)
|
||||
|
||||
total_servers = 0
|
||||
|
||||
|
|
@ -250,11 +242,9 @@ if __name__ == '__main__':
|
|||
|
||||
ray_start = time.time()
|
||||
|
||||
results = ray.get([getservers.get_alive_servers.remote(server)
|
||||
for server in alive_servers])
|
||||
results = ray.get([getservers.get_alive_servers.remote(server) for server in alive_servers])
|
||||
|
||||
print(
|
||||
f"duration = {time.time() - ray_start}.\nprocessed servers: {len(results)}")
|
||||
print(f"duration = {time.time() - ray_start}.\nprocessed servers: {len(results)}")
|
||||
|
||||
# get current total servers and users, get users from every software
|
||||
|
||||
|
|
@ -262,8 +252,7 @@ if __name__ == '__main__':
|
|||
|
||||
# get last check values and write current total ones
|
||||
|
||||
evo_servers, evo_users, evo_mau = db.last_values(
|
||||
total_servers, total_users, total_mau)
|
||||
evo_servers, evo_users, evo_mau = db.last_values(total_servers, total_users, total_mau)
|
||||
|
||||
# write evo values
|
||||
|
||||
|
|
@ -284,14 +273,11 @@ if __name__ == '__main__':
|
|||
###############################################################################
|
||||
# generate graphs
|
||||
|
||||
plt.plot([-6, -5, -4, -3, -2, -1, 0], [servers_plots[6], servers_plots[5], servers_plots[4], servers_plots[3],
|
||||
servers_plots[2], servers_plots[1], servers_plots[0]], marker='o', color='mediumseagreen')
|
||||
plt.plot([-6, -5, -4, -3, -2, -1, 0], [servers_plots[6], servers_plots[5], servers_plots[4], servers_plots[3], servers_plots[2], servers_plots[1], servers_plots[0]], marker='o', color='mediumseagreen')
|
||||
|
||||
plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_servers, max_servers, max_servers,
|
||||
max_servers, max_servers, max_servers, max_servers], color='red')
|
||||
plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_servers, max_servers, max_servers, max_servers, max_servers, max_servers, max_servers], color='red')
|
||||
|
||||
plt.title('fediverse: total alive servers (max: ' +
|
||||
str(f"{max_servers:,}" + ')'), loc='right', color='blue')
|
||||
plt.title('fediverse: total alive servers (max: ' + str(f"{max_servers:,}" + ')'), loc='right', color='blue')
|
||||
|
||||
plt.xlabel('Last seven days')
|
||||
|
||||
|
|
@ -305,17 +291,13 @@ if __name__ == '__main__':
|
|||
|
||||
plt.close()
|
||||
|
||||
plt.plot([-6, -5, -4, -3, -2, -1, 0], [mau_plots[6], mau_plots[5], mau_plots[4],
|
||||
mau_plots[3], mau_plots[2], mau_plots[1], mau_plots[0]], marker='o', color='royalblue')
|
||||
plt.plot([-6, -5, -4, -3, -2, -1, 0], [mau_plots[6], mau_plots[5], mau_plots[4], mau_plots[3], mau_plots[2], mau_plots[1], mau_plots[0]], marker='o', color='royalblue')
|
||||
|
||||
plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_mau, max_mau, max_mau,
|
||||
max_mau, max_mau, max_mau, max_mau], color='red')
|
||||
plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_mau, max_mau, max_mau, max_mau, max_mau, max_mau, max_mau], color='red')
|
||||
|
||||
plt.title('fediverse: total MAU (max: ' +
|
||||
str(f"{max_mau:,}" + ')'), loc='right', color='royalblue')
|
||||
plt.title('fediverse: total MAU (max: ' + str(f"{max_mau:,}" + ')'), loc='right', color='royalblue')
|
||||
|
||||
plt.legend(('mau', 'max'), shadow=True, loc=(
|
||||
0.01, 0.80), handlelength=1.5, fontsize=10)
|
||||
plt.legend(('mau', 'max'), shadow=True, loc=(0.01, 0.80), handlelength=1.5, fontsize=10)
|
||||
|
||||
plt.xlabel('Last seven days')
|
||||
|
||||
|
|
@ -328,9 +310,9 @@ if __name__ == '__main__':
|
|||
plt.close()
|
||||
|
||||
df = pd.DataFrame({'date': np.array(global_week),
|
||||
#'servers': np.array(global_servers),
|
||||
'users': np.array(global_users),
|
||||
'mau': np.array(global_mau)})
|
||||
#'servers': np.array(global_servers),
|
||||
'users': np.array(global_users),
|
||||
'mau': np.array(global_mau)})
|
||||
|
||||
df['date'] = pd.to_datetime(df['date'])
|
||||
|
||||
|
|
@ -367,30 +349,25 @@ if __name__ == '__main__':
|
|||
|
||||
if evo_servers >= 0:
|
||||
|
||||
toot_text += "alive servers: " + \
|
||||
str(f"{total_servers:,}") + \
|
||||
" (+" + str(f"{evo_servers:,}") + ") \n"
|
||||
toot_text += "alive servers: " + str(f"{total_servers:,}") + " (+"+ str(f"{evo_servers:,}") + ") \n"
|
||||
|
||||
toot_text += "max: " + str(f"{max_servers:,}") + "\n"
|
||||
|
||||
elif evo_servers < 0:
|
||||
|
||||
toot_text += "alive servers: " + \
|
||||
str(f"{total_servers:,}") + " (" + str(f"{evo_servers:,}") + ") \n"
|
||||
toot_text += "alive servers: " + str(f"{total_servers:,}") + " ("+ str(f"{evo_servers:,}") + ") \n"
|
||||
|
||||
toot_text += "max: " + str(f"{max_servers:,}") + "\n"
|
||||
|
||||
if evo_mau >= 0:
|
||||
|
||||
toot_text += "total MAU: " + \
|
||||
str(f"{total_mau:,}") + " (+" + str(f"{evo_mau:,}") + ") \n"
|
||||
toot_text += "total MAU: " + str(f"{total_mau:,}") + " (+"+ str(f"{evo_mau:,}") + ") \n"
|
||||
|
||||
toot_text += "max: " + str(f"{max_mau:,}") + "\n"
|
||||
|
||||
elif evo_mau < 0:
|
||||
|
||||
toot_text += "total MAU: " + \
|
||||
str(f"{total_mau:,}") + " (" + str(f"{evo_mau:,}") + ") \n"
|
||||
toot_text += "total MAU: " + str(f"{total_mau:,}") + " ("+ str(f"{evo_mau:,}") + ") \n"
|
||||
|
||||
toot_text += "max: " + str(f"{max_mau:,}") + "\n"
|
||||
|
||||
|
|
@ -420,17 +397,13 @@ if __name__ == '__main__':
|
|||
|
||||
print(toot_text)
|
||||
|
||||
servers_image_id = mastodon.media_post(
|
||||
'servers.png', "image/png", description='servers graph').id
|
||||
servers_image_id = mastodon.media_post('servers.png', "image/png", description='servers graph').id
|
||||
|
||||
mau_image_id = mastodon.media_post(
|
||||
'mau.png', "image/png", description='MAU graph').id
|
||||
mau_image_id = mastodon.media_post('mau.png', "image/png", description='MAU graph').id
|
||||
|
||||
global_image_id = mastodon.media_post(
|
||||
'global.png', "image/png", description='global graph').id
|
||||
global_image_id = mastodon.media_post('global.png', "image/png", description='global graph').id
|
||||
|
||||
mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={
|
||||
servers_image_id, mau_image_id, global_image_id})
|
||||
mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, mau_image_id, global_image_id})
|
||||
|
||||
db.delete_dead_servers()
|
||||
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
requests
|
||||
psycopg2-binary
|
||||
pytz
|
||||
ray
|
||||
Mastodon.py
|
||||
matplotlib
|
||||
pandas
|
||||
humanfriendly
|
||||
pytest
|
||||
ipdb
|
||||
|
|
@ -4,6 +4,7 @@ from datetime import datetime
|
|||
import pytz
|
||||
from mastodon import Mastodon
|
||||
from mastodon.Mastodon import MastodonMalformedEventError, MastodonNetworkError, MastodonReadTimeout, MastodonAPIError, MastodonIllegalArgumentError
|
||||
import pdb
|
||||
|
||||
class Setup():
|
||||
|
||||
|
|
@ -78,7 +79,7 @@ class Setup():
|
|||
client_secret = response[1]
|
||||
|
||||
mastodon = Mastodon(client_id = client_id, client_secret = client_secret, api_base_url = self.mastodon_hostname)
|
||||
|
||||
|
||||
token = mastodon.log_in(
|
||||
self.mastodon_user,
|
||||
self.mastodon_password,
|
||||
|
|
@ -1 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
from federation import Database
|
||||
|
||||
def test__database_connection():
|
||||
db = Database(config_file='config/test_db_config.txt', fediverse_db_host='localhost', fediverse_db_port='5432')
|
||||
|
||||
# noexcept
|
||||
db.check_connection()
|
||||
|
||||
|
||||
def test__database_connection__noservers():
|
||||
db = Database(config_file='config/test_db_config.txt', fediverse_db_host='localhost', fediverse_db_port='5432')
|
||||
|
||||
checked_server = db.get_last_checked_servers()
|
||||
|
||||
assert checked_server == []
|
||||
Loading…
Referencia en una nova incidència