Comparar commits

..

No hay commits en común. 'master' y 'master' tienen historias completamente diferentes.

S'han modificat 13 arxius amb 48 adicions i 189 eliminacions

75
.gitignore vendido
Veure arxiu

@ -1,75 +0,0 @@
secrets/secrets.txt
config/config.txt
config/db_config.txt
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
env/
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
*.egg-info/
.installed.cfg
*.egg
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*,cover
.hypothesis/
.noseids
# Translations
*.mo
*.pot
# Django stuff:
*.log
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Ipython Notebook
.ipynb_checkpoints
# vim swap files
.*sw?
# IDEs
.vscode/
*.code-workspace

Veure arxiu

@ -1,3 +0,0 @@
fediverse_db: fediverse_test
fediverse_db_user: tester
fediverse_db_user_password: tester

Veure arxiu

@ -6,6 +6,7 @@ from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
import uuid
from datetime import datetime
import pytz
import pdb
tz = pytz.timezone('Europe/Madrid')
@ -13,14 +14,12 @@ class Database():
name = 'fediverse database library'
def __init__(self, config_file=None, fediverse_db=None, fediverse_db_user=None, fediverse_db_user_password=None, fediverse_db_host='/var/run/postgresql', fediverse_db_port='6432'):
def __init__(self, config_file=None, fediverse_db=None, fediverse_db_user=None, fediverse_db_user_password=None):
self.config_file = config_file or "config/db_config.txt"
self.fediverse_db = fediverse_db or self.__get_parameter("fediverse_db", self.config_file)
self.fediverse_db_user = fediverse_db_user or self.__get_parameter("fediverse_db_user", self.config_file)
self.fediverse_db_user_password = fediverse_db_user_password or self.__get_parameter("fediverse_db_user_password", self.config_file)
self.fediverse_db_host = fediverse_db_host
self.fediverse_db_port = fediverse_db_port
self.config_file = "config/db_config.txt"
self.fediverse_db = self.__get_parameter("fediverse_db", self.config_file)
self.fediverse_db_user = self.__get_parameter("fediverse_db_user", self.config_file)
self.fediverse_db_user_password = self.__get_parameter("fediverse_db_user_password", self.config_file)
db_setup = self.__check_dbsetup(self)
@ -44,7 +43,7 @@ class Database():
try:
conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = self.fediverse_db_host, port = self.fediverse_db_port)
conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = "/var/run/postgresql", port = "6432")
cur = conn.cursor()
@ -74,7 +73,7 @@ class Database():
try:
conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = self.fediverse_db_host, port = self.fediverse_db_port)
conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = "/var/run/postgresql", port = "6432")
cur = conn.cursor()
@ -744,7 +743,7 @@ class Database():
conn = None
conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = self.fediverse_db_host, port = self.fediverse_db_port)
conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = "/var/run/postgresql", port = "6432")
cur = conn.cursor()
@ -788,7 +787,7 @@ class Database():
conn = None
conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = self.fediverse_db_host, port = self.fediverse_db_port)
conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = "/var/run/postgresql", port = "6432")
cur = conn.cursor()
@ -936,13 +935,6 @@ class Database():
conn.close()
def check_connection(self):
try:
conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = self.fediverse_db_host, port = self.fediverse_db_port)
finally:
conn.close()
@staticmethod
def __check_dbsetup(self):
@ -952,7 +944,7 @@ class Database():
conn = None
conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = self.fediverse_db_host, port = self.fediverse_db_port)
conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = "/var/run/postgresql", port = "6432")
db_setup = True
@ -1031,7 +1023,7 @@ class Database():
try:
conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = self.fediverse_db_host, port = self.fediverse_db_port)
conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = "/var/run/postgresql", port = "6432")
cur = conn.cursor()

Veure arxiu

@ -1,3 +0,0 @@
#!/usr/bin/env python3
from .database import Database

Veure arxiu

@ -6,7 +6,7 @@ import urllib3
import requests
import socket
from setup import Setup
from federation import Database
from database import Database
from mastodon import Mastodon
from matplotlib import pyplot as plt
import matplotlib.dates as mdates
@ -31,8 +31,7 @@ plt.rc('figure', titlesize=BIGGER_SIZE) # fontsize of the figure title
mdates.set_epoch('2000-01-01T00:00:00')
y_formatter = ScalarFormatter(useOffset=False)
ray.init(num_cpus=25) # Specify this system CPUs.
ray.init(num_cpus = 25) # Specify this system CPUs.
class Server:
@ -55,8 +54,7 @@ class Server:
try:
data = requests.get('https://' + self + api,
headers=setup.user_agent, timeout=3)
data = requests.get('https://' + self + api, headers = setup.user_agent, timeout=3)
nodeinfo_json = data.json()
@ -67,8 +65,7 @@ class Server:
users = 1
mau = nodeinfo_json.get('usage').get(
'users').get('activeMonth') or '0'
mau = nodeinfo_json.get('usage').get('users').get('activeMonth') or '0'
if software == 'socialhome':
@ -118,8 +115,7 @@ class Server:
if soft_version != "" and soft_version is not None:
print(
f'\n** Server {self} ({software} {soft_version}) is alive! **')
print(f'\n** Server {self} ({software} {soft_version}) is alive! **')
else:
@ -127,13 +123,11 @@ class Server:
if software != 'birdsitelive':
db.write_alive_server(
self, software, soft_version, alive, api, users, downs, first_checked_at, mau)
db.write_alive_server(self, software, soft_version, alive, api, users, downs, first_checked_at, mau)
else:
db.write_blocked_software(
self, software, soft_version, alive, api, users, downs, first_checked_at)
db.write_blocked_software(self, software, soft_version, alive, api, users, downs, first_checked_at)
except urllib3.exceptions.ProtocolError as protoerr:
@ -211,12 +205,10 @@ class Server:
return (self, software, soft_version, alive, api, users, downs, first_checked_at, mau)
def print_dead(server):
print(f'\nServer {server} is dead :-(')
if __name__ == '__main__':
db = Database()
@ -234,8 +226,8 @@ if __name__ == '__main__':
now = start
mastodon = Mastodon(
access_token=setup.mastodon_app_token,
api_base_url=setup.mastodon_hostname
access_token = setup.mastodon_app_token,
api_base_url= setup.mastodon_hostname
)
total_servers = 0
@ -250,11 +242,9 @@ if __name__ == '__main__':
ray_start = time.time()
results = ray.get([getservers.get_alive_servers.remote(server)
for server in alive_servers])
results = ray.get([getservers.get_alive_servers.remote(server) for server in alive_servers])
print(
f"duration = {time.time() - ray_start}.\nprocessed servers: {len(results)}")
print(f"duration = {time.time() - ray_start}.\nprocessed servers: {len(results)}")
# get current total servers and users, get users from every software
@ -262,8 +252,7 @@ if __name__ == '__main__':
# get last check values and write current total ones
evo_servers, evo_users, evo_mau = db.last_values(
total_servers, total_users, total_mau)
evo_servers, evo_users, evo_mau = db.last_values(total_servers, total_users, total_mau)
# write evo values
@ -284,14 +273,11 @@ if __name__ == '__main__':
###############################################################################
# generate graphs
plt.plot([-6, -5, -4, -3, -2, -1, 0], [servers_plots[6], servers_plots[5], servers_plots[4], servers_plots[3],
servers_plots[2], servers_plots[1], servers_plots[0]], marker='o', color='mediumseagreen')
plt.plot([-6, -5, -4, -3, -2, -1, 0], [servers_plots[6], servers_plots[5], servers_plots[4], servers_plots[3], servers_plots[2], servers_plots[1], servers_plots[0]], marker='o', color='mediumseagreen')
plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_servers, max_servers, max_servers,
max_servers, max_servers, max_servers, max_servers], color='red')
plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_servers, max_servers, max_servers, max_servers, max_servers, max_servers, max_servers], color='red')
plt.title('fediverse: total alive servers (max: ' +
str(f"{max_servers:,}" + ')'), loc='right', color='blue')
plt.title('fediverse: total alive servers (max: ' + str(f"{max_servers:,}" + ')'), loc='right', color='blue')
plt.xlabel('Last seven days')
@ -305,17 +291,13 @@ if __name__ == '__main__':
plt.close()
plt.plot([-6, -5, -4, -3, -2, -1, 0], [mau_plots[6], mau_plots[5], mau_plots[4],
mau_plots[3], mau_plots[2], mau_plots[1], mau_plots[0]], marker='o', color='royalblue')
plt.plot([-6, -5, -4, -3, -2, -1, 0], [mau_plots[6], mau_plots[5], mau_plots[4], mau_plots[3], mau_plots[2], mau_plots[1], mau_plots[0]], marker='o', color='royalblue')
plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_mau, max_mau, max_mau,
max_mau, max_mau, max_mau, max_mau], color='red')
plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_mau, max_mau, max_mau, max_mau, max_mau, max_mau, max_mau], color='red')
plt.title('fediverse: total MAU (max: ' +
str(f"{max_mau:,}" + ')'), loc='right', color='royalblue')
plt.title('fediverse: total MAU (max: ' + str(f"{max_mau:,}" + ')'), loc='right', color='royalblue')
plt.legend(('mau', 'max'), shadow=True, loc=(
0.01, 0.80), handlelength=1.5, fontsize=10)
plt.legend(('mau', 'max'), shadow=True, loc=(0.01, 0.80), handlelength=1.5, fontsize=10)
plt.xlabel('Last seven days')
@ -367,30 +349,25 @@ if __name__ == '__main__':
if evo_servers >= 0:
toot_text += "alive servers: " + \
str(f"{total_servers:,}") + \
" (+" + str(f"{evo_servers:,}") + ") \n"
toot_text += "alive servers: " + str(f"{total_servers:,}") + " (+"+ str(f"{evo_servers:,}") + ") \n"
toot_text += "max: " + str(f"{max_servers:,}") + "\n"
elif evo_servers < 0:
toot_text += "alive servers: " + \
str(f"{total_servers:,}") + " (" + str(f"{evo_servers:,}") + ") \n"
toot_text += "alive servers: " + str(f"{total_servers:,}") + " ("+ str(f"{evo_servers:,}") + ") \n"
toot_text += "max: " + str(f"{max_servers:,}") + "\n"
if evo_mau >= 0:
toot_text += "total MAU: " + \
str(f"{total_mau:,}") + " (+" + str(f"{evo_mau:,}") + ") \n"
toot_text += "total MAU: " + str(f"{total_mau:,}") + " (+"+ str(f"{evo_mau:,}") + ") \n"
toot_text += "max: " + str(f"{max_mau:,}") + "\n"
elif evo_mau < 0:
toot_text += "total MAU: " + \
str(f"{total_mau:,}") + " (" + str(f"{evo_mau:,}") + ") \n"
toot_text += "total MAU: " + str(f"{total_mau:,}") + " ("+ str(f"{evo_mau:,}") + ") \n"
toot_text += "max: " + str(f"{max_mau:,}") + "\n"
@ -420,17 +397,13 @@ if __name__ == '__main__':
print(toot_text)
servers_image_id = mastodon.media_post(
'servers.png', "image/png", description='servers graph').id
servers_image_id = mastodon.media_post('servers.png', "image/png", description='servers graph').id
mau_image_id = mastodon.media_post(
'mau.png', "image/png", description='MAU graph').id
mau_image_id = mastodon.media_post('mau.png', "image/png", description='MAU graph').id
global_image_id = mastodon.media_post(
'global.png', "image/png", description='global graph').id
global_image_id = mastodon.media_post('global.png', "image/png", description='global graph').id
mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={
servers_image_id, mau_image_id, global_image_id})
mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, mau_image_id, global_image_id})
db.delete_dead_servers()

Veure arxiu

@ -1,10 +0,0 @@
requests
psycopg2-binary
pytz
ray
Mastodon.py
matplotlib
pandas
humanfriendly
pytest
ipdb

Veure arxiu

@ -4,6 +4,7 @@ from datetime import datetime
import pytz
from mastodon import Mastodon
from mastodon.Mastodon import MastodonMalformedEventError, MastodonNetworkError, MastodonReadTimeout, MastodonAPIError, MastodonIllegalArgumentError
import pdb
class Setup():

Veure arxiu

@ -1 +0,0 @@
#!/usr/bin/env python3

Veure arxiu

@ -1,15 +0,0 @@
from federation import Database
def test__database_connection():
db = Database(config_file='config/test_db_config.txt', fediverse_db_host='localhost', fediverse_db_port='5432')
# noexcept
db.check_connection()
def test__database_connection__noservers():
db = Database(config_file='config/test_db_config.txt', fediverse_db_host='localhost', fediverse_db_port='5432')
checked_server = db.get_last_checked_servers()
assert checked_server == []