Comparar commits

..

4 commits

S'han modificat 13 arxius amb 189 adicions i 48 eliminacions

75
.gitignore vendido Normal file
Veure arxiu

@ -0,0 +1,75 @@
secrets/secrets.txt
config/config.txt
config/db_config.txt
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
env/
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
*.egg-info/
.installed.cfg
*.egg
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*,cover
.hypothesis/
.noseids
# Translations
*.mo
*.pot
# Django stuff:
*.log
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Ipython Notebook
.ipynb_checkpoints
# vim swap files
.*sw?
# IDEs
.vscode/
*.code-workspace

Veure arxiu

@ -0,0 +1,3 @@
fediverse_db: fediverse_test
fediverse_db_user: tester
fediverse_db_user_password: tester

3
federation/__init__.py Normal file
Veure arxiu

@ -0,0 +1,3 @@
#!/usr/bin/env python3
from .database import Database

Veure arxiu

@ -6,7 +6,6 @@ from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
import uuid import uuid
from datetime import datetime from datetime import datetime
import pytz import pytz
import pdb
tz = pytz.timezone('Europe/Madrid') tz = pytz.timezone('Europe/Madrid')
@ -14,12 +13,14 @@ class Database():
name = 'fediverse database library' name = 'fediverse database library'
def __init__(self, config_file=None, fediverse_db=None, fediverse_db_user=None, fediverse_db_user_password=None): def __init__(self, config_file=None, fediverse_db=None, fediverse_db_user=None, fediverse_db_user_password=None, fediverse_db_host='/var/run/postgresql', fediverse_db_port='6432'):
self.config_file = "config/db_config.txt" self.config_file = config_file or "config/db_config.txt"
self.fediverse_db = self.__get_parameter("fediverse_db", self.config_file) self.fediverse_db = fediverse_db or self.__get_parameter("fediverse_db", self.config_file)
self.fediverse_db_user = self.__get_parameter("fediverse_db_user", self.config_file) self.fediverse_db_user = fediverse_db_user or self.__get_parameter("fediverse_db_user", self.config_file)
self.fediverse_db_user_password = self.__get_parameter("fediverse_db_user_password", self.config_file) self.fediverse_db_user_password = fediverse_db_user_password or self.__get_parameter("fediverse_db_user_password", self.config_file)
self.fediverse_db_host = fediverse_db_host
self.fediverse_db_port = fediverse_db_port
db_setup = self.__check_dbsetup(self) db_setup = self.__check_dbsetup(self)
@ -43,7 +44,7 @@ class Database():
try: try:
conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = "/var/run/postgresql", port = "6432") conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = self.fediverse_db_host, port = self.fediverse_db_port)
cur = conn.cursor() cur = conn.cursor()
@ -73,7 +74,7 @@ class Database():
try: try:
conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = "/var/run/postgresql", port = "6432") conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = self.fediverse_db_host, port = self.fediverse_db_port)
cur = conn.cursor() cur = conn.cursor()
@ -743,7 +744,7 @@ class Database():
conn = None conn = None
conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = "/var/run/postgresql", port = "6432") conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = self.fediverse_db_host, port = self.fediverse_db_port)
cur = conn.cursor() cur = conn.cursor()
@ -787,7 +788,7 @@ class Database():
conn = None conn = None
conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = "/var/run/postgresql", port = "6432") conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = self.fediverse_db_host, port = self.fediverse_db_port)
cur = conn.cursor() cur = conn.cursor()
@ -935,6 +936,13 @@ class Database():
conn.close() conn.close()
def check_connection(self):
try:
conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = self.fediverse_db_host, port = self.fediverse_db_port)
finally:
conn.close()
@staticmethod @staticmethod
def __check_dbsetup(self): def __check_dbsetup(self):
@ -944,7 +952,7 @@ class Database():
conn = None conn = None
conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = "/var/run/postgresql", port = "6432") conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = self.fediverse_db_host, port = self.fediverse_db_port)
db_setup = True db_setup = True
@ -1023,7 +1031,7 @@ class Database():
try: try:
conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = "/var/run/postgresql", port = "6432") conn = psycopg2.connect(database = self.fediverse_db, user = self.fediverse_db_user, password = self.fediverse_db_user_password, host = self.fediverse_db_host, port = self.fediverse_db_port)
cur = conn.cursor() cur = conn.cursor()

Veure arxiu

@ -4,7 +4,6 @@ from datetime import datetime
import pytz import pytz
from mastodon import Mastodon from mastodon import Mastodon
from mastodon.Mastodon import MastodonMalformedEventError, MastodonNetworkError, MastodonReadTimeout, MastodonAPIError, MastodonIllegalArgumentError from mastodon.Mastodon import MastodonMalformedEventError, MastodonNetworkError, MastodonReadTimeout, MastodonAPIError, MastodonIllegalArgumentError
import pdb
class Setup(): class Setup():

10
requirements-dev.txt Normal file
Veure arxiu

@ -0,0 +1,10 @@
requests
psycopg2-binary
pytz
ray
Mastodon.py
matplotlib
pandas
humanfriendly
pytest
ipdb

Veure arxiu

@ -6,7 +6,7 @@ import urllib3
import requests import requests
import socket import socket
from setup import Setup from setup import Setup
from database import Database from federation import Database
from mastodon import Mastodon from mastodon import Mastodon
from matplotlib import pyplot as plt from matplotlib import pyplot as plt
import matplotlib.dates as mdates import matplotlib.dates as mdates
@ -31,7 +31,8 @@ plt.rc('figure', titlesize=BIGGER_SIZE) # fontsize of the figure title
mdates.set_epoch('2000-01-01T00:00:00') mdates.set_epoch('2000-01-01T00:00:00')
y_formatter = ScalarFormatter(useOffset=False) y_formatter = ScalarFormatter(useOffset=False)
ray.init(num_cpus = 25) # Specify this system CPUs. ray.init(num_cpus=25) # Specify this system CPUs.
class Server: class Server:
@ -54,7 +55,8 @@ class Server:
try: try:
data = requests.get('https://' + self + api, headers = setup.user_agent, timeout=3) data = requests.get('https://' + self + api,
headers=setup.user_agent, timeout=3)
nodeinfo_json = data.json() nodeinfo_json = data.json()
@ -65,7 +67,8 @@ class Server:
users = 1 users = 1
mau = nodeinfo_json.get('usage').get('users').get('activeMonth') or '0' mau = nodeinfo_json.get('usage').get(
'users').get('activeMonth') or '0'
if software == 'socialhome': if software == 'socialhome':
@ -115,7 +118,8 @@ class Server:
if soft_version != "" and soft_version is not None: if soft_version != "" and soft_version is not None:
print(f'\n** Server {self} ({software} {soft_version}) is alive! **') print(
f'\n** Server {self} ({software} {soft_version}) is alive! **')
else: else:
@ -123,11 +127,13 @@ class Server:
if software != 'birdsitelive': if software != 'birdsitelive':
db.write_alive_server(self, software, soft_version, alive, api, users, downs, first_checked_at, mau) db.write_alive_server(
self, software, soft_version, alive, api, users, downs, first_checked_at, mau)
else: else:
db.write_blocked_software(self, software, soft_version, alive, api, users, downs, first_checked_at) db.write_blocked_software(
self, software, soft_version, alive, api, users, downs, first_checked_at)
except urllib3.exceptions.ProtocolError as protoerr: except urllib3.exceptions.ProtocolError as protoerr:
@ -205,10 +211,12 @@ class Server:
return (self, software, soft_version, alive, api, users, downs, first_checked_at, mau) return (self, software, soft_version, alive, api, users, downs, first_checked_at, mau)
def print_dead(server): def print_dead(server):
print(f'\nServer {server} is dead :-(') print(f'\nServer {server} is dead :-(')
if __name__ == '__main__': if __name__ == '__main__':
db = Database() db = Database()
@ -226,8 +234,8 @@ if __name__ == '__main__':
now = start now = start
mastodon = Mastodon( mastodon = Mastodon(
access_token = setup.mastodon_app_token, access_token=setup.mastodon_app_token,
api_base_url= setup.mastodon_hostname api_base_url=setup.mastodon_hostname
) )
total_servers = 0 total_servers = 0
@ -242,9 +250,11 @@ if __name__ == '__main__':
ray_start = time.time() ray_start = time.time()
results = ray.get([getservers.get_alive_servers.remote(server) for server in alive_servers]) results = ray.get([getservers.get_alive_servers.remote(server)
for server in alive_servers])
print(f"duration = {time.time() - ray_start}.\nprocessed servers: {len(results)}") print(
f"duration = {time.time() - ray_start}.\nprocessed servers: {len(results)}")
# get current total servers and users, get users from every software # get current total servers and users, get users from every software
@ -252,7 +262,8 @@ if __name__ == '__main__':
# get last check values and write current total ones # get last check values and write current total ones
evo_servers, evo_users, evo_mau = db.last_values(total_servers, total_users, total_mau) evo_servers, evo_users, evo_mau = db.last_values(
total_servers, total_users, total_mau)
# write evo values # write evo values
@ -273,11 +284,14 @@ if __name__ == '__main__':
############################################################################### ###############################################################################
# generate graphs # generate graphs
plt.plot([-6, -5, -4, -3, -2, -1, 0], [servers_plots[6], servers_plots[5], servers_plots[4], servers_plots[3], servers_plots[2], servers_plots[1], servers_plots[0]], marker='o', color='mediumseagreen') plt.plot([-6, -5, -4, -3, -2, -1, 0], [servers_plots[6], servers_plots[5], servers_plots[4], servers_plots[3],
servers_plots[2], servers_plots[1], servers_plots[0]], marker='o', color='mediumseagreen')
plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_servers, max_servers, max_servers, max_servers, max_servers, max_servers, max_servers], color='red') plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_servers, max_servers, max_servers,
max_servers, max_servers, max_servers, max_servers], color='red')
plt.title('fediverse: total alive servers (max: ' + str(f"{max_servers:,}" + ')'), loc='right', color='blue') plt.title('fediverse: total alive servers (max: ' +
str(f"{max_servers:,}" + ')'), loc='right', color='blue')
plt.xlabel('Last seven days') plt.xlabel('Last seven days')
@ -291,13 +305,17 @@ if __name__ == '__main__':
plt.close() plt.close()
plt.plot([-6, -5, -4, -3, -2, -1, 0], [mau_plots[6], mau_plots[5], mau_plots[4], mau_plots[3], mau_plots[2], mau_plots[1], mau_plots[0]], marker='o', color='royalblue') plt.plot([-6, -5, -4, -3, -2, -1, 0], [mau_plots[6], mau_plots[5], mau_plots[4],
mau_plots[3], mau_plots[2], mau_plots[1], mau_plots[0]], marker='o', color='royalblue')
plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_mau, max_mau, max_mau, max_mau, max_mau, max_mau, max_mau], color='red') plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_mau, max_mau, max_mau,
max_mau, max_mau, max_mau, max_mau], color='red')
plt.title('fediverse: total MAU (max: ' + str(f"{max_mau:,}" + ')'), loc='right', color='royalblue') plt.title('fediverse: total MAU (max: ' +
str(f"{max_mau:,}" + ')'), loc='right', color='royalblue')
plt.legend(('mau', 'max'), shadow=True, loc=(0.01, 0.80), handlelength=1.5, fontsize=10) plt.legend(('mau', 'max'), shadow=True, loc=(
0.01, 0.80), handlelength=1.5, fontsize=10)
plt.xlabel('Last seven days') plt.xlabel('Last seven days')
@ -349,25 +367,30 @@ if __name__ == '__main__':
if evo_servers >= 0: if evo_servers >= 0:
toot_text += "alive servers: " + str(f"{total_servers:,}") + " (+"+ str(f"{evo_servers:,}") + ") \n" toot_text += "alive servers: " + \
str(f"{total_servers:,}") + \
" (+" + str(f"{evo_servers:,}") + ") \n"
toot_text += "max: " + str(f"{max_servers:,}") + "\n" toot_text += "max: " + str(f"{max_servers:,}") + "\n"
elif evo_servers < 0: elif evo_servers < 0:
toot_text += "alive servers: " + str(f"{total_servers:,}") + " ("+ str(f"{evo_servers:,}") + ") \n" toot_text += "alive servers: " + \
str(f"{total_servers:,}") + " (" + str(f"{evo_servers:,}") + ") \n"
toot_text += "max: " + str(f"{max_servers:,}") + "\n" toot_text += "max: " + str(f"{max_servers:,}") + "\n"
if evo_mau >= 0: if evo_mau >= 0:
toot_text += "total MAU: " + str(f"{total_mau:,}") + " (+"+ str(f"{evo_mau:,}") + ") \n" toot_text += "total MAU: " + \
str(f"{total_mau:,}") + " (+" + str(f"{evo_mau:,}") + ") \n"
toot_text += "max: " + str(f"{max_mau:,}") + "\n" toot_text += "max: " + str(f"{max_mau:,}") + "\n"
elif evo_mau < 0: elif evo_mau < 0:
toot_text += "total MAU: " + str(f"{total_mau:,}") + " ("+ str(f"{evo_mau:,}") + ") \n" toot_text += "total MAU: " + \
str(f"{total_mau:,}") + " (" + str(f"{evo_mau:,}") + ") \n"
toot_text += "max: " + str(f"{max_mau:,}") + "\n" toot_text += "max: " + str(f"{max_mau:,}") + "\n"
@ -397,13 +420,17 @@ if __name__ == '__main__':
print(toot_text) print(toot_text)
servers_image_id = mastodon.media_post('servers.png', "image/png", description='servers graph').id servers_image_id = mastodon.media_post(
'servers.png', "image/png", description='servers graph').id
mau_image_id = mastodon.media_post('mau.png', "image/png", description='MAU graph').id mau_image_id = mastodon.media_post(
'mau.png', "image/png", description='MAU graph').id
global_image_id = mastodon.media_post('global.png', "image/png", description='global graph').id global_image_id = mastodon.media_post(
'global.png', "image/png", description='global graph').id
mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, mau_image_id, global_image_id}) mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={
servers_image_id, mau_image_id, global_image_id})
db.delete_dead_servers() db.delete_dead_servers()

1
tests/__init__.py Normal file
Veure arxiu

@ -0,0 +1 @@
#!/usr/bin/env python3

15
tests/test_database.py Normal file
Veure arxiu

@ -0,0 +1,15 @@
from federation import Database
def test__database_connection():
db = Database(config_file='config/test_db_config.txt', fediverse_db_host='localhost', fediverse_db_port='5432')
# noexcept
db.check_connection()
def test__database_connection__noservers():
db = Database(config_file='config/test_db_config.txt', fediverse_db_host='localhost', fediverse_db_port='5432')
checked_server = db.get_last_checked_servers()
assert checked_server == []