Totally recoded
This commit is contained in:
spla
pare
d82ad780d2
commit
c104d250d5
22
README.md
22
README.md
|
|
@ -1,5 +1,5 @@
|
|||
# Fediverse Stats
|
||||
This code gets all peers from running Mastodon, Pleroma and Lemmy host servers and then all peers from host server's peers. Goal is to collect maximum number
|
||||
This code gets all peers from mastodon.social. Goal is to collect maximum number
|
||||
of alive fediverse's servers and then query their API to obtain their registered users (if their API provide such information).
|
||||
At the end it post the results to host server bot account.
|
||||
|
||||
|
|
@ -7,7 +7,7 @@ At the end it post the results to host server bot account.
|
|||
|
||||
- **Python 3**
|
||||
- Postgresql server
|
||||
- Mastodon or Pleroma running server.
|
||||
- Mastodon running server.
|
||||
|
||||
### Usage:
|
||||
|
||||
|
|
@ -15,19 +15,11 @@ Within Python Virtual Environment:
|
|||
|
||||
1. Run `pip install -r requirements.txt` to install needed libraries.
|
||||
|
||||
2. Run `python db-setup.py` to setup and create new Postgresql database and needed tables in it.
|
||||
2. Run `python fetchservers.py` to add servers to alive servers database.
|
||||
|
||||
3. Run `python setup.py` to get your bot's access token of your Mastodon or Pleroma server existing account. It will be saved to 'secrets/secrets.txt' for further use.
|
||||
3. Run `python fediverse.py` to query world alive servers API. It gets data from server's nodeinfo.
|
||||
|
||||
4. Run `python getpeers.py` to get all peers from your host and the whole world of fediverse's servers (or almost the whole world).
|
||||
|
||||
5. Run `python fetchservers.py` to add servers to alive servers database.
|
||||
|
||||
6. Run `python fediverse.py` to query world alive servers API. It gets data from server's nodeinfo.
|
||||
|
||||
7. Run `python uptime_setup.py` to get your Uptime bot's access token of your Mastodon or Pleroma server existing account. It will be saved to 'secrets/uptime_secrets.txt' for further use.
|
||||
|
||||
8. Use your favourite scheduling method to set `python fediverse.py` to run twice daily, `python fetchservers.py` one time daily, `python getworld.py` to run monthly and `python uptime.py` (choose your desired frequency) if you want to publish best fediverse's servers uptime.
|
||||
4. Use your favourite scheduling method to set `python fediverse.py` to run twice daily, `python fetchservers.py` one time daily.
|
||||
|
||||
18.2.2021 - New feature! Added [Lemmy project](https://join.lemmy.ml)
|
||||
12.5.2021 - New feature! Added Wordpress support. The code can now detect Wordpress instances with ActivityPub enabled plugin.
|
||||
|
|
@ -35,4 +27,6 @@ Within Python Virtual Environment:
|
|||
21.8.2021 - New feature! Added Best Fediverse's servers Uptime publishing bot.
|
||||
22.10.2021 - New feature! Added [Funkwhale](https://funkwhale.audio) support.
|
||||
26.10.2021 - New feature! Added [Socialhome](https://socialhome.network) support.
|
||||
2.3.2022 - Improved server nodeinfo detection
|
||||
2.3.2022 - Improved server nodeinfo detection.
|
||||
4.1.2023 - Refactored.
|
||||
4.1.2023 - Now all peers are obtained from mastodon.social's peers list.
|
||||
|
|
|
|||
1070
fediverse.py
1070
fediverse.py
La diferencia del archivo ha sido suprimido porque es demasiado grande
Cargar Diff
259
fetchservers.py
259
fetchservers.py
|
|
@ -4,13 +4,15 @@ import os
|
|||
import json
|
||||
import sys
|
||||
import os.path
|
||||
import psycopg2
|
||||
from setup import Setup
|
||||
from database import Database
|
||||
import requests
|
||||
import urllib3
|
||||
import socket
|
||||
import ray
|
||||
import pdb
|
||||
|
||||
ray.init(num_cpus = 32) # Specify this system CPUs.
|
||||
#ray.init(num_cpus = 25) # Specify this system CPUs.
|
||||
|
||||
from ray.exceptions import (
|
||||
RaySystemError,
|
||||
|
|
@ -19,18 +21,18 @@ from ray.exceptions import (
|
|||
ObjectStoreFullError,
|
||||
)
|
||||
|
||||
apis = ['/api/v1/instance?',
|
||||
'/api/v1/nodeinfo?',
|
||||
'/nodeinfo/2.0?',
|
||||
'/nodeinfo/2.0.json?',
|
||||
'/nodeinfo/2.1.json?',
|
||||
'/main/nodeinfo/2.0?',
|
||||
'/api/statusnet/config?',
|
||||
'/api/nodeinfo/2.0.json?',
|
||||
'/api/nodeinfo?',
|
||||
'/wp-json/nodeinfo/2.0?',
|
||||
'/api/v1/instance/nodeinfo/2.0?',
|
||||
'/.well-known/x-nodeinfo2?'
|
||||
apis = ['/api/v1/instance',
|
||||
'/api/v1/nodeinfo',
|
||||
'/nodeinfo/2.0',
|
||||
'/nodeinfo/2.0.json',
|
||||
'/nodeinfo/2.1.json',
|
||||
'/main/nodeinfo/2.0',
|
||||
'/api/statusnet/config',
|
||||
'/api/nodeinfo/2.0.json',
|
||||
'/api/nodeinfo',
|
||||
'/wp-json/nodeinfo/2.0',
|
||||
'/api/v1/instance/nodeinfo/2.0',
|
||||
'/.well-known/x-nodeinfo2'
|
||||
]
|
||||
|
||||
def is_json(myjson):
|
||||
|
|
@ -41,45 +43,6 @@ def is_json(myjson):
|
|||
return False
|
||||
return True
|
||||
|
||||
def write_api(server, software, users, alive, api, soft_version):
|
||||
|
||||
fediverse_db, fediverse_db_user = get_db_config()
|
||||
|
||||
insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
|
||||
|
||||
conn = None
|
||||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
print(f'Writing {server} nodeinfo data...')
|
||||
|
||||
cur.execute(insert_sql, (server, now, software, users, alive, api, soft_version))
|
||||
|
||||
cur.execute(
|
||||
"UPDATE fediverse SET updated_at=(%s), software=(%s), users=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)",
|
||||
(now, software, users, alive, api, soft_version, server)
|
||||
)
|
||||
|
||||
cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
|
||||
|
||||
conn.commit()
|
||||
|
||||
cur.close()
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print(error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
@ray.remote
|
||||
def getsoft(server):
|
||||
|
||||
|
|
@ -91,8 +54,6 @@ def getsoft(server):
|
|||
return
|
||||
if server.find(":") != -1:
|
||||
return
|
||||
if server == 'z.fedipen.xyz':
|
||||
return
|
||||
|
||||
soft = ''
|
||||
|
||||
|
|
@ -100,11 +61,9 @@ def getsoft(server):
|
|||
|
||||
url = 'https://' + server
|
||||
|
||||
user_agent = {'User-agent': "fediverse's stats (fediverse@mastodont.cat)"}
|
||||
|
||||
try:
|
||||
|
||||
response = requests.get(url + '/.well-known/nodeinfo', headers = user_agent, timeout=3)
|
||||
response = requests.get(url + '/.well-known/nodeinfo', headers = setup.user_agent, timeout=3)
|
||||
|
||||
if response.status_code == 200:
|
||||
|
||||
|
|
@ -112,11 +71,17 @@ def getsoft(server):
|
|||
|
||||
response_json = response.json()
|
||||
|
||||
nodeinfo = response_json['links'][0]['href'].replace(f'https://{server}','')
|
||||
if len(response_json['links']) == 1:
|
||||
|
||||
nodeinfo = response_json['links'][0]['href'].replace(f'https://{server}','')
|
||||
|
||||
elif len(response_json['links']) == 2:
|
||||
|
||||
nodeinfo = response_json['links'][1]['href'].replace(f'https://{server}','')
|
||||
|
||||
try:
|
||||
|
||||
nodeinfo_data = requests.get(url + nodeinfo, headers = user_agent, timeout=3)
|
||||
nodeinfo_data = requests.get(url + nodeinfo, headers = setup.user_agent, timeout=3)
|
||||
|
||||
if nodeinfo_data.status_code == 200:
|
||||
|
||||
|
|
@ -126,7 +91,7 @@ def getsoft(server):
|
|||
|
||||
else:
|
||||
|
||||
print(f"Server {server}'s nodeinfo not responding: error code {nodeinfo_data.status_code}")
|
||||
print(f"{nodeinfo} not responding: error code {nodeinfo_data.status_code}")
|
||||
|
||||
except:
|
||||
|
||||
|
|
@ -134,7 +99,7 @@ def getsoft(server):
|
|||
|
||||
except:
|
||||
|
||||
print(f'Server {server} not responding: error code {response.status_code}')
|
||||
print(f'{server} is not responding: error code {response.status_code}')
|
||||
print('*********************************************************************')
|
||||
|
||||
pass
|
||||
|
|
@ -144,39 +109,41 @@ def getsoft(server):
|
|||
|
||||
try:
|
||||
|
||||
response = requests.get(url + api, headers = user_agent, timeout=3)
|
||||
response = requests.get(url + api, headers = setup.user_agent, timeout=3)
|
||||
|
||||
if is_json(response.text):
|
||||
if response.status_code == 200:
|
||||
|
||||
nodeinfo_json = response.json()
|
||||
if is_json(response.text):
|
||||
|
||||
if 'software' in nodeinfo_json:
|
||||
nodeinfo_json = response.json()
|
||||
|
||||
nodeinfo = api
|
||||
|
||||
is_nodeinfo = True
|
||||
|
||||
break
|
||||
|
||||
elif 'title' in nodeinfo_json:
|
||||
|
||||
if nodeinfo_json['title'] == 'Zap':
|
||||
if 'software' in nodeinfo_json:
|
||||
|
||||
nodeinfo = api
|
||||
|
||||
is_nodeinfo = True
|
||||
|
||||
soft = 'zap'
|
||||
|
||||
break
|
||||
|
||||
elif 'version' in nodeinfo_json:
|
||||
elif 'title' in nodeinfo_json:
|
||||
|
||||
nodeinfo = api
|
||||
if nodeinfo_json['title'] == 'Zap':
|
||||
|
||||
is_nodeinfo = True
|
||||
nodeinfo = api
|
||||
|
||||
break
|
||||
is_nodeinfo = True
|
||||
|
||||
soft = 'zap'
|
||||
|
||||
break
|
||||
|
||||
elif 'version' in nodeinfo_json:
|
||||
|
||||
nodeinfo = api
|
||||
|
||||
is_nodeinfo = True
|
||||
|
||||
break
|
||||
|
||||
except:
|
||||
|
||||
|
|
@ -233,12 +200,13 @@ def getsoft(server):
|
|||
soft = nodeinfo_json['software']['name']
|
||||
soft = soft.lower()
|
||||
soft_version = nodeinfo_json['software']['version']
|
||||
users = nodeinfo_json['usage']['users']['total']
|
||||
users = nodeinfo_json.get('usage').get('users').get('total') or '0'
|
||||
if users > 1000000:
|
||||
return
|
||||
|
||||
alive = True
|
||||
|
||||
write_api(server, soft, users, alive, nodeinfo, soft_version)
|
||||
db.write_api(server, soft, users, alive, nodeinfo, soft_version)
|
||||
|
||||
print(f"Server {server} ({soft} {soft_version}) is alive!")
|
||||
print('*********************************************************************')
|
||||
|
|
@ -259,11 +227,12 @@ def getsoft(server):
|
|||
users = nodeinfo_json['usage']['users']['total']
|
||||
if users > 1000000:
|
||||
return
|
||||
|
||||
alive = True
|
||||
|
||||
if soft == 'socialhome':
|
||||
|
||||
write_api(server, soft, users, alive, nodeinfo, soft_version)
|
||||
db.write_api(server, soft, users, alive, nodeinfo, soft_version)
|
||||
|
||||
print('*********************************************************************')
|
||||
print(f"Server {serve}r ({soft} {soft_version}) is alive!")
|
||||
|
|
@ -301,7 +270,7 @@ def getsoft(server):
|
|||
|
||||
alive = True
|
||||
|
||||
write_api(server, soft, users, alive, nodeinfo, soft_version)
|
||||
db.write_api(server, soft, users, alive, nodeinfo, soft_version)
|
||||
|
||||
print('*********************************************************************')
|
||||
print(f"Server {server} ({soft}) is alive!")
|
||||
|
|
@ -323,106 +292,6 @@ def getsoft(server):
|
|||
print(f'Server {server} is dead')
|
||||
print('*********************************************************************')
|
||||
|
||||
def get_world_servers():
|
||||
|
||||
world_servers = []
|
||||
|
||||
try:
|
||||
|
||||
conn = None
|
||||
|
||||
conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
# get world servers list
|
||||
|
||||
cur.execute("select server from world where checked='f'")
|
||||
|
||||
rows = cur.fetchall()
|
||||
|
||||
for row in rows:
|
||||
|
||||
world_servers.append(row[0])
|
||||
|
||||
cur.close()
|
||||
|
||||
print("Remaining servers: " + str(len(world_servers)))
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print(error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
return world_servers
|
||||
|
||||
def save_time(program, start, finish):
|
||||
|
||||
insert_sql = "INSERT INTO execution_time(program, start, finish) VALUES(%s,%s,%s) ON CONFLICT DO NOTHING"
|
||||
|
||||
conn = None
|
||||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute(insert_sql, (program, start, finish,))
|
||||
|
||||
cur.execute("UPDATE execution_time SET start=(%s), finish=(%s) where program=(%s)", (start, finish, program))
|
||||
|
||||
conn.commit()
|
||||
|
||||
cur.close()
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print(error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
def get_parameter(parameter, file_path):
|
||||
|
||||
# Check if secrets file exists
|
||||
if not os.path.isfile(file_path):
|
||||
print("File %s not found, exiting." % file_path)
|
||||
sys.exit(0)
|
||||
|
||||
# Find parameter in file
|
||||
with open(file_path) as f:
|
||||
for line in f:
|
||||
if line.startswith(parameter):
|
||||
return line.replace(parameter + ":", "").strip()
|
||||
|
||||
# Cannot find parameter, exit
|
||||
print(file_path + " Missing parameter %s " % parameter)
|
||||
sys.exit(0)
|
||||
|
||||
def get_config():
|
||||
|
||||
# Load configuration from config file
|
||||
config_filepath = "config/config.txt"
|
||||
mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
|
||||
return mastodon_hostname
|
||||
|
||||
def get_db_config():
|
||||
|
||||
# Load database config from db_config file
|
||||
db_config_filepath = "config/db_config.txt"
|
||||
fediverse_db = get_parameter("fediverse_db", db_config_filepath)
|
||||
fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath)
|
||||
return (fediverse_db, fediverse_db_user)
|
||||
|
||||
###############################################################################
|
||||
# main
|
||||
|
||||
|
|
@ -430,7 +299,13 @@ if __name__ == '__main__':
|
|||
|
||||
## name: fetchservers.py
|
||||
|
||||
fediverse_db, fediverse_db_user = get_db_config()
|
||||
setup = Setup()
|
||||
|
||||
db = Database()
|
||||
|
||||
res = requests.get('https://' + 'mastodon.social' + setup.peers_api, headers = setup.user_agent, timeout=3)
|
||||
|
||||
hostname_peers = res.json()
|
||||
|
||||
start = datetime.now()
|
||||
|
||||
|
|
@ -438,19 +313,19 @@ if __name__ == '__main__':
|
|||
|
||||
finish = start
|
||||
|
||||
save_time(program, start, finish)
|
||||
db.save_time(program, start, finish)
|
||||
|
||||
now = start
|
||||
|
||||
mastodon_hostname = get_config()
|
||||
|
||||
world_servers = get_world_servers()
|
||||
#world_servers = db.get_world_servers()
|
||||
|
||||
ray_start = time.time()
|
||||
|
||||
try:
|
||||
|
||||
results = ray.get([getsoft.remote(server) for server in world_servers])
|
||||
#results = ray.get([getsoft.remote(server) for server in world_servers])
|
||||
results = ray.get([getsoft.remote(server) for server in hostname_peers])
|
||||
#[getsoft(server) for server in world_servers]
|
||||
|
||||
print(f"duration = {time.time() - ray_start}.\nprocessed servers: {len(results)}")
|
||||
|
||||
|
|
@ -460,4 +335,4 @@ if __name__ == '__main__':
|
|||
|
||||
finish = datetime.now()
|
||||
|
||||
save_time(program, start, finish)
|
||||
db.save_time(program, start, finish)
|
||||
|
|
|
|||
177
getpeers.py
177
getpeers.py
|
|
@ -1,177 +0,0 @@
|
|||
import os
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime
|
||||
import requests
|
||||
import json
|
||||
import psycopg2
|
||||
import ray
|
||||
import pdb
|
||||
|
||||
ray.init(num_cpus = 32) # Specify this system CPUs.
|
||||
|
||||
def write_server(server, federated_with):
|
||||
|
||||
insert_sql = "INSERT INTO world(server, federated_with, updated_at, saved_at, checked) VALUES(%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
|
||||
|
||||
conn = None
|
||||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute(insert_sql, (server, federated_with, now, now, 'f'))
|
||||
|
||||
print(f'writing {server} to world database')
|
||||
|
||||
conn.commit()
|
||||
|
||||
cur.close()
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print(error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
@ray.remote
|
||||
def get_peers(peer):
|
||||
|
||||
try:
|
||||
|
||||
user_agent = {'User-agent': "fediverse's stats (fediverse@mastodont.cat)"}
|
||||
|
||||
domain_res = requests.get('https://' + peer + '/api/v1/instance', headers = user_agent, timeout=3)
|
||||
|
||||
domain_res_json = domain_res.json()
|
||||
|
||||
if domain_res.status_code == 200:
|
||||
|
||||
domain_uri = domain_res_json['uri'].replace('https://', '')
|
||||
|
||||
if domain_uri != peer:
|
||||
|
||||
print(f'{peer} is an aliased domain of {domain_uri}!')
|
||||
|
||||
else:
|
||||
|
||||
response = requests.get('https://' + peer + peers_api, headers = user_agent, timeout=3)
|
||||
|
||||
response_json = response.json()
|
||||
|
||||
if response.status_code == 200:
|
||||
|
||||
try:
|
||||
|
||||
print(f"Server: {peer}, federated with {str(len(response_json))} servers")
|
||||
|
||||
for peer_peer in response_json:
|
||||
|
||||
write_server(peer_peer, peer)
|
||||
|
||||
except:
|
||||
|
||||
pass
|
||||
|
||||
except:
|
||||
|
||||
pass
|
||||
|
||||
def save_time(program, start, finish):
|
||||
|
||||
insert_sql = "INSERT INTO execution_time(program, start, finish) VALUES(%s,%s,%s) ON CONFLICT DO NOTHING"
|
||||
|
||||
conn = None
|
||||
|
||||
try:
|
||||
|
||||
conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute(insert_sql, (program, start, finish,))
|
||||
|
||||
cur.execute("UPDATE execution_time SET start=(%s), finish=(%s) where program=(%s)", (start, finish, program))
|
||||
|
||||
conn.commit()
|
||||
|
||||
cur.close()
|
||||
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
|
||||
print(error)
|
||||
|
||||
finally:
|
||||
|
||||
if conn is not None:
|
||||
|
||||
conn.close()
|
||||
|
||||
def get_parameter( parameter, file_path ):
|
||||
|
||||
# Check if secrets file exists
|
||||
if not os.path.isfile(file_path):
|
||||
print("File %s not found, exiting."%file_path)
|
||||
sys.exit(0)
|
||||
|
||||
# Find parameter in file
|
||||
with open( file_path ) as f:
|
||||
for line in f:
|
||||
if line.startswith( parameter ):
|
||||
return line.replace(parameter + ":", "").strip()
|
||||
|
||||
# Cannot find parameter, exit
|
||||
print(file_path + " Missing parameter %s "%parameter)
|
||||
sys.exit(0)
|
||||
|
||||
###############################################################################
|
||||
# main
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
now = datetime.now()
|
||||
|
||||
peers_api = '/api/v1/instance/peers?'
|
||||
|
||||
# Load configuration from config file
|
||||
config_filepath = "config/config.txt"
|
||||
mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
|
||||
|
||||
# Load database config from db_config file
|
||||
db_config_filepath = "config/db_config.txt"
|
||||
fediverse_db = get_parameter("fediverse_db", db_config_filepath)
|
||||
fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath)
|
||||
|
||||
user_agent = {'User-agent': "fediverse's stats (fediverse@mastodont.cat)"}
|
||||
|
||||
res = requests.get('https://' + mastodon_hostname + peers_api, headers = user_agent, timeout=3)
|
||||
|
||||
hostname_peers = res.json()
|
||||
|
||||
start = datetime.now()
|
||||
|
||||
program = 'getpeers'
|
||||
|
||||
finish = start
|
||||
|
||||
save_time(program, start, finish)
|
||||
|
||||
for peer in hostname_peers:
|
||||
|
||||
write_server(peer, mastodon_hostname)
|
||||
|
||||
results = ray.get([get_peers.remote(server) for server in hostname_peers])
|
||||
|
||||
finish = datetime.now()
|
||||
|
||||
print(f"duration = {finish - start}.\nprocessed servers: {len(results)}")
|
||||
|
||||
save_time(program, start, finish)
|
||||
|
||||
|
||||
|
|
@ -1,9 +1,7 @@
|
|||
Mastodon.py>=1.5.1
|
||||
psycopg2-binary>=2.8.4
|
||||
aiohttp>=3.6.2
|
||||
aiodns>=2.0.0
|
||||
matplotlib>=3.3.4
|
||||
humanfriendly>=9.2
|
||||
urllib3>=1.26.8
|
||||
requests>=2.27.1
|
||||
ray>=1.11.0
|
||||
requests
|
||||
psycopg2-binary
|
||||
pytz
|
||||
ray
|
||||
Mastodon.py
|
||||
matplotlib
|
||||
pandas
|
||||
|
|
|
|||
273
setup.py
273
setup.py
|
|
@ -1,226 +1,157 @@
|
|||
import getpass
|
||||
from mastodon import Mastodon
|
||||
from mastodon.Mastodon import MastodonMalformedEventError, MastodonNetworkError, MastodonReadTimeout, MastodonAPIError, MastodonIllegalArgumentError
|
||||
import fileinput,re
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime
|
||||
import pytz
|
||||
from mastodon import Mastodon
|
||||
from mastodon.Mastodon import MastodonMalformedEventError, MastodonNetworkError, MastodonReadTimeout, MastodonAPIError, MastodonIllegalArgumentError
|
||||
import pdb
|
||||
|
||||
def create_dir():
|
||||
class Setup():
|
||||
|
||||
if not os.path.exists('secrets'):
|
||||
os.makedirs('secrets')
|
||||
name = 'fediverse setup'
|
||||
|
||||
def create_file():
|
||||
def __init__(self, config_file=None, mastodon_hostname=None, peers_api=None, user_agent=None, secrets_filepath=None, mastodon_app_token=None):
|
||||
|
||||
if not os.path.exists('secrets/secrets.txt'):
|
||||
with open('secrets/secrets.txt', 'w'): pass
|
||||
print(secrets_filepath + " created!")
|
||||
self.config_file = "config/config.txt"
|
||||
self.mastodon_hostname = self.__get_parameter("mastodon_hostname", self.config_file)
|
||||
self.peers_api = '/api/v1/instance/peers?'
|
||||
self.user_agent = {'User-agent': "fediverse's stats (fediverse@mastodont.cat)"}
|
||||
|
||||
def create_config():
|
||||
self.secrets_filepath = 'secrets/secrets.txt'
|
||||
|
||||
if not os.path.exists('config'):
|
||||
os.makedirs('config')
|
||||
if not os.path.exists(config_filepath):
|
||||
print(config_filepath + " created!")
|
||||
with open('config/config.txt', 'w'): pass
|
||||
is_setup = self.__check_mastodon_setup(self)
|
||||
|
||||
def write_params():
|
||||
if is_setup:
|
||||
|
||||
with open(secrets_filepath, 'a') as the_file:
|
||||
print("Writing secrets parameter names to " + secrets_filepath)
|
||||
the_file.write('uc_client_id: \n'+'uc_client_secret: \n'+'uc_access_token: \n')
|
||||
self.mastodon_app_token = self.__get_mastodon_parameter("mastodon_app_token", self.secrets_filepath)
|
||||
|
||||
def write_config():
|
||||
else:
|
||||
|
||||
with open(config_filepath, 'a') as the_file:
|
||||
the_file.write('mastodon_hostname: \n')
|
||||
print("adding parameter name 'mastodon_hostname' to "+ config_filepath)
|
||||
self.mastodon_app_token = self.mastodon_setup(self)
|
||||
|
||||
def read_client_lines(self):
|
||||
@staticmethod
|
||||
def __check_mastodon_setup(self):
|
||||
|
||||
client_path = 'app_clientcred.txt'
|
||||
is_setup = False
|
||||
|
||||
with open(client_path) as fp:
|
||||
if not os.path.isfile(self.secrets_filepath):
|
||||
print(f"File {self.secrets_filepath} not found, running setup.")
|
||||
else:
|
||||
is_setup = True
|
||||
|
||||
line = fp.readline()
|
||||
cnt = 1
|
||||
while line:
|
||||
if cnt == 1:
|
||||
print("Writing client id to " + secrets_filepath)
|
||||
modify_file(secrets_filepath, "uc_client_id: ", value=line.rstrip())
|
||||
elif cnt == 2:
|
||||
print("Writing client secret to " + secrets_filepath)
|
||||
modify_file(secrets_filepath, "uc_client_secret: ", value=line.rstrip())
|
||||
line = fp.readline()
|
||||
cnt += 1
|
||||
return is_setup
|
||||
|
||||
def read_token_line(self):
|
||||
@staticmethod
|
||||
def mastodon_setup(self):
|
||||
|
||||
token_path = 'app_usercred.txt'
|
||||
if not os.path.exists('secrets'):
|
||||
os.makedirs('secrets')
|
||||
|
||||
with open(token_path) as fp:
|
||||
self.mastodon_user = input("Mastodon user login? ")
|
||||
self.mastodon_password = input("Mastodon user password? ")
|
||||
self.app_name = 'fediverse'
|
||||
|
||||
line = fp.readline()
|
||||
print("Writing access token to " + secrets_filepath)
|
||||
modify_file(secrets_filepath, "uc_access_token: ", value=line.rstrip())
|
||||
self.mastodon_app_token = self.mastodon_log_in()
|
||||
|
||||
def read_config_line():
|
||||
if not os.path.exists(self.secrets_filepath):
|
||||
with open(self.secrets_filepath, 'w'): pass
|
||||
print(f"{self.secrets_filepath} created!")
|
||||
|
||||
with open(config_filepath) as fp:
|
||||
with open(self.secrets_filepath, 'a') as the_file:
|
||||
print("Writing Mastodon parameters to " + self.secrets_filepath)
|
||||
the_file.write(f'mastodon_app_token: {self.mastodon_app_token}')
|
||||
|
||||
line = fp.readline()
|
||||
modify_file(config_filepath, "mastodon_hostname: ", value=hostname)
|
||||
modify_file(config_filepath, "bot_username: ", value=bot_username)
|
||||
return self.mastodon_app_token
|
||||
|
||||
def log_in():
|
||||
def mastodon_log_in(self):
|
||||
|
||||
error = 0
|
||||
token = ''
|
||||
|
||||
try:
|
||||
try:
|
||||
|
||||
global hostname
|
||||
hostname = input("Enter Mastodon hostname: ")
|
||||
user_name = input("User name, ex. user@" + hostname +"? ")
|
||||
user_password = getpass.getpass("User password? ")
|
||||
bot_username = input("Bot's username, ex. fediverse: ")
|
||||
app_name = input("This app name? ")
|
||||
Mastodon.create_app(
|
||||
app_name,
|
||||
scopes=["read","write"],
|
||||
to_file="app_clientcred.txt",
|
||||
api_base_url=hostname
|
||||
)
|
||||
mastodon = Mastodon(client_id = "app_clientcred.txt", api_base_url = hostname)
|
||||
mastodon.log_in(
|
||||
user_name,
|
||||
user_password,
|
||||
scopes = ["read", "write"],
|
||||
to_file = "app_usercred.txt"
|
||||
response = Mastodon.create_app(
|
||||
self.app_name,
|
||||
scopes=["read","write"],
|
||||
to_file=None,
|
||||
api_base_url=self.mastodon_hostname
|
||||
)
|
||||
client_id = response[0]
|
||||
client_secret = response[1]
|
||||
|
||||
mastodon = Mastodon(client_id = client_id, client_secret = client_secret, api_base_url = self.mastodon_hostname)
|
||||
|
||||
token = mastodon.log_in(
|
||||
self.mastodon_user,
|
||||
self.mastodon_password,
|
||||
scopes = ["read", "write"],
|
||||
to_file = None
|
||||
)
|
||||
|
||||
except MastodonIllegalArgumentError as i_error:
|
||||
print('Log in succesful!')
|
||||
|
||||
error = 1
|
||||
except MastodonIllegalArgumentError as i_error:
|
||||
|
||||
if os.path.exists("secrets/secrets.txt"):
|
||||
print("Removing secrets/secrets.txt file..")
|
||||
os.remove("secrets/secrets.txt")
|
||||
if os.path.exists("app_clientcred.txt"):
|
||||
print("Removing app_clientcred.txt file..")
|
||||
os.remove("app_clientcred.txt")
|
||||
sys.exit(i_error)
|
||||
sys.stdout.write(f'\n{str(i_error)}\n')
|
||||
|
||||
except MastodonNetworkError as n_error:
|
||||
except MastodonNetworkError as n_error:
|
||||
|
||||
error = 1
|
||||
sys.stdout.write(f'\n{str(n_error)}\n')
|
||||
|
||||
if os.path.exists("secrets/secrets.txt"):
|
||||
print("Removing secrets/secrets.txt file..")
|
||||
os.remove("secrets/secrets.txt")
|
||||
if os.path.exists("app_clientcred.txt"):
|
||||
print("Removing app_clientcred.txt file..")
|
||||
os.remove("app_clientcred.txt")
|
||||
sys.exit(n_error)
|
||||
except MastodonReadTimeout as r_error:
|
||||
|
||||
except MastodonReadTimeout as r_error:
|
||||
sys.stdout.write(f'\n{str(r_error)}\n')
|
||||
|
||||
error = 1
|
||||
except MastodonAPIError as a_error:
|
||||
|
||||
if os.path.exists("secrets/secrets.txt"):
|
||||
print("Removing secrets/secrets.txt file..")
|
||||
os.remove("secrets/secrets.txt")
|
||||
if os.path.exists("app_clientcred.txt"):
|
||||
print("Removing app_clientcred.txt file..")
|
||||
os.remove("app_clientcred.txt")
|
||||
sys.exit(r_error)
|
||||
sys.stdout.write(f'\n{str(a_error)}\n')
|
||||
|
||||
except MastodonAPIError as a_error:
|
||||
finally:
|
||||
|
||||
error = 1
|
||||
return token
|
||||
|
||||
if os.path.exists("secrets/secrets.txt"):
|
||||
print("Removing secrets/secrets.txt file..")
|
||||
os.remove("secrets/secrets.txt")
|
||||
if os.path.exists("app_clientcred.txt"):
|
||||
print("Removing app_clientcred.txt file..")
|
||||
os.remove("app_clientcred.txt")
|
||||
sys.exit(a_error)
|
||||
def __get_parameter(self, parameter, config_file):
|
||||
|
||||
finally:
|
||||
if not os.path.isfile(config_file):
|
||||
print(f"File {config_file} not found..")
|
||||
|
||||
if error == 0:
|
||||
self.mastodon_hostname = input("\nMastodon hostname: ")
|
||||
|
||||
create_dir()
|
||||
create_file()
|
||||
write_params()
|
||||
client_path = 'app_clientcred.txt'
|
||||
read_client_lines(client_path)
|
||||
token_path = 'app_usercred.txt'
|
||||
read_token_line(token_path)
|
||||
if os.path.exists("app_clientcred.txt"):
|
||||
print("Removing app_clientcred.txt temp file..")
|
||||
os.remove("app_clientcred.txt")
|
||||
if os.path.exists("app_usercred.txt"):
|
||||
print("Removing app_usercred.txt temp file..")
|
||||
os.remove("app_usercred.txt")
|
||||
print("Secrets setup done!\n")
|
||||
self.__create_config(self)
|
||||
self.__write_config(self)
|
||||
|
||||
def modify_file(file_name,pattern,value=""):
|
||||
with open( self.config_file ) as f:
|
||||
for line in f:
|
||||
if line.startswith( parameter ):
|
||||
return line.replace(parameter + ":", "").strip()
|
||||
|
||||
fh=fileinput.input(file_name,inplace=True)
|
||||
for line in fh:
|
||||
replacement=pattern + value
|
||||
line=re.sub(pattern,replacement,line)
|
||||
sys.stdout.write(line)
|
||||
fh.close()
|
||||
def __get_mastodon_parameter(self, parameter, secrets_filepath):
|
||||
|
||||
def get_parameter( parameter, file_path ):
|
||||
# Check if secrets file exists
|
||||
if not os.path.isfile(file_path):
|
||||
print("File %s not found, creating it."%file_path)
|
||||
log_in()
|
||||
if not os.path.isfile(secrets_filepath):
|
||||
print(f"File {secrets_filepath} not found..")
|
||||
|
||||
# Find parameter in file
|
||||
with open( file_path ) as f:
|
||||
for line in f:
|
||||
if line.startswith( parameter ):
|
||||
return line.replace(parameter + ":", "").strip()
|
||||
self.sign_in()
|
||||
|
||||
# Cannot find parameter, exit
|
||||
print(file_path + " Missing parameter %s "%parameter)
|
||||
sys.exit(0)
|
||||
with open( self.secrets_filepath ) as f:
|
||||
for line in f:
|
||||
if line.startswith( parameter ):
|
||||
return line.replace(parameter + ":", "").strip()
|
||||
|
||||
def get_hostname( parameter, config_filepath ):
|
||||
# Check if secrets file exists
|
||||
if not os.path.isfile(config_filepath):
|
||||
print("File %s not found, creating it."%config_filepath)
|
||||
create_config()
|
||||
@staticmethod
|
||||
def __create_config(self):
|
||||
|
||||
# Find parameter in file
|
||||
with open( config_filepath ) as f:
|
||||
for line in f:
|
||||
if line.startswith( parameter ):
|
||||
return line.replace(parameter + ":", "").strip()
|
||||
if not os.path.exists('config'):
|
||||
|
||||
# Cannot find parameter, exit
|
||||
print(config_filepath + " Missing parameter %s "%parameter)
|
||||
write_config()
|
||||
read_config_line()
|
||||
print("setup done!")
|
||||
sys.exit(0)
|
||||
os.makedirs('config')
|
||||
|
||||
###############################################################################
|
||||
# main
|
||||
if not os.path.exists(self.config_file):
|
||||
|
||||
if __name__ == '__main__':
|
||||
print(self.config_file + " created!")
|
||||
with open(self.config_file, 'w'): pass
|
||||
|
||||
# Load secrets from secrets file
|
||||
secrets_filepath = "secrets/secrets.txt"
|
||||
uc_client_id = get_parameter("uc_client_id", secrets_filepath)
|
||||
uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
|
||||
uc_access_token = get_parameter("uc_access_token", secrets_filepath)
|
||||
@staticmethod
|
||||
def __write_config(self):
|
||||
|
||||
# Load configuration from config file
|
||||
config_filepath = "config/config.txt"
|
||||
mastodon_hostname = get_hostname("mastodon_hostname", config_filepath)
|
||||
bot_username = get_parameter("bot_username", config_filepath)
|
||||
with open(self.config_file, 'a') as the_file:
|
||||
|
||||
the_file.write(f'mastodon_hostname: {self.mastodon_hostname}')
|
||||
print(f"adding parameters to {self.config_file}\n")
|
||||
|
|
|
|||
Carregant…
Referencia en una nova incidència