This code gets all peers from running Mastodon, Pleroma or Lemmy host server and then all peers from host server's peers. Goal is to collect maximum number of alive fediverse's servers by querying their API and then post servers and registered users.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

153 lines
3.8 KiB

  1. import os
  2. import sys
  3. import time
  4. from datetime import datetime
  5. import requests
  6. import threading
  7. import json
  8. import psycopg2
  9. import pdb
  10. def write_servers(world_servers):
  11. i = 0
  12. while i < len(world_servers):
  13. saved_at = datetime.now()
  14. insert_sql = "INSERT INTO world(server, federated_with, updated_at, saved_at) VALUES(%s,%s,%s,%s) ON CONFLICT DO NOTHING"
  15. conn = None
  16. try:
  17. conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")
  18. cur = conn.cursor()
  19. cur.execute(insert_sql, (world_servers[i], peer, updated_at, saved_at,))
  20. print('writing to database ' + str(i) + ' of ' + str(len(world_servers)) + ': ' + world_servers[i])
  21. conn.commit()
  22. cur.close()
  23. except (Exception, psycopg2.DatabaseError) as error:
  24. print(error)
  25. finally:
  26. if conn is not None:
  27. conn.close()
  28. i += 1
  29. def get_peers(peer):
  30. try:
  31. response = requests.get('https://' + peer + peers_api, timeout=2)
  32. response_json = response.json()
  33. if response.status_code == 200:
  34. try:
  35. print("Server: " + peer + ", " + "federated with " + str(len(response_json)) + " servers")
  36. for peer_peer in response_json:
  37. exist_count = world_peers.count(peer_peer)
  38. if exist_count == 0:
  39. world_peers.append(peer_peer)
  40. except:
  41. pass
  42. except:
  43. pass
  44. class Peerthreads(threading.Thread):
  45. def __init__(self, peer, peers_semaphore):
  46. threading.Thread.__init__(self)
  47. self.sql_conn = None
  48. self.peer = peer
  49. self.peers_semaphore = peers_semaphore
  50. def run(self):
  51. self.peers_semaphore.acquire()
  52. get_peers(self.peer)
  53. self.peers_semaphore.release()
  54. # Returns the parameter from the specified file
  55. def get_parameter( parameter, file_path ):
  56. # Check if secrets file exists
  57. if not os.path.isfile(file_path):
  58. print("File %s not found, exiting."%file_path)
  59. sys.exit(0)
  60. # Find parameter in file
  61. with open( file_path ) as f:
  62. for line in f:
  63. if line.startswith( parameter ):
  64. return line.replace(parameter + ":", "").strip()
  65. # Cannot find parameter, exit
  66. print(file_path + " Missing parameter %s "%parameter)
  67. sys.exit(0)
  68. ###############################################################################
  69. # main
  70. if __name__ == '__main__':
  71. start_time = time.time()
  72. updated_at = datetime.now()
  73. peers_api = '/api/v1/instance/peers?'
  74. lemmy_api = '/api/v2/site?'
  75. # Load configuration from config file
  76. config_filepath = "config/config.txt"
  77. mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
  78. # Load database config from db_config file
  79. db_config_filepath = "config/db_config.txt"
  80. fediverse_db = get_parameter("fediverse_db", db_config_filepath)
  81. fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath)
  82. world_peers = []
  83. res = requests.get('https://' + mastodon_hostname + peers_api)
  84. hostname_peers = res.json()
  85. for peer in hostname_peers:
  86. exist_count = world_peers.count(peer)
  87. if exist_count == 0:
  88. world_peers.append(peer)
  89. peers_semaphore = threading.Semaphore(100)
  90. peer_threads = []
  91. for peer in hostname_peers:
  92. p_thread = Peerthreads(peer, peers_semaphore)
  93. peer_threads.append(p_thread)
  94. p_thread.start()
  95. for p_thread in peer_threads:
  96. p_thread.join()
  97. write_servers(world_peers)
  98. exec_time = str(round((time.time() - start_time), 2))
  99. print(exec_time)