This code gets all peers from running Mastodon, Pleroma or Lemmy host server and then all peers from host server's peers. Goal is to collect maximum number of alive fediverse's servers by querying their API and then post servers and registered users.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

225 lines
6.1 KiB

  1. import time
  2. from datetime import datetime
  3. import os
  4. import json
  5. import sys
  6. import os.path
  7. import psycopg2
  8. from multiprocessing import Pool, Manager
  9. import aiohttp
  10. import asyncio
  11. import socket
  12. apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?',
  13. '/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?', '/wp-json/nodeinfo/2.0?']
  14. client_exceptions = (
  15. aiohttp.ClientResponseError,
  16. aiohttp.ClientConnectionError,
  17. aiohttp.ClientConnectorError,
  18. aiohttp.ClientError,
  19. asyncio.TimeoutError,
  20. socket.gaierror,
  21. )
  22. def is_json(myjson):
  23. try:
  24. json_object = json.loads(myjson)
  25. except ValueError as e:
  26. return False
  27. return True
  28. def write_api(server, software, users, alive, api, soft_version):
  29. insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
  30. conn = None
  31. try:
  32. conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql",
  33. port="5432")
  34. cur = conn.cursor()
  35. cur.execute(insert_sql, (server, now, software, users, alive, api, soft_version))
  36. cur.execute(
  37. "UPDATE fediverse SET updated_at=(%s), software=(%s), users=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)",
  38. (now, software, users, alive, api, soft_version, server))
  39. cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
  40. conn.commit()
  41. cur.close()
  42. except (Exception, psycopg2.DatabaseError) as error:
  43. print(error)
  44. finally:
  45. if conn is not None:
  46. conn.close()
  47. async def getsoft(server):
  48. try:
  49. socket.gethostbyname(server)
  50. except socket.gaierror:
  51. pass
  52. return
  53. soft = ''
  54. url = 'https://' + server
  55. timeout = aiohttp.ClientTimeout(total=3)
  56. async with aiohttp.ClientSession(timeout=timeout) as session:
  57. for api in apis:
  58. try:
  59. async with session.get(url + api) as response:
  60. if response.status == 200:
  61. try:
  62. response_json = await response.json()
  63. except:
  64. pass
  65. except aiohttp.ClientConnectorError as err:
  66. pass
  67. else:
  68. if response.status == 200 and api != '/api/v1/instance?':
  69. try:
  70. soft = response_json['software']['name']
  71. soft = soft.lower()
  72. soft_version = response_json['software']['version']
  73. users = response_json['usage']['users']['total']
  74. if users > 1000000:
  75. return
  76. alive = True
  77. write_api(server, soft, users, alive, api, soft_version)
  78. print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
  79. return
  80. except:
  81. pass
  82. if response.status == 200 and soft == '' and api == "/api/v1/instance?":
  83. soft = 'mastodon'
  84. users = response_json['stats']['user_count']
  85. soft_version = response_json['version']
  86. if users > 1000000:
  87. return
  88. alive = True
  89. write_api(server, soft, users, alive, api)
  90. print("Server " + server + " (" + soft + ") is alive!")
  91. def getserver(server, x):
  92. server = server[0].rstrip('.').lower()
  93. if server.find(".") == -1:
  94. return
  95. if server.find("@") != -1:
  96. return
  97. if server.find("/") != -1:
  98. return
  99. if server.find(":") != -1:
  100. return
  101. try:
  102. loop = asyncio.get_event_loop()
  103. coroutines = [getsoft(server)]
  104. soft = loop.run_until_complete(asyncio.gather(*coroutines, return_exceptions=True))
  105. except:
  106. pass
  107. # Returns the parameter from the specified file
  108. def get_parameter(parameter, file_path):
  109. # Check if secrets file exists
  110. if not os.path.isfile(file_path):
  111. print("File %s not found, exiting." % file_path)
  112. sys.exit(0)
  113. # Find parameter in file
  114. with open(file_path) as f:
  115. for line in f:
  116. if line.startswith(parameter):
  117. return line.replace(parameter + ":", "").strip()
  118. # Cannot find parameter, exit
  119. print(file_path + " Missing parameter %s " % parameter)
  120. sys.exit(0)
  121. # Load configuration from config file
  122. config_filepath = "config/config.txt"
  123. mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
  124. # Load database config from db_config file
  125. db_config_filepath = "config/db_config.txt"
  126. fediverse_db = get_parameter("fediverse_db", db_config_filepath)
  127. fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath)
  128. ###############################################################################
  129. # main
  130. if __name__ == '__main__':
  131. now = datetime.now()
  132. start_time = time.time()
  133. world_servers = []
  134. try:
  135. conn = None
  136. conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
  137. cur = conn.cursor()
  138. # get world servers list
  139. cur.execute("select server from world where checked='f'")
  140. rows = cur.fetchall()
  141. for row in rows:
  142. world_servers.append(row[0])
  143. cur.close()
  144. print("Remaining servers: " + str(len(world_servers)))
  145. except (Exception, psycopg2.DatabaseError) as error:
  146. print(error)
  147. finally:
  148. if conn is not None:
  149. conn.close()
  150. ###########################################################################
  151. # multiprocessing!
  152. m = Manager()
  153. q = m.Queue()
  154. z = zip(world_servers)
  155. serv_number = len(world_servers)
  156. pool_tuple = [(x, q) for x in z]
  157. with Pool(processes=64) as pool:
  158. pool.starmap(getserver, pool_tuple)
  159. print('Done.')