This code gets all peers from running Mastodon, Pleroma or Lemmy host server and then all peers from host server's peers. Goal is to collect maximum number of alive fediverse's servers by querying their API and then post servers and registered users.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

893 lines
26 KiB

4 months ago
  1. import time
  2. import urllib3
  3. from datetime import datetime
  4. from mastodon import Mastodon
  5. import os
  6. import json
  7. import sys
  8. import os.path
  9. import requests
  10. import psycopg2
  11. from itertools import product
  12. import multiprocessing
  13. import aiohttp
  14. import asyncio
  15. import socket
  16. import matplotlib.pyplot as plt
  17. plt.style.use('seaborn')
  18. start_time = time.time()
  19. apis = ['/nodeinfo/2.0?', '/nodeinfo/2.0.json?', '/main/nodeinfo/2.0?', '/api/statusnet/config?', '/api/nodeinfo/2.0.json?', '/api/nodeinfo?', '/api/v1/instance?', '/wp-json/nodeinfo/2.0?']
  20. client_exceptions = (
  21. aiohttp.ClientResponseError,
  22. aiohttp.ClientConnectionError,
  23. aiohttp.ClientConnectorError,
  24. aiohttp.ClientError,
  25. asyncio.TimeoutError,
  26. socket.gaierror,
  27. )
  28. def is_json(myjson):
  29. try:
  30. json_object = json.loads(myjson)
  31. except ValueError as e:
  32. print(e)
  33. return False
  34. return True
  35. def get_alive_servers(server):
  36. serv_api = ''
  37. serv_soft = ''
  38. soft_version = ''
  39. try:
  40. conn = None
  41. conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
  42. cur = conn.cursor()
  43. cur.execute("select alive, software, users_api, version from fediverse where server=(%s)", (server,))
  44. row = cur.fetchone()
  45. if row is not None:
  46. was_alive = row[0]
  47. serv_soft = row[1]
  48. serv_api = row[2]
  49. soft_version = row[3]
  50. cur.close()
  51. except (Exception, psycopg2.DatabaseError) as error:
  52. print(error)
  53. finally:
  54. if conn is not None:
  55. conn.close()
  56. alive = False
  57. try:
  58. data = requests.get('https://' + server + serv_api, timeout=3)
  59. if serv_soft == "mastodon":
  60. if serv_api == '/nodeinfo/2.0?':
  61. try:
  62. users = data.json()['usage']['users']['total']
  63. soft_version = data.json()['software']['version']
  64. alive = True
  65. except:
  66. users = 0
  67. soft_version = ""
  68. if serv_api == '/nodeinfo/2.0.json?':
  69. try:
  70. users = data.json()['usage']['users']['total']
  71. soft_version = data.json()['software']['version']
  72. alive = True
  73. except:
  74. users = 0
  75. soft_version = ""
  76. elif serv_api == '/api/v1/instance?':
  77. try:
  78. users = data.json()['stats']['user_count']
  79. soft_version = data.json()['version']
  80. alive = True
  81. except:
  82. users = 0
  83. soft_version = ""
  84. if serv_soft == "pleroma" or serv_soft == "diaspora" or serv_soft == "peertube" or serv_soft == "pixelfed" or serv_soft == "hubzilla" or serv_soft == "writefreely" or serv_soft == "friendica":
  85. try:
  86. users = data.json()['usage']['users']['total']
  87. soft_version = data.json()['software']['version']
  88. alive = True
  89. except:
  90. users = 0
  91. soft_version = ""
  92. if serv_soft == "gnusocialv2" or serv_soft == "gnusocial":
  93. try:
  94. users = data.json()['usage']['users']['total']
  95. if users == 0:
  96. users = data.json()['usage']['users']['activeHalfyear']
  97. soft_version = data.json()['software']['version']
  98. alive = True
  99. except:
  100. users = 0
  101. soft_version = ""
  102. if serv_soft == "plume" or serv_soft == 'red' or serv_soft == "misskey" or serv_soft == "zap" or serv_soft == "prismo" or serv_soft == "ravenvale" or serv_soft == "osada" or serv_soft == "groundpolis":
  103. try:
  104. users = data.json()['usage']['users']['total']
  105. soft_version = data.json()['software']['version']
  106. alive = True
  107. except:
  108. users = 0
  109. soft_version = ""
  110. if serv_soft == "ganggo" or serv_soft == "squs" or serv_soft == "dolphin" or serv_soft == "lemmy" or serv_soft == "wordpress":
  111. try:
  112. users = data.json()['usage']['users']['total']
  113. soft_version = data.json()['software']['version']
  114. if serv_soft == "wordpress" and "activitypub" in data.json()['protocols']:
  115. alive = True
  116. elif serv_soft == "wordpress" and "activitypub" not in data.json()['protocols']:
  117. alive = False
  118. else:
  119. alive = True
  120. except:
  121. users = 0
  122. soft_version = ""
  123. if alive:
  124. if soft_version != "" and soft_version is not None:
  125. print("Server " + str(server) + " (" + serv_soft + " " + soft_version + ") is alive!")
  126. else:
  127. print("Server " + str(server) + " (" + serv_soft + ") is alive!")
  128. insert_sql = "INSERT INTO fediverse(server, users, updated_at, software, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
  129. conn = None
  130. try:
  131. conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
  132. cur = conn.cursor()
  133. cur.execute(insert_sql, (server, users, now, serv_soft, alive, serv_api, soft_version))
  134. cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s), software=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)", (users, now, serv_soft, alive, serv_api, soft_version, server))
  135. cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
  136. conn.commit()
  137. cur.close()
  138. except (Exception, psycopg2.DatabaseError) as error:
  139. print(error)
  140. finally:
  141. if conn is not None:
  142. conn.close()
  143. except urllib3.exceptions.ProtocolError as protoerr:
  144. print(protoerr)
  145. print("Server " + server + " is dead :-(")
  146. alive = False
  147. pass
  148. except requests.exceptions.ChunkedEncodingError as chunkerr:
  149. print(chunkerr)
  150. print("Server " + server + " is dead :-(")
  151. alive = False
  152. pass
  153. except KeyError as e:
  154. print(e)
  155. print("Server " + server + " is dead :-(")
  156. alive = False
  157. pass
  158. except ValueError as verr:
  159. print(verr)
  160. print("Server " + server + " is dead :-(")
  161. alive = False
  162. pass
  163. except requests.exceptions.SSLError as errssl:
  164. print(errssl)
  165. print("Server " + server + " is dead :-(")
  166. alive = False
  167. pass
  168. except requests.exceptions.HTTPError as errh:
  169. print(errh)
  170. print("Server " + server + " is dead :-(")
  171. alive = False
  172. pass
  173. except requests.exceptions.ConnectionError as errc:
  174. print(errc)
  175. print("Server " + server + " is dead :-(")
  176. alive = False
  177. pass
  178. except requests.exceptions.Timeout as errt:
  179. print(errt)
  180. print("Server " + server + " is dead :-(")
  181. alive = False
  182. pass
  183. except requests.exceptions.RequestException as err:
  184. print(err)
  185. print("Server " + server + " is dead :-(")
  186. alive = False
  187. pass
  188. except socket.gaierror as gai_error:
  189. print(gai_error)
  190. pass
  191. if alive is False:
  192. conn = None
  193. try:
  194. conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="",
  195. host="/var/run/postgresql", port="5432")
  196. cur = conn.cursor()
  197. cur.execute("UPDATE fediverse SET updated_at=(%s), alive=(%s) where server=(%s)", (now, alive, server))
  198. cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
  199. conn.commit()
  200. cur.close()
  201. except (Exception, psycopg2.DatabaseError) as error:
  202. print(error)
  203. finally:
  204. if conn is not None:
  205. conn.close()
  206. def write_api(server, software, users, alive, api, soft_version):
  207. insert_sql = "INSERT INTO fediverse(server, updated_at, software, users, alive, users_api, version) VALUES(%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING"
  208. conn = None
  209. try:
  210. conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
  211. cur = conn.cursor()
  212. cur.execute(insert_sql, (server, now, software, users, alive, api, soft_version))
  213. cur.execute("UPDATE fediverse SET updated_at=(%s), software=(%s), users=(%s), alive=(%s), users_api=(%s), version=(%s) where server=(%s)", (now, software, users, alive, api, soft_version, server))
  214. cur.execute("UPDATE world SET checked='t' where server=(%s)", (server,))
  215. conn.commit()
  216. cur.close()
  217. except (Exception, psycopg2.DatabaseError) as error:
  218. print(error)
  219. finally:
  220. if conn is not None:
  221. conn.close()
  222. async def getsoft(server):
  223. try:
  224. socket.gethostbyname(server)
  225. except socket.gaierror:
  226. pass
  227. return
  228. soft = ''
  229. url = 'https://' + server
  230. timeout = aiohttp.ClientTimeout(total=3)
  231. async with aiohttp.ClientSession(timeout=timeout) as session:
  232. for api in apis:
  233. try:
  234. async with session.get(url+api) as response:
  235. if response.status == 200:
  236. try:
  237. response_json = await response.json()
  238. except:
  239. pass
  240. except aiohttp.ClientConnectorError as err:
  241. print(err)
  242. pass
  243. else:
  244. if response.status == 200 and api != '/api/v1/instance?':
  245. try:
  246. soft = response_json['software']['name']
  247. soft = soft.lower()
  248. soft_version = response_json['software']['version']
  249. users = response_json['usage']['users']['total']
  250. if users > 1000000:
  251. return
  252. alive = True
  253. write_api(server, soft, users, alive, api, soft_version)
  254. print("Server " + server + " (" + soft + " " + soft_version + ") is alive!")
  255. return
  256. except:
  257. pass
  258. if response.status == 200 and soft == '' and api == "/api/v1/instance?":
  259. soft = 'mastodon'
  260. users = response_json['stats']['user_count']
  261. soft_version = response_json['version']
  262. if users > 1000000:
  263. return
  264. alive = True
  265. write_api(server, soft, users, alive, api)
  266. print("Server " + server + " (" + soft + ") is alive!")
  267. def getserver(server, x):
  268. server = server[0].rstrip('.').lower()
  269. if server.find(".") == -1:
  270. return
  271. if server.find("@") != -1:
  272. return
  273. if server.find("/") != -1:
  274. return
  275. if server.find(":") != -1:
  276. return
  277. try:
  278. loop = asyncio.get_event_loop()
  279. coroutines = [getsoft(server)]
  280. soft = loop.run_until_complete(asyncio.gather(*coroutines, return_exceptions=True))
  281. except:
  282. pass
  283. def set_world_servers_check_to_false():
  284. ############################################################################
  285. # set all world servers's checked column to False
  286. try:
  287. conn = None
  288. conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
  289. cur = conn.cursor()
  290. cur.execute("UPDATE world SET checked='f'")
  291. conn.commit()
  292. cur.close()
  293. except (Exception, psycopg2.DatabaseError) as error:
  294. print(error)
  295. finally:
  296. if conn is not None:
  297. conn.close()
  298. def get_last_checked_servers():
  299. ############################################################################
  300. # get last checked servers from fediverse DB
  301. alive_servers = []
  302. try:
  303. conn = None
  304. conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
  305. cur = conn.cursor()
  306. # get world servers list
  307. cur.execute("select server from world where server in (select server from fediverse where users_api != '')")
  308. alive_servers = []
  309. for row in cur:
  310. alive_servers.append(row[0])
  311. cur.close()
  312. except (Exception, psycopg2.DatabaseError) as error:
  313. print(error)
  314. finally:
  315. if conn is not None:
  316. conn.close()
  317. return alive_servers
  318. def mastodon():
  319. # Load secrets from secrets file
  320. secrets_filepath = "secrets/secrets.txt"
  321. uc_client_id = get_parameter("uc_client_id", secrets_filepath)
  322. uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
  323. uc_access_token = get_parameter("uc_access_token", secrets_filepath)
  324. # Load configuration from config file
  325. config_filepath = "config/config.txt"
  326. mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
  327. # Initialise Mastodon API
  328. mastodon = Mastodon(
  329. client_id=uc_client_id,
  330. client_secret=uc_client_secret,
  331. access_token=uc_access_token,
  332. api_base_url='https://' + mastodon_hostname,
  333. )
  334. # Initialise access headers
  335. headers = {'Authorization': 'Bearer %s'%uc_access_token}
  336. return (mastodon, mastodon_hostname)
  337. def db_config():
  338. # Load db configuration from config file
  339. config_filepath = "config/db_config.txt"
  340. fediverse_db = get_parameter("fediverse_db", config_filepath)
  341. fediverse_db_user = get_parameter("fediverse_db_user", config_filepath)
  342. return (fediverse_db, fediverse_db_user)
  343. def usage():
  344. print('usage: python ' + sys.argv[0] + ' --multi' + ' (multiprocessing, fast)')
  345. print('usage: python ' + sys.argv[0] + ' --mono' + ' (one process, slow)')
  346. # Returns the parameter from the specified file
  347. def get_parameter(parameter, file_path):
  348. # Check if secrets file exists
  349. if not os.path.isfile(file_path):
  350. print("File %s not found, exiting."%file_path)
  351. sys.exit(0)
  352. # Find parameter in file
  353. with open(file_path) as f:
  354. for line in f:
  355. if line.startswith(parameter):
  356. return line.replace(parameter + ":", "").strip()
  357. # Cannot find parameter, exit
  358. print(file_path + " Missing parameter %s "%parameter)
  359. sys.exit(0)
  360. ###############################################################################
  361. # main
  362. if __name__ == '__main__':
  363. # usage modes
  364. if len(sys.argv) == 1:
  365. usage()
  366. elif len(sys.argv) == 2:
  367. if sys.argv[1] == '--multi':
  368. now = datetime.now()
  369. mastodon, mastodon_hostname = mastodon()
  370. fediverse_db, fediverse_db_user = db_config()
  371. total_servers = 0
  372. total_users = 0
  373. set_world_servers_check_to_false()
  374. alive_servers = get_last_checked_servers()
  375. ###########################################################################
  376. # multiprocessing!
  377. nprocs = multiprocessing.cpu_count()
  378. with multiprocessing.Pool(processes=64) as pool:
  379. results = pool.starmap(get_alive_servers, product(alive_servers))
  380. elif sys.argv[1] == '--mono':
  381. now = datetime.now()
  382. mastodon, mastodon_hostname = mastodon()
  383. fediverse_db, fediverse_db_user = db_config()
  384. total_servers = 0
  385. total_users = 0
  386. set_world_servers_check_to_false()
  387. alive_servers = get_last_checked_servers()
  388. i = 0
  389. while i < len(alive_servers):
  390. get_alive_servers(alive_servers[i])
  391. i += 1
  392. else:
  393. usage()
  394. ###########################################################################
  395. # get current total servers and users, get users from every software
  396. gettotals_sql = "select count(server), sum(users) from fediverse where alive"
  397. get_soft_totals_sql = "select software, sum(users) as users, count(server) as servers from fediverse where users != 0 and alive group by software order by users desc"
  398. soft_total_project = []
  399. soft_total_users = []
  400. soft_total_servers = []
  401. try:
  402. conn = None
  403. conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
  404. cur = conn.cursor()
  405. cur.execute(gettotals_sql)
  406. row = cur.fetchone()
  407. total_servers = row[0]
  408. total_users = row[1]
  409. cur.execute(get_soft_totals_sql)
  410. rows = cur.fetchall()
  411. for row in rows:
  412. soft_total_project.append(row[0])
  413. soft_total_users.append(row[1])
  414. soft_total_servers.append(row[2])
  415. cur.close()
  416. except (Exception, psycopg2.DatabaseError) as error:
  417. print(error)
  418. finally:
  419. if conn is not None:
  420. conn.close()
  421. ###########################################################################
  422. # get last check values and write current total ones
  423. select_sql = "select total_servers, total_users from totals order by datetime desc limit 1"
  424. insert_sql = "INSERT INTO totals(datetime, total_servers, total_users) VALUES(%s,%s,%s)"
  425. try:
  426. conn = None
  427. conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
  428. cur = conn.cursor()
  429. cur.execute(select_sql)
  430. row = cur.fetchone()
  431. if row is not None:
  432. servers_before = row[0]
  433. users_before = row[1]
  434. else:
  435. servers_before = 0
  436. users_before = 0
  437. cur.execute(insert_sql, (now, total_servers, total_users))
  438. conn.commit()
  439. cur.close()
  440. evo_servers = total_servers - servers_before
  441. evo_users = total_users - users_before
  442. except (Exception, psycopg2.DatabaseError) as error:
  443. print(error)
  444. finally:
  445. if conn is not None:
  446. conn.close()
  447. ################################################################################
  448. # write evo values
  449. insert_sql = "INSERT INTO evo(datetime, servers, users) VALUES(%s,%s,%s)"
  450. conn = None
  451. try:
  452. conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
  453. cur = conn.cursor()
  454. cur.execute(insert_sql, (now, evo_servers, evo_users))
  455. conn.commit()
  456. cur.close()
  457. except (Exception, psycopg2.DatabaseError) as error:
  458. print(error)
  459. finally:
  460. if conn is not None:
  461. conn.close()
  462. ##############################################################################
  463. # get world's last update datetime
  464. conn = None
  465. try:
  466. conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
  467. cur = conn.cursor()
  468. cur.execute("select updated_at from world order by updated_at desc limit 1")
  469. row = cur.fetchone()
  470. last_update = row[0]
  471. last_update = last_update.strftime('%m/%d/%Y, %H:%M:%S')
  472. cur.close()
  473. except (Exception, psycopg2.DatabaseError) as error:
  474. print(error)
  475. finally:
  476. if conn is not None:
  477. conn.close()
  478. ##############################################################################
  479. # get max servers and users
  480. conn = None
  481. try:
  482. conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
  483. cur = conn.cursor()
  484. cur.execute("select MAX(total_servers) from totals")
  485. row = cur.fetchone()
  486. if row is not None:
  487. max_servers = row[0]
  488. else:
  489. max_servers = 0
  490. cur.execute("select MAX(total_users) from totals")
  491. row = cur.fetchone()
  492. if row is not None:
  493. max_users = row[0]
  494. else:
  495. max_users = 0
  496. cur.close()
  497. except (Exception, psycopg2.DatabaseError) as error:
  498. print(error)
  499. finally:
  500. if conn is not None:
  501. conn.close()
  502. ###############################################################################
  503. # get plots
  504. servers_plots = []
  505. users_plots = []
  506. conn = None
  507. try:
  508. conn = psycopg2.connect(database=fediverse_db, user=fediverse_db_user, password="", host="/var/run/postgresql", port="5432")
  509. cur = conn.cursor()
  510. cur.execute("select total_servers, total_users from totals order by datetime desc limit 14")
  511. rows = cur.fetchall()
  512. for row in rows:
  513. servers_plots.append(row[0])
  514. users_plots.append(row[1])
  515. cur.close()
  516. except (Exception, psycopg2.DatabaseError) as error:
  517. print(error)
  518. finally:
  519. if conn is not None:
  520. conn.close()
  521. ###############################################################################
  522. # generate graphs
  523. plt.plot([-6, -5, -4, -3, -2, -1, 0], [servers_plots[6], servers_plots[5], servers_plots[4], servers_plots[3], servers_plots[2], servers_plots[1], servers_plots[0]], marker='o', color='mediumseagreen')
  524. plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_servers, max_servers, max_servers, max_servers, max_servers, max_servers, max_servers], color='red')
  525. plt.title('fediverse: total alive servers (max: ' + str(f"{max_servers:,}" + ')'), loc='right', color='blue')
  526. plt.xlabel('Last seven days')
  527. plt.ylabel('fediverse alive servers')
  528. plt.legend(('servers', 'max'), shadow=True, loc=(0.01, 1.00), handlelength=1.5, fontsize=10)
  529. plt.savefig('servers.png')
  530. plt.close()
  531. plt.plot([-6, -5, -4, -3, -2, -1, 0], [users_plots[6], users_plots[5], users_plots[4], users_plots[3], users_plots[2], users_plots[1], users_plots[0]], marker='o', color='royalblue')
  532. plt.plot([-6, -5, -4, -3, -2, -1, 0], [max_users, max_users, max_users, max_users, max_users, max_users, max_users], color='red')
  533. plt.title('fediverse: total registered users (max: ' + str(f"{max_users:,}" + ')'), loc='right', color='royalblue')
  534. plt.legend(('users', 'max'), shadow=True, loc=(0.01, 0.80), handlelength=1.5, fontsize=10)
  535. plt.xlabel('Last seven days')
  536. plt.ylabel('Registered users')
  537. plt.savefig('users.png')
  538. plt.close()
  539. ###############################################################################
  540. # T O O T !
  541. toot_text = "#fediverse alive servers stats" + " \n"
  542. toot_text += "\n"
  543. if evo_servers >= 0:
  544. toot_text += "alive servers: " + str(f"{total_servers:,}") + " (+"+ str(f"{evo_servers:,}") + ") \n"
  545. toot_text += "max: " + str(f"{max_servers:,}") + "\n"
  546. elif evo_servers < 0:
  547. toot_text += "alive servers: " + str(f"{total_servers:,}") + " ("+ str(f"{evo_servers:,}") + ") \n"
  548. toot_text += "max: " + str(f"{max_servers:,}") + "\n"
  549. if evo_users >= 0:
  550. toot_text += "total users: " + str(f"{total_users:,}") + " (+"+ str(f"{evo_users:,}") + ") \n"
  551. toot_text += "max: " + str(f"{max_users:,}") + "\n"
  552. elif evo_users < 0:
  553. toot_text += "total users: " + str(f"{total_users:,}") + " ("+ str(f"{evo_users:,}") + ") \n"
  554. toot_text += "max: " + str(f"{max_users:,}") + "\n"
  555. toot_text += "\n"
  556. toot_text += "top ten (soft users servers):" + " \n"
  557. toot_text += "\n"
  558. i = 0
  559. while i < 10:
  560. project_soft = soft_total_project[i]
  561. project_users = soft_total_users[i]
  562. project_servers = soft_total_servers[i]
  563. len_pr_soft = len(project_soft)
  564. if project_soft == 'writefreely':
  565. str_len = 11
  566. else:
  567. str_len = 13
  568. toot_text += f"{':'+project_soft+':':<11}" + f"{project_users:>{str_len},}" + " " + f"{project_servers:>5,}" + "\n"
  569. i += 1
  570. print("Tooting...")
  571. print(toot_text)
  572. servers_image_id = mastodon.media_post('servers.png', "image/png").id
  573. users_image_id = mastodon.media_post('users.png', "image/png").id
  574. mastodon.status_post(toot_text, in_reply_to_id=None, media_ids={servers_image_id, users_image_id})