#!/usr/bin/env python
# -*- coding: utf-8 -*-

import time
start_time = time.time()
import urllib3
from urllib3 import exceptions
from datetime import datetime
import pytz
from subprocess import call
from mastodon import Mastodon
import threading
import os
import json
import signal
import sys
import os.path
import requests
from requests import exceptions
import operator
import calendar
import psycopg2
from itertools import product

from multiprocessing import Pool, Lock, Process, Queue, current_process, Manager
import queue
import multiprocessing
import tqdm

from decimal import *
getcontext().prec = 2

###############################################################################
# INITIALISATION
###############################################################################

def is_json(myjson):
  try:
    json_object = json.loads(myjson)
  except ValueError as e:
    return False
  return True

def getserver(server):

  #server = server[0].rstrip('.').lower()

  if server.find(".") == -1:
      #print(server + "is Not a real domain!")
      return
  if server.find("@") != -1:
     #print(server + "is Not a real domain!")
     return
  if server.find("/") != -1:
     #print("Removing substring after first / from " + server)
     server = server.split("/", 1)[0]
  if server.find(":") != -1:
     #print("Removing substring after first : from " + server)
     server = server.split(":", 1)[0]

  global mastodont
  global pleroma
  global gnusocial
  global zap
  global plume
  global hubzilla
  global misskey
  global prismo
  global osada
  global groundpolis
  global ganggo
  global squs
  global peertube
  global friendica
  global pixelfed
  global writefreely
  global ravenvale
  global diaspora

  global pl_users
  global total_pl_users
  global mast_users
  global total_mast_users
  global gs_users
  global total_gs_users
  global usuaris_zap
  global total_zap_users
  global usuaris_plume
  global total_plume_users
  global usuaris_hubzilla
  global total_hubzilla_users
  global usuaris_misskey
  global total_misskey_users
  global usuaris_prismo
  global total_prismo_users
  global usuaris_osada
  global total_osada_users
  global usuaris_gpolis
  global total_gpolis_users
  global usuaris_ggg
  global total_ggg_users
  global usuaris_squs
  global total_squs_users
  global peertube_users
  global total_peertube_users
  global diaspora_users
  global total_diaspora_users
  global usuaris_friendica
  global total_friendica_users
  global usuaris_pixelfed
  global total_pixelfed_users
  global usuaris_writefreely
  global total_writefreely_users
  global ravenvale_users
  global total_ravenvale_users
  global usuaris_total

  check_diaspora = False
  check_peertube = False
  check_zap = False
  check_plume = False
  check_hubzilla = False
  check_misskey = False
  check_prismo = False
  check_osada = False
  check_groundpolis = False
  check_ganggo = False
  check_squs = False
  check_gnusocial = False
  check_friendica = False
  check_pixelfed = False
  check_writefreely = False
  check_raven = False
  check_gnusocial2 = False

  usuaris = 0

  try:

    res = requests.get('https://' + server + '/api/v1/instance?',timeout=3)
    res_friendica = requests.get('https://' + server + '/nodeinfo/2.0?',timeout=3)
    check_pix = "compatible; Pixelfed" in res.text
    check_ravenvale = "ravenvale" in res.text
    check_friendica = "friendica" in res_friendica.text
    if check_friendica == False:
      check_friendica = "Friendica" in res_friendica.text

    if (res.ok) and check_pix == False and check_ravenvale == False and check_friendica == False:

      if is_json(res.text) == True and server in res.text:

        nodeinfo = requests.get('https://' + server + '/nodeinfo/2.0.json?',timeout=3)
        check_pleroma = "pleroma" in nodeinfo.text.lower()

        if nodeinfo.status_code == 200 and check_pleroma == True:

          #print("Servidor Pleroma: ")
          pl_users = nodeinfo.json()['usage']['users']['total']
          total_pl_users = total_pl_users + pl_users
          usuaris = pl_users
          #print(server, pl_users)

        else:

          if type(res.json()) != int:

              if res.json().get('stats') != None:
                  #print("Servidor Mastodon: ")
                  mast_users = res.json()['stats']['user_count']
              else:
                  mast_users = 0
          else:

              mast_users = 0

          if mast_users != None:
            total_mast_users = total_mast_users + mast_users
          usuaris = mast_users
          #print(server, mast_users)

        #print("\n")

        insert_sql = "INSERT INTO fediverse(server, users, updated_at) VALUES(%s,%s,%s) ON CONFLICT DO NOTHING"
        conn = None

        try:

          conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")

          cur = conn.cursor()

          cur.execute(insert_sql, (server, usuaris, now))

          cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s) where server=(%s)", (usuaris, now, server))

          conn.commit()

          cur.close()

        except (Exception, psycopg2.DatabaseError) as error:
          print(error)

        finally:

          if conn is not None:
            conn.close()
          return

      else:

        return
 
  except urllib3.exceptions.ProtocolError as protoerr:

      pass

  except requests.exceptions.ChunkedEncodingError as chunkerr:

      pass

  except KeyError as e:

      pass

  except ValueError as verr:

      pass

  except requests.exceptions.SSLError as errssl:

      pass

  except requests.exceptions.HTTPError as errh:

      pass

  except requests.exceptions.ConnectionError as errc:

      pass

  except requests.exceptions.Timeout as errt:

      pass

  except requests.exceptions.RequestException as err:

      pass

  else:

    try:

      soft = ""
      others_nodeinfo = requests.get('https://' + server + '/nodeinfo/2.0?',timeout=3)
      gs_nodeinfo = requests.get('https://' + server + '/main/nodeinfo/2.0?',timeout=3)
      nodeinfo = requests.get('https://' + server + '/nodeinfo/2.0.json?',timeout=3)
      statusnet = requests.get('https://' + server + '/api/statusnet/config?',timeout=3)
      px_nodeinfo = requests.get('https://' + server + '/api/nodeinfo/2.0.json?',timeout=3)
      wf_nodeinfo = requests.get('https://' + server + '/api/nodeinfo?',timeout=3)

      try:

        if is_json(nodeinfo.text) == True:
          if nodeinfo.json() != None:
            soft = nodeinfo.json()['software']['name']
            if soft == "peertube":
              check_peertube = True
            elif soft == "prismo":
              check_prismo = True
            elif soft == "zap":
              check_zap = True
            elif soft == "osada":
              check_osada = True
            elif soft == "ravenvale":
              check_raven = True
            elif soft == "diaspora":
              check_diaspora = True

        if is_json(others_nodeinfo.text) == True:
          soft = others_nodeinfo.json()['software']['name']
          if soft == "plume":
            check_plume = True
          elif soft == "hubzilla":
            check_hubzilla = True
          elif soft == "misskey":
            check_misskey = True
          elif soft == "groundpolis":
            check_groundpolis = True
          elif soft == "ganggo":
            check_ganggo = True
          elif soft == "squs":
            check_squs = True
          elif soft == "friendica" or soft == "Friendica":
            check_friendica = True

        if is_json(wf_nodeinfo.text) == True and soft == "":
          if wf_nodeinfo.json().get('error') != None and wf_nodeinfo.json().get('status') == None:
            soft = wf_nodeinfo.json()['software']['name']
            if soft == "writefreely":
              check_writefreely = True

        if is_json(gs_nodeinfo.text) == True:
          soft = gs_nodeinfo.json()['software']['name']
          if soft == "gnusocial":
            check_gnusocial = True

        if is_json(statusnet.text) == True and soft == "":
          if soft != "zap" and soft != "osada":
            soft = statusnet.json()['site']['friendica']['FRIENDICA_PLATFORM']
            if soft == "Friendica":
              check_friendica = True

        if is_json(px_nodeinfo.text) == True and soft == "":
           if soft != "Friendica" and soft != "hubzilla":
             soft = px_nodeinfo.json()['software']['name']
             if soft == "pixelfed":
               check_pixelfed = True
             if soft == "gnusocial":
               check_gnusocial2 = True

      except :

          pass

      if check_diaspora == False and check_peertube == False and check_zap == False and check_plume == False and check_hubzilla == False and check_misskey == False and check_prismo == False and check_osada == False and check_groundpolis == False and check_ganggo == False and check_squs == False and check_gnusocial == False and check_friendica == False and check_pixelfed == False and check_writefreely == False and check_raven == False and check_gnusocial2 == False:

        api_reply = 0
        pass

      elif check_diaspora == True or check_peertube == True or check_zap == True or check_plume == True or check_hubzilla == True or check_misskey == True or check_prismo == True or check_osada == True or check_groundpolis == True or check_ganggo == True or check_squs == True or check_gnusocial == True or check_friendica == True or check_pixelfed == True or check_writefreely == True or check_raven == True or check_gnusocial2 == True:

        api_reply = 1

        if nodeinfo.ok == True and check_peertube == True:

          #print("Servidor Peertube")
          peertube_users = nodeinfo.json()['usage']['users']['total']
          total_peertube_users = total_peertube_users + peertube_users
          usuaris = peertube_users
          #print(server, peertube_users)
          #print("\n")

        if nodeinfo.ok == True and check_diaspora == True:

          #print("Servidor Diaspora")
          diaspora_users = nodeinfo.json()['usage']['users']['total']
          total_diaspora_users = total_diaspora_users + diaspora_users
          usuaris = diaspora_users
          #print(server, diaspora_users)
          #print("\n")

        elif nodeinfo.ok == True and check_raven == True:

          #print("Servidor Ravenvale")
          ravenvale_users = nodeinfo.json()['usage']['users']['total']
          total_ravenvale_users = total_ravenvale_users + ravenvale_users
          usuaris = ravenvale_users
          #print(server, ravenvale_users)
          #print("\n")

        elif others_nodeinfo.ok == True and check_zap == True:

          #print("Servidor Zap")
          usuaris_zap = others_nodeinfo.json()['usage']['users']['total']
          total_zap_users = total_zap_users + usuaris_zap
          usuaris = usuaris_zap
          #print(server, usuaris_zap)
          #print("\n")

        elif others_nodeinfo.ok == True and check_plume == True:

          #print("Servidor Plume")
          usuaris_plume = others_nodeinfo.json()['usage']['users']['total']
          total_plume_users = total_plume_users + usuaris_plume
          usuaris = usuaris_plume
          #print(server, usuaris_plume)
          #print("\n")

        elif others_nodeinfo.ok == True and check_hubzilla == True:

          #print("Servidor Hubzilla")
          usuaris_hubzilla = others_nodeinfo.json()['usage']['users']['total']
          total_hubzilla_users = total_hubzilla_users + usuaris_hubzilla
          usuaris = usuaris_hubzilla
          #print(server, usuaris_hubzilla)
          #print("\n")

        elif others_nodeinfo.ok == True and check_misskey == True:

          #print("Servidor Misskey")
          usuaris_misskey = 0
          total_misskey_users = total_misskey_users + usuaris_misskey
          usuaris = usuaris_misskey
          #print(server, usuaris_misskey)
          #print("\n")

        elif others_nodeinfo.ok == True and check_prismo == True:

          #print("Servidor Prismo")
          usuaris_prismo = others_nodeinfo.json()['usage']['users']['total']
          total_prismo_users = total_prismo_users + usuaris_prismo
          usuaris = usuaris_prismo
          #print(server, usuaris_prismo)
          #print("\n")

        elif others_nodeinfo.ok == True and check_osada == True:

          #print("Servidor Osada")
          usuaris_osada = others_nodeinfo.json()['usage']['users']['total']
          total_osada_users = total_osada_users + usuaris_osada
          usuaris = usuaris_osada
          #print(server, usuaris_osada)
          #print("\n")

        elif others_nodeinfo.ok == True and check_groundpolis == True:

          #print("Servidor Groundpolis")
          usuaris_gpolis = others_nodeinfo.json()['usage']['users']['total']
          total_gpolis_users = total_gpolis_users + usuaris_gpolis
          usuaris = usuaris_gpolis
          #print(server, usuaris_gpolis)
          #print("\n")

        elif others_nodeinfo.ok == True and check_ganggo == True:

          #print("Servidor Ganggo")
          usuaris_ggg = others_nodeinfo.json()['usage']['users']['total']
          total_ggg_users = total_ggg_users + usuaris_ggg
          usuaris = usuaris_ggg
          #print(server, usuaris_ggg)
          #print("\n")

        elif others_nodeinfo.ok == True and check_squs == True:

          #print("Servidor Squs")
          usuaris_squs = others_nodeinfo.json()['usage']['users']['total']
          total_squs_users = total_squs_users + usuaris_squs
          usuaris = usuaris_squs
          #print(server, usuaris_squs)
          #print("\n")

        elif others_nodeinfo.ok == True and check_friendica == True:

          #print("Servidor Friendica")
          if others_nodeinfo.json()['usage'] != []:
            usuaris_friendica = others_nodeinfo.json()['usage']['users']['total']
          else:
            usuaris_friendica = 0
          total_friendica_users = total_friendica_users + usuaris_friendica
          usuaris = usuaris_friendica
          #print(server, usuaris_friendica)
          #print("\n")
          check_friendica = False

        elif gs_nodeinfo.ok == True and check_gnusocial == True:

          #print("Servidor GNU Social")
          gs_users = gs_nodeinfo.json()['usage']['users']['total']
          if gs_users == 0:
            gs_users = gs_nodeinfo.json()['usage']['users']['activeHalfyear']
          total_gs_users = total_gs_users + gs_users
          usuaris = gs_users
          #print(server, gs_users)
          #print("\n")

        elif statusnet.ok == True and check_friendica == True: #statusnet.json()['site']['friendica']['FRIENDICA_PLATFORM'] == "Friendica":

          #print("Servidor Friendica")
          usuaris_friendica = 0
          total_friendica_users = total_friendica_users + usuaris_friendica
          usuaris = usuaris_friendica
          instancies = 0
          #print(server, usuaris_friendica)
          #print("\n")

        elif px_nodeinfo.ok == True and check_pixelfed == True:

          #print("Servidor Pixelfed")
          usuaris_pixelfed = px_nodeinfo.json()['usage']['users']['total']
          total_pixelfed_users = total_pixelfed_users + usuaris_pixelfed
          usuaris = usuaris_pixelfed
          #print(server, usuaris_pixelfed)
          #print("\n")

        elif px_nodeinfo.ok == True and check_gnusocial2 == True:

          #print("Servidor GNU Social 2.x")
          gs_users = px_nodeinfo.json()['usage']['users']['total']
          if gs_users == 0:
            gs_users = px_nodeinfo.json()['usage']['users']['activeHalfyear']
          total_gs_users = total_gs_users + gs_users
          usuaris = gs_users
          #print(server, gs_users)
          #print("\n")

        elif wf_nodeinfo.ok == True and check_writefreely == True:

          #print("Servidor WriteFreely")
          usuaris_writefreely = wf_nodeinfo.json()['usage']['users']['total']
          total_writefreely_users = total_writefreely_users + usuaris_writefreely
          usuaris = usuaris_writefreely
          #print(server, usuaris_writefreely)
          #print("\n")

        else:

          api_reply = 0

      if api_reply == 1 and others_nodeinfo.ok or gs_nodeinfo.ok or nodeinfo.ok or statusnet.ok or px_nodeinfo.ok or wf_nodeinfo.ok:

        insert_sql = "INSERT INTO fediverse(server, users, updated_at) VALUES(%s,%s,%s) ON CONFLICT DO NOTHING"
        conn = None

        try:

          conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")

          cur = conn.cursor()

          cur.execute(insert_sql, (server, usuaris, now))

          cur.execute("UPDATE fediverse SET users=(%s), updated_at=(%s) where server=(%s)", (usuaris, now, server))

          conn.commit()

          cur.close()
          return

        except (Exception, psycopg2.DatabaseError) as error:

          #print(error)
          return

        finally:

          if conn is not None:

            conn.close()
        return

    except KeyError as e:

        pass

    except ValueError as verr:

        pass

    except requests.exceptions.ChunkedEncodingError as chunkerr:

        #print(server)
        pass

    except requests.exceptions.ConnectionError as errc:

        pass

    except requests.exceptions.Timeout as errt:

        pass

    except requests.exceptions.ConnectionError as errc:

        pass

# Returns the parameter from the specified file
def get_parameter( parameter, file_path ):
    # Check if secrets file exists
    if not os.path.isfile(file_path):
        print("File %s not found, exiting."%file_path)
        sys.exit(0)

    # Find parameter in file
    with open( file_path ) as f:
        for line in f:
            if line.startswith( parameter ):
                return line.replace(parameter + ":", "").strip()

    # Cannot find parameter, exit
    print(file_path + "  Missing parameter %s "%parameter)
    sys.exit(0)

# Load secrets from secrets file
secrets_filepath = "secrets/secrets.txt"
uc_client_id     = get_parameter("uc_client_id",     secrets_filepath)
uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
uc_access_token  = get_parameter("uc_access_token",  secrets_filepath)

# Load configuration from config file
config_filepath = "config.txt"
mastodon_hostname = get_parameter("mastodon_hostname", config_filepath) # E.g., mastodon.social

# Load database config from db_config file
db_config_filepath = "config/db_config.txt"
fediverse_db = get_parameter("fediverse_db", db_config_filepath)
fediverse_db_user = get_parameter("fediverse_db_user", db_config_filepath)

# Initialise Mastodon API
mastodon = Mastodon(
    client_id = uc_client_id,
    client_secret = uc_client_secret,
    access_token = uc_access_token,
    api_base_url = 'https://' + mastodon_hostname,
)

# Initialise access headers
headers={ 'Authorization': 'Bearer %s'%uc_access_token }

###############################################################################
# main

pl_users = 0
total_pl_users = 0
mast_users = 0
total_mast_users = 0
gs_users = 0
total_gs_users = 0
usuaris_zap = 0
total_zap_users = 0
usuaris_plume = 0
total_plume_users = 0
usuaris_hubzilla = 0
total_hubzilla_users = 0
usuaris_misskey = 0
total_misskey_users = 0
usuaris_prismo = 0
total_prismo_users = 0
usuaris_osada = 0
total_osada_users = 0
usuaris_gpolis = 0
total_gpolis_users = 0
usuaris_ggg = 0
total_ggg_users = 0
usuaris_squs = 0
total_squs_users = 0
peertube_users = 0
total_peertube_users = 0
diaspora_users = 0
total_diaspora_users = 0
usuaris_friendica = 0
total_friendica_users = 0
usuaris_pixelfed = 0
total_pixelfed_users = 0
usuaris_writefreely = 0
total_writefreely_users = 0
ravenvale_users = 0
total_ravenvale_users = 0
usuaris_total = 0

total_servers = 0
total_users = 0

now = datetime.now()

try:

    conn = None
    conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")

    cur = conn.cursor()

    ### get world servers list

    cur.execute("select server from world")

    world_servers = []

    for row in cur:

        world_servers.append(row[0])

    cur.close()

except (Exception, psycopg2.DatabaseError) as error:

    print(error)

finally:

    if conn is not None:

        conn.close()

###########################################################################

nprocs = multiprocessing.cpu_count()
pool = Pool(processes=nprocs)
for _ in tqdm.tqdm(pool.imap_unordered(getserver, world_servers), total=len(world_servers)):
    pass

###########################################################################
# get current total servers and users

select_sql = "select count(server), sum(users) from fediverse"

try:

    conn = None
    conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")

    cur = conn.cursor()

    cur.execute(select_sql)

    row = cur.fetchone()

    total_servers = row[0]
    total_users = row[1]

    cur.close()

except (Exception, psycopg2.DatabaseError) as error:

    print(error)

finally:

    if conn is not None:

        conn.close()

#########################################################################################
#  get last server and users values and calc evolution. Write current values to database

select_sql = "select total_servers, total_users from totals order by datetime desc limit 1"
insert_sql = "INSERT INTO totals(datetime, total_servers, total_users) VALUES(%s,%s,%s)"

try:

    conn = None
    conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")

    cur = conn.cursor()

    cur.execute(select_sql)

    row = cur.fetchone()

    if row != None:

        servers_before = row[0]
        users_before = row[1]

    else:

        servers_before = 0
        users_before = 0

    cur.execute(insert_sql, (now, total_servers, total_users))

    conn.commit()

    cur.close()

    evo_servers = total_servers - servers_before
    evo_users = total_users - users_before

except (Exception, psycopg2.DatabaseError) as error:

    print(error)

finally:

    if conn is not None:

        conn.close()

################################################################################
#  escriure en la bbdd evo

insert_sql = "INSERT INTO evo(datetime, servers, users) VALUES(%s,%s,%s)"

conn = None

try:

    conn = psycopg2.connect(database = fediverse_db, user = fediverse_db_user, password = "", host = "/var/run/postgresql", port = "5432")

    cur = conn.cursor()

    cur.execute(insert_sql, (now, evo_servers, evo_users))

    conn.commit()

    cur.close()

except (Exception, psycopg2.DatabaseError) as error:

    print(error)

finally:

    if conn is not None:

        conn.close()

###############################################################################
# T  O  O  T !

toot_text = "#fediverse live servers stats" + " \n"
toot_text += "\n"
if evo_servers > 0:
    toot_text += "Live servers: " + str(total_servers) + " (+"+ str(evo_servers) + ") \n"
else:
    toot_text += "Live servers: " + str(total_servers) + " ("+ str(evo_servers) + ") \n"
toot_text += "\n"
if evo_users > 0:
    toot_text += "Total Users: " + str(total_users) + " (+"+ str(evo_users) + ") \n"
else:
    toot_text += "Total Users: " + str(total_users) + " ("+ str(evo_users) + ") \n"
toot_text += "\n"
toot_text += "Exec. time %s seconds" % (time.time() - start_time)

print("Tooting...")
print(toot_text)

mastodon.status_post(toot_text, in_reply_to_id=None, )