#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2013-2019
# This file is part of Shinken Enterprise, all rights reserved.

import collections
from inspect import getargspec

from shinkensolutions.lib_checks.common import *

API_CONNECTION_WARNING_LEVEL = 2

DEFAULT_SCHEDULER_LATENCY_THRESHOLD = 1.5  # s
DEFAULT_NB_CHECK_IN_TIMEOUT_TOLERATE = 0  # check


# Allow \n in the parser output
class MyParser(optparse.OptionParser):
    def format_epilog(self, formatter):
        return self.epilog
    
    
    def format_help(self, formatter=None):
        if formatter is None:
            formatter = self.formatter
        result = []
        if self.usage:
            result.append(self.get_usage() + "\n")
        if self.description:
            result.append(self.format_description(formatter) + "\n")
        result.append(self.format_option_help(formatter))
        result.append(self.format_epilog(formatter))
        return "".join(result)


# Call get_raw_stats and raise the exception if there is one.
def get_raw_stat_with_exp(param='', _uri=''):
    sparam = "?param=%s" % param if param else ''
    _uri = _uri if _uri else uri
    buf, connexion_time = Utils.request_get(result, _uri, '/get_raw_stats%s' % sparam, timeout=opts.timeout, raise_exp=True)
    data = json.loads(buf)
    data['connexion_time'] = connexion_time
    return data


def get_raw_stat():
    sparam = ''
    if last_check:
        sparam = "?param=%s" % last_check
    # Now all others need raw stats, so get it
    buf, connexion_time = ShinkenUtils.request_get_daemon(result, daemon_type, uri, '/get_raw_stats%s' % sparam, timeout=opts.timeout)
    try:
        data = json.loads(buf)
        data['connexion_time'] = connexion_time
        return data
    except:
        result.hard_exit(EXIT_STATUS.WARNING, 'Cannot load performance data from daemon: %s' % buf)


def get_modes_help():
    modes_help = 'Mode, depends of the -t/--daemon-type parameter:\n'
    for (k, d) in MODES.iteritems():
        modes_help += '\t%s:\n' % k
        keys = d.keys()
        keys.sort()
        for k2 in keys:
            modes_help += '\t\t%s %s\n' % (k2, d[k2]['help'])
    
    return modes_help


def get_option_parser():
    parser = MyParser(option_class=ShinkenOption, epilog=get_modes_help())
    parser.add_option(u'-H', u'--hostname', dest=u'hostname', default=u'127.0.0.1')
    parser.add_option(u'-p', u'--port', dest=u'port', type=int)
    parser.add_option(u'-t', u'--daemon-type', dest=u'daemon_type', default=u'arbiter')
    parser.add_option(u'-T', u'--timeout', dest=u'timeout', type=int, default=3)
    parser.add_option(u'-m', u'--mode', dest=u'mode', default=u'alive', help=u'')
    parser.add_option(u'-l', u'--last_check', dest=u'last_check', default=u'', help=u'')
    parser.add_option(u'-n', u'--module-name', dest=u'module_name', default=u'', help=u'')
    parser.add_option(u'-w', u'--cpu_stolen_warning', dest=u'thresold_cpu_stolen_warning', default=5, type=int, help=u'Threshold for the warning of cpu stolen')
    parser.add_option(u'-c', u'--cpu_stolen_critical', dest=u'thresold_cpu_stolen_critical', default=10, type=int, help=u'Threshold for the critical of cpu stolen')
    parser.add_option(u'', u'--livedata_warning', dest=u'livedata_warning', default=1, type=int)
    parser.add_option(u'', u'--livedata_error_displayed_limit', dest=u'livedata_error_limit', default=5, type=int)
    parser.add_option(u'', u'--active_poller_latency', dest=u'active_poller_latency', type=float, default=0.5, help=u'Threshold of latency between a scheduler and a active poller in second.')
    parser.add_option(u'', u'--active_reactionner_latency', dest=u'active_reactionner_latency', type=float, default=0.5, help=u'Threshold of latency between a scheduler and a active reactionner in second.')
    parser.add_option(u'', u'--passive_poller_latency', dest=u'passive_poller_latency', type=float, default=0.5, help=u'Threshold of latency between a scheduler and a passive poller in second.')
    parser.add_option(u'', u'--scheduler_too_old_retention_save_margin', dest=u'scheduler_too_old_retention_save_margin', type=int, default=5,
                      help=u'Number of minutes of margin to add to retention_interval before raise a WARNING about out of delay retention save')
    parser.add_option(u'', u'--check_tolerate', type=int, dest=u'check_tolerate', default=0, help=u'Number of checks in timeout before the Poller Running Well change this state to WARNING.')
    parser.add_option(u'', u'--shinkenversion', dest=u'shinken_supervisor_version', default='', help=u'This shinken version number to compare with the monitored shinken.')
    return parser


def basic_check(daemon_type):
    data = None
    try:
        data = get_raw_stat_with_exp()
    except Exception as exp:
        # get raw stat failed, try a ping and return different error depending on the ping
        try:
            ShinkenUtils.request_get_daemon(result, daemon_type, uri, '/ping', timeout=opts.timeout)
        except Exception as exp_ping:
            output = "Impossible to contact the %s at %s." % (daemon_type, uri)
            long_output = "%s" % exp_ping
            result.hard_exit(EXIT_STATUS.CRITICAL, output, long_output)
        output = "Impossible to get stats the %s at %s." % (daemon_type, uri)
        long_output = "%s" % exp
        result.hard_exit(EXIT_STATUS.CRITICAL, output, long_output)
    
    # if data contain only conn time,
    if len(data) == 1 and 'connexion_time' in data:
        output = "Connection established in %.3fs but failed to get stats from the %s at %s. The %s probably needs to be updated" % (data['connexion_time'], daemon_type, uri, daemon_type)
        result.hard_exit(EXIT_STATUS.WARNING, output)
    
    ShinkenUtils.minimal_check(result, data, daemon_type, shinken_supervisor_version)
    
    # add the daemon type as data
    data['daemon_type'] = daemon_type
    return data


def alive():
    # simply check if the daemon is alive and run the basic checks
    start_time = time.time()
    data = basic_check(daemon_type)
    connexion_time = time.time() - start_time
    
    ShinkenUtils.check_master_information(data, result)
    ShinkenUtils.check_hypervisor_vm_cpu_stolen(data, opts.thresold_cpu_stolen_warning, opts.thresold_cpu_stolen_critical, result, output_in_li=True)
    result.set_perf_data({'connexion_time': connexion_time})
    result.set_spare_info(data)
    ShinkenUtils.check_arbiter_connection(result, uri, daemon_type, timeout=opts.timeout)  # Look for arbiter conflict, and arbiter did connect recently
    ShinkenUtils.add_http_error_count_message(result, data)
    ShinkenUtils.add_warning_module_restart(result, data)
    
    if result.status == EXIT_STATUS.OK:
        result.add_check(EXIT_STATUS.OK, u'The daemon is running well:', title=True)
    else:
        result.add_check(EXIT_STATUS.WARNING, u'The daemon have some issues:', title=True)
    
    version = data.get(u'daemon_version', None)
    if version:
        result.add_check(EXIT_STATUS.OK, HTMLList.one_bullet_list(u'Version [%s].' % get_version_for_output(version)), no_new_line=True)
    result.add_check(EXIT_STATUS.OK, HTMLList.one_bullet_list(u'Connection established in %.3fs.' % connexion_time), no_new_line=True)
    
    ShinkenUtils.add_module_info(result, data)
    result.exit()


def api_connection():
    data = basic_check(daemon_type)
    
    # Then pure performance data things
    start_time = time.time()
    ShinkenUtils.request_get_daemon(result, daemon_type, uri, '/get_lock', timeout=opts.timeout)
    get_lock_time = time.time() - start_time
    perfs = {'get_lock_time': get_lock_time}
    result.set_perf_data(perfs)
    
    result.set_spare_info(data, daemon_type)
    
    if get_lock_time > API_CONNECTION_WARNING_LEVEL:
        result.add_check(status=EXIT_STATUS.WARNING,
                         output='API Connexion was too long (%.3f > %ds). It can be a sign that your daemon is overloaded.' % (
                             get_lock_time, API_CONNECTION_WARNING_LEVEL))
    else:
        result.add_check(status=EXIT_STATUS.OK, output='API Connection is in the good range (%.3f < %ds).' % (
            get_lock_time, API_CONNECTION_WARNING_LEVEL))
    result.exit()


def arbiter_alive():
    # Same alive as other but without the check_arbiter_connection
    start_time = time.time()
    data = basic_check(daemon_type)
    connexion_time = time.time() - start_time
    result.set_perf_data({u'connexion_time': connexion_time})
    
    result.set_spare_info(data)
    ShinkenUtils.add_warning_module_restart(result, data)
    ShinkenUtils.add_http_error_count_message(result, data)
    
    if result.status == EXIT_STATUS.OK:
        result.add_check(EXIT_STATUS.OK, u'Your arbiter is running well.', title=True)
    else:
        result.add_check(EXIT_STATUS.WARNING, u'Your arbiter has some issues.', title=True)
    
    version = data.get(u'daemon_version', None)
    if version:
        result.add_check(EXIT_STATUS.OK, u'Version [%s].' % get_version_for_output(version))
    result.add_check(EXIT_STATUS.OK, u'Connection established in %.3fs.' % connexion_time)
    ShinkenUtils.add_module_info(result, data)
    result.exit()


def synchronizer_alive():
    # Same alive as other but without the check_arbiter_connection and spare check
    start_time = time.time()
    data = basic_check(daemon_type)
    connexion_time = time.time() - start_time
    result.set_perf_data({u'connexion_time': connexion_time})
    stats_data = get_raw_stat()
    
    result.set_spare_info(data)
    ShinkenUtils.add_warning_module_restart(result, data)
    ShinkenUtils.add_http_error_count_message(result, data)
    
    if result.status == EXIT_STATUS.OK:
        result.add_check(EXIT_STATUS.OK, u'The synchronizer is running well.', title=True)
    else:
        result.add_check(EXIT_STATUS.WARNING, u'The synchronizer have some issues.', title=True)
    
    version = data.get(u'daemon_version', None)
    if version:
        result.add_check(EXIT_STATUS.OK, u'Version [%s].' % get_version_for_output(version))
    result.add_check(EXIT_STATUS.OK, u'Connection established in %.3fs.' % connexion_time)
    
    ShinkenUtils.check_hypervisor_vm_cpu_stolen(stats_data, opts.thresold_cpu_stolen_warning, opts.thresold_cpu_stolen_critical, result)
    ShinkenUtils.add_module_info(result, data)
    result.exit()


def arbiter_stats():
    raw_data = None
    connexion_time = 0
    arbiter_api_version = ''
    arbiter_version = ''
    
    try:
        buf, connexion_time = ShinkenUtils.request_get_daemon(result, daemon_type, uri, '/get_daemon_infos', timeout=opts.timeout)
        raw_data = json.loads(buf)
    except Exception as e:
        result.hard_exit(EXIT_STATUS.WARNING, 'Cannot load daemon information from arbiter [%s]' % e)
    
    try:
        arbiter = raw_data.get('arbiter', None)
        if arbiter:
            arbiter_version = arbiter.get('version', None)
            arbiter_api_version = arbiter.get('api_version', None)
            output = ShinkenUtils._check_versions(daemon_type, arbiter_api_version, arbiter_version, arbiter_version, shinken_supervisor_version)
            if output:
                result.hard_exit(EXIT_STATUS.WARNING, output)
        else:
            result.hard_exit(EXIT_STATUS.WARNING, 'Your %s is alive but not up to date. Please update.' % daemon_type)
    except AttributeError:
        result.hard_exit(EXIT_STATUS.WARNING, 'Your %s is alive but not up to date. Please update.' % daemon_type)
    
    # The arbiter is letting us know if it's a master or a spare that is running (a master IS running)
    is_activated = arbiter.get('activated', None)
    is_spare = arbiter.get('spare', None)
    if is_spare is None or is_activated is None:
        result.hard_exit(EXIT_STATUS.WARNING, 'Your %s is alive but not up to date. Please update.' % daemon_type)
    connected_satellites = raw_data.get('satellites', [])
    connected_satellites.sort(key=lambda sat: sat[u'type'])
    stats_data = get_raw_stat()
    result.add_check(output='<br>Arbiter answered in %.3fs.' % connexion_time, title=True)
    
    ShinkenUtils.check_hypervisor_vm_cpu_stolen(stats_data, opts.thresold_cpu_stolen_warning, opts.thresold_cpu_stolen_critical, result)
    
    if not is_spare or (is_spare and is_activated):
        in_warning = False
        in_error = False
        list_daemons = []
        for e in connected_satellites:
            state = HTMLTag.color_text('[OK]', COLOR.GREEN)
            state_text = ''
            show_version = True
            daemon_version = e.get('daemon_version', None)
            output_version = ShinkenUtils._check_versions(e.get('type', 'daemon'), arbiter_api_version, daemon_version, arbiter_version, shinken_supervisor_version)
            if output_version:
                state = HTMLTag.color_text('[WARNING]', COLOR.ORANGE)
                in_warning = True
                show_version = False
                state_text = output_version
            diff_time_with_arbiter = abs(e.get('diff_time_with_arbiter', 0))
            if diff_time_with_arbiter > 30:
                state = HTMLTag.color_text('[ERROR]', COLOR.RED)
                in_error = True
                state_text += ' => connection OK but server times are different, time shift of %s' % HTMLTag.color_text('%s' % Utils.print_period(diff_time_with_arbiter), color=COLOR.ORANGE)
            if not e.get('reachable', True):
                state = HTMLTag.color_text('[WARNING]', COLOR.ORANGE)
                state_text = 'couldn\'t be reached during the last ping'
                in_warning = True
                show_version = False
            if not e['alive']:
                state = HTMLTag.color_text('[WARNING]', COLOR.ORANGE)
                state_text = 'connection %s' % HTMLTag.color_text('KO', color=COLOR.ORANGE)
                in_warning = True
                show_version = False
            
            if show_version:
                list_daemons.append(
                    u'%s: %s [%s] Version : [%s] %s' % (state, e['type'], e['display_name'], get_version_for_output(daemon_version), state_text))
            else:
                list_daemons.append(u'%s: %s [%s] %s' % (state, e['type'], e['display_name'], state_text))
        
        output = HTMLList.header_list('Connected to daemons', list_daemons)
        status = EXIT_STATUS.WARNING if in_warning else EXIT_STATUS.OK
        if in_error:
            status = EXIT_STATUS.CRITICAL
        result.add_check(status=status, output=output)
    else:  # sleeping spare
        # if the arbiter has no connection to any daemons that probably means that he is a spare
        # call the raw stats to be sure about that
        stats_data = get_raw_stat()
        result.set_spare_info(stats_data, daemon_type=daemon_type)
    
    result.add_check(output=u'Version [%s]' % get_version_for_output(arbiter_version))
    result.exit()


# Spec at SEF-1053
def poller_alive(opts):
    start_time = time.time()
    data = basic_check(u'poller')
    connexion_time = time.time() - start_time
    
    # Threshold of latency between a scheduler and a active poller
    active_poller_latency = opts.active_poller_latency
    # Number of checks in timeout before the Poller Running Well change this state to WARNING.
    check_tolerate = opts.check_tolerate
    
    style = '<style type="text/css">' \
            '.skn-pahc {' \
            'font-style:italic!important;' \
            'color:#7F7F7F!important;' \
            '}' \
            '</style>'
    
    s_schedulers_ok = []
    as_print_scheduler_info = False
    version = data.get(u'daemon_version', None)
    nb_check_in_timeout = data.get(u'nb_check_in_timeout', 0)
    executor_in_overload = data.get(u'executor_in_overload', False)
    nb_action_done_per_sec = data.get(u'nb_action_done_per_sec', 0)
    poller_type = data.get(u'type', u'MISSING_TYPE')
    poller_tag = data.get(u'tags', [u'MISSING TAGS INFO'])
    keep_timeout_time = data.get(u'keep_timeout_time', 1200)
    workers_restarts = data.get(u'workers_restarts', {})
    dead_workers = data.get(u'dead_worker_stat', {})
    
    stats = {u'nb_check_in_timeout': nb_check_in_timeout}
    result.set_perf_data(stats)
    
    result.set_spare_info(data)
    
    if workers_restarts:
        list_dead_workers = []
        for woker_type, restart_data in workers_restarts.iteritems():
            woker_type = 'default worker type' if woker_type == 'fork' else woker_type
            list_dead_workers.append("'%s' restarted %s times" % (woker_type, len(restart_data)))
        output = HTMLList.header_list('In the last 24 hours, some workers died but were restarted:', list_dead_workers)
        result.add_check(EXIT_STATUS.WARNING, output)
    
    elif dead_workers:
        list_dead_workers = []
        for woker_type, restart_count in dead_workers.iteritems():
            woker_type = 'default worker type' if woker_type == 'fork' else woker_type
            list_dead_workers.append("'%s' restarted %s times" % (woker_type, restart_count))
        output = HTMLList.header_list('Since last restart, some workers died but were restarted:', list_dead_workers)
        result.add_check(EXIT_STATUS.WARNING, output)
    
    ShinkenUtils.check_arbiter_connection(result, uri, daemon_type, timeout=opts.timeout)  # Look for arbiter conflict, and arbiter did connect recently
    ShinkenUtils.add_warning_module_restart(result, data)
    ShinkenUtils.add_http_error_count_message(result, data)
    
    if executor_in_overload:
        result.add_check(EXIT_STATUS.WARNING,
                         'Your poller is overloaded, it cannot make more than %d check by sec.%sYou should add a new poller%s' % (
                             nb_action_done_per_sec, BREAK_LINE, BREAK_LINE))
    
    if not poller_type == 'PASSIVE' and data.get('schedulers', None):
        s_schedulers_ko = []
        s_schedulers_ko_info = []
        all_ko = True
        one_ko = False
        for scheduler in data.get('schedulers', []):
            s_scheduler = "%s [%s]" % (scheduler.get('name', 'MISSING_NAME'), scheduler.get('addr', 'MISSING_ADDRESS'))
            
            if scheduler.get('con', False):
                all_ko = False
                s_schedulers_ok.append(HTMLTag.color_text(s_scheduler, COLOR.GREEN, bold=False))
            else:
                one_ko = True
                s_schedulers_ko.append(HTMLTag.color_text(s_scheduler, COLOR.RED, bold=False))
                s_scheduler_ko_info = '%s&nbsp;:&nbsp;<br/>%s<br/>-------<br/>' % (
                    scheduler.get('name', 'MISSING_NAME'), scheduler.get('info', 'MISSING_INFO'))
                s_schedulers_ko_info.append(s_scheduler_ko_info)
        
        if all_ko:
            output = HTMLList.header_list('The poller cannot join the schedulers', s_schedulers_ko)
            long_output = ''.join(s_schedulers_ko_info)
            result.add_check(EXIT_STATUS.CRITICAL, output=output, long_output=long_output)
            as_print_scheduler_info = True
        elif one_ko:
            output = ''.join((HTMLList.header_list('The poller cannot join the scheduler', s_schedulers_ko),
                              HTMLList.header_list('But it joins the schedulers', s_schedulers_ok)))
            result.add_check(EXIT_STATUS.WARNING, output=output, long_output=''.join(s_schedulers_ko_info))
            as_print_scheduler_info = True
        else:
            s_schedulers_ok = []
            s_schedulers_ko = []
            
            one_ko = False
            for scheduler in data.get('schedulers', []):
                latency = scheduler.get('latency', -1)
                s_scheduler = "%s [%s]&nbsp;:&nbsp;%sms" % (
                    scheduler.get('name', 'MISSING_NAME'), scheduler.get('addr', 'MISSING_ADDRESS'),
                    round(latency * 1000, 2))
                
                if latency > active_poller_latency:
                    s_scheduler += " (&nbsp;>&nbsp;%dms&nbsp;)" % (active_poller_latency * 1000)
                    s_schedulers_ko.append(HTMLTag.color_text(s_scheduler, COLOR.RED, bold=False))
                    one_ko = True
                else:
                    s_schedulers_ok.append(HTMLTag.color_text(s_scheduler, COLOR.GREEN, bold=False))
            
            if one_ko:
                output = HTMLList.header_list('The latency between the poller and schedulers is too high',
                                              s_schedulers_ko + s_schedulers_ok)
                result.add_check(EXIT_STATUS.WARNING, output)
                as_print_scheduler_info = True
    
    if nb_check_in_timeout > check_tolerate:
        checks_in_timeout = []
        for check_in_timeout in data.get('checks_in_timeout', []):
            commands = check_in_timeout[0].split('-//-')
            
            if len(commands) > 2:
                host_name = commands[0]
                check_name = commands[1]
            elif len(commands) > 1:
                host_name = commands[0]
                check_name = '<div class="skn-pahc"> check command( %s )</div>' % commands[1]
            else:
                host_name = commands[0]
                check_name = ''
            
            timeout = check_in_timeout[1]
            at = datetime.datetime.fromtimestamp(check_in_timeout[2]).strftime('%H:%M:%S')
            nb = check_in_timeout[3]
            
            checks_in_timeout.append((nb, at, host_name, check_name, timeout))
        
        output = ''.join((style,
                          'You have %d (&nbsp;>&nbsp;%d tolerate&nbsp;) check(s) that timed out in the last %s.<br/>The last actions that timed out&nbsp;:&nbsp;<br/>' % (
                              nb_check_in_timeout, check_tolerate, Utils.print_period(keep_timeout_time)),
                          HTMLTable.table(('Nb', 'Last', 'Host', 'Check', 'Timeout'), checks_in_timeout) + BREAK_LINE))
        result.add_check(EXIT_STATUS.WARNING, output)
    
    executor_type_display = HTMLTag.color_text('[&nbsp;PASSIVE MODE&nbsp;]',
                                               COLOR.BLACK) if poller_type == 'PASSIVE' else ''
    result.add_check(output='Poller %s' % executor_type_display)
    result.add_check(output=HTMLList.simple_list(('tags&nbsp;:&nbsp;%s' % ', '.join(poller_tag),)), no_new_line=True)
    
    if not poller_type == 'PASSIVE' and data.get('schedulers', None) and not as_print_scheduler_info:
        output = HTMLList.header_list('The latency between the poller and the schedulers are', s_schedulers_ok)
        result.add_check(output=output, no_new_line=True)
    
    if result.status == EXIT_STATUS.OK:
        result.add_check(EXIT_STATUS.OK, 'Your poller is running well.', title=True)
    else:
        result.add_check(EXIT_STATUS.WARNING, 'Your poller have some issues.', title=True)
    
    if version:
        result.add_check(output=u'Version [%s]' % get_version_for_output(version))
    result.add_check(EXIT_STATUS.OK, u'Connection established in %.3fs.' % connexion_time)
    
    ShinkenUtils.add_module_info(result, data)
    result.exit()


# Spec at SEF-1053
def poller_stats():
    data = basic_check('poller')
    
    stats_data = get_raw_stat()
    nb_action_done_per_sec = data.get('nb_action_done_per_sec', 0)
    executor_load = data.get('executor_load', 0)
    cpu_usage = data.get('cpu_usage', 0.) or 0.
    nb_cpus = data.get('nb_cpus', 0)
    ram_usage = int(round(data.get('ram_usage', 0.), 0))
    max_ram_usage = data.get('max_ram_usage', 0.)
    cpu_running_queue = int(data.get('cpu_running_queue', 0))
    max_cpu_queue_per_cpu = data.get('max_cpu_queue_par_cpu', 0.)
    platform = data.get('platform', '')
    
    checks_top_usage = data.get('checks_top_usage', [])
    exec_stats = data.get('exec_stats', {})
    
    cpu_usage_percent = 0 if nb_cpus == 0 else int(round(cpu_usage * 100 / nb_cpus, 0))
    poller_load_percent = 100 if executor_load > EXECUTOR_LOAD_LIMIT else 0
    stats = {
        'nb_action_done_per_sec': nb_action_done_per_sec,
        'poller_load_state'     : poller_load_percent,
        'cpu_usage_percent'     : cpu_usage_percent,
        'cpu_running_queue'     : cpu_running_queue,
        'used_ram_percent'      : ram_usage
    }
    
    style = '<style type="text/css">' \
            '.poller-stats-table, .poller-stats-table td, .poller-stats-table th {' \
            '   border:              1px solid #000000   !important;' \
            '   border-collapse:     collapse            !important;' \
            '   color:               #000000             !important;' \
            '}' \
            '.poller-stats-table {' \
            '    width:              90%                 !important;' \
            '}' \
            '.poller-stats-table-th {' \
            '    background-color :  #DDDDDD             !important;' \
            '    width:              auto                !important;' \
            '    max-width:          20%                 !important;' \
            '    padding:            2px                 !important;' \
            '    word-break:         break-word          !important;' \
            '    background-color:   #FFFFFF             !important;' \
            '}' \
            '.poller-stats-table-td {' \
            '    padding:            2px                 !important;' \
            '    width:              auto                !important;' \
            '    max-width:          20%                 !important;' \
            '    font-weight:        normal              !important;' \
            '    word-break:         break-word          !important;' \
            '    background-color:   #FFFFFF             !important;' \
            '}' \
            '.poller-stats-host-command {' \
            '    font-style:         italic              !important;' \
            '    color:              #7F7F7F             !important;' \
            '}' \
            '.poller-stats-table-center {' \
            '    text-align:         center;' \
            '}' \
            '</style>'
    
    poller_stats_infos = []
    result.set_spare_info(data)
    
    status_cpu_ready = EXIT_STATUS.OK
    message_by_status, exit_status_code, _hypervisor_perfdatas = ShinkenUtils.check_hypervisor_vm_cpu_stolen(stats_data, opts.thresold_cpu_stolen_warning, opts.thresold_cpu_stolen_critical)
    
    result.set_perf_data(_hypervisor_perfdatas)
    if message_by_status:
        status_cpu_ready = exit_status_code
        poller_stats_infos.append(message_by_status)
    if nb_action_done_per_sec:
        poller_stats_infos.append('[&nbsp;%s&nbsp;] Checks done per second.' % round(nb_action_done_per_sec, 2))
    else:
        poller_stats_infos.append('The poller has not launched any check in the last 60 seconds.')
    if platform == 'nt':
        poller_stats_infos.append('This poller is running on Windows, impossible to get CPU informations.')
    else:
        if executor_load:
            poller_stats_infos.append('[&nbsp;%s&nbsp;] Poller load.' % (HTMLTag.load_tag(executor_load)))
        else:
            poller_stats_infos.append('Poller load is unavailable, please wait for data.')
        if cpu_usage:
            poller_stats_infos.append(
                '[&nbsp;%s%%&nbsp;] Average CPU running of the checks on %d core.' % (cpu_usage_percent, nb_cpus))
        else:
            poller_stats_infos.append('Average CPU running time of the checks is unavailable, please wait for data.')
    if ram_usage:
        poller_stats_infos.append('[&nbsp;%-.0f%%&nbsp;&nbsp;%s] Average server RAM usage.' % (
            ram_usage, HTMLTag.ram_tag(ram_usage, max_ram_usage)))
    else:
        poller_stats_infos.append('Average RAM usage of the poller is unavailable, please wait for data.')
    if platform != 'nt':
        if cpu_running_queue:
            poller_stats_infos.append('[%i] Processes in the CPU Queue. %s' % (
                cpu_running_queue, HTMLTag.cpu_queue_tag(cpu_running_queue, max_cpu_queue_per_cpu, nb_cpus)))
        else:
            poller_stats_infos.append('CPU running queue of the poller can\'t be compute.')
    
    output = HTMLList.header_list('Poller statistics', poller_stats_infos)
    
    lines = []
    for check_top_usage in checks_top_usage:
        commands = check_top_usage[0].split('-//-')
        
        if len(commands) > 2:
            host_name = commands[0]
            check_name = commands[1]
        elif len(commands) > 1:
            host_name = commands[0]
            check_name = '<div class="poller-stats-host-command"> check command( %s )</div>' % commands[1]
        else:
            host_name = commands[0]
            check_name = ''
        
        cpu_time = check_top_usage[1] * 1000
        
        lines.append((host_name, check_name, "%dms" % cpu_time))
    
    long_output = style
    
    if lines and nb_action_done_per_sec:
        long_output += HTMLTable.table(('Hosts', 'Check', 'Cpu time'), lines, 'Top 5 checks')
    
    if exec_stats and nb_action_done_per_sec:
        long_output += '<br>'
        lines = []
        headers = []
        
        start_range = 0
        exec_stat_ranges = exec_stats.keys()
        exec_stat_ranges = [int(k) for k in exec_stat_ranges]
        exec_stat_ranges.remove(-1)
        exec_stat_ranges.sort()
        for _range in exec_stat_ranges:
            exec_stat = exec_stats[str(_range)]
            headers.append('%s&nbsp;-&nbsp;%sms' % (start_range, _range))
            lines.append('%s<br/>(&nbsp;%s%%&nbsp;)' % (exec_stat[0], exec_stat[1]))
            stats['checks_per_cpu_running_time_%s_%s_ms' % (start_range, _range)] = exec_stat[0]
            start_range = _range
        
        exec_stat = exec_stats['-1']
        headers.append('+&nbsp;%sms' % start_range)
        lines.append('%s<br/>(&nbsp;%s%%&nbsp;)' % (exec_stat[0], exec_stat[1]))
        stats['checks_per_cpu_running_time_%s_ms_and_more' % start_range] = exec_stat[0]
        long_output += HTMLTable.table(headers, [lines], 'Nb Checks per CPU running time')
    
    result.set_perf_data(stats)
    result.add_check(output=output, status=status_cpu_ready, long_output=long_output)
    result.exit()


def poller_api_connection():
    result.hard_exit(EXIT_STATUS.WARNING,
                     'The mode "api_connection" is deprecated for this plugin version. Please upgrade shinken template.')


def poller_cpu_load():
    result.hard_exit(EXIT_STATUS.WARNING,
                     'The mode "cpu_load" is deprecated for this plugin version. Please upgrade shinken template.')


def poller_overload_protection():
    result.hard_exit(EXIT_STATUS.WARNING,
                     'The mode "overload_protection" is deprecated for this plugin version. Please upgrade shinken template.')


# Spec at SEF-1053
def reactionner_alive(opts):
    start_time = time.time()
    data = basic_check('reactionner')
    connexion_time = time.time() - start_time
    
    # Threshold of latency between a scheduler and a active reactionner
    active_reactionner_latency = opts.active_reactionner_latency
    # Number of checks in timeout before the reactionner Running Well change this state to WARNING.
    check_tolerate = opts.check_tolerate
    
    s_schedulers_ok = []
    as_print_scheduler_info = False
    version = data.get(u'daemon_version', None)
    reactionner_type = data.get(u'type', u'MISSING_TYPE')
    reactionner_tag = data.get(u'tags', [u'MISSING TAGS INFO'])
    keep_timeout_time = data.get(u'keep_timeout_time', 1200)
    nb_check_in_timeout = data.get(u'nb_check_in_timeout', 0)
    executor_in_overload = data.get(u'executor_in_overload', False)
    nb_action_done_per_sec = data.get(u'nb_action_done_per_sec', 0)
    
    stats = {u'nb_check_in_timeout': nb_check_in_timeout}
    result.set_perf_data(stats)
    
    result.set_spare_info(data)
    ShinkenUtils.check_arbiter_connection(result, uri, daemon_type, timeout=opts.timeout)
    ShinkenUtils.add_warning_module_restart(result, data)
    ShinkenUtils.add_http_error_count_message(result, data)
    
    workers_restarts = data.get('workers_restarts', {})
    dead_workers = data.get('dead_worker_stat', {})
    
    if workers_restarts:
        list_dead_workers = []
        for woker_type, restart_data in workers_restarts.iteritems():
            woker_type = 'default worker type' if woker_type == 'fork' else woker_type
            list_dead_workers.append("'%s' restarted %s times" % (woker_type, len(restart_data)))
        output = HTMLList.header_list('In the last 24 hours, some workers died but were restarted:', list_dead_workers)
        result.add_check(EXIT_STATUS.WARNING, output)
    
    elif dead_workers:
        # Here the healthcheck ask a daemon who haven't the workers_restarts in get_raw_stats (deprecated)
        list_dead_workers = []
        for woker_type, restart_count in dead_workers.iteritems():
            woker_type = 'default worker type' if woker_type == 'fork' else woker_type
            list_dead_workers.append("'%s' restarted %s times" % (woker_type, restart_count))
        output = HTMLList.header_list('Since last restart, some workers died but were restarted:', list_dead_workers)
        result.add_check(EXIT_STATUS.WARNING, output)
    
    if executor_in_overload:
        result.add_check(EXIT_STATUS.WARNING,
                         'Your reactionner overload, it cannot make more than %d check by sec.%sYou should add a new reactionner%s' % (
                             nb_action_done_per_sec, BREAK_LINE, BREAK_LINE))
    
    if not reactionner_type == 'PASSIVE' and data.get('schedulers', None):
        s_schedulers_ko = []
        s_schedulers_ko_info = []
        all_ko = True
        one_ko = False
        for scheduler in data.get('schedulers', []):
            s_scheduler = "%s [%s]" % (scheduler.get('name', 'MISSING_NAME'), scheduler.get('addr', 'MISSING_ADDRESS'))
            
            if scheduler.get('con', False):
                all_ko = False
                s_schedulers_ok.append(HTMLTag.color_text(s_scheduler, COLOR.GREEN, bold=False))
            else:
                one_ko = True
                s_schedulers_ko.append(HTMLTag.color_text(s_scheduler, COLOR.RED, bold=False))
                s_scheduler_ko_info = '%s&nbsp;:&nbsp;<br/>%s<br/>-------<br/>' % (
                    scheduler.get('name', 'MISSING_NAME'), scheduler.get('info', 'MISSING_INFO'))
                s_schedulers_ko_info.append(s_scheduler_ko_info)
        if all_ko:
            output = HTMLList.header_list("The reactionner cannot join the schedulers)", s_schedulers_ko)
            long_output = "%s" % (''.join(s_schedulers_ko_info))
            result.add_check(EXIT_STATUS.CRITICAL, output=output, long_output=long_output)
            as_print_scheduler_info = True
        elif one_ko:
            output = ''.join((
                HTMLList.header_list('The reactionner cannot join the scheduler', s_schedulers_ko),
                HTMLList.header_list('But it joins the schedulers', s_schedulers_ok)))
            result.add_check(EXIT_STATUS.WARNING, output=output, long_output=''.join(s_schedulers_ko_info))
            as_print_scheduler_info = True
        else:
            s_schedulers_ok = []
            s_schedulers_ko = []
            
            one_ko = False
            for scheduler in data.get('schedulers', []):
                latency = scheduler.get('latency', -1)
                s_scheduler = "%s [%s]&nbsp;:&nbsp;%sms" % (
                    scheduler.get('name', 'MISSING_NAME'), scheduler.get('addr', 'MISSING_ADDRESS'),
                    round(latency * 1000, 2))
                
                if latency > active_reactionner_latency:
                    s_scheduler += " (&nbsp;>&nbsp;%dms&nbsp;)" % (active_reactionner_latency * 1000)
                    s_schedulers_ko.append(HTMLTag.color_text(s_scheduler, COLOR.RED, bold=False))
                    one_ko = True
                else:
                    s_schedulers_ok.append(HTMLTag.color_text(s_scheduler, COLOR.GREEN, bold=False))
            
            if one_ko:
                output = HTMLList.header_list('The latency between the reactionner and schedulers is too high',
                                              s_schedulers_ko + s_schedulers_ok)
                result.add_check(EXIT_STATUS.WARNING, output)
                as_print_scheduler_info = True
    
    if nb_check_in_timeout > check_tolerate:
        checks_in_timeout = []
        for check_in_timeout in data.get('checks_in_timeout', []):
            action_name = check_in_timeout[0]
            timeout = check_in_timeout[1]
            at = datetime.datetime.fromtimestamp(check_in_timeout[2]).strftime('%H:%M:%S')
            nb = check_in_timeout[3]
            checks_in_timeout.append((nb, at, action_name, timeout))
        
        output = ''.join((
            'You have %d (&nbsp;>&nbsp;%d tolerate&nbsp;) action(s) that timed out in the last %s.<br/>The last actions that timed out&nbsp;:&nbsp;<br/>' % (
                nb_check_in_timeout, check_tolerate, Utils.print_period(keep_timeout_time)),
            HTMLTable.table(('Nb', 'At', 'Actions', 'Timeout'), checks_in_timeout) + BREAK_LINE))
        result.add_check(EXIT_STATUS.WARNING, output)
    
    executor_type_display = '[&nbsp;PASSIVE MODE&nbsp;]' if reactionner_type == 'PASSIVE' else ''
    result.add_check(output='Reactionner %s' % executor_type_display)
    result.add_check(output=HTMLList.simple_list(('tags&nbsp;:&nbsp;%s' % ', '.join(reactionner_tag),)),
                     no_new_line=True)
    
    if not reactionner_type == 'PASSIVE' and data.get('schedulers', None) and not as_print_scheduler_info:
        output = HTMLList.header_list('The latency between the reactionner and the schedulers are', s_schedulers_ok)
        result.add_check(output=output, no_new_line=True)
    
    if result.status == EXIT_STATUS.OK:
        result.add_check(EXIT_STATUS.OK, 'Your reactionner is running well.', title=True)
    else:
        result.add_check(EXIT_STATUS.WARNING, 'Your reactionner have some issues.', title=True)
    
    if version:
        result.add_check(output=u'Version [%s]' % get_version_for_output(version))
    result.add_check(EXIT_STATUS.OK, u'Connection established in %.3fs.' % connexion_time)
    
    ShinkenUtils.add_module_info(result, data)
    result.exit()


# Spec at SEF-1053
def reactionner_stats():
    data = basic_check('reactionner')
    stats_data = get_raw_stat()
    
    nb_action_done_per_sec = data.get('nb_action_done_per_sec', 0)
    cpu_usage = data.get('cpu_usage', 0)
    last_action_launch_time = data.get('last_action_launch_time', -1)
    checks_top_usage = data.get('checks_top_usage', [])
    
    stats = {'nb_action_done_per_sec': nb_action_done_per_sec, 'cpu_usage': cpu_usage}
    result.set_perf_data(stats)
    
    result.set_spare_info(data)
    
    reactionner_stats_infos = []
    if nb_action_done_per_sec == 0:
        if last_action_launch_time == -1:
            reactionner_stats_infos.append('The reactionner has not launched any action since the daemon start.')
        else:
            reactionner_stats_infos.append(
                'The reactionner has not launched any action since %s.' % Utils.print_time(last_action_launch_time))
    else:
        nb_action_done_per_sec = round(nb_action_done_per_sec, 2)
        if nb_action_done_per_sec < 1:
            reactionner_stats_infos.append('Less than 1 action done per second')
        else:
            reactionner_stats_infos.append('[&nbsp;%s&nbsp;] Actions done per second.' % nb_action_done_per_sec)
        
        if cpu_usage == 0:
            reactionner_stats_infos.append(
                'CPU Percentage used among the server available resources for actions execution is unavailable, please wait for data.')
        else:
            cpu_usage = round(cpu_usage, 2)
            if cpu_usage < 1:
                reactionner_stats_infos.append(
                    'Less than 1% CPU used among the server available resources for checks execution.')
            else:
                reactionner_stats_infos.append(
                    '[&nbsp;%s%%&nbsp;] CPU Percentage used among the server available resources for checks execution.' % cpu_usage)
    output = HTMLList.header_list('Reactionner statistics', reactionner_stats_infos, compact=True)
    
    result.add_check(output=output, no_new_line=True)
    
    ShinkenUtils.check_hypervisor_vm_cpu_stolen(stats_data, opts.thresold_cpu_stolen_warning, opts.thresold_cpu_stolen_critical, result, output_in_li=True)
    lines = []
    for check_top_usage in checks_top_usage:
        lines.append((check_top_usage[0], '%dms' % (check_top_usage[1] * 1000)))
    
    if lines and nb_action_done_per_sec != -1.0:
        long_output = HTMLTable.table(('Actions', 'Cpu time'), lines, 'Top 5 actions')
        result.add_check(long_output=long_output)
    
    result.exit()


def reactionner_api_connection():
    result.hard_exit(EXIT_STATUS.WARNING,
                     'The mode "api_connection" is deprecated for this plugin version. Please upgrade shinken template.')


def reactionner_cpu_load():
    result.hard_exit(EXIT_STATUS.WARNING,
                     'The mode "cpu_load" is deprecated for this plugin version. Please upgrade shinken template.')


def reactionner_overload_protection():
    result.hard_exit(EXIT_STATUS.WARNING,
                     'The mode "overload_protection" is deprecated for this plugin version. Please upgrade shinken template.')


# Spec at SEF-1143
def scheduler_alive(opts):
    start_time = time.time()
    data = basic_check(u'scheduler')
    connexion_time = time.time() - start_time
    
    # Threshold of latency between a scheduler and a active poller
    passive_poller_latency = opts.passive_poller_latency
    
    version = data.get(u'daemon_version', None)
    hosts = data[u'nb_hosts']
    clusters = data[u'nb_clusters']
    checks = data[u'nb_checks']
    passive_pollers = data.get(u'passive_pollers', [])
    realm = data[u'realm']
    total = hosts + clusters + checks
    checks_n_actions_stats = data.get(u'checks_n_actions_stats', None)
    
    if checks_n_actions_stats is None:
        # DEPRECATED : we are checking an old version : only late_check is available
        nb_late_checks = data[u'late_checks']
        nb_late_notifs = 0
        nb_late_event = 0
        late_checks_by_tags = data.get(u'late_checks_by_tags', {})
        average_checks_latency = data[u'average_latency']
    else:
        nb_late_checks = checks_n_actions_stats[u'check'][u'nb_late']
        nb_late_notifs = checks_n_actions_stats[u'notification'][u'nb_late']
        nb_late_event = checks_n_actions_stats[u'eventhandler'][u'nb_late']
        late_checks_by_tags = checks_n_actions_stats[u'check'][u'late_by_tags']
        average_checks_latency = checks_n_actions_stats[u'check'][u'latency']
    
    stats = {
        u'nb_late_checks'        : nb_late_checks,
        u'nb_late_notifications' : nb_late_notifs,
        u'nb_late_event_handlers': nb_late_event,
        u'nb_checks'             : checks,
        u'nb_clusters'           : clusters,
        u'nb_hosts'              : hosts
    }
    result.set_perf_data(stats)
    
    result.set_spare_info(data)
    ShinkenUtils.check_arbiter_connection(result, uri, daemon_type, timeout=opts.timeout)
    ShinkenUtils.add_warning_module_restart(result, data)
    ShinkenUtils.add_http_error_count_message(result, data)
    
    if nb_late_checks:
        percent_check = nb_late_checks * 100 / total
        list_late_checks_by_tags = [u'Tag %s&nbsp;:&nbsp;%s checks' % (HTMLTag.color_text(tag), nb_lates) for tag, nb_lates in late_checks_by_tags.iteritems()]
        if average_checks_latency:
            latency_info = u'In the last 5 minutes, the executed checks had an average full waiting time ( in Scheduler + Poller ) of %ss before being launched' % round(average_checks_latency, 2)
        else:
            latency_info = u'In the last 5 minutes, no checks were executed by a Poller.'
        
        late_checks_info = ''.join((
            u'CHECKS : There are %s late(s) ( %s%% )!' % (HTMLTag.color_text(nb_late_checks, COLOR.ORANGE), percent_check),
            BREAK_LINE,
            HTMLList.header_list(u'Late checks grouped by poller tags&nbsp;', list_late_checks_by_tags),
            latency_info,
            BREAK_LINE,
        
        ))
        result.add_check(EXIT_STATUS.WARNING, late_checks_info)
    
    if nb_late_notifs:
        list_late_by_tags = [u'Tag <b>%s</b>&nbsp;:&nbsp;%s notifications' % (HTMLTag.color_text(tag), nb_lates) for tag, nb_lates in checks_n_actions_stats[u'notification'][u'late_by_tags'].iteritems()]
        latency = checks_n_actions_stats[u'notification'][u'latency']
        if latency:
            latency_info = u'In the last 5 minutes, the executed notifications had an average waiting time of %ss in the Scheduler before being launched by a Reactionner.' % round(latency, 2)
        else:
            latency_info = u'In the last 5 minutes, no notifications were executed by a Reactionner.'
        
        late_notifs_info = ''.join((
            u'NOTIFICATIONS : There are %s late(s)!' % (HTMLTag.color_text(nb_late_notifs, COLOR.ORANGE)),
            BREAK_LINE,
            HTMLList.header_list(u'Late notifications grouped by reactionner tags&nbsp;', list_late_by_tags),
            latency_info,
            BREAK_LINE,
        ))
        result.add_check(EXIT_STATUS.WARNING, late_notifs_info)
    
    if nb_late_event:
        list_late_by_tags = [u'Tag %s&nbsp;:&nbsp;%s event handler' % (HTMLTag.color_text(tag), nb_lates) for tag, nb_lates in checks_n_actions_stats[u'eventhandler'][u'late_by_tags'].iteritems()]
        latency = checks_n_actions_stats[u'eventhandler'][u'latency']
        if latency:
            latency_info = u'In the last 5 minutes, the executed event had an average waiting time of %ss in the Scheduler before being launched by a Reactionner.' % round(latency, 2)
        else:
            latency_info = u'In the last 5 minutes, no events were executed by a Reactionner.'
        
        late_event_info = ''.join((
            u'EVENTS : There are %s late(s)!' % (HTMLTag.color_text(nb_late_event, COLOR.ORANGE)),
            BREAK_LINE,
            HTMLList.header_list(u'Late events grouped by reactionner tags&nbsp;', list_late_by_tags),
            latency_info,
            BREAK_LINE,
        ))
        result.add_check(EXIT_STATUS.WARNING, late_event_info)
    
    if passive_pollers:
        s_pollers_con_ok = []
        s_pollers_con_ko = []
        s_pollers_latency_ok = []
        s_pollers_latency_ko = []
        s_pollers_con_ko_info = []
        
        one_latency_ko = False
        for poller in passive_pollers:
            latency = poller.get(u'latency', -1)
            con = poller.get(u'con', False)
            s_poller_con = u'%s [%s]' % (poller.get(u'name', u'MISSING_NAME'), poller.get(u'addr', u'MISSING_ADDRESS'))
            s_poller_latency = u'%s [%s]&nbsp;:&nbsp;%sms' % (
                poller.get(u'name', u'MISSING_NAME'), poller.get(u'addr', u'MISSING_ADDRESS'), round(latency * 1000, 2))
            
            if con:
                s_pollers_con_ok.append(HTMLTag.color_text(s_poller_con, COLOR.GREEN, bold=False))
            else:
                s_pollers_con_ko.append(HTMLTag.color_text(s_poller_con, COLOR.ORANGE, bold=False))
                s_poller_ko_info = u'%(poller_name)s&nbsp;:&nbsp;%(break_line)s%(poller_info)s%(break_line)s-------%(break_line)s' % {
                    u'poller_name': poller.get(u'name', u'MISSING_NAME'), u'poller_info': poller.get(u'info', u'MISSING_INFO'), u'break_line': BREAK_LINE}
                s_pollers_con_ko_info.append(s_poller_ko_info)
            
            if latency > passive_poller_latency:
                s_poller_latency += u' (&nbsp;>&nbsp;%dms&nbsp;)' % (passive_poller_latency * 1000)
                s_pollers_latency_ko.append(HTMLTag.color_text(s_poller_latency, COLOR.ORANGE))
                one_latency_ko = True
            elif latency > -1:
                s_pollers_latency_ok.append(HTMLTag.color_text(s_poller_latency, COLOR.GREEN, bold=False))
        
        if one_latency_ko:
            text = HTMLList.header_list(u'The latency is too high between some passive pollers and the scheduler',
                                        s_pollers_latency_ko + s_pollers_latency_ok)
            result.add_check(EXIT_STATUS.WARNING, output=text, long_output=''.join(s_pollers_con_ko_info))
        elif s_pollers_latency_ok:
            latency_info = HTMLList.header_list(u'The latency between the scheduler and the passive pollers are',
                                                s_pollers_latency_ok)
            result.add_check(output=latency_info)
    
    satellites = None
    satellites_query_time = time.time()
    my_name = u''
    try:
        satellites, satellites_query_time = ShinkenUtils.get_satellites_connection(result, uri, timeout=opts.timeout, raise_excepton=True)
    except Exception as e:
        satellites_query_time = time.time() - satellites_query_time
        result.add_check(EXIT_STATUS.WARNING, u'No answer from Scheduler, unable to fetch connectivity data with cause : ( %s )' % HTMLTag.color_text(str(e), COLOR.ORANGE))
    if satellites and len(satellites) > 1:
        satellites_info = {u'satellites': [], u'in_error': {}, u'raw_stats': {}}
        nb_schedulers = 0
        ok_tag = HTMLTag.tag_value(u'OK', COLOR.GREEN)
        error_tag = HTMLTag.tag_value(u'WARNING', COLOR.ORANGE)
        unreachable_timeout_nb = 0
        unreachable_timeout_required = opts.timeout
        for satellite in satellites:
            
            if satellite[u'type'] != u'scheduler':
                satellite_type = u'%s%s(&nbsp;passive&nbsp;)' % (satellite[u'type'], BREAK_LINE)
                
                if satellite[u'type'] not in satellites_info[u'raw_stats']:
                    satellites_info[u'raw_stats'][satellite[u'type']] = {}
                    for item in data.get(u'info_%ss' % satellite[u'type'], []):
                        if item.get(u'realm', u'') and item.get(u'realm') == realm:
                            satellites_info[u'raw_stats'][satellite[u'type']][item.get(u'name', u'')] = item.get(u'tags', u'').replace(u',', u' ')
                tags = satellites_info[u'raw_stats'][satellite[u'type']].get(satellite[u'name'], u'')  # beware if the satellite is missing, another part detect it
            else:
                nb_schedulers = nb_schedulers + 1
                tags = u''
                satellite_type = satellite[u'type']
                if satellite[u'address'] == opts.hostname and satellite[u'port'] == opts.port:
                    my_name = satellite[u'name']
            
            if tags:
                tags = u'&nbsp;tags: %s' % HTMLTag.color_text(tags, bold=True)
            
            if satellite[u'status'] == u'ok':
                status = ok_tag
                err_msg = u''
            else:
                status = error_tag
                err_msg = satellite[u'status']
                if opts.timeout < satellite[u'timeout']:
                    if satellite[u'status'] == u'timed out':
                        unreachable_timeout_nb = unreachable_timeout_nb + 1
                        if satellite[u'timeout'] > unreachable_timeout_required:
                            unreachable_timeout_required = satellite[u'timeout']
                        err_msg = u'%s, BUT this result may not be reliable => [ because the check timeout (%ss) is shorter than %s configured timeout (%ss) ] %%(hint)s' % (
                            satellite[u'status'], opts.timeout, satellite[u'name'], satellite[u'timeout'])
                if satellite[u'type'] not in satellites_info[u'in_error']:
                    satellites_info[u'in_error'][satellite[u'type']] = []
                
                satellites_info[u'in_error'][satellite[u'type']].append(u'%s%s' % (HTMLTag.color_text(u'%s [%s:%s]' % (satellite[u'name'], satellite[u'address'], satellite[u'port']), COLOR.ORANGE), tags))
                err_msg = u' Unreachable %(type)s at %(uri)s ( %(err)s&nbsp;)' % {u'err': err_msg, u'type': satellite[u'type'], u'uri': satellite[u'uri']}
            
            if tags:
                tags = u'%s(%s&nbsp;)' % (BREAK_LINE, tags)
            satellites_info[u'satellites'].append((u'%s%s' % (satellite[u'name'], tags), satellite_type, u'%ss' % satellite[u'timeout'], u'%s%s' % (status, err_msg)))
        
        if unreachable_timeout_nb > 0:
            unreachable_timeout_required = unreachable_timeout_required + 1
            if unreachable_timeout_nb > 1:
                timeout_msg = u'Some daemons have a higher timeout than THIS check timeout ( See long result for details )'
            else:
                timeout_msg = u'A daemon have a higher timeout than THIS check timeout ( See long result for details )'
            timeout_msg = HTMLTag.tag_value(timeout_msg, COLOR.ORANGE)
            timeout_msg = HTMLList.header_list(header=timeout_msg, items=[HTMLTag.color_text(u'Timeout of this check should be increased to %ss' % unreachable_timeout_required, COLOR.ORANGE)], compact=False)
            tmp_info = []
            hint = u'=> Please increase timeout of THIS check to at least %ss' % unreachable_timeout_required
            for sat_name, sat_type, sat_timeout, sat_status in satellites_info[u'satellites']:
                tmp_info.append((sat_name, sat_type, sat_timeout, sat_status % {u'hint': hint}))
            satellites_info[u'satellites'] = tmp_info
        else:
            timeout_msg = u''
        
        satellites = HTMLTable.table((u'Name', u'Type', u'Timeout', u'Status'), satellites_info[u'satellites'], u'Connectivity status')
        result.add_check(long_output=satellites)
        if timeout_msg:
            result.add_check(EXIT_STATUS.WARNING, output=timeout_msg)
        
        if satellites_info[u'in_error']:
            for satellite_type in sorted(satellites_info[u'in_error'].keys()):
                nb_hs = len(satellites_info[u'in_error'][satellite_type])
                cpt = u''
                cluster_computation_warn = u''
                unreachable_list = []
                status = EXIT_STATUS.WARNING
                if satellites_info[u'raw_stats'].get(satellite_type, {}):
                    nb_available = len(satellites_info[u'raw_stats'][satellite_type])
                elif satellite_type == u'scheduler':
                    nb_available = nb_schedulers
                    if clusters > 0:
                        cluster_computation_warn = (u' %s' % HTMLTag.tag_value(u'cluster computation may be wrong', COLOR.ORANGE))
                else:
                    nb_available = 0
                if nb_available:
                    cpt = u' (%s/%s)' % (nb_hs, nb_available)
                unreachable_list.extend(satellites_info[u'in_error'][satellite_type])
                result.add_check(status,
                                 output=HTMLList.header_list(u'Unreachable %s%s%s%s' % (satellite_type, u's' if nb_hs > 1 else u'', cpt, cluster_computation_warn), unreachable_list),
                                 no_new_line=True)
    
    if result.status == EXIT_STATUS.OK:
        result.add_check(EXIT_STATUS.OK, u'Your scheduler is running well.', title=True)
    else:
        result.add_check(EXIT_STATUS.WARNING, u'Your scheduler have some issues.', title=True)
    
    elements_info = HTMLTable.table((u'Hosts', u'Clusters', u'Checks', u'Total'), [(hosts, clusters, checks, total)], u'Element in the scheduler')
    
    if not my_name:
        my_name = uri
    infos_header = u'Data retrieved from [ %s ] on realm %s.' % (my_name, realm)
    infos = []
    if version:
        infos.append(u'Version [ %s ]' % get_version_for_output(version))
    infos.append(u'Raw Data fetched in %.3fs' % connexion_time)
    infos.append(u'Connectivity to other daemons in %.3fs' % satellites_query_time)
    
    result.add_check(output=elements_info)
    result.add_check(EXIT_STATUS.OK, output=HTMLList.header_list(header=infos_header, items=infos, compact=True))
    
    ShinkenUtils.add_module_info(result, data)
    result.exit()


# Spec at SEF-1143
def scheduler_stats(opts):
    scheduler_too_old_retention_save_margin = opts.scheduler_too_old_retention_save_margin  # from parameter, and scheduler template
    data = basic_check(u'scheduler')
    stats_data = get_raw_stat()
    
    color_orange = u'<div style="color:#FF8F00;font-weight: bold;">%s</div>'
    style = u'''<style>.scheduler-stats-heading {font-size: 1.2em !important;font-weight: bold !important;margin: 15px 0 10px 0 !important;} .scheduler-stats-table, .scheduler-stats-table td, .scheduler-stats-table th {border: 1px solid #000000 !important;border-collapse: collapse !important;word-break: break-word !important;color: #000000 !important;margin-top: 4px !important;} .scheduler-stats-table {width: 100% !important;} .scheduler-stats-table th {text-align: center} .scheduler-stats-table th, .scheduler-stats-table td {background-color: #FFFFFF !important;} .scheduler-stats-table.same-with-col th, .scheduler-stats-table.same-with-col td {width: 11% !important} .scheduler-stats-table th.back-grey-sub-info {background-color: #E8E7E7 !important;} .scheduler-stats-table td {background-color: #FFFFFF !important;font-weight: normal !important;padding-left: 5px !important;padding-right: 5px !important;} .scheduler-stats-table .back-grey, .scheduler-stats-table .back-grey td, .scheduler-stats-table .back-grey th {background-color: #DDDDDD !important;} .scheduler-stats-table .invert-color td {background-color: #000000 !important;color: #FFFFFF !important;}</style>'''
    
    result.set_spare_info(data)
    
    result.add_check(output=style, no_new_line=True)
    
    arbiter_uri = data.get(u'arbiter_uri', u'')
    checks_todo_by_sec = data.get(u'checks_todo_by_sec', 0)
    notifications_todo_by_sec = data.get(u'notifications_todo_by_sec', 0)
    event_handlers_todo_by_sec = data.get(u'event_handlers_todo_by_sec', 0)
    checks_n_actions_stats = data.get(u'checks_n_actions_stats', None)
    if checks_n_actions_stats and u'check' in checks_n_actions_stats:
        average_latency = checks_n_actions_stats[u'check'].get(u'latency', 0)
    else:
        average_latency = data.get(u'average_latency', 0)
    info_pollers = data.get(u'info_pollers', [])
    info_reactionners = data.get(u'info_reactionners', [])
    loop_turn_time_avg = data.get(u'loop_turn_time_avg', 0)
    rogue_pollers = data.get(u'rogue_pollers', {})
    rogue_reactionners = data.get(u'rogue_reactionners', {})
    checks_warning_threshold_cpu_usage = data.get(u'checks_warning_threshold_cpu_usage', {})
    checks_warning_threshold_cpu_usage_nb = data.get(u'checks_warning_threshold_cpu_usage_nb', {})
    save_retention_time = data.get(u'save_retention_time', 0)
    save_retention_error = data.get(u'save_retention_error', u'')
    last_retention_save_epoch = data.get(u'last_retention_save', 0)  # date of the last save
    last_retention_save_try_epoch = data.get(u'last_retention_save_try', 0)  # date of the last try of save
    retention_save_interval = data.get(u'retention_save_interval', 0)
    last_retention_load_epoch = data.get(u'last_retention_load_epoch', 0)
    last_retention_load_duration = data.get(u'last_retention_load_duration', 0)
    
    avg_checks_cause_by_sec = {
        u'schedule'  : data.get(u'avg_checks_received_schedule_by_sec', 0),
        u'force'     : data.get(u'avg_checks_received_force_by_sec', 0),
        u'retry'     : data.get(u'avg_checks_received_retry_by_sec', 0),
        u'dependency': data.get(u'avg_checks_received_dependency_by_sec', 0),
    }
    avg_checks_cause_by_sec = collections.OrderedDict(avg_checks_cause_by_sec)
    
    nb_poller = len(info_pollers)
    nb_reactionner = len(info_reactionners)
    nb_poller_in_overload = 0
    nb_reactionner_in_overload = 0
    checks_done_by_sec = 0
    notifications_and_event_handlers_done_by_sec = 0
    error_message = ''
    info_satellites = {}
    
    exit_status = EXIT_STATUS.OK
    msg_scheduler_cpu_usage = [u'<div class="scheduler-stats-heading">Scheduler performance&nbsp;:&nbsp;</div>', BREAK_LINE]
    # Check for global time loop. If more than 1s, warn the user, but display all in percent
    if (loop_turn_time_avg * 100) > 100:
        exit_status = EXIT_STATUS.WARNING
        msg_scheduler_cpu_usage.extend((
            u'- Average scheduler CPU usage: %d%%' % min(loop_turn_time_avg * 100, 100),
            color_orange % u' + %d%% estimated CPU usage overload.' % (loop_turn_time_avg * 100 - 100),
            BREAK_LINE
        ))
    else:
        msg_scheduler_cpu_usage.extend(
            (u'- Average scheduler CPU usage: %d%%' % min(loop_turn_time_avg * 100, 100), BREAK_LINE))
    
    msg_scheduler_cpu_usage.extend(
        (u'- In the last 5 minutes, the executed checks had an average full waiting time ( in Scheduler + Poller ) of %ss before being launched' % round(average_latency, 2), BREAK_LINE))
    msg_scheduler_cpu_usage.append(u'- Retention save:%s' % BREAK_LINE)
    if last_retention_save_epoch and isinstance(last_retention_save_epoch, int):
        last_retention_age = abs(int(time.time()) - last_retention_save_epoch)
        last_retention_age_str = Utils.print_period(last_retention_age)
        last_retention_save_message = u'&nbsp;&nbsp;&nbsp;- was successfully saved at %s (age=%s)' % (time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(last_retention_save_epoch)), last_retention_age_str)
        if retention_save_interval:  # NOTE: old schedulers (before v02.08.01-Patched-06.02, SEF-8099) don't give this so no check
            
            if last_retention_age > retention_save_interval * 60 + scheduler_too_old_retention_save_margin * 60:  # retention interval + 5min margin
                if exit_status != EXIT_STATUS.CRITICAL:
                    exit_status = EXIT_STATUS.WARNING
                _error_msg = u' => it\'s %s (expected at least every %s minutes).' % (
                    HTMLTag.color_text(u'too old', COLOR.ORANGE), retention_save_interval)
                last_retention_save_message = u'&nbsp;&nbsp;&nbsp;- was successfully saved at %s, it is %s old' % (
                    time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(last_retention_save_epoch)), HTMLTag.color_text(last_retention_age_str, COLOR.ORANGE))
                last_retention_save_message = u'%s %s' % (last_retention_save_message, _error_msg)
        msg_scheduler_cpu_usage.append(u'%s %s' % (last_retention_save_message, BREAK_LINE))
        if last_retention_save_epoch != last_retention_save_try_epoch and isinstance(last_retention_save_try_epoch, int):
            msg_scheduler_cpu_usage.append(u'&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;- last save attempt was at %s.%s' % (time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(last_retention_save_try_epoch)), BREAK_LINE))
    
    if save_retention_error:
        exit_status = EXIT_STATUS.CRITICAL
        error_msg = '- Last retention save try was in %s: %s' % (HTMLTag.color_text('ERROR', COLOR.RED), BREAK_LINE)
        msg_scheduler_cpu_usage.append(error_msg)
        error_msg = ' => %s.%s' % (save_retention_error, BREAK_LINE)
        msg_scheduler_cpu_usage.append(HTMLTag.color_text(error_msg, COLOR.RED))
    
    elif save_retention_time == -1:
        msg_scheduler_cpu_usage.append('&nbsp;&nbsp;&nbsp;- save in progress %s' % BREAK_LINE)
    elif save_retention_time > 0:
        _msg_ret = Utils.print_period(save_retention_time)
        _msg_ret = 'less than 1s' if _msg_ret == '0s' else _msg_ret
        if save_retention_time > 100:
            exit_status = EXIT_STATUS.WARNING
            _msg_ret = HTMLTag.color_text(_msg_ret, COLOR.ORANGE)
        msg_scheduler_cpu_usage.append('&nbsp;&nbsp;&nbsp;- save time : %s.' % _msg_ret)
        if save_retention_time > 100:
            msg_scheduler_cpu_usage.append(' Your retention save was long.')
        msg_scheduler_cpu_usage.append(BREAK_LINE)
    
    # Load retention time
    if last_retention_load_epoch:
        msg_scheduler_cpu_usage.append('- Retention load:%s' % (BREAK_LINE))
        msg_scheduler_cpu_usage.append('&nbsp;&nbsp;&nbsp;- was at %s %s' % (time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(last_retention_load_epoch)), BREAK_LINE))
    if last_retention_load_duration:
        _msg_ret = Utils.print_period(last_retention_load_duration)
        _msg_ret = 'less than 1s' if _msg_ret == '0s' else _msg_ret
        msg_scheduler_cpu_usage.append('&nbsp;&nbsp;&nbsp;- load time : %s %s' % (_msg_ret, BREAK_LINE))
    
    message_by_status, exit_status_code, _hypervisor_perfdatas = ShinkenUtils.check_hypervisor_vm_cpu_stolen(stats_data, opts.thresold_cpu_stolen_warning, opts.thresold_cpu_stolen_critical)
    result.set_perf_data(_hypervisor_perfdatas)
    if message_by_status:
        msg_scheduler_cpu_usage.append('- %s' % message_by_status)
    else:
        exit_status_code = EXIT_STATUS.OK
    if (exit_status == EXIT_STATUS.WARNING and exit_status_code == EXIT_STATUS.OK) or (exit_status == EXIT_STATUS.CRITICAL and exit_status_code == EXIT_STATUS.OK):
        final_exist_status = exit_status
    elif exit_status == EXIT_STATUS.CRITICAL and exit_status_code == EXIT_STATUS.WARNING:
        final_exist_status = exit_status
    else:
        final_exist_status = exit_status_code
    
    result.add_check(final_exist_status, output=''.join(msg_scheduler_cpu_usage))
    
    if checks_warning_threshold_cpu_usage:
        msg_checks_warning_threshold_cpu_usage = '<div class="scheduler-stats-heading">There are %s checks that take too much CPU time. Last 5 checks in warning : </div><br/>' % checks_warning_threshold_cpu_usage_nb
        msg_checks_warning_threshold_cpu_usage += '<table class="scheduler-stats-table">' \
                                                  '<tr>' \
                                                  '<th class="back-grey">Command name</th>' \
                                                  '<th class="back-grey">CPU consumed (in sec)</th>' \
                                                  '<th class="back-grey">CPU threshold (in sec)</th>' \
                                                  '<th class="back-grey">At</th>' \
                                                  '</tr>'
        for entry in checks_warning_threshold_cpu_usage:
            msg_checks_warning_threshold_cpu_usage += '<tr>' \
                                                      ' <td>%s</td>' \
                                                      ' <td class="number">%s</td>' \
                                                      ' <td class="number">%s</td>' \
                                                      ' <td>%s</td>' \
                                                      '</tr>' % (entry[0], entry[1], entry[2], Utils.print_time(entry[3]))
        msg_checks_warning_threshold_cpu_usage += "</table>"
        result.add_check(EXIT_STATUS.WARNING, output=msg_checks_warning_threshold_cpu_usage)
    
    # Some unknown satellites talk to us, this shouldn't be a normal case
    if rogue_pollers or rogue_reactionners:
        message = ['Some unknown daemons are currently contacting this Scheduler.',
                   'This may be a temporary problem that can be fixed by restarting the listed daemons, or be an another unrelated issue (network problems for exemple)']
        if rogue_pollers:
            message.append('<table class="scheduler-stats-table" >%s</table>' % ''.join(
                _table_rogue_stats('pollers', rogue_pollers)))
        if rogue_reactionners:
            message.append('<table class="scheduler-stats-table" >%s</table>' % ''.join(
                _table_rogue_stats('reactionners', rogue_reactionners)))
        result.add_check(EXIT_STATUS.WARNING, BREAK_LINE.join(message))
    
    scheduler_satellite = ['<div class="scheduler-stats-heading">Scheduler Satellite&nbsp;:&nbsp;</div><br/>']
    if arbiter_uri:
        arbiter_uri = arbiter_uri.replace("127.0.0.1", address)
        arbiter_uri = arbiter_uri.replace("localhost", address)
        try:
            buf, _ = Utils.request_get(result, arbiter_uri, '/get_satellites', timeout=opts.timeout, raise_exp=True)
            info_satellites = json.loads(buf)
        except:
            error_message = 'Failed to connect to [%s]<br>Note : <div style="font-style: italic;">Address retrieved from arbiter configuration.</div>' % arbiter_uri
            scheduler_satellite.append(error_message)
    else:
        error_message = 'Missing arbiter uri. please upgrade your scheduler'
        scheduler_satellite.append(error_message)
    
    if error_message:
        result.add_check(EXIT_STATUS.WARNING, output=''.join(scheduler_satellite))
    else:
        if nb_poller == 0:
            scheduler_satellite.append('No pollers found for this scheduler! Please wait pollers will connect to the scheduler.<br/>')
        else:
            checks_done_by_sec = sum([info_poller['done_by_sec'] for info_poller in info_pollers])
            lines, executor_in_overload, unreachable_pollers = _table_executor_stats(checks_done_by_sec, checks_todo_by_sec, info_pollers, info_satellites, nb_poller, 'poller')
            scheduler_satellite.append('<table class="scheduler-stats-table" >%s</table><br/>' % ''.join(lines))
            if executor_in_overload:
                nb_poller_in_overload += 1
            if unreachable_pollers:
                result.add_check(EXIT_STATUS.WARNING, output=HTMLList.header_list('The check failed to connect to poller(s)', unreachable_pollers), no_new_line=True)
        
        if nb_reactionner == 0:
            scheduler_satellite.append('No reactionner found for this scheduler! Please wait reactionners will connect to the scheduler.<br/>')
        else:
            notifications_and_event_handlers_done_by_sec = sum([info_reactionner['done_by_sec'] for info_reactionner in info_reactionners])
            lines, executor_in_overload, unreachable_reactionners = _table_executor_stats(notifications_and_event_handlers_done_by_sec, notifications_todo_by_sec + event_handlers_todo_by_sec, info_reactionners, info_satellites, nb_reactionner,
                                                                                          'reactionner')
            scheduler_satellite.append('<table class="scheduler-stats-table">%s</table><br/>' % ''.join(lines))
            if executor_in_overload:
                nb_reactionner_in_overload += 1
            if unreachable_reactionners:
                result.add_check(EXIT_STATUS.WARNING, output=HTMLList.header_list('The check failed to connect to reactionner(s)', unreachable_reactionners), no_new_line=True)
        
        result.add_check(EXIT_STATUS.OK, output=''.join(scheduler_satellite))
    
    check_by_cause = ''
    # add the check cause by seconds
    check_by_cause += '<div class="scheduler-stats-heading">Scheduled checks per second classified by causes&nbsp;:&nbsp;</div>'
    check_by_cause += '<table class="scheduler-stats-table">' \
                      '<tr>' \
                      '<th class="back-grey">Causes</th>' \
                      '<th class="back-grey">Checks per second</th>' \
                      '</tr>'
    total = 0
    for check_cause, avg_per_sec in avg_checks_cause_by_sec.iteritems():
        avg_round = round(avg_per_sec, 2)
        total += avg_round
        check_by_cause += '<tr>' \
                          ' <td>%s</td>' \
                          ' <td class="number">%0.2f/s</td>' \
                          '</tr>' % (check_cause, avg_round)
    check_by_cause += '<tr class="invert-color">' \
                      ' <td>Total</td>' \
                      ' <td>%0.2f/s</td>' \
                      '</tr>' % total
    check_by_cause += "</table>"
    result.add_check(output=check_by_cause)
    
    stats = {
        'checks_todo_by_sec'                          : checks_todo_by_sec,
        'checks_done_by_sec'                          : checks_done_by_sec,
        'notifications_todo_by_sec'                   : notifications_todo_by_sec,
        'event_handlers_todo_by_sec'                  : event_handlers_todo_by_sec,
        'notifications_and_event_handlers_done_by_sec': notifications_and_event_handlers_done_by_sec,
        'average_scheduler_cpu_usage'                 : min(100, loop_turn_time_avg * 100),
        'average_scheduler_cpu_estimated_overload'    : max(0, (loop_turn_time_avg * 100) - 100),
        'nb_pollers'                                  : nb_poller,
        'nb_pollers_in_overload'                      : nb_poller_in_overload,
        'nb_reactionners'                             : nb_reactionner,
        'nb_reactionners_in_overload'                 : nb_reactionner_in_overload,
    }
    if save_retention_time > 0:
        stats['save_retention_time'] = save_retention_time
    if last_retention_load_duration > 0:
        stats['load_retention_time'] = last_retention_load_duration
    result.set_perf_data(stats)
    
    if result.status == EXIT_STATUS.OK:
        result.add_check(EXIT_STATUS.OK, 'The scheduler is running well.', title=True)
    else:
        result.add_check(EXIT_STATUS.WARNING, 'The scheduler have some issues.', title=True)
    
    result.exit()


def _table_rogue_stats(executor_type, info_rogue_satellites):
    lines = []
    first_line = '<tr>' \
                 '<th class="back-grey">rogue %s name</th>' \
                 '<th class="back-grey">last check date</th>' \
                 '</tr>' % (executor_type,)
    lines.append(first_line)
    for name, last_check_time in info_rogue_satellites.iteritems():
        line = '<tr><td>%s</td><td>%s</td></tr>'
        line = line % (name, Utils.print_time(last_check_time))
        lines.append(line)
    return lines


def _table_executor_stats(actions_done_by_sec, actions_todo_by_sec, info_executors, info_satellites, nb_executor, executor_type):
    action_name = 'checks' if executor_type == 'poller' else 'notifications & event handlers'
    lines = []
    executor_in_overload = False
    unreachable_executor = []
    first_line = '<tr>' \
                 '<th class="back-grey" rowspan="2" style="width: 8%% !important;">%s name</th>' \
                 '<th class="back-grey" rowspan="2" style="width: 8%% !important;">realm</th>' \
                 '<th class="back-grey" rowspan="2" style="width: 8%% !important;">tags</th>' \
                 '<th class="back-grey" rowspan="2" style="width: 8%% !important;">%s todo</th>' \
                 '<th class="back-grey" rowspan="2" style="width: 8%% !important;">%s done</th>' \
                 '<th class="back-grey" colspan="2" style="width: 45%% !important;">CPU</th>' \
                 '<th class="back-grey" rowspan="1" style="width: 15%% !important;">RAM</th>' \
                 '<th class="back-grey" rowspan="1" style="width: 15%% !important;">LOAD</th>' \
                 '</tr>' % (executor_type, action_name, action_name)
    lines.append(first_line)
    seconde_line = '<tr>' \
                   '<th class="back-grey-sub-info" style="width: 15%% !important;">available</th>' \
                   '<th class="back-grey-sub-info" style="width: 15%% !important;">used by the daemon %s</th>' \
                   '<th class="back-grey-sub-info" style="width: 15%% !important;">%% used on the server</th>' \
                   '<th class="back-grey-sub-info" style="width: 15%% !important;"> on the server</th>' \
                   '</tr>' % (executor_type)
    lines.append(seconde_line)
    
    for line_number, info_executor in enumerate(info_executors):
        if line_number == 0:
            line = '<tr class="%s">' \
                   '<td>%s</td>' \
                   '<td>%s</td>' \
                   '<td>%s</td>' \
                   '<td rowspan="' + str(nb_executor) + '" class="number">' + '%0.2f' % (round(actions_todo_by_sec, 2)) + \
                   '/s</td>' \
                   '<td class="number">%s/s</td>' \
                   '<td>%s</td>' \
                   '<td>%s</td>' \
                   '<td>%s</td>' \
                   '<td>%s</td>' \
                   '</tr>'
            windows_line = '<tr class="%s">' \
                           '<td>%s</td>' \
                           '<td>%s</td>' \
                           '<td>%s</td>' \
                           '<td rowspan="' + str(nb_executor) + '" class="number">' + '%0.2f' % (round(actions_todo_by_sec, 2)) + \
                           '<td class="number">%s/s</td>' \
                           '<td colspan="3" style="text-align: center;">%s</td>' \
                           '<td>%s</td>' \
                           '</tr>'
        else:
            line = '<tr class="%s">' \
                   '<td>%s</td>' \
                   '<td>%s</td>' \
                   '<td>%s</td>' \
                   '<td class="number">%s/s</td>' \
                   '<td>%s</td>' \
                   '<td>%s</td>' \
                   '<td>%s</td>' \
                   '<td>%s</td>' \
                   '</tr>'
            windows_line = '<tr class="%s">' \
                           '<td>%s</td>' \
                           '<td>%s</td>' \
                           '<td>%s</td>' \
                           '<td class="number">%s/s</td>' \
                           '<td colspan="3" style="text-align: center;">%s</td>' \
                           '<td>%s</td>' \
                           '</tr>'
        nb_cpus = -1
        cpu_usage = -1
        executor_load = -1.0
        realm = info_executor.get('realm', '')
        tags = info_executor.get('tags', '')
        name = info_executor['name']
        cpu_running_queue = -1
        max_cpu_queue_per_cpu = 4
        ram_usage = -1.0
        max_ram_usage = 100.0
        error_message = ''
        executor_uri = next((x for x in info_satellites if x['name'] == info_executor['name'] and x['type'] == executor_type), {}).get('uri', '')
        if executor_uri:
            executor_uri = executor_uri.replace("127.0.0.1", address)
            executor_uri = executor_uri.replace("localhost", address)
            try:
                raw_info_from_executor = get_raw_stat_with_exp(_uri=executor_uri)
                daemon_api_version = raw_info_from_executor.get(u'api_version', None)
                arbiter_version = raw_info_from_executor.get(u'arbiter_version', None)
                daemon_version = raw_info_from_executor.get(u'daemon_version', None)
                nb_cpus = raw_info_from_executor.get(u'nb_cpus', -1)
                cpu_usage = raw_info_from_executor.get(u'cpu_usage', -1)
                cpu_running_queue = raw_info_from_executor.get(u'cpu_running_queue', -1)
                max_cpu_queue_per_cpu = raw_info_from_executor.get(u'max_cpu_queue_par_cpu', -1.0)
                ram_usage = raw_info_from_executor.get(u'ram_usage', -1.0)
                max_ram_usage = raw_info_from_executor.get(u'max_ram_usage', -1.0)
                executor_load = raw_info_from_executor.get(u'executor_load', -1.0)
                realm = raw_info_from_executor.get(u'realm', '')
                tags = ','.join(raw_info_from_executor.get(u'tags', []))
                schedulers = raw_info_from_executor.get(u'schedulers', None)
                platform = raw_info_from_executor.get(u'platform', '')
                spare = raw_info_from_executor.get(u'spare', False)
                activated = raw_info_from_executor.get(u'activated', True)
                have_conf = raw_info_from_executor.get(u'have_conf', False)
                if spare:
                    name = "%s <b>(SPARE)" % name
                    if activated:
                        name += "(RUNNING)"
                    name += '</b>'
                if not have_conf:
                    error_message = "No configuration given by an Arbiter for now."
            except:
                error_message = 'UNREACHABLE : Failed to connect to [%s]<br>Note : <div style="font-style: italic;">Address retrieved from %s configuration.</div>' % (executor_uri, executor_type)
                unreachable_executor.append('%s at %s' % (name, executor_uri))
        else:
            error_message = 'UNREACHABLE : Failed to get URI of the %s [%s]' % (executor_type, info_executor['name'])
        if error_message:
            error_message = HTMLTag.tag_value(error_message, COLOR.RED)
            if line_number == 0:
                line = '<tr>' \
                       '<td>%s</td>' \
                       '<td>%s</td>' \
                       '<td>%s</td>' \
                       '<td rowspan="' + str(nb_executor) + '" class="number">' + '%0.2f' % round(actions_todo_by_sec, 2) + \
                       '/s</td>' \
                       '<td colspan="5" style="text-align: center;">' + error_message + \
                       '</td>' \
                       '</tr>'
            else:
                line = '<tr>' \
                       '<td>%s</td>' \
                       '<td>%s</td>' \
                       '<td>%s</td>' \
                       '<td colspan="5" style="text-align: center;">' + error_message + \
                       '</td>' \
                       '</tr>'
            realm = realm or '-'
            tags = tags or '-'
            line = line % (name, realm, tags)
            lines.append(line)
            continue
        
        message = ShinkenUtils._check_versions(executor_type, daemon_api_version, daemon_version, arbiter_version, shinken_supervisor_version)
        
        if message:
            warning_message = 'WARNING : ' + message
            warning_message = HTMLTag.tag_value(warning_message, COLOR.ORANGE)
            if line_number == 0:
                line = '<tr>' \
                       '<td>%s</td>' \
                       '<td>%s</td>' \
                       '<td>%s</td>' \
                       '<td rowspan="' + str(nb_executor) + '" class="number">' + '%0.2f' % (round(actions_todo_by_sec, 2)) + \
                       '/s</td>' \
                       '<td colspan="5" style="text-align: center;">' + warning_message + \
                       '</td>' \
                       '</tr>'
            else:
                line = '<tr>' \
                       '<td>%s</td>' \
                       '<td>%s</td>' \
                       '<td>%s</td>' \
                       '<td colspan="5" style="text-align: center;">' + warning_message + \
                       '</td>' \
                       '</tr>'
            realm = realm or '-'
            tags = tags or '-'
            line = line % (name, realm, tags)
            lines.append(line)
            continue
        
        # special case for windows for ex, cpu-usage is not available but give us 0.0
        if info_executor['done_by_sec'] and cpu_usage == 0.0:
            cpu_usage = None
            executor_load = -1
        
        executor_in_overload = executor_load > EXECUTOR_LOAD_LIMIT
        executor_in_overram = ram_usage > max_ram_usage
        cpu_usage = '&#216;' if cpu_usage is None or cpu_usage == -1 else int(round(cpu_usage * 100 / nb_cpus, 0))
        cpu_usage = '%s%% maximal usable (of %d core)' % (
            cpu_usage, nb_cpus) if executor_in_overload else '%s%% (of %d core)' % (cpu_usage, nb_cpus)
        line_class = 'overload' if executor_in_overload or executor_in_overram else ''
        executor_load = -1 if executor_load == -1 else round(executor_load, 2)
        
        if cpu_running_queue and cpu_running_queue != -1:
            cpu_running_queue_text = HTMLTag.cpu_queue_tag(cpu_running_queue, max_cpu_queue_per_cpu,
                                                           nb_cpus) + '<br>' + '%s Processes in the queue (limit : %s)' % (
                                         cpu_running_queue, (max_cpu_queue_per_cpu * nb_cpus))
        else:
            cpu_running_queue_text = HTMLTag.cpu_queue_tag(cpu_running_queue, max_cpu_queue_per_cpu,
                                                           nb_cpus) + '<br>' + 'CPU running queue of the %s can\'t be compute.' % executor_type
        
        ram_usage_text = '&#216;' if ram_usage is None or ram_usage == -1.0 else '%.0f%%' % ram_usage
        if platform == 'nt':
            line = windows_line % (
                line_class,
                name,
                realm,
                tags,
                round(info_executor['done_by_sec'], 2),
                'This poller is running on Windows, impossible to get CPU informations.',
                HTMLTag.ram_tag(ram_usage, max_ram_usage) + '<br>' + ram_usage_text + ' (limit :%.0f%%)' % max_ram_usage,
            )
        else:
            line = line % (
                line_class,
                name,
                realm,
                tags,
                '%0.2f' % round(info_executor['done_by_sec'], 2),
                HTMLTag.load_tag(executor_load),
                cpu_usage,
                HTMLTag.ram_tag(ram_usage, max_ram_usage) + '<br>' + ram_usage_text + ' (limit :%.0f%%)' % max_ram_usage,
                cpu_running_queue_text,
            )
        lines.append(line)
    last_line = '<tr class="invert-color">' \
                '<td>Total</td>' \
                '<td colspan="2"></td>' \
                '<td>%s/s</td>' \
                '<td>%s/s</td>' \
                '<td colspan=4></td>' \
                '</tr>' % \
                ('%0.2f' % round(actions_todo_by_sec, 2), '%0.2f' % round(actions_done_by_sec, 2))
    lines.append(last_line)
    return lines, executor_in_overload, unreachable_executor


def scheduler_latency():
    result.hard_exit(EXIT_STATUS.WARNING, 'The mode "latency" is deprecated for this plugin version. Please upgrade shinken template. Info will be available in Scheduler Running Well check.')


def scheduler_late_checks():
    result.hard_exit(EXIT_STATUS.WARNING, 'The mode "latency" is deprecated for this plugin version. Please upgrade shinken template. Info will be available in Scheduler Running Well check.')


def scheduler_top10_total():
    result.hard_exit(EXIT_STATUS.WARNING, 'The mode "top10_total" is deprecated for this plugin version. Please upgrade shinken template. You can see the check usage in Poller Performance check.')


def scheduler_top10_average():
    result.hard_exit(EXIT_STATUS.WARNING, 'The mode "top10_total" is deprecated for this plugin version. Please upgrade shinken template. You can see the check usage in Poller Performance check.')


def broker_modules_queue():
    perfs = {}
    data = basic_check('broker')
    
    modules = data.get('modules', [])
    for m_ in modules:
        mname = m_['module_name']
        l_ = m_['queue_size']
        perfs['module.%s.data_queue' % mname] = l_
    
    result.set_spare_info(data, daemon_type)
    
    result.add_check(EXIT_STATUS.OK, 'Modules performance is going well')
    
    module_stats = data.get('module_stats', {})
    for (module_type, module_type_entry) in module_stats.iteritems():
        for module_name, module_entry in module_type_entry.iteritems():
            if module_entry is None:
                continue
            workers = module_entry.get('workers', None)
            if workers is None:
                continue
            
            status = EXIT_STATUS.OK
            # Set a table with workers informations into the long output
            header = ('Worker', 'Managed hosts')
            total_number_of_managed_hosts = 0
            tab_lines = []
            for (worker_id, worker) in sorted(workers.iteritems()):
                number_of_managed_hosts = worker.get('number_of_managed_hosts', 0)
                queue_size = worker.get('main_process_to_worker_queue_size', 0)
                total_number_of_managed_hosts += number_of_managed_hosts
                perfs['module.%s.data_queue' % module_name] = queue_size
                perfs['module.%s.managed_hosts' % module_name] = number_of_managed_hosts
                tab_lines.append(('%s Worker - %s' % (module_name, worker_id), number_of_managed_hosts))
            nb_workers = len(workers)
            output = '<ul><li>'
            if nb_workers == 0:
                output += '%s The module %s do not have any workers' % (HTMLTag.CRITICAL, module_name)
                long_output = ''
                status = EXIT_STATUS.CRITICAL
            else:
                output += '%s The module %s have <span style="font-style:italic;color: black;">%d workers</span>' % (HTMLTag.OK, module_name, nb_workers)
                long_output = HTMLTable.table(header, tab_lines, 'Module %s workers' % module_name, compact_title=True)  # We do not want a space between the title and the table
            output += '</li></ul>'
            result.add_check(status=status, output=output, long_output=long_output)
    
    result.set_perf_data(perfs)
    result.exit()


def broker_sla():
    data = basic_check('broker')
    
    sla_module_stats = data.get('module_stats', {}).get('sla', {})
    perfs = {}
    
    # if not stats, means no such module
    if sla_module_stats == {}:
        result.add_check(EXIT_STATUS.CRITICAL, 'Your daemon does not have any sla module.')
    else:
        for (module_name, stats) in sla_module_stats.iteritems():
            update_sla_per_second = stats.get('update_sla_per_second', 0)
            perfs['%s.update_check_per_second' % module_name] = update_sla_per_second
            result.set_perf_data(perfs)
            
            msg = "The SLA module processes %s checks per second." % update_sla_per_second
            
            result.add_check(EXIT_STATUS.OK if update_sla_per_second else EXIT_STATUS.WARNING, msg)
            
            now = time.time()
            last_archive_sla_time = stats.get('last_archive_sla_time', -1)
            if last_archive_sla_time != -1 and last_archive_sla_time is not None:
                if last_archive_sla_time > 30 * 60:
                    result.add_check(EXIT_STATUS.CRITICAL, 'Archiving sla takes too much time ( %s ), Please upgrade your machine.' % Utils.print_period(int(last_archive_sla_time)))
                elif last_archive_sla_time > 10 * 60:
                    msg = 'Archiving sla was long ( %s ) it should take under 10 minute, please check the shinken mongo base activity.' % Utils.print_period(int(last_archive_sla_time))
                    result.add_check(EXIT_STATUS.WARNING, msg)
                else:
                    result.add_check(output='%s Archiving SLA works well at %s' % (HTMLTag.state_tag(HTMLTag.STATE_OK), Utils.print_period(int(last_archive_sla_time))))
            
            last_archive_sla = stats.get('last_archive_sla', -1)
            if last_archive_sla != -1 and last_archive_sla is not None:
                periode_last_archive = (now - last_archive_sla)  # 60m x 60s
                if periode_last_archive > 25 * 3 * 3600:
                    msg = 'Your last archive was too old ( done %s ago ), please check the shinken mongo base availability.' % Utils.print_period(periode_last_archive)
                    result.add_check(EXIT_STATUS.CRITICAL, msg)
                elif periode_last_archive > 24 * 3600:
                    msg = 'Your last archive was done %s ago, the archiving should be done daily, please restart your shinken to force SLA archiving.' % Utils.print_period(periode_last_archive)
                    result.add_check(EXIT_STATUS.WARNING, msg)
                else:
                    result.add_check(output='%s Your last archive was done %s ago.' % (HTMLTag.state_tag(HTMLTag.STATE_OK), Utils.print_period(periode_last_archive)))
    
    if result.status == EXIT_STATUS.OK:
        result.add_check(EXIT_STATUS.OK, 'The SLA module is running well.', title=True)
    else:
        result.add_check(EXIT_STATUS.WARNING, 'The SLA module has some issues.', title=True)
    
    result.exit()


def broker_livedata():
    data = basic_check(daemon_type)
    result.set_spare_info(data)
    info = {}
    perfs = {}
    module_name = getattr(opts, 'module_name', '')
    if not module_name:
        module_name = "broker-module-livedata"
    modules = data.get('modules_info')
    for module in modules:
        if module['name'] == module_name:
            info = module
            break
    
    # if no info, means no such module
    if not info:
        result.add_check(EXIT_STATUS.CRITICAL, '%s is not running' % module_name)
    else:
        uri = '%s:%s' % (opts.hostname, info['port'])
        uri = uri.decode('utf-8')
        if info['https'] is True:
            result.add_check(EXIT_STATUS.OK, "The module is using <span style=\"color: black;\">HTTPS</span>")
            conn = Utils._http_get_conn(uri, timeout=3, use_ssl=(info['https'] is True), ssl_version=ssl.PROTOCOL_TLSv1_2)
        else:
            result.add_check(EXIT_STATUS.OK, "The module is using <span style=\"color: black;\">HTTP</span>")
            conn = Utils._http_get_conn(uri, timeout=3, use_ssl=(info['https'] is True))
        try:
            conn.request("GET", "/api/v1/ping")
            r1 = conn.getresponse()
            buf = r1.read()
        except Exception as e:
            result.add_check(EXIT_STATUS.CRITICAL, "Your module is activated, but we can't communicate with the API : %s" % e)
        else:
            if "Not found" in buf or "pong" not in buf:
                result.add_check(EXIT_STATUS.CRITICAL, "Your module is activated, but we can't communicate with the API.")
        
        if info['token'] == 2:
            result.add_check(EXIT_STATUS.WARNING, "There is no token in the configuration, it's not safe")
        
        if info['token'] == 1:
            result.add_check(EXIT_STATUS.WARNING, "You are using the default token, it's recommended to change it")
        
        info_for_list = []
        perfs["livedata_nb_error_last_hour"] = info['error_per_hour']
        result.add_check(EXIT_STATUS.OK, 'The module handled <span style="color: black;">%d</span> request(s) in the last hour' % info['request_per_hour'])
        
        if info['error_per_hour'] >= opts.livedata_warning > 0:
            result.add_check(EXIT_STATUS.WARNING, 'The module handled <span style="color: %s;">%d</span> request error(s) in the last hour' % (COLOR.ORANGE, info['error_per_hour']))
        elif info['error_per_hour'] > 0:
            info_for_list.append('Error(s) handled: <span style="font-style:italic;color: black;">%d</span>' % info['error_per_hour'])
        else:
            info_for_list.append('No error handled')
        perfs["livedata_nb_request_last_hour"] = info['request_per_hour']
        
        if info['best_in_one_second']['nb'] != 0:
            info_for_list.append('Connection peak : <span style="font-style:italic;color: black;">%d request(s) at %s</span>' % (info['best_in_one_second']['nb'], info['best_in_one_second']['time']))
        
        if 'average_response_time' in info.keys() and info['average_response_time']:
            info_for_list.append('Average response time: <span style="font-style:italic;color: black;">%.4fs</span>' % info['average_response_time'])
            perfs["livedata_average_response_time"] = info['average_response_time']
        else:
            info_for_list.append('You need to make at least one call to the api for informations')
        
        if 'worst_response_time' in info.keys() and info['worst_response_time']:
            info_for_list.append(
                'Max response time: <span style="font-style:italic;color: black;">%.4fs at %s</span>' % (info['worst_response_time']['response_time'], datetime.datetime.fromtimestamp(info['worst_response_time']['time']).strftime('%H:%M:%S')))
        
        result.add_check(EXIT_STATUS.OK, HTMLList.simple_list(info_for_list))
        if info['request_per_hour'] > 0:
            perfs["livedata_error_percent"] = (float(info['error_per_hour']) / float(info['request_per_hour'])) * 100
        else:
            perfs["livedata_error_percent"] = 0
        
        if len(info['errors']) > 0:
            info['errors'].reverse()
            number = 0
            tab_lines = []
            header = ('Date', 'Error')
            for index in range(0, opts.livedata_error_limit):
                if index >= len(info['errors']):
                    break
                number += 1
                hour = datetime.datetime.fromtimestamp(info['errors'][index]['time']).strftime('%H:%M:%S')
                tab_lines.append((hour, info['errors'][index]['error']))
            result.add_check(long_output=HTMLTable.table(header, tab_lines, 'Last %d errors' % number))
    result.set_perf_data(perfs)
    
    if result.status == EXIT_STATUS.OK:
        result.add_check(EXIT_STATUS.OK, 'Livedata %s module is running well.' % module_name, title=True)
    elif result.status == EXIT_STATUS.WARNING:
        result.add_check(EXIT_STATUS.WARNING, 'Livedata %s module has some issues.' % module_name, title=True)
    elif result.status == EXIT_STATUS.CRITICAL:
        result.add_check(EXIT_STATUS.CRITICAL, 'Livedata %s module has critical issues.' % module_name, title=True)
    
    result.exit()


# Available mode for the check based on the daemon types
MODES = {
    'arbiter'     : {
        'alive'         : {'methode': arbiter_alive, 'help': ''},
        'api_connection': {'methode': api_connection, 'help': ''},
        'stats'         : {'methode': arbiter_stats, 'help': ''},
    },
    'scheduler'   : {
        'alive'         : {'methode': scheduler_alive, 'help': ''},
        'api_connection': {'methode': api_connection, 'help': '[DEPRECATED]'},
        'latency'       : {'methode': scheduler_latency, 'help': '[DEPRECATED]'},
        'late_checks'   : {'methode': scheduler_late_checks, 'help': '[DEPRECATED]'},
        'top10_total'   : {'methode': scheduler_top10_total, 'help': '[DEPRECATED]'},
        'top10_average' : {'methode': scheduler_top10_average, 'help': '[DEPRECATED]'},
        'stats'         : {'methode': scheduler_stats, 'help': ''},
    },
    'poller'      : {
        'alive'              : {'methode': poller_alive, 'help': ''},
        'api_connection'     : {'methode': poller_api_connection, 'help': '[DEPRECATED]'},
        'cpu_load'           : {'methode': poller_cpu_load, 'help': '[DEPRECATED]'},
        'overload_protection': {'methode': poller_overload_protection, 'help': '[DEPRECATED]'},
        'stats'              : {'methode': poller_stats, 'help': ''},
    },
    'reactionner' : {
        'alive'              : {'methode': reactionner_alive, 'help': ''},
        'api_connection'     : {'methode': reactionner_api_connection, 'help': '[DEPRECATED]'},
        'cpu_load'           : {'methode': reactionner_cpu_load, 'help': '[DEPRECATED]'},
        'overload_protection': {'methode': reactionner_overload_protection, 'help': '[DEPRECATED]'},
        'stats'              : {'methode': reactionner_stats, 'help': ''},
    },
    'broker'      : {
        'alive'         : {'methode': alive, 'help': ''},
        'api_connection': {'methode': api_connection, 'help': ''},
        'modules_queue' : {'methode': broker_modules_queue, 'help': ''},
        'sla'           : {'methode': broker_sla, 'help': ''},
        'livedata'      : {'methode': broker_livedata, 'help': ''},
    },
    'receiver'    : {
        'alive'         : {'methode': alive, 'help': ''},
        'api_connection': {'methode': api_connection, 'help': ''},
    },
    'synchronizer': {
        'alive'         : {'methode': synchronizer_alive, 'help': ''},
        'api_connection': {'methode': api_connection, 'help': ''},
    },
}

DAEMON_DEFAULT_PORT = {
    u'arbiter'     : 7770,
    u'scheduler'   : 7768,
    u'poller'      : 7771,
    u'reactionner' : 7769,
    u'broker'      : 7772,
    u'receiver'    : 7773,
    u'synchronizer': 7765
}

daemon_type = u''
mode = u''
uri = u''
address = u''
shinken_supervisor_version = u''
last_check = None
to_print = u''  # use for testing the output of the check
result = Result()
opts = None


def _main():
    global daemon_type, mode, uri, last_check, address, opts, shinken_supervisor_version
    
    parser = get_option_parser()
    opts, args = parser.parse_args()
    
    daemon_type = opts.daemon_type
    
    mode = opts.mode
    last_check = opts.last_check
    address = opts.hostname
    shinken_supervisor_version = opts.shinken_supervisor_version
    
    _port = opts.port
    if not _port:
        _port = DAEMON_DEFAULT_PORT[daemon_type] if daemon_type in DAEMON_DEFAULT_PORT else _port
    
    uri = u'%s:%s' % (opts.hostname, _port)
    uri = uri.decode(u'utf-8')
    
    # Check if the daemon type is known
    if daemon_type not in MODES:
        result.hard_exit(EXIT_STATUS.CRITICAL, u'The daemon type %s is unknown' % daemon_type)
    
    # And if check mode choose is known for this daemon too
    if mode not in MODES[daemon_type]:
        result.hard_exit(EXIT_STATUS.CRITICAL, u'The mode %s is unknown for this daemon type' % mode)
    
    if opts.timeout <= 0:
        result.hard_exit(EXIT_STATUS.CRITICAL, u'The --timeout option (%s) must be greater than 0' % opts.timeout)
    
    methode_to_call = MODES[daemon_type][mode][u'methode']
    if callable(methode_to_call):
        arg, _, _, _, = getargspec(methode_to_call)
        if len(arg) > 0:
            methode_to_call(opts)
        else:
            methode_to_call()


if __name__ == u'__main__':
    _main()
