#!/usr/bin/env python3
# -*- coding: utf-8; py-indent-offset: 4 -*-
#
# Author:  Linuxfabrik GmbH, Zurich, Switzerland
# Contact: info (at) linuxfabrik (dot) ch
#          https://www.linuxfabrik.ch/
# License: The Unlicense, see LICENSE file.

# https://github.com/Linuxfabrik/monitoring-plugins/blob/main/CONTRIBUTING.md

"""See the check's README for more details."""

import argparse
import ipaddress
import os
import sys
import urllib.parse

import lib.args
import lib.base
import lib.db_sqlite
import lib.human
import lib.lftest
import lib.url
from lib.globals import STATE_OK, STATE_UNKNOWN

__author__ = 'Linuxfabrik GmbH, Zurich/Switzerland'
__version__ = '2026082501'

DESCRIPTION = """Monitors Apache httpd via the mod_status endpoint (server-status?auto).
Reports worker slot usage, worker and connection states, request and traffic rates, mean
request duration, CPU usage and system load averages. Alerts when the percentage of
occupied worker slots exceeds the warning or critical threshold. Cumulative counters are
converted into per-second rates against the previous check run, so the first run after an
installation and the first run after an httpd restart report no rates yet. Metrics that
the queried httpd version, MPM or ExtendedStatus setting does not provide are left out
instead of failing."""

DEFAULT_CRIT = '95'  # %
DEFAULT_INSECURE = False
DEFAULT_NO_PROXY = False
DEFAULT_TIMEOUT = 8
DEFAULT_URL = 'http://localhost/server-status'
DEFAULT_WARN = '80'  # %

# Scoreboard flags, complete as of mod_status.c (`status_flags[]`). Listed in the order
# mod_status documents them, which follows a worker's life cycle, rather than
# alphabetically.
#
# The length of the string is the number of worker slots. On httpd 2.4 that is exactly
# MaxRequestWorkers, because mod_status marks every slot beyond the configured limit
# SERVER_DISABLED and leaves those out of the string. Verified on Rocky 9 / httpd 2.4.62
# for MaxRequestWorkers 50, 150, 256 and 400.
#
# httpd 2.2 has no SERVER_DISABLED and writes the whole ServerLimit grid, so there the
# length is ServerLimit rather than MaxClients. Verified on CentOS 6 / httpd 2.2.15:
# ServerLimit 256 with MaxClients 50 still yields a 256 character scoreboard. Nothing in
# the response distinguishes 2.2 from an equally terse 2.4.6, so this is documented in
# the README rather than guessed at here.
SCOREBOARD_FLAGS = [
    # flag, perfdata label, table label
    ('_', 'waiting', 'Waiting for connection'),
    ('S', 'starting', 'Starting up'),
    ('R', 'reading', 'Reading request'),
    ('W', 'sending', 'Sending reply'),
    ('K', 'keepalive', 'Keepalive (read)'),
    ('D', 'dns_lookup', 'DNS lookup'),
    ('C', 'closing', 'Closing connection'),
    ('L', 'logging', 'Logging'),
    ('G', 'finishing', 'Gracefully finishing'),
    ('I', 'idle_cleanup', 'Idle cleanup of worker'),
    ('.', 'free', 'Open slot'),
]

# A slot counts as occupied unless it is an open slot ('.') or waits for a connection
# ('_'). Everything else holds a request, is about to hold one, or is draining one.
UNOCCUPIED_FLAGS = '._'

# mod_status keys reported as they are, as (key, perfdata label, table label). Which of
# them a host sends depends on its MPM (the process and connection counters need the
# event MPM, the only one that answers AP_MPMQ_IS_ASYNC with 1), on its httpd version,
# and on the ExtendedStatus setting. The version ladder, measured across httpd 2.2.15
# through 2.4.68:
#
#   2.4.0    ConnsTotal and the ConnsAsync* trio
#   2.4.13   Load1/5/15, alongside the whole ServerVersion identity block
#   2.4.35   Processes and Stopping
#   2.4.58   GracefulWorkers
#   2.4.63   ConnsAsyncWaitIO
#
# Distributions backport, so the ladder does not say what a given host reports: Rocky 8
# has GracefulWorkers on httpd 2.4.37. Never gate on a version, only on the key. A key
# nobody sent stays absent from the parsed data and yields None, and
# lib.base.get_perfdata() drops a None, so none of these needs a guard of its own.
GAUGES = [
    ('BusyWorkers', 'busy_workers', 'Workers processing a request'),
    ('GracefulWorkers', 'graceful_workers', 'Workers finishing gracefully'),
    ('IdleWorkers', 'idle_workers', 'Workers ready for a request'),
    ('Processes', 'processes', 'Child processes'),
    ('Stopping', 'processes_stopping', 'Child processes shutting down'),
    ('ConnsTotal', 'connections_total', 'Connections'),
    ('ConnsAsyncWaitIO', 'connections_wait_io', 'Connections waiting for I/O'),
    ('ConnsAsyncWriting', 'connections_writing', 'Connections writing'),
    ('ConnsAsyncKeepAlive', 'connections_keepalive', 'Connections in keepalive'),
    ('ConnsAsyncClosing', 'connections_closing', 'Connections closing'),
    # Whole-machine values, not Apache's share: mod_status fills these from
    # getloadavg(3), which on Linux is /proc/loadavg. They are also raw, where the
    # `load` check reports the same averages normalised per CPU. Both facts are in the
    # name so nobody reads a 1.00 here as "Apache saturates this host".
    ('Load1', 'system_load1', 'System load average, 1 minute'),
    ('Load5', 'system_load5', 'System load average, 5 minutes'),
    ('Load15', 'system_load15', 'System load average, 15 minutes'),
]

# The four cumulative CPU counters mod_status reports with ExtendedStatus on, since
# httpd 2.4.13. Its own CPULoad, which older versions report instead, is the average
# over the whole httpd uptime and therefore converges to a straight line in a graph, so
# the plugin adds these up itself and turns the sum into a percentage of the current
# check interval.
CPU_KEYS = ['CPUUser', 'CPUSystem', 'CPUChildrenUser', 'CPUChildrenSystem']

# Deliberately unused, for the next reader who finds them in the raw output:
# BytesPerReq, BytesPerSec, CPULoad, DurationPerReq and ReqPerSec are averages httpd
# computes over its whole uptime. A graph of them converges to a constant line within
# minutes and says nothing about the moment of measurement, so the plugin recomputes
# what it needs from the underlying counters instead (CONTRIBUTING.md, "Plugin
# Performance Data").


def parse_args():
    """Parse command line arguments using argparse."""
    parser = argparse.ArgumentParser(
        description=DESCRIPTION,
        epilog=lib.args.epilog(__file__),
        formatter_class=lib.args.HelpFormatter,
    )

    parser.add_argument(
        '-V',
        '--version',
        action='version',
        version=f'%(prog)s: v{__version__} by {__author__}',
    )

    parser.add_argument(
        '--always-ok',
        help=lib.args.help('--always-ok'),
        dest='ALWAYS_OK',
        action='store_true',
        default=False,
    )

    parser.add_argument(
        '-c',
        '--critical',
        help='CRIT threshold for the percentage of occupied worker slots. '
        'Supports Nagios ranges. '
        'Default: %(default)s',
        dest='CRIT',
        default=DEFAULT_CRIT,
    )

    parser.add_argument(
        '--insecure',
        help=lib.args.help('--insecure'),
        dest='INSECURE',
        action='store_true',
        default=DEFAULT_INSECURE,
    )

    parser.add_argument(
        '--no-perfdata',
        help=lib.args.help('--no-perfdata'),
        dest='NO_PERFDATA',
        action='store_true',
        default=False,
    )

    parser.add_argument(
        '--no-proxy',
        help=lib.args.help('--no-proxy'),
        dest='NO_PROXY',
        action='store_true',
        default=DEFAULT_NO_PROXY,
    )

    parser.add_argument(
        '--proxy',
        help=lib.args.help('--proxy'),
        dest='PROXY',
        default=None,
    )

    parser.add_argument(
        '--test',
        help=lib.args.help('--test'),
        dest='TEST',
        type=lib.args.csv,
    )

    parser.add_argument(
        '--timeout',
        help=lib.args.help('--timeout') + ' Default: %(default)s (seconds)',
        dest='TIMEOUT',
        type=int,
        default=DEFAULT_TIMEOUT,
    )

    parser.add_argument(
        '-u',
        '--url',
        help='Apache Server Status URL. The plugin appends the "auto" query '
        'parameter itself. '
        'Default: %(default)s',
        dest='URL',
        default=DEFAULT_URL,
    )

    parser.add_argument(
        '-w',
        '--warning',
        help='WARN threshold for the percentage of occupied worker slots. '
        'Supports Nagios ranges. '
        'Default: %(default)s',
        dest='WARN',
        default=DEFAULT_WARN,
    )

    args, _ = parser.parse_known_args()
    return args


def get_status_url(url):
    """Return `url` with mod_status' `auto` query parameter attached.

    Appends `auto` to whatever query string the admin configured instead of
    concatenating `?auto`, so a URL that already carries parameters stays intact.
    """
    parts = urllib.parse.urlsplit(url)
    if 'auto' in parts.query.split('&'):
        return url
    query = f'{parts.query}&auto' if parts.query else 'auto'
    return urllib.parse.urlunsplit(parts._replace(query=query))


def get_cpu_count(url):
    """Return the CPU count the load averages can be normalized with, or None.

    mod_status reports the load averages of the machine httpd runs on, but never how
    many CPUs that machine has, and the value is raw rather than per CPU. Dividing is
    therefore only correct when the check runs on that same machine. A loopback URL is
    the one case where that is certain; against any other host the local CPU count
    belongs to the monitoring host instead, and would turn a correct raw number into a
    confidently wrong one.
    """
    host = urllib.parse.urlsplit(url).hostname or ''
    if host not in ('localhost', 'ip6-localhost', 'ip6-loopback'):
        try:
            if not ipaddress.ip_address(host).is_loopback:
                return None
        except ValueError:
            return None
    return os.cpu_count()


def parse_status(raw):
    """Parse the `server-status?auto` response into (server name, dict of values).

    mod_status writes one `Key: value` line per metric, preceded by a bare line holding
    the server name (since httpd 2.4.13). Afterwards it calls `ap_run_status_hook()`,
    which lets other modules append blocks of their own: mod_ssl, mod_cache_socache,
    mod_md and mod_proxy do so upstream, third-party modules as well. Each of those
    blocks opens with a bare marker line, for example `TLSSessionCacheStatus` or
    `ModCacheSocacheStatus`, and the keys below it belong to that module rather than to
    mod_status. Parsing therefore stops at the first bare line after the first key, so a
    foreign `CacheType` can never overwrite a mod_status value and a marker can never be
    mistaken for the server name.

    Splitting on the first `': '` rather than on every occurrence matches how
    httpd's own test suite reads the format
    (`test/modules/http2/test_008_ranges.py`).
    """
    server_name = ''
    data = {}
    for line in raw.splitlines():
        line = line.strip()
        if not line:
            continue
        key, separator, value = line.partition(': ')
        if not separator:
            if not data and not server_name:
                server_name = line
                continue
            break
        data[key] = value
    return server_name, data


def get_rates(url, data):
    """Return the per-second rates of the cumulative mod_status counters, or None.

    Returns None while no comparable previous measurement exists, which is the case on
    the first run and after an httpd restart has reset the counters. A rate whose source
    counter this httpd does not report is left out of the returned mapping rather than
    handed back as a zero, so an older httpd does not look like a server that answers
    every request in no time at all.
    """
    if 'Total Accesses' not in data:
        # ExtendedStatus is off, so httpd reports no counters at all. Recording zeroes
        # here would make every rate read as a genuine zero once ExtendedStatus is
        # turned on.
        return None

    # Always hand over the same set of counters, filling in absent ones with 0.
    # per_second_deltas() derives the cache table schema from these keys, and a set that
    # changes between runs makes it rebuild the table and lose the baseline every time.
    # Which of them carry real data is decided afterwards, on the keys httpd actually
    # sent: `Total Duration` needs httpd 2.4.35, the CPU breakdown needs 2.4.13, and
    # httpd 2.2 reports neither.
    rates = lib.db_sqlite.per_second_deltas(
        'linuxfabrik-monitoring-plugins-apache-httpd-status.db',
        url,
        {
            'accesses': int(data['Total Accesses']),
            'bytes': int(data.get('Total kBytes', 0)) * 1024,
            # milliseconds, as reported
            'duration': int(data.get('Total Duration', 0)),
            # CPU seconds as milliseconds, because the cache stores integers
            'cpu': int(sum(float(data.get(key, 0)) for key in CPU_KEYS) * 1000),
        },
    )
    if rates is None:
        return None
    if 'Total Duration' not in data:
        del rates['duration']
    if not any(key in data for key in CPU_KEYS):
        del rates['cpu']
    return rates


def main():
    """The main function. This is where the magic happens."""

    # parse the command line
    try:
        args = parse_args()
    except SystemExit:
        sys.exit(STATE_UNKNOWN)

    # fetch data
    if args.TEST is None:
        if not args.URL.startswith(('http://', 'https://')):
            lib.base.cu('--url has to start with "http://" or "https://".')
        stdout = lib.base.coe(
            lib.url.fetch(
                get_status_url(args.URL),
                insecure=args.INSECURE,
                no_proxy=args.NO_PROXY,
                proxy=args.PROXY,
                timeout=args.TIMEOUT,
            )
        )
    else:
        # do not call the command, put in test data
        stdout, _stderr, _retc = lib.lftest.test(args.TEST)

    # init some vars
    msg = ''
    state = STATE_OK
    perfdata = ''
    table_data = []

    # analyze data
    server_name, data = parse_status(stdout)
    scoreboard = data.get('Scoreboard')
    if not scoreboard:
        lib.base.cu(
            f'{args.URL} did not answer with Apache mod_status data. '
            'Load mod_status and point --url at a location that has '
            '"SetHandler server-status".',
            traceback=False,
        )

    # The scoreboard holds one character per worker slot and is the only source that
    # stays comparable across MPMs, httpd versions and ExtendedStatus settings. Its
    # length is MaxRequestWorkers, so it is also the denominator an admin sizes the
    # server by. BusyWorkers is not usable for that: mod_status skips every process that
    # is quiescing, so during a graceful restart the slots draining the old generation
    # count as neither busy nor graceful. Measured on Rocky 9 with 20 requests in flight
    # across a `httpd -k graceful`: 20 slots showed 'G', BusyWorkers reported 1 and
    # GracefulWorkers 0.
    slots = len(scoreboard)
    workers = {label: scoreboard.count(flag) for flag, label, _ in SCOREBOARD_FLAGS}
    occupied = slots - sum(scoreboard.count(flag) for flag in UNOCCUPIED_FLAGS)
    occupied_percent = round(occupied / slots * 100, 1)

    state = lib.base.get_state(
        occupied_percent, args.WARN, args.CRIT, _operator='range'
    )
    rates = get_rates(args.URL, data)
    cpu_count = get_cpu_count(args.URL)
    uptime = data.get('ServerUptimeSeconds', data.get('Uptime'))

    # build the message
    msg += f'{server_name}: ' if server_name else ''
    msg += (
        f'{occupied_percent}% worker usage'
        f' ({occupied}/{slots})'
        f'{lib.base.state2str(state, prefix=" ")}'
    )
    if rates is not None:
        # The mean request duration stays out of the summary and lives in the table
        # below, because it is the one rate here that legitimately spikes. httpd credits
        # a request's whole runtime at the moment it finishes
        # (`ws->duration += ws->stop_time - ws->start_time` on STOP_PREQUEST in
        # server/scoreboard.c), so a single long download completing inside a one-minute
        # window can push the mean into the minutes while the server is idle. That is
        # real signal, but not a number to lead a notification with.
        msg += (
            f', {round(rates["accesses"], 1)} req/s'
            f', {lib.human.bytes2human(rates["bytes"])}/s'
        )
    if uptime is not None:
        msg += f', up {lib.human.seconds2human(int(uptime))}'

    perfdata += lib.base.get_perfdata(
        'apache_workers_occupied_percent',
        occupied_percent,
        uom='%',
        warn=args.WARN,
        crit=args.CRIT,
        _min=0,
        _max=100,
    )
    perfdata += lib.base.get_perfdata(
        'apache_workers_occupied',
        occupied,
        uom=None,
        _min=0,
        _max=slots,
    )
    perfdata += lib.base.get_perfdata(
        'apache_workers_total',
        slots,
        uom=None,
        _min=0,
    )
    for _flag, label, _description in SCOREBOARD_FLAGS:
        perfdata += lib.base.get_perfdata(
            f'apache_workers_{label}',
            workers[label],
            uom=None,
            _min=0,
            _max=slots,
        )
    for key, label, _description in GAUGES:
        perfdata += lib.base.get_perfdata(
            f'apache_{label}',
            data.get(key),
            uom=None,
            _min=0,
        )
    if cpu_count:
        # Only alongside the raw values, never instead of them: the raw number is what
        # httpd reported and stays comparable across hosts with different CPU counts.
        for key, label in [
            ('Load1', 'load1'),
            ('Load5', 'load5'),
            ('Load15', 'load15'),
        ]:
            raw = data.get(key)
            perfdata += lib.base.get_perfdata(
                f'apache_system_{label}_per_cpu',
                None if raw is None else round(float(raw) / cpu_count, 2),
                uom=None,
                _min=0,
            )
    if rates is not None:
        perfdata += lib.base.get_perfdata(
            'apache_requests_per_second',
            round(rates['accesses'], 2),
            uom=None,
            _min=0,
        )
        perfdata += lib.base.get_perfdata(
            'apache_bytes_per_second',
            round(rates['bytes'], 2),
            uom='B',
            _min=0,
        )
        if 'cpu' in rates:
            perfdata += lib.base.get_perfdata(
                'apache_cpu_percent',
                # rate of CPU milliseconds per second, so a full core is 1000ms/s = 100%
                round(rates['cpu'] / 10, 2),
                uom='%',
                _min=0,
            )
        if 'duration' in rates and rates['accesses'] > 0:
            perfdata += lib.base.get_perfdata(
                'apache_seconds_per_request',
                round(rates['duration'] / rates['accesses'] / 1000, 4),
                uom='s',
                _min=0,
            )

    # build table output
    for flag, label, description in SCOREBOARD_FLAGS:
        table_data.append(
            {
                'flag': flag,
                'state': description,
                'slots': workers[label],
                'usage': f'{round(workers[label] / slots * 100, 1)}%',
            }
        )
    msg += '\n\n' + lib.base.get_table(
        table_data,
        ['flag', 'state', 'slots', 'usage'],
        header=['Flag', 'Worker State', 'Slots', 'Usage'],
    )

    # Everything mod_status reports beyond the scoreboard, as a second table. This is
    # the deployment's own record of what it is running and how it is behaving, and an
    # admin reading a check result wants all of it, so none of it hides behind a
    # verbosity switch.
    details = []

    def detail(key, value, indent=0):
        """Append one row, skipping a value the server did not report."""
        if value is not None:
            details.append({'key': '  ' * indent + key, 'value': value})

    detail('Server Name', server_name or None)
    detail('Server Version', data.get('ServerVersion'))
    detail('Server MPM', data.get('ServerMPM'))
    detail('Server Built', data.get('Server Built'))
    detail('Current Time', data.get('CurrentTime'))
    detail('Restart Time', data.get('RestartTime'))
    detail(
        'Uptime', lib.human.seconds2human(int(uptime)) if uptime is not None else None
    )
    detail('Parent Server ConfigGeneration', data.get('ParentServerConfigGeneration'))
    detail('Parent Server MPMGeneration', data.get('ParentServerMPMGeneration'))

    if rates is not None:
        detail('Requests', f'{round(rates["accesses"], 1)}/s')
        detail('Traffic', f'{lib.human.bytes2human(rates["bytes"])}/s')
        if 'duration' in rates and rates['accesses'] > 0:
            detail(
                'Request Duration',
                f'{round(rates["duration"] / rates["accesses"], 1)}ms mean',
            )
        if 'cpu' in rates:
            detail('CPU Usage', f'{round(rates["cpu"] / 10, 2)}%')
    for key in CPU_KEYS:
        if key in data:
            # Seconds since the last restart, kept as a plain number so it compares
            # directly against the CPU time a process viewer reports. float() also
            # normalises httpd's leading-dot notation ('.03') into something an admin
            # reads as a number.
            detail(key, f'{float(data[key])}s', indent=1)

    detail('Connections', data.get('ConnsTotal'))
    detail('Async Wait I/O', data.get('ConnsAsyncWaitIO'), indent=1)
    detail('Async Writing', data.get('ConnsAsyncWriting'), indent=1)
    detail('Async KeepAlive', data.get('ConnsAsyncKeepAlive'), indent=1)
    detail('Async Closing', data.get('ConnsAsyncClosing'), indent=1)

    detail('Processes', data.get('Processes'))
    detail('Stopping', data.get('Stopping'), indent=1)

    detail('Workers Total', slots)
    detail('Occupied', f'{occupied} ({occupied_percent}%)', indent=1)
    detail('Busy', data.get('BusyWorkers'), indent=1)
    detail('Graceful', data.get('GracefulWorkers'), indent=1)
    detail('Idle', data.get('IdleWorkers'), indent=1)

    if 'Load1' in data:
        detail(
            'System Load',
            f'whole machine, {cpu_count} CPUs' if cpu_count else 'whole machine, raw',
        )
        for key in ['Load1', 'Load5', 'Load15']:
            raw = data.get(key)
            if raw is None:
                continue
            value = raw
            if cpu_count:
                value = f'{raw} ({float(raw) / cpu_count:.2f} per CPU)'
            detail(key, value, indent=1)

    # A single newline: get_table() already ends its output with one, so the usual
    # '\n\n' would leave two blank lines between the two tables instead of one.
    msg += '\n' + lib.base.get_table(
        details,
        ['key', 'value'],
        header=['Key', 'Value'],
        strip=False,
    )

    # over and out
    lib.base.oao(
        msg, state, perfdata, always_ok=args.ALWAYS_OK, no_perfdata=args.NO_PERFDATA
    )


if __name__ == '__main__':
    try:
        main()
    except Exception:
        lib.base.cu()
