#!/usr/bin/env python3
# -*- coding: utf-8; py-indent-offset: 4 -*-
#
# Author:  Linuxfabrik GmbH, Zurich, Switzerland
# Contact: info (at) linuxfabrik (dot) ch
#          https://www.linuxfabrik.ch/
# License: The Unlicense, see LICENSE file.

# https://github.com/Linuxfabrik/monitoring-plugins/blob/main/CONTRIBUTING.md

"""See the check's README for more details."""

import argparse
import datetime
import os
import re
import stat
import sys

import lib.args
import lib.base
import lib.db_sqlite
import lib.disk
import lib.human
import lib.logmatch
import lib.logsource
import lib.net
import lib.txt
from lib.globals import STATE_CRIT, STATE_OK, STATE_UNKNOWN, STATE_WARN

__author__ = 'Linuxfabrik GmbH, Zurich/Switzerland'
__version__ = '2026083101'

DESCRIPTION = """Scans the Apache HTTP Server error log for the events an administrator
has to act on: children that died on a signal, a server that ran out of workers, processes
it failed to fork, backends a reverse proxy could not reach, and stapling switched on for a
certificate it cannot work for. Startups, restarts and shutdowns are counted alongside
them, so a server that keeps restarting is visible.
Alerts when one of those events shows up, when the lines one client provokes cross
the rates the thresholds set, and when a line arrives at a level `--critical-level`
or `--warning-level` covers.
What Apache logged about one request and one client - a denied access, a password that did
not match, a request line it refused to parse - is counted within `--lookback` and judged
by how many of them arrived, not by the fact that they did: one is a bot or a bad link,
hundreds within ten minutes is somebody walking the site or guessing passwords. Those
lines are counted there and nowhere else, so the background noise every internet-facing
server produces does not keep the check permanently yellow.
What is left is what Apache said about itself, and that is counted by the level it wrote
at the head of the line: `emerg`, `alert` and `crit` return CRITICAL, `error` returns
WARNING, and `--critical-level` and `--warning-level` move that split. The events named
above carry their own state and are counted there and nowhere else, because the level says
nothing about what happened and Apache logs some of them at `notice` anyway. A
message that merely contains the word "error" never counts, and a `LogLevel` below `warn`
hides a line from this check just as it hides it from the file.
The log is read either from a file, from a systemd unit (`systemd:`) or from a container
(`docker:`/`podman:`/`kubectl:`). `--server-log` may be given several times, and everything
named is then read as one window. Without it the check follows the configuration from the
main file through its `Include`/`IncludeOptional` files and reads the server's `ErrorLog`
together with every `ErrorLog` a virtual host sets, so a host whose sites log to their own
files is watched where the sites write and not only where the server does; where no
configuration can be read, the common locations of the distributions are probed instead.
The journal of the Apache unit is read along with them, because a server which fails to
start writes to its standard error instead of into the error log, and a rejected
configuration or an address already in use is in the journal only. What the sources hold in
common is counted once. The most recent rotated file is read along with the live one, so
the window does not end where logrotate last ran.
Requires root or sudo."""

# What `--critical-level` and `--warning-level` accept on top of the levels
# themselves: no level at all raises that state, which leaves the named events as
# the only way to reach it.
LEVEL_NONE = 'none'

# The rates a host collects from the internet are judged by how many arrived,
# not by the fact that they did, and the defaults below assume an intrusion
# prevention system in front of this check: one that reads the same log, counts
# what a single source fails within ten minutes, and blocks that source once it
# passes a handful of them. Five per source in ten minutes is what such a system
# commonly allows, and six is the first number above it, so these thresholds
# report what gets past the blocking rather than what it is already handling.
# The window they are counted in is `--lookback`, whose default is those same ten
# minutes. A host without such a system collects far more and wants them raised;
# the README says by how much.
DEFAULT_AUTH_FAILURES_CRITICAL = 60
DEFAULT_AUTH_FAILURES_WARNING = 6
DEFAULT_CLIENT_DENIALS_CRITICAL = 60
DEFAULT_CLIENT_DENIALS_WARNING = 6
DEFAULT_CRITICAL_LEVEL = 'crit'
DEFAULT_ICINGA_CALLBACK = False
DEFAULT_INSECURE = True
DEFAULT_LOOKBACK = 600  # seconds
DEFAULT_NO_MATCH_SEVERITY = 'ok'
DEFAULT_NO_PROXY = False
DEFAULT_PER_SOURCE = True
DEFAULT_PROXY_FAILURES_CRITICAL = 100
DEFAULT_PROXY_FAILURES_WARNING = 10
DEFAULT_REQUEST_ERRORS_CRITICAL = 60
DEFAULT_REQUEST_ERRORS_WARNING = 6
DEFAULT_TIMEOUT = 8
DEFAULT_WARNING_LEVEL = 'error'

ACK_RETENTION_DAYS = 30
MAXLINES = 30000  # maximum log lines to consider from the end of the source
# Rotated predecessors of the log file to read along with it. The distributions
# rotate this log daily or weekly, so without one the window would end at the
# last rotation and an event from before it would stop being reported the moment
# logrotate runs. One covers a daily rotation with room to spare.
ROTATED_FILES = 1

# The unit the distributions run the Apache HTTP Server as. Only the fixed locations systemd
# reads unit files from are looked at, and only for the names the distributions
# use, so nothing a log or a configuration file holds can point the check at
# another unit.
# The timestamp Apache puts at the head of a line in the error log, which it does
# not write when the log goes to syslog: a bracket holding a year, as opposed to
# the `[core:error]` that follows it.
LEADING_DATE_REGEX = re.compile(r'^\[[^\]]*\d{4}\]\s*')

# What the journal calls the server, for telling its own lines from the ones
# something else logged into the same place.
IDENTIFIERS = ('httpd', 'apache2')

UNIT_CANDIDATES = ('httpd.service', 'apache2.service')
UNIT_DIRECTORIES = (
    '/etc/systemd/system',
    '/usr/lib/systemd/system',
    '/lib/systemd/system',
)

# How long a path may be in the summary before it is abbreviated for display.
# The paths the distributions use fit within it untouched; a host serving a
# dozen sites names a dozen logs in one directory, and there the directory is
# the part that says nothing.
SOURCE_PATH_MAX_LEN = 32

# The main server configuration of the distributions. Only these fixed locations
# are read, and only the `ErrorLog` outside of a `<VirtualHost>` counts, so
# neither an included file nor a virtual host can steer the check somewhere else:
# the RHEL family ships `ErrorLog logs/ssl_error_log` inside the virtual host of
# `conf.d/ssl.conf`, and the Debian family ships one inside every site in
# `sites-enabled/`. An `ErrorLog` that only an included file sets is therefore
# not found, and the probe below takes over.
CONFIG_FILES = (
    '/etc/httpd/conf/httpd.conf',
    '/etc/apache2/apache2.conf',
    '/etc/apache2/httpd.conf',
    '/usr/local/apache2/conf/httpd.conf',
)

# The Debian family keeps the log directory out of the configuration and in a
# shell snippet its init script and its logrotate configuration source, so
# `ErrorLog ${APACHE_LOG_DIR}/error.log` only resolves with this file at hand.
ENVVARS_FILES = ('/etc/apache2/envvars',)

# Apache refuses a configuration that nests includes deeper than this.
MAX_INCLUDE_DEPTH = 16

# Directives are matched case-insensitively, the way Apache itself compares them.
# A `<VirtualHost>` is the only section that can carry an `ErrorLog` of its own;
# every other one (`<IfModule>` for example) passes it on to the main server, so
# only this section is skipped.
CONFIG_COMMENT_REGEX = re.compile(r'^\s*#')
DIRECTIVE_REGEX = re.compile(
    r'^\s*(ErrorLog|LogLevel|ServerRoot|IncludeOptional|Include)\s+(.+?)\s*$',
    re.IGNORECASE,
)
VIRTUALHOST_REGEX = re.compile(r'^\s*<\s*(/?)\s*VirtualHost\b', re.IGNORECASE)
# `export NAME=value` and a bare `NAME=value`, which is what a shell snippet
# holds. Anything else in it (conditionals, sourced files) is out of reach and
# leaves the variable unset, which is what a shell does with it as well.
ENVVAR_REGEX = re.compile(r'^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)=(.*)$')
VARIABLE_REGEX = re.compile(
    r'\$\{([A-Za-z_][A-Za-z0-9_]*)\}|\$([A-Za-z_][A-Za-z0-9_]*)'
)

# What Apache accepts instead of a path. `syslog` optionally names a facility
# (`syslog:local7`), and a value starting with a pipe hands the log to a program,
# `rotatelogs` for example. Neither is a file this check can open.
SYSLOG_TARGET = 'syslog'
PIPE_TARGET = 'pipe'

# Where the distributions keep the error log when the configuration does not say.
# The last entry is the path a source build compiles in.
LOG_FILE_CANDIDATES = (
    '/var/log/httpd/error_log',
    '/var/log/apache2/error.log',
    '/var/log/apache2/error_log',
    '/var/log/httpd/error.log',
    '/usr/local/apache2/logs/error_log',
)

# What a virtual host's log is called where the configuration could not be read
# and the probe above had to take over. `<servername>-error.log` next to the
# server's own log is what our configuration management writes; a site set up by
# hand commonly follows the same shape. Only these fixed directories are looked
# in, so this stays as bounded as the list above - it is not a pattern anybody
# can hand to the check.
VHOST_LOG_CANDIDATES = (
    '/var/log/httpd/*-error.log',
    '/var/log/apache2/*-error.log',
    '/var/log/apache2/*-error_log',
    '/var/log/httpd/*-error_log',
)

# Apache writes `[Fri Aug 28 17:11:44.101220 2026] [mpm_event:notice] [pid 1835:tid
# 1835] AH00489: message`, and `ErrorLogFormat` lets an administrator rearrange
# that head. The level is therefore looked for wherever it stands rather than at
# a fixed offset, which also carries the lines through unchanged when the source
# is a unit and journalctl puts its own timestamp and unit name in front of them.
# The module in front of the level can be empty (`[:error]`), which is what
# Apache writes for a message that reaches it without one - the standard error
# of a CGI script and what `mod_security` logs both arrive that way.
# Verified against httpd 2.4.37 on Rocky 8, 2.4.62 on Rocky 9 and 2.4.68 on
# Debian 12.
LEVEL_REGEX = re.compile(
    r'\[(?:[^\[\]\s:]*:)?'
    r'(emerg|alert|crit|error|warn|notice|info|debug|trace[1-8])\]'
)

# The timestamp Apache puts in front of a line, `[Fri Aug 28 17:12:07.128496
# 2026]`, in the local time of the host and in English regardless of the locale,
# because Apache formats it in the C locale. Python parses it in the C locale as
# well, as nothing here calls `setlocale()`.
TIMESTAMP_REGEX = re.compile(
    r'\[(\w{3} \w{3} [ \d]\d \d{2}:\d{2}:\d{2})(?:\.\d+)? (\d{4})\]'
)
TIMESTAMP_FORMAT = '%a %b %d %H:%M:%S %Y'

# Apache leaves its own timestamp out as soon as the log goes through syslog,
# which is what a server logging into the journal does: the line then starts at
# `[ssl:error] [pid ...]`. What stands in front of it there is journalctl's
# timestamp, and reading that one is `lib.logsource.timestamp()`'s job.
# Every message Apache logs carries a unique code since 2.4, and the lines it
# writes while starting up carry nothing else: neither timestamp nor level, so
# `AH00558: httpd: Could not reliably determine the server's fully qualified
# domain name` stands bare in the log of every host without a `ServerName`. A
# line is therefore Apache's if it holds a level or a code.
CODE_REGEX = re.compile(r'\bAH\d{5}\b')

# The levels that say something is wrong, worst first. `notice` and everything
# below it is what a healthy server writes on every start, so those are read but
# not counted: alerting on them would alert on `resuming normal operations`.
#
# Which of them raise which state is left to `--critical-level` and
# `--warning-level`, because Apache's `error` is not the word an administrator
# reads into it. Most of what arrives at that level is about one request and one
# client rather than about the server: `AH01630: client denied by server
# configuration` for every bot that probes a path the configuration protects,
# `AH01276: Cannot serve directory`, and `AH01071: Got error 'PHP message: ...'`
# for every notice a PHP application prints. Getting somebody out of bed at 02:00
# for those teaches them to ignore the check, so `error` warns by default and
# only `crit` and above alerts. What is genuinely about the server and still
# logged below `crit` is named in EVENTS and raises its own state regardless of
# where these two thresholds sit.
LEVELS = (
    {'level': 'emerg', 'perfdata': 'apache_httpd_emerg_lines'},
    {'level': 'alert', 'perfdata': 'apache_httpd_alert_lines'},
    {'level': 'crit', 'perfdata': 'apache_httpd_crit_lines'},
    {'level': 'error', 'perfdata': 'apache_httpd_error_lines'},
    {'level': 'warn', 'perfdata': 'apache_httpd_warn_lines'},
)
LEVEL_CHOICES = [item['level'] for item in LEVELS] + [LEVEL_NONE]

# The events worth naming on their own, because the level alone does not say what
# happened - and because Apache logs some of them below `warn`, where nothing but
# the message itself makes them visible. Everything else Apache writes is covered
# by the per-level counts above. Each entry carries the state the situation
# deserves on its own, which is then taken next to the state the level asks for.
#
# Every code below was read off the httpd `2.4.x` branch, which is what the
# distributions ship, rather than off `trunk`, where a message can carry a code
# and a wording it does not have in any release yet. Message and level of all of
# them are the same in `2.4.37` (the oldest release an enterprise distribution
# still ships) as in the current branch, and the ones that decide a state were
# reproduced on httpd 2.4.37, 2.4.62 and 2.4.68.
EVENTS = (
    {
        'key': 'child_crashes',
        'label': 'child crash',
        'suffix': 'es',
        'perfdata': 'apache_httpd_child_crashes',
        # AH00051 and AH00052 report a child that died on a signal. Apache logs
        # them at `notice`, so without naming them here a segfaulting module
        # would leave the check green. It writes neither for SIGTERM, SIGHUP,
        # SIGKILL nor for its own graceful-shutdown signal, so every line that
        # does appear is a child that went down on a fault or was killed from
        # outside. AH00050 and AH00060 are the same thing hitting the parent.
        'regex': re.compile(r'\bAH(?:00050|00051|00052|00060)\b'),
        'state': STATE_WARN,
        'recommendation': (
            'Children died on a signal Apache did not send them; look for a core'
            ' dump, a faulty module, or the OOM killer in the kernel log'
        ),
    },
    {
        'key': 'worker_saturations',
        'label': 'worker saturation',
        'perfdata': 'apache_httpd_worker_saturations',
        # `server reached MaxRequestWorkers setting` for the three MPMs of the
        # Unix family, plus `scoreboard is full, not at MaxRequestWorkers`, which
        # is the same wall reached through a ServerLimit that is too low. Apache
        # logs each of them once per generation, so the count says how often the
        # situation returned after a restart rather than how many requests waited.
        'regex': re.compile(r'\bAH(?:00161|00286|00484|00288|03490)\b'),
        'state': STATE_CRIT,
        'recommendation': (
            'The server ran out of workers; raise `MaxRequestWorkers` (and'
            ' `ServerLimit` with it) or shorten the requests, otherwise clients'
            ' wait in the listen queue'
        ),
    },
    {
        'key': 'worker_pressure',
        'label': 'worker pressure warning',
        'perfdata': 'apache_httpd_worker_pressure',
        # `server is within MinSpareThreads of MaxRequestWorkers`, the leading
        # indicator of the saturation above. Its prefork counterpart, `server
        # seems busy`, is logged at `info` and therefore invisible at the default
        # `LogLevel warn`; it is left out rather than promised and never seen.
        # The event MPM only learned to say this after 2.4.37, so a RHEL 8 host
        # reaches its limit without the warning ahead of it. Its worker MPM
        # counterpart is as old as the branch.
        'regex': re.compile(r'\bAH(?:00287|10159)\b'),
        'state': STATE_WARN,
        'recommendation': (
            'The server came within `MinSpareThreads` of `MaxRequestWorkers`;'
            ' raise the limit before the next traffic peak reaches it'
        ),
    },
    {
        'key': 'fork_failures',
        'label': 'fork failure',
        'perfdata': 'apache_httpd_fork_failures',
        # `fork: Unable to fork new process` for the three MPMs, and the alert
        # Apache raises when a shortage kept every child from coming up.
        'regex': re.compile(r'\bAH(?:00159|00283|00481|02324|02325)\b'),
        'state': STATE_CRIT,
        'recommendation': (
            'Apache could not fork a process; check the memory of the host and'
            ' the process and file-descriptor limits of the service unit'
        ),
    },
    {
        'key': 'stapling_failures',
        'label': 'stapling failure',
        'perfdata': 'apache_httpd_stapling_failures',
        # `SSLUseStapling on` for a certificate stapling cannot be set up for.
        # Apache says so once per certificate on every start and then serves TLS
        # without stapling, which is why the level alone leaves an administrator
        # looking for a certificate problem that is not one. Since Let's Encrypt
        # stopped publishing OCSP responders this reaches every host that kept
        # the directive on. All four reasons `ssl_stapling_init_cert()` can fail
        # are here - no OCSP URI in the certificate (AH02218, and AH02814 where
        # another virtual host already parsed it), no issuer certificate to ask
        # about (AH02217), and a request that could not be built (AH02815) -
        # together with the line its caller writes per certificate (AH02604).
        # What the module logs about a responder that answered badly is left
        # out: that is a working setup with a failing responder, and a different
        # problem with a different fix.
        'regex': re.compile(r'\bAH(?:02217|02218|02604|02814|02815)\b'),
        'state': STATE_WARN,
        'recommendation': (
            'Stapling is switched on for a certificate whose issuer publishes no'
            ' OCSP responder, so it does nothing; turn `SSLUseStapling off` for'
            ' that host, or point `SSLStaplingForceURL` at a responder that'
            ' answers for it'
        ),
    },
    {
        'key': 'startup_failures',
        'label': 'startup failure',
        'perfdata': 'apache_httpd_startup_failures',
        # A rejected configuration, a port already taken, and a log Apache could
        # not open. These reach the standard error instead of the error log, so
        # they show up only when the source is the unit or the container - which
        # is exactly why they are named: there the check is the one thing that
        # sees why the server is gone. The last alternative is matched by its
        # text because no release carries a code for it: the whole `2.4.x`
        # branch logs that line without one, and only `trunk` has since given
        # it one.
        'regex': re.compile(
            r'\bAH(?:00015|00072|00526)\b|no listening sockets available'
        ),
        'state': STATE_CRIT,
        'recommendation': (
            'Apache refused to start; `apachectl configtest` names a rejected'
            ' directive, and a port that is already taken names the process'
            ' holding it in `ss --listening --processes`'
        ),
    },
)

# What Apache writes once per rejected request rather than once per problem: an
# access somebody is not allowed, and a password that did not match. One of them
# is a misconfigured link or a bot, and no reason to get anybody out of bed;
# hundreds of them in ten minutes is somebody walking the site or guessing
# passwords. They are therefore counted against a rate rather than reported line
# by line, and they are kept out of the per-level counts above, so the background
# noise every internet-facing server produces leaves the check green.
#
# `mod_auth_digest` carries no message codes at all and is not matched; it is
# also not what a site set up today uses.
#
# The last entry catches everything else Apache logged about one request, which
# is what the two named ones are a special case of. A reverse proxy on the open
# internet collects those all day: a client that sends one host name in SNI and
# another in the `Host` header, a scanner posting `/cgi-bin/.%2e/.%2e/bin/sh`,
# a request line Apache refuses to parse. Naming each code would be an endless
# list, so the scope decides instead of the code.
RATE_EVENTS = (
    {
        'key': 'client_denials',
        'label': 'client denial',
        'perfdata': 'apache_httpd_client_denials',
        # `client denied by server configuration` from mod_authz_core and from
        # the deprecated mod_access_compat.
        'regex': re.compile(r'\bAH(?:01630|01797)\b'),
        'critical': 'CLIENT_DENIALS_CRITICAL',
        'warning': 'CLIENT_DENIALS_WARNING',
        'recommendation': (
            'Clients are being denied access in bulk; the paths in the log say'
            ' whether somebody is walking the site, and an intrusion prevention'
            ' system or the firewall is where that is answered'
        ),
    },
    {
        'key': 'auth_failures',
        'label': 'authentication failure',
        'perfdata': 'apache_httpd_auth_failures',
        # `authentication failure ... Password Mismatch` and `user ... not found`
        # from mod_auth_basic and mod_auth_form (AH01617, AH01618, AH01807,
        # AH01808), a client offering a scheme the server does not accept
        # (AH01614), and a user who authenticated and is still not allowed in
        # (AH01631, `mod_authz_core`). The last two belong here rather than in
        # the generic request counter, because all five are somebody working on
        # a login. What `mod_authz_dbm` and `mod_authz_owner` log about the same
        # situation carries codes of its own and is rare enough to leave to the
        # generic counter.
        'regex': re.compile(r'\bAH(?:01614|01617|01618|01631|01807|01808)\b'),
        'critical': 'AUTH_FAILURES_CRITICAL',
        'warning': 'AUTH_FAILURES_WARNING',
        'recommendation': (
            'Passwords are failing in bulk, which is what a guessing run looks'
            ' like; the user names in the log say whether real accounts are'
            ' being targeted'
        ),
    },
    {
        'key': 'proxy_failures',
        'label': 'proxy failure',
        'perfdata': 'apache_httpd_proxy_failures',
        # Judged by the total rather than per peer: this counter is about
        # backends that could not be reached, and whether that was one backend
        # ten times or ten backends once, what matters is how many requests fell
        # through. The peer of such a line is the backend anyway, not a client.
        'per_source': False,
        # Everything `mod_proxy` and its protocol modules log when they cannot
        # reach a backend or lose its reply: a refused connection over TCP
        # (AH00939, AH00957), through a CONNECT proxy (AH00958) or over a Unix
        # socket (AH02454), the same from the AJP, FastCGI, SCGI, uwsgi,
        # WebSocket and HTTP modules (AH00896, AH01079, AH00866, AH10101,
        # AH02452, AH01114), a worker taken out of the pool over it (AH00940,
        # AH00959), and a reply that broke off (AH00860, AH00898, AH01102,
        # AH01110). Counted as a rate rather than reported one by one, because
        # one of these is a backend being restarted - a `systemctl reload
        # php-fpm` leaves exactly one line - while a backend that is really gone
        # writes one per request. `mod_proxy_ftp` is left out: its EPSV and PASV
        # failures are about an FTP data channel and a firewall, not about a
        # backend of the web server. So is `AH10404`, which Apache itself logs
        # at `warn` and which the level count picks up.
        'regex': re.compile(
            r'\bAH(?:00860|00866|00896|00898|00939|00940|00957|00958|00959'
            r'|01079|01102|01110|01114|02452|02454|10101)\b'
        ),
        'critical': 'PROXY_FAILURES_CRITICAL',
        'warning': 'PROXY_FAILURES_WARNING',
        'recommendation': (
            'Backends behind the reverse proxy could not be reached; check that'
            ' they are running and that Apache may open the connection to them,'
            ' and whether the count matches the times one of them was restarted'
        ),
    },
    {
        'key': 'request_errors',
        'label': 'request error',
        'perfdata': 'apache_httpd_request_errors',
        # Matched by scope rather than by a pattern, see SCOPE_REGEX.
        'regex': None,
        'critical': 'REQUEST_ERRORS_CRITICAL',
        'warning': 'REQUEST_ERRORS_WARNING',
        'recommendation': (
            'Requests are failing in bulk; the paths and host names in the log'
            ' say whether that is a scanner, a client sending a host name the'
            ' server has no matching site for, or an application behind the'
            ' server that stopped answering'
        ),
        # This one is matched by scope rather than by a pattern, so the command
        # that shows its lines has to be spelled out. Apache writes the level
        # before the request scope in `ap_log_error_core()`, which is what lets
        # one expression carry both conditions. It finds the lines a named
        # counter claimed as well, because those are request-scoped too, hence
        # the note the renderer puts next to it.
        # No command can express this one: it is every line at `error` or `warn`
        # carrying a request scope, minus those a named counter above already
        # claimed. A `grep` for the scope alone would print a larger number than
        # the one it stands next to.
        'criterion': (
            'lines at level error or warn carrying a `[client ...]` or'
            ' `[remote ...]` field, minus the named counters above'
        ),
    },
)

# Apache prints this in front of the message whenever the line is about one
# request, and never otherwise: `[client 198.51.100.7:4000]`, or `[remote ...]`
# when the peer is a backend rather than a browser. It is not a heuristic but the
# boundary in Apache's own logging API - `ap_log_rerror()` has a request and adds
# the field, `ap_log_error()` has none and cannot. A line carrying it says
# something went wrong with that one request, which is worth a rate and not an
# alert of its own.
SCOPE_REGEX = re.compile(r'\[(?:client|remote) [^\]]*\]')

# Where the peer of a request-scoped line stands. Apache builds the field as
# `[%s %s:%d]` in `ap_log_error_core()` of `server/log.c` - the kind, the
# address and the port joined by a colon. An older version writes the address
# without a port, and another puts an IPv6 address in brackets of its own, so
# the field is taken as it stands and `get_source()` decides what of it is the
# address.
CLIENT_REGEX = re.compile(r'\[(?:client|remote) (\[[^\]]+\](?::\d+)?|[^\]\s]+)\]')

# The levels at which a request-scoped line is counted as a rate instead of by
# its level. `crit` and above stay level-driven whatever their scope: across the
# whole of Apache only a couple of dozen request-scoped messages are logged that
# high, and they are broken LDAP, Lua and FastCGI backends rather than anything a
# client can provoke.
# Apache's levels from the quietest to the most verbose, for deciding whether a
# level lets `notice` through. `LogLevel` names the last one that is still
# logged, so anything up to and including the configured one is written.
LEVEL_LADDER = (
    'emerg',
    'alert',
    'crit',
    'error',
    'warn',
    'notice',
    'info',
    'debug',
)

SCOPED_LEVELS = ('error', 'warn')

# The rate event that takes the request-scoped lines no named one claimed.
SCOPE_FALLBACK_KEY = 'request_errors'

# Lifecycle markers, counted but never alerting on. A restart re-reads the
# configuration and logs `resuming normal operations` a second time, so it shows
# up as a restart and as a startup.
LIFECYCLE = (
    {
        'key': 'startups',
        'label': 'startup',
        'perfdata': 'apache_httpd_startups',
        # `<version> configured -- resuming normal operations`, per MPM.
        'regex': re.compile(r'\bAH(?:00163|00292|00489)\b'),
        'verb': 'detected',
    },
    {
        'key': 'restarts',
        'label': 'restart',
        'perfdata': 'apache_httpd_restarts',
        # `Doing graceful restart` and `SIGHUP received.  Attempting to restart`,
        # per MPM. logrotate reloads Apache to make it reopen the file, so the
        # nightly rotation shows up here.
        'regex': re.compile(r'\bAH(?:00171|00173|00297|00298|00493|00494)\b'),
        'verb': 'detected',
    },
    {
        'key': 'shutdowns',
        'label': 'shutdown',
        'perfdata': 'apache_httpd_shutdowns',
        # `caught SIGTERM, shutting down` and `caught SIGWINCH, shutting down
        # gracefully`, per MPM. The RHEL family stops the service with SIGWINCH,
        # so both are needed to see an ordinary `systemctl stop`.
        'regex': re.compile(r'\bAH(?:00169|00170|00295|00296|00491|00492)\b'),
        'verb': 'detected',
    },
)


def parse_args():
    """Parse command line arguments using argparse."""
    parser = argparse.ArgumentParser(
        description=DESCRIPTION,
        epilog=lib.args.epilog(__file__),
        formatter_class=lib.args.HelpFormatter,
    )

    parser.add_argument(
        '-V',
        '--version',
        action='version',
        version=f'%(prog)s: v{__version__} by {__author__}',
    )

    parser.add_argument(
        '--always-ok',
        help=lib.args.help('--always-ok'),
        dest='ALWAYS_OK',
        action='store_true',
        default=False,
    )

    parser.add_argument(
        '--auth-failures-critical',
        help='Number of authentication failures within `--lookback` that '
        'returns CRITICAL. '
        '0 turns the threshold off. '
        'Example: `--auth-failures-critical=200`. '
        'Default: %(default)s',
        dest='AUTH_FAILURES_CRITICAL',
        type=int,
        default=DEFAULT_AUTH_FAILURES_CRITICAL,
    )

    parser.add_argument(
        '--auth-failures-warning',
        help='Number of authentication failures within `--lookback` that '
        'returns WARNING. '
        '0 turns the threshold off. '
        'Example: `--auth-failures-warning=20`. '
        'Default: %(default)s',
        dest='AUTH_FAILURES_WARNING',
        type=int,
        default=DEFAULT_AUTH_FAILURES_WARNING,
    )

    parser.add_argument(
        '--client-denials-critical',
        help='Number of denied requests within `--lookback` that returns '
        'CRITICAL. '
        '0 turns the threshold off. '
        'Example: `--client-denials-critical=200`. '
        'Default: %(default)s',
        dest='CLIENT_DENIALS_CRITICAL',
        type=int,
        default=DEFAULT_CLIENT_DENIALS_CRITICAL,
    )

    parser.add_argument(
        '--client-denials-warning',
        help='Number of denied requests within `--lookback` that returns '
        'WARNING. '
        '0 turns the threshold off. '
        'Example: `--client-denials-warning=50`. '
        'Default: %(default)s',
        dest='CLIENT_DENIALS_WARNING',
        type=int,
        default=DEFAULT_CLIENT_DENIALS_WARNING,
    )

    # Developer-only hook, like `--test` on the plugins that inject command
    # output: it prefixes every path the check opens, so the fixture tests can
    # stand a directory tree in for the host's file system. Deliberately not
    # named `--test-...`, because argparse would then resolve a bare `--test`
    # into it.
    parser.add_argument(
        '--config-root',
        help=argparse.SUPPRESS,
        dest='CONFIG_ROOT',
        default='',
    )

    parser.add_argument(
        '--critical-level',
        help='Least severe Apache log level that returns CRITICAL. '
        'Each level includes everything more severe than itself, so `error` '
        'covers `crit`, `alert` and `emerg` as well. '
        'Case-sensitive. '
        '`none` lets no level return CRITICAL, which leaves the events this '
        'check names as the only way to reach it. '
        'Example: `--critical-level=error`. '
        'Default: %(default)s',
        dest='CRITICAL_LEVEL',
        choices=LEVEL_CHOICES,
        default=DEFAULT_CRITICAL_LEVEL,
    )

    parser.add_argument(
        '--icinga-callback',
        help=lib.args.help('--icinga-callback'),
        dest='ICINGA_CALLBACK',
        action='store_true',
        default=DEFAULT_ICINGA_CALLBACK,
    )

    parser.add_argument(
        '--icinga-password',
        help=lib.args.help('--icinga-password'),
        dest='ICINGA_PASSWORD',
    )

    parser.add_argument(
        '--icinga-service-name',
        help=lib.args.help('--icinga-service-name'),
        dest='ICINGA_SERVICE_NAME',
    )

    parser.add_argument(
        '--icinga-url',
        help=lib.args.help('--icinga-url'),
        dest='ICINGA_URL',
    )

    parser.add_argument(
        '--icinga-username',
        help=lib.args.help('--icinga-username'),
        dest='ICINGA_USERNAME',
    )

    parser.add_argument(
        '--ignore',
        help='Ignore a log line matching this Python regular expression. '
        'The log line is lowercased before matching, so write the pattern in '
        'lowercase (or use the `(?i)` flag). '
        'Can be specified multiple times. '
        "Example: `--ignore='ah01630'`.",
        action='append',
        default=None,
        dest='IGNORE',
    )

    parser.add_argument(
        '--insecure',
        help='Applies to the connection to the monitoring server that `--icinga-callback` makes, which is the only network connection this check opens. '
        + lib.args.help('--insecure'),
        dest='INSECURE',
        action='store_true',
        default=DEFAULT_INSECURE,
    )

    parser.add_argument(
        '--lookback',
        help='Denied requests, failed passwords and failed requests are counted '
        'within this window rather than reported one by one. '
        + lib.args.help('--lookback')
        + ' Example: `--lookback=3600`. '
        'Default: %(default)s (seconds)',
        dest='LOOKBACK',
        type=int,
        default=DEFAULT_LOOKBACK,
    )

    parser.add_argument(
        '--match',
        help='Only consider a log line matching this Python regular expression. '
        'The log line is lowercased before matching, so write the pattern in '
        'lowercase (or use the `(?i)` flag). '
        'Can be specified multiple times. '
        + lib.args.MATCH_IGNORE_PRECEDENCE
        + " Example: `--match='\\[ssl:'`.",
        action='append',
        default=None,
        dest='MATCH',
    )

    parser.add_argument(
        '--no-insecure',
        help='Applies to the connection to the monitoring server that `--icinga-callback` makes, which is the only network connection this check opens. '
        + lib.args.help('--no-insecure'),
        dest='INSECURE',
        action='store_false',
        default=DEFAULT_INSECURE,
    )

    parser.add_argument(
        '--no-match-severity',
        help=lib.args.help('--no-match-severity') + ' Default: %(default)s',
        dest='NO_MATCH_SEVERITY',
        choices=['ok', 'warn', 'crit', 'unknown'],
        default=DEFAULT_NO_MATCH_SEVERITY,
    )

    parser.add_argument(
        '--no-per-source',
        help=lib.args.help('--no-per-source'),
        dest='PER_SOURCE',
        action='store_false',
        default=DEFAULT_PER_SOURCE,
    )

    parser.add_argument(
        '--no-perfdata',
        help=lib.args.help('--no-perfdata'),
        dest='NO_PERFDATA',
        action='store_true',
        default=False,
    )

    parser.add_argument(
        '--no-proxy',
        help='Applies to the connection to the monitoring server that `--icinga-callback` makes, which is the only network connection this check opens. '
        + lib.args.help('--no-proxy'),
        dest='NO_PROXY',
        action='store_true',
        default=DEFAULT_NO_PROXY,
    )

    parser.add_argument(
        '--per-source',
        help=lib.args.help('--per-source') + ' Default: %(default)s',
        dest='PER_SOURCE',
        action='store_true',
        default=DEFAULT_PER_SOURCE,
    )

    parser.add_argument(
        '--proxy',
        help='Applies to the connection to the monitoring server that `--icinga-callback` makes, which is the only network connection this check opens. '
        + lib.args.help('--proxy'),
        dest='PROXY',
        default=None,
    )

    parser.add_argument(
        '--proxy-failures-critical',
        help='Number of unreachable backends within `--lookback` that returns '
        'CRITICAL. '
        '0 turns the threshold off. '
        'Example: `--proxy-failures-critical=50`. '
        'Default: %(default)s',
        dest='PROXY_FAILURES_CRITICAL',
        type=int,
        default=DEFAULT_PROXY_FAILURES_CRITICAL,
    )

    parser.add_argument(
        '--proxy-failures-warning',
        help='Number of unreachable backends within `--lookback` that returns '
        'WARNING. '
        'One of them is a backend being restarted, so this counts how many '
        'arrived rather than that any did. '
        '0 turns the threshold off. '
        'Example: `--proxy-failures-warning=1`. '
        'Default: %(default)s',
        dest='PROXY_FAILURES_WARNING',
        type=int,
        default=DEFAULT_PROXY_FAILURES_WARNING,
    )

    parser.add_argument(
        '--request-errors-critical',
        help='Number of failed requests within `--lookback` that returns '
        'CRITICAL. '
        'Counts what Apache logged about one request and one client, denied '
        'requests and failed passwords excluded, as those have counters of '
        'their own. '
        '0 turns the threshold off. '
        'Example: `--request-errors-critical=200`. '
        'Default: %(default)s',
        dest='REQUEST_ERRORS_CRITICAL',
        type=int,
        default=DEFAULT_REQUEST_ERRORS_CRITICAL,
    )

    parser.add_argument(
        '--request-errors-warning',
        help='Number of failed requests within `--lookback` that returns '
        'WARNING. '
        'Counts what Apache logged about one request and one client, denied '
        'requests and failed passwords excluded, as those have counters of '
        'their own. '
        '0 turns the threshold off. '
        'Example: `--request-errors-warning=50`. '
        'Default: %(default)s',
        dest='REQUEST_ERRORS_WARNING',
        type=int,
        default=DEFAULT_REQUEST_ERRORS_WARNING,
    )

    parser.add_argument(
        '--server-log',
        help='Log source to read from. '
        'Accepts a file path, `docker:CONTAINER`, `podman:CONTAINER`, '
        '`kubectl:CONTAINER` or `systemd:UNITNAME`. '
        'Can be specified multiple times, and everything named is then read as '
        'one window; a source named twice is read once. '
        'If omitted, the check reads the `ErrorLog` of the main Apache '
        'configuration file and of every virtual host it configures, falls back '
        'to the common locations of the distributions, and reads the journal of '
        'the Apache unit along with them; what they share is counted once. '
        'Example: `--server-log=systemd:httpd.service`.',
        action='append',
        default=None,
        dest='SERVER_LOG',
    )

    parser.add_argument(
        '--timeout',
        help=lib.args.help('--timeout') + ' Default: %(default)s (seconds)',
        dest='TIMEOUT',
        type=int,
        default=DEFAULT_TIMEOUT,
    )

    parser.add_argument(
        '--warning-level',
        help='Least severe Apache log level that returns WARNING. '
        'Each level includes everything more severe than itself, and a level '
        'that `--critical-level` already covers returns CRITICAL instead. '
        'Case-sensitive. '
        '`none` lets no level return WARNING, which leaves the events this '
        'check names as the only way to reach it. '
        'Example: `--warning-level=warn`. '
        'Default: %(default)s',
        dest='WARNING_LEVEL',
        choices=LEVEL_CHOICES,
        default=DEFAULT_WARNING_LEVEL,
    )

    args, _ = parser.parse_known_args()
    return args


def get_source(log_line):
    """Return the peer of a request-scoped line, or None where it names none.

    Only a line Apache logged about one request carries the field, which is the
    same boundary the rate counters rest on: a line about the server itself has
    no peer to attribute it to and is judged by its level instead.

    The port behind the address is taken off, and only where there is one: an
    IPv6 address is written with a colon between every group, so cutting at the
    last one unasked turns `2001:db8::1:2` into `2001:db8::1` and counts one
    client as two. Whether the whole field already is an address is what decides,
    and the brackets a version puts around an IPv6 address come off with it.
    """
    match = CLIENT_REGEX.search(log_line)
    if not match:
        return None
    field = match.group(1)
    address = lib.net.normalize_address(field)
    if address:
        return address
    if field.startswith('[') and ']' in field:
        return lib.net.normalize_address(field[1 : field.index(']')])
    return lib.net.normalize_address(field.rsplit(':', 1)[0])


def split_alternatives(pattern):
    """Split a pattern at every `|` that is not inside a group."""
    parts = []
    depth = 0
    current = ''
    escaped = False
    for char in pattern:
        if escaped:
            current += char
            escaped = False
            continue
        if char == '\\':
            current += char
            escaped = True
            continue
        if char == '(':
            depth += 1
        elif char == ')':
            depth -= 1
        elif char == '|' and depth == 0:
            parts.append(current)
            current = ''
            continue
        current += char
    parts.append(current)
    return parts


def to_grep_args(pattern):
    """Return a Python pattern as `-e` arguments for `grep`, or None where it has
    no form this can take apart.

    Not one expression with alternation, because `lib.base.oao()` replaces every
    `|` in the message with `!`: the character separates the message from the
    performance data and cannot survive in it. A command carrying `AH(00860!...)`
    would be printed, copied and find nothing, which is worse than printing no
    command at all. `grep` takes each alternative as its own `-e` instead and
    needs no `|` anywhere.

    Every pattern here is a list of alternatives, each of them either a literal
    or of the shape `prefix(?:a|b|c)suffix`. Anything else returns None and the
    caller names the source without a command, rather than printing one that
    looks right and matches something else.
    """
    args = []
    for part in split_alternatives(pattern):
        if '(' not in part:
            args.append(part)
            continue
        match = re.fullmatch(r'([^()|]*)\(\?:([^()|]*(?:\|[^()|]*)*)\)([^()|]*)', part)
        if not match:
            return None
        prefix, alternatives, suffix = match.groups()
        args.extend(f'{prefix}{item}{suffix}' for item in alternatives.split('|'))
    return args


def get_origins(sources):
    """Map every log line to the source it was read from.

    `read_many()` drops a line an earlier source already delivered and keeps that
    earlier one, so the first source holding a line is the one it was counted
    from, and the one an administrator finds it in.
    """
    origins = {}
    for source in sources:
        for line in source['lines']:
            origins.setdefault(line, source['label'])
    return origins


def describe_matching(event, hits, origins):
    """Return the lines that say where an event was counted and what matched it.

    A count nobody can reproduce is a number to be believed rather than checked.
    The sources named are the ones the counted lines actually came from, not
    every source read: a reverse proxy reads a dozen logs and the events are
    usually in one of them.

    Each source kind gets the command that reads it, because a path handed to
    `grep` and a unit handed to `journalctl` are not interchangeable, and an
    event whose criterion is not a pattern at all gets its criterion spelled out
    instead of a command that would count something else.
    """
    files = []
    units = []
    for line in hits:
        origin = origins.get(line)
        if origin is None:
            continue
        success, parsed = lib.logsource.parse(origin)
        if not success:
            continue
        kind, _, target = parsed
        bucket = files if kind == lib.logsource.KIND_FILE else units
        if target not in bucket:
            bucket.append(target)
    if not files and not units:
        return []

    label = lib.txt.pluralize(event['label'], len(hits), event.get('suffix', 's'))
    heading = f'{len(hits)} {label}'
    where = ', '.join(f'`{item}`' for item in files + [f'systemd:{u}' for u in units])

    # A criterion that is not a pattern is described rather than run: the
    # request-scoped fallback holds the lines no named counter claimed, and no
    # single command can express that subtraction.
    if event.get('criterion'):
        return [f'{heading} in {where}: {event["criterion"]}']

    args = to_grep_args(event['regex'].pattern) if event['regex'] else None
    if not args:
        return [f'{heading} in {where}']
    expression = ' '.join(f"-e '{item}'" for item in args)
    commands = []
    if files:
        commands.append(f'grep -n {expression} {" ".join(files)}')
    commands.extend(
        f'journalctl --unit={unit} | grep -n {expression}' for unit in units
    )
    return [f'{heading}:'] + [f'  {item}' for item in commands]


def parse_timestamp(log_line):
    """Return the moment Apache stamped a line with, or None when it carries none.

    Only Apache's own timestamp; `lib.logsource.timestamp()` falls back to the one
    the transport prefixed. A line without either is not an error:
    `ErrorLogFormat` can leave it out, a log going through syslog has none, and
    the lines Apache writes while starting up never carry one.
    """
    match = TIMESTAMP_REGEX.search(log_line)
    if not match:
        return None
    try:
        return datetime.datetime.strptime(
            f'{match.group(1)} {match.group(2)}', TIMESTAMP_FORMAT
        )
    except ValueError:
        return None


def get_level_states(critical_level, warning_level):
    """Map every counted level to the state it raises.

    A level raises a state when it is at least as severe as the level the
    parameter names, and CRITICAL wins wherever both would apply.
    """
    ranks = {item['level']: rank for rank, item in enumerate(LEVELS)}
    states = {}
    for level, rank in ranks.items():
        if critical_level != LEVEL_NONE and rank <= ranks[critical_level]:
            states[level] = STATE_CRIT
        elif warning_level != LEVEL_NONE and rank <= ranks[warning_level]:
            states[level] = STATE_WARN
        else:
            states[level] = STATE_OK
    return states


def unquote(value):
    """Strip one pair of surrounding quotes, which is what Apache does with a value."""
    if len(value) > 1 and value[0] == value[-1] and value[0] in ('"', "'"):
        return value[1:-1]
    return value


def get_env_vars(config_root=''):
    """Read the shell snippet the Debian family keeps its log directory in.

    Values are expanded against what the file has defined so far, and a variable
    it never defines expands to nothing, the way a shell expands an unset one.
    """
    env = {}
    for candidate in ENVVARS_FILES:
        success, content = lib.disk.read_file(f'{config_root}{candidate}')
        if not success or not content:
            continue
        for line in content.splitlines():
            if CONFIG_COMMENT_REGEX.match(line):
                continue
            match = ENVVAR_REGEX.match(line)
            if not match:
                continue
            env[match.group(1)] = expand(unquote(match.group(2).strip()), env, '')
    return env


def expand(value, env, fallback=None):
    """Replace `${NAME}` and `$NAME` with what `env` says, or with `fallback`.

    Returns None when a variable is unknown and no fallback is given, so the
    caller can fall back to probing rather than act on a half-resolved path.
    """
    unresolved = []

    def substitute(match):
        name = match.group(1) or match.group(2)
        if name in env:
            return env[name]
        if fallback is None:
            unresolved.append(name)
            return ''
        return fallback

    expanded = VARIABLE_REGEX.sub(substitute, value)
    return None if unresolved else expanded


def parse_config(content):
    """Return what one configuration file says, in the order it says it.

    A list of `(directive, value, in_virtualhost)`, so the caller can keep the
    last `ServerRoot` and the last main-server `ErrorLog` the way Apache does,
    collect the `ErrorLog` of every virtual host, and expand an `Include` at the
    point it stands rather than after everything else.
    """
    found = []
    in_virtualhost = False
    for line in content.splitlines():
        section = VIRTUALHOST_REGEX.match(line)
        if section:
            in_virtualhost = not section.group(1)
            continue
        if CONFIG_COMMENT_REGEX.match(line):
            continue
        match = DIRECTIVE_REGEX.match(line)
        if not match:
            continue
        found.append((match.group(1).lower(), unquote(match.group(2)), in_virtualhost))
    return found


def walk_config(filename, config_root, env, depth=0, server_root=None):
    """Yield what a configuration file and the files it includes say, in order.

    Apache reads an `Include` at the point the directive stands, so the caller
    sees the directives in the order the server does and the last value of a
    keyword is the one that counts. A pattern that matches nothing is not an
    error - `IncludeOptional` is what every distribution uses for its drop-in
    directories, and an empty one is the normal state of a fresh installation.

    `ServerRoot` is what a relative path resolves against, and it keeps applying
    inside the files that are included: a virtual host in a drop-in directory
    writes `ErrorLog logs/site-error.log` and means the server's log directory,
    not one below the drop-in directory. Where the configuration sets no
    `ServerRoot` at all - the Debian family leaves it to the value the package
    was compiled with - the directory holding the main configuration file is
    that value.
    """
    if depth > MAX_INCLUDE_DEPTH:
        return
    success, content = lib.disk.read_file(filename)
    if not success or not content:
        return
    if server_root is None:
        server_root = os.path.dirname(filename[len(config_root) :])
    for directive, value, in_virtualhost in parse_config(content):
        if directive == 'serverroot' and not in_virtualhost:
            server_root = expand(value, env) or server_root
        if directive not in ('include', 'includeoptional'):
            yield directive, value, in_virtualhost, filename, server_root
            continue
        for pattern in (expand(value, env) or '').split():
            if not pattern.startswith('/'):
                pattern = os.path.join(server_root, pattern)
            for included in lib.disk.glob(f'{config_root}{pattern}'):
                yield from walk_config(
                    included, config_root, env, depth + 1, server_root
                )


def resolve_log(value, env, server_root, config_root):
    """Turn the value of an `ErrorLog` into a path, or into what it is instead.

    `syslog` and a piped logger come back as markers rather than as paths,
    because the caller has to report them instead of opening them. None where a
    variable this check cannot resolve would leave a path pointing somewhere
    else entirely.
    """
    error_log = expand(value, env)
    if not error_log:
        return None
    if error_log.startswith('|'):
        return PIPE_TARGET
    if error_log.lower() == SYSLOG_TARGET or error_log.lower().startswith(
        f'{SYSLOG_TARGET}:'
    ):
        return SYSLOG_TARGET
    if not error_log.startswith('/'):
        error_log = os.path.join(server_root, error_log)
    return f'{config_root}{error_log}'


def get_configured_log_files(config_root=''):
    """Return the main server's `ErrorLog` and the ones its virtual hosts name.

    As a tuple of the two, because they are not interchangeable: the main log is
    where the server writes about itself, a virtual host's is where the requests
    to one site end up. A server that hosts several sites keeps a log per site
    and writes almost nothing but its own lifecycle into the main one, so
    reading that alone would report a quiet server while the sites it serves are
    being walked - and where the main log goes somewhere this check cannot
    follow, the sites are still worth reading. Which files those are is taken
    from the configuration rather than from a pattern, so nothing but what Apache
    itself was told to write is opened.

    The values `syslog` and a piped logger come back as markers rather than as
    paths, and only for the main server: a virtual host logging into either is
    simply one this check cannot follow, and the run goes on with the others.

    Only the fixed configuration locations of the distributions are read as a
    starting point, so nothing the log itself contains can steer the check to
    another file. What those files include is followed from there, which is what
    finds the drop-in directories every distribution and our own configuration
    management put the virtual hosts in.
    """
    env = get_env_vars(config_root)
    for candidate in CONFIG_FILES:
        for main in lib.disk.glob(f'{config_root}{candidate}'):
            main_log = None
            vhost_logs = []
            seen = set()
            for directive, value, in_virtualhost, _, root in walk_config(
                main, config_root, env
            ):
                if directive != 'errorlog':
                    continue
                resolved = resolve_log(value, env, root, config_root)
                if not resolved:
                    continue
                if not in_virtualhost:
                    # The last one the configuration sets is the one Apache uses.
                    main_log = resolved
                    continue
                if resolved in (SYSLOG_TARGET, PIPE_TARGET):
                    continue
                # The virtual hosts the distributions ship point at the file the
                # main server already writes, so the same path turns up twice.
                identity = os.path.realpath(resolved)
                if identity in seen:
                    continue
                seen.add(identity)
                vhost_logs.append(resolved)
            if not main_log and not vhost_logs:
                continue
            if main_log in (SYSLOG_TARGET, PIPE_TARGET) or not main_log:
                return main_log, vhost_logs
            seen_main = os.path.realpath(main_log)
            return main_log, [
                log for log in vhost_logs if os.path.realpath(log) != seen_main
            ]
    return None, []


def get_log_level(config_root=''):
    """Return the `LogLevel` the main server runs with, or None where it sets none.

    Only the value outside every `<VirtualHost>` counts, because that is the one
    the server logs its own lifecycle at.
    """
    env = get_env_vars(config_root)
    for candidate in CONFIG_FILES:
        for main in lib.disk.glob(f'{config_root}{candidate}'):
            level = None
            for directive, value, in_virtualhost, _, _ in walk_config(
                main, config_root, env
            ):
                if directive == 'loglevel' and not in_virtualhost:
                    # `LogLevel warn ssl:info` sets the default first and then
                    # per module; the first word is the one that applies here.
                    level = expand(value, env).split()[0].lower()
            if level:
                return level
    return None


def get_log_file_real_paths(config_root=''):
    """Probe the locations the distributions keep the Apache error log in.

    The server's own log first, and next to it what looks like the log of a
    virtual host, so a host whose configuration could not be read is still
    watched where the sites write rather than only where the server does.
    """
    found = []
    seen = set()
    for candidate in LOG_FILE_CANDIDATES + VHOST_LOG_CANDIDATES:
        for filename in lib.disk.glob(f'{config_root}{candidate}'):
            if not lib.disk.file_exists(filename, allow_empty=True):
                continue
            identity = os.path.realpath(filename)
            if identity in seen:
                continue
            seen.add(identity)
            found.append(filename)
        if found and candidate in LOG_FILE_CANDIDATES:
            # The server's log is the first of the fixed names that exists; the
            # rest of that list are the other distributions' spellings of it.
            break
    return found


def get_units(config_root=''):
    """Return the the Apache HTTP Server units this host has a unit file for.

    A unit file two names point at is one unit: the MariaDB package ships
    `mysqld.service` and `mysql.service` as symlinks to `mariadb.service`, and
    reading that journal three times would count every line three times.
    """
    found = []
    seen = set()
    for candidate in UNIT_CANDIDATES:
        for directory in UNIT_DIRECTORIES:
            for filename in lib.disk.glob(f'{config_root}{directory}/{candidate}'):
                if not lib.disk.file_exists(filename, allow_empty=True):
                    continue
                identity = os.path.realpath(filename)
                if identity in seen:
                    continue
                seen.add(identity)
                found.append(os.path.basename(filename))
    return found


def dedup_key(line):
    """Identify the event in a line, whichever transport delivered it.

    The error log and the journal of the unit are both read, and where
    `ErrorLog` hands the log to syslog every line is in both. Apache writes the
    same message either way; only what stands in front of it differs, a
    bracketed timestamp in the file and the syslog prefix in the journal.
    """
    written_at = lib.logsource.timestamp(line, parse_timestamp)
    body = LEADING_DATE_REGEX.sub('', lib.logsource.strip_syslog_prefix(line), count=1)
    return (
        written_at.replace(microsecond=0) if written_at else None,
        body,
    )


def main():
    """The main function. This is where the magic happens."""

    # parse the command line
    try:
        args = parse_args()
    except SystemExit:
        sys.exit(STATE_UNKNOWN)

    # set default values for append parameters that were not specified
    if args.IGNORE is None:
        args.IGNORE = []
    if args.MATCH is None:
        args.MATCH = []
    if args.SERVER_LOG is None:
        args.SERVER_LOG = []

    if args.ICINGA_CALLBACK and not all(
        (
            args.ICINGA_URL,
            args.ICINGA_PASSWORD,
            args.ICINGA_USERNAME,
            args.ICINGA_SERVICE_NAME,
        )
    ):
        lib.base.cu(
            '`--icinga-callback` requires `--icinga-url`, `--icinga-password`, '
            '`--icinga-username` and `--icinga-service-name`.'
        )

    # fetch data
    server_logs = args.SERVER_LOG
    # Where the server's own log went when it went somewhere this check cannot
    # open. Only worth saying once the sites' logs carry the run, which is why it
    # is a sentence in the summary rather than the abort further down.
    main_log_notice = ''
    main_log_recommendation = ''
    if not server_logs:
        main_log, vhost_logs = get_configured_log_files(args.CONFIG_ROOT)
        if main_log is None:
            # No `ErrorLog` outside the virtual hosts, or no configuration this
            # check could read at all: Apache then writes where it was compiled
            # to, which is where the probe looks. What else the probe finds is a
            # site's log, and only of interest where the configuration named
            # none - it knows the sites better than a file name does.
            probed = get_log_file_real_paths(args.CONFIG_ROOT)
            main_log = probed[0] if probed else None
            vhost_logs = vhost_logs or probed[1:]
        if main_log in (SYSLOG_TARGET, PIPE_TARGET) and vhost_logs:
            # Named precisely, because the journal of the unit is read as well
            # and the summary would otherwise contradict itself. What a server
            # logging this way loses is its lifecycle: Apache writes a `notice`
            # line into a log file whatever `LogLevel` says, but through syslog
            # the level applies, so the default `warn` drops every start,
            # restart and shutdown. Measured on Rocky 9 (httpd 2.4.62, rsyslog
            # 8.2510): with `LogLevel warn` a denied request (AH01630, `error`)
            # reaches the journal while AH00489, AH00493 and AH00491 reach
            # neither the journal, nor rsyslog's files, nor a file of their own;
            # with `LogLevel notice` they are all there.
            target = 'syslog' if main_log == SYSLOG_TARGET else 'a program'
            main_log_notice = (
                f'`ErrorLog` hands the log the server writes about itself to '
                f'{target}, so this run covers the sites and the unit but not '
                f'that log'
            )
            level = get_log_level(args.CONFIG_ROOT) or 'warn'
            if main_log == SYSLOG_TARGET and LEVEL_LADDER.index(
                level if level in LEVEL_LADDER else 'warn'
            ) < LEVEL_LADDER.index('notice'):
                main_log_notice += (
                    f' - and `LogLevel {level}` drops the `notice` lines there, '
                    f'so no startup, restart or shutdown can be seen at all'
                )
                main_log_recommendation = (
                    'Set `LogLevel notice` to see the server start, restart and '
                    'stop: through syslog the level applies to those lines, '
                    'where a log file would carry them whatever `LogLevel` says'
                )
            main_log = None
        # The unit alongside the files, not instead of them: a server that failed
        # to start wrote why to its standard error and never reached the error
        # log, and where `ErrorLog` hands the log to syslog the journal is the
        # only place the server's own lines are. What both carry is counted once.
        server_logs = ([main_log] if main_log else []) + vhost_logs
        server_logs += [f'systemd:{unit}' for unit in get_units(args.CONFIG_ROOT)]
    if not server_logs:
        lib.base.cu(
            'Found no Apache error log. Set `ErrorLog` in the main Apache '
            "configuration file, or name the log with the check's "
            '`--server-log` parameter.'
        )
    if server_logs == [SYSLOG_TARGET]:
        lib.base.cu(
            '`ErrorLog` hands the log to syslog, so it is not a file this check '
            'can open. Point `--server-log` at the systemd unit '
            '(`--server-log=systemd:httpd.service`) or at the file the syslog '
            'daemon writes.'
        )
    if server_logs == [PIPE_TARGET]:
        lib.base.cu(
            '`ErrorLog` pipes the log into a program, so it is not a file this '
            'check can open. Name the file that program writes with '
            '`--server-log`, or point the parameter at the systemd unit '
            '(`--server-log=systemd:httpd.service`).'
        )

    # A size is known for plain on-disk files only, so the size fact and its
    # perfdata series stay out of the output for a unit or a container. Where
    # several files are read, the series is what they add up to.
    sizes = {}
    for server_log in server_logs:
        kind, _, target = lib.base.coe(lib.logsource.parse(server_log))
        if kind != lib.logsource.KIND_FILE:
            continue
        # Report a path that is not a regular file before reading it, so the
        # admin hears about a directory or a device rather than about an I/O
        # error further down. With one source that is the whole answer; with
        # several it is one source of many and the run goes on.
        log_stat = lib.disk.stat(target)
        if log_stat is None or not stat.S_ISREG(log_stat.st_mode):
            if len(server_logs) == 1:
                lib.base.oao(
                    f'Logging seems to be configured, but `{target}` does not'
                    f' seem to be an existing regular file. Check the path and'
                    f' file permissions, or provide the `--server-log`'
                    f' parameter.',
                    STATE_WARN,
                )
            continue
        sizes[server_log] = log_stat.st_size
    if len(server_logs) == 1 and sizes.get(server_logs[0]) == 0:
        # An empty log file is a deterministic "no events observed" state,
        # not an unknown one - typical right after logrotate fires.
        lib.base.oao(
            f'Log file `{server_logs[0]}` is empty. Assuming log-rotation.',
            STATE_OK,
        )

    # The plugin runs as root via sudo, so restrict an on-disk log to the system
    # log directory. The path the Apache configuration reports is deliberately
    # not trusted beyond that: a configuration file an unprivileged user may edit
    # could otherwise point the check at any file on the host. The plugin's own
    # unit-test/ directory is allowed so the fixture tests work; it does not
    # exist next to the flat, root-owned plugin in production. Bind-mount a
    # custom log directory under /var/log to include it (see the README).
    allowed_roots = [
        '/var/log',
        os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), 'unit-test'),
    ]
    # No position is kept on purpose: this check reports on a window of the log,
    # counting startups, restarts and shutdowns next to the problems, so every
    # run has to see the whole window rather than only what is new.
    result = lib.base.coe(
        lib.logsource.read_many(
            server_logs,
            allowed_roots=allowed_roots,
            dedup_key=dedup_key,
            max_lines=MAXLINES,
            rotated=ROTATED_FILES,
            timeout=args.TIMEOUT,
        )
    )
    # What to call the whole of what was read, for the messages that have to
    # name it before the summary below spells every source out.
    # Abbreviated here, because this one shares a line with a sentence; the
    # listing further down gives the path in full.
    source_label = (
        '`'
        + lib.disk.shorten_path(
            result['sources'][0]['label'], max_len=SOURCE_PATH_MAX_LEN, truncate=False
        )
        + '`'
        if len(result['sources']) == 1
        else f'the {len(result["sources"])} sources read'
    )

    # init some vars
    state = STATE_OK
    sections = []
    facts = []
    # All recommendations from all WARN/CRIT paths land here and render once at
    # the end as a `Recommendations:\n* ...` bulleted block, regardless of which
    # combinations of paths fire.
    recommendations = []
    if main_log_recommendation:
        recommendations.append(main_log_recommendation)
    level_states = get_level_states(args.CRITICAL_LEVEL, args.WARNING_LEVEL)
    levels = {item['level']: [] for item in LEVELS}
    found = {item['key']: [] for item in EVENTS}
    found.update({item['key']: [] for item in RATE_EVENTS})
    found.update({item['key']: [] for item in LIFECYCLE})
    rate_since = datetime.datetime.now() - datetime.timedelta(seconds=args.LOOKBACK)
    considered_cnt = 0
    recognized_cnt = 0
    unrecognized_cnt = 0
    suppressed_cnt = 0
    compiled_ignore = [
        lib.base.coe(item) for item in lib.txt.compile_regex(args.IGNORE, '--ignore')
    ]
    compiled_match = [
        lib.base.coe(item) for item in lib.txt.compile_regex(args.MATCH, '--match')
    ]

    # Persisted acknowledgements are only needed when the callback is in use.
    # Each unique combination of filters gets its own state database, so two
    # services watching the same error log for different things do not share what
    # has been acknowledged. Only the filters actually set take part in the
    # identifier, so adding a filter later does not orphan the state of a service
    # that does not use it.
    acked_fingerprints = set()
    ack_conn = None
    if args.ICINGA_CALLBACK:
        # Only what the operator set takes part in the identifier, so a host
        # that gains a second source - a unit next to the file - keeps the
        # acknowledgements it already has. `server_log` is in there when it was
        # named, because two services then deliberately watch different logs.
        instance_payload = {}
        for name, value in (
            ('ignore_regex', args.IGNORE),
            ('match', args.MATCH),
            ('server_log', args.SERVER_LOG),
        ):
            if value:
                instance_payload[name] = value
        ack_conn = lib.base.coe(
            lib.logmatch.connect(
                'apache-httpd-logfile', lib.logmatch.instance_id(instance_payload)
            )
        )
        # Drop acknowledgements older than the retention: by that age the line has
        # left the window this check reads and can no longer re-appear.
        lib.base.coe(lib.logmatch.prune(ack_conn, retention=ACK_RETENTION_DAYS))
        acked_fingerprints = lib.base.coe(lib.logmatch.suppressed(ack_conn))

    # analyze data
    for log_line in result['lines']:
        haystack = log_line.lower()
        if any(item.search(haystack) for item in compiled_ignore):
            continue
        # `--match` (include) is applied first, then `--ignore` (exclude), so a
        # line hit by `--ignore` is dropped even if it also matches `--match`.
        if compiled_match and not any(item.search(haystack) for item in compiled_match):
            continue
        considered_cnt += 1
        # Drop a line an operator has already taken on. The whole log line is
        # keyed, timestamp included, so the very same message logged again later
        # is a new event and alerts again.
        if acked_fingerprints and lib.logmatch.key(log_line) in acked_fingerprints:
            suppressed_cnt += 1
            continue
        identifier = lib.logsource.syslog_identifier(log_line)
        if identifier is not None and identifier not in IDENTIFIERS:
            # Somebody else writing into the same place: systemd's own
            # bookkeeping about the unit ("Starting ...", "Started ..."), and
            # whatever else a syslog daemon put into the file this check reads.
            # It is not Apache's line, and counting it as one Apache did not
            # write would put a permanent "lines not written by Apache" into the
            # summary of every healthy host.
            continue
        level_match = LEVEL_REGEX.search(log_line)
        hits = [event for event in EVENTS if event['regex'].search(log_line)]
        if (
            not level_match
            and not CODE_REGEX.search(log_line)
            and not hits
            and identifier is None
        ):
            # Not a line Apache wrote. Counted rather than dropped silently,
            # because a log full of them means the check is pointed at the wrong
            # file. An event counts as Apache's as well, because the one line it
            # writes about listening sockets carries neither level nor code, and
            # so does a line the journal says Apache wrote: `Server configured,
            # listening on: port 80` comes from the server through `mod_systemd`
            # and carries neither either.
            unrecognized_cnt += 1
            continue
        recognized_cnt += 1
        rate_hits = [
            event
            for event in RATE_EVENTS
            if event['regex'] and event['regex'].search(log_line)
        ]
        level = level_match.group(1) if level_match else None
        # Everything else Apache logged about one request goes to the rate
        # counter the named ones are a special case of, rather than to its level.
        if not rate_hits and level in SCOPED_LEVELS and SCOPE_REGEX.search(log_line):
            rate_hits = [
                event for event in RATE_EVENTS if event['key'] == SCOPE_FALLBACK_KEY
            ]
        for event in rate_hits:
            found[event['key']].append(log_line)
        # A line one of the catalogs owns is left out of the per-level counts.
        # A rate event is judged by how often it arrives, and counting it by its
        # level too would put every internet-facing server permanently into
        # WARNING for the bots that probe it. A named event carries its own
        # state and says what happened, which the level cannot; reporting the
        # same eight lines once as "8 error lines" and once as "8 stapling
        # failures" says the same thing twice and doubles them in the graphs.
        if not rate_hits and not hits and level in levels:
            levels[level].append(log_line)
        for event in hits:
            found[event['key']].append(log_line)
        for item in LIFECYCLE:
            if item['regex'].search(log_line):
                found[item['key']].append(log_line)

    # Everything collected is put back into the order it was written in. The
    # sources are read one after the other, so a line from a site's log sits
    # before a newer one from the journal however late it was written, and "the
    # last one" would name something days old.
    for key, lines in found.items():
        found[key] = lib.logsource.sort_by_time(lines, parse_timestamp)
    for level, lines in levels.items():
        levels[level] = lib.logsource.sort_by_time(lines, parse_timestamp)

    # `--match` narrowed the run down to nothing, so this run looked at no line at
    # all rather than at a quiet log. Only reachable when the operator set a
    # filter, so a log that simply has nothing to report stays OK.
    if compiled_match and result['lines'] and not considered_cnt:
        if ack_conn is not None:
            lib.db_sqlite.close(ack_conn)
        lib.base.oao(
            f'Nothing checked: `--match` dropped all {len(result["lines"])} '
            f'{lib.txt.pluralize("line", len(result["lines"]))} of '
            f'{source_label}.',
            lib.base.str2state(args.NO_MATCH_SEVERITY),
            always_ok=args.ALWAYS_OK,
        )

    # Every line Apache writes carries its level or its message code, so a window
    # without a single one is not a quiet Apache but a source that holds something
    # else, the access log for example. Saying so beats reporting the green state
    # a check would otherwise keep on the wrong file forever.
    if unrecognized_cnt and not recognized_cnt:
        if ack_conn is not None:
            lib.db_sqlite.close(ack_conn)
        lib.base.oao(
            f'None of the {unrecognized_cnt} '
            f'{lib.txt.pluralize("line", unrecognized_cnt)} read from '
            f'{source_label} carries an Apache log level or message code, so '
            f'this does not look like an Apache error log. Check `--server-log` '
            f'and the `ErrorLog` directive of the Apache configuration.',
            STATE_UNKNOWN,
        )

    # build the message
    # What was read, and where from. The size stands next to the live file
    # rather than next to the line count, because it describes that one file and
    # nothing else - right after logrotate a 34 KiB file said to hold 30000
    # lines reads like a miscount, when almost all of them came from the rotated
    # predecessor. Naming the predecessor says where they did come from.
    # The count matters because everything below is counted within it: a run
    # that reports no startup at all is telling the truth about the window rather
    # than about the day.
    # And where the window stopped at the cap this check reads rather than at
    # the start of the log, it says so, because a busy host then reports on the
    # last few hours rather than on the day.
    # Which stretch of time those lines cover, because the count alone does not
    # say whether the window is an hour or a week: on a busy host the cap is
    # reached within hours, on a quiet one the same 30000 lines reach back
    # months, and every count below has to be read against that. Taken from the
    # ends of each source rather than from the ends of everything read: the
    # sources arrive one after the other, so the newest line of the first one
    # sits in the middle. A source that stamps no line simply leaves it out.
    covered = ''
    window_from, window_to = lib.logsource.covered_window(
        [item['lines'] for item in result['sources']], parse_timestamp
    )
    if window_from is not None and window_to is not None:
        span = int((window_to - window_from).total_seconds())
        covered = (
            f'{window_from:%Y-%m-%d %H:%M} .. {window_to:%Y-%m-%d %H:%M}'
            f' ({lib.human.seconds2human(span)}): '
        )
    line_cnt = len(result['lines'])
    lines_read = (
        f'{lib.human.number2human(line_cnt)} {lib.txt.pluralize("line", line_cnt)}'
    )
    if result['truncated']:
        lines_read = f'the most recent {lines_read}'
    # The sources get a section of their own, the way the lines behind every
    # count do: a reverse proxy serving a dozen sites reads a dozen logs, and
    # naming them in the summary would bury the verdict under a paragraph of
    # paths - and the summary line is what a monitoring server shows in a list.
    # The paths are not abbreviated there: a bullet has the room, and that is
    # where an administrator copies them from.
    described = [
        lib.logsource.describe(item, sizes.get(item['label']))
        for item in result['sources']
    ]
    sources = f'{len(described)} {lib.txt.pluralize("source", len(described))}'

    if result['duplicates']:
        # Named rather than silently dropped: an administrator who sees two
        # sources read and one count has to be able to tell that the check knows
        # they hold the same events.
        shared = result['duplicates']
        source_fact_tail = (
            f', {lib.human.number2human(shared)} '
            f'{lib.txt.pluralize("line", shared)} they share counted once'
        )
    else:
        source_fact_tail = ''
    source_heading = f'Read {lines_read} from {sources}{source_fact_tail}:'

    # The counted events. Their state comes from how many of them arrived within
    # the rate window, not from the fact that they arrived at all. Their facts
    # are collected before the level counts are rendered, because a run that has
    # something to say about them is not the quiet one the literal below claims.
    rate_counts = {}
    rate_facts = []
    for event in RATE_EVENTS:
        hits = found[event['key']]
        # Counted per client where the lines name one, because that is what
        # makes the number mean something: a handful of denied requests from one
        # address within the window is somebody working on this server, the same
        # number spread over as many addresses is the open network going past.
        # The state follows the busiest single client, which is the quantity an
        # intrusion prevention system counts before it blocks one, so the
        # thresholds compare against the same thing. `--no-per-source` goes back
        # to judging everything that arrived.
        window = lib.logsource.count_within(
            hits,
            rate_since,
            parse_line=parse_timestamp,
            # A counter that is not about who caused the lines judges the
            # total: a backend that failed and a server that ran out of slots
            # are one situation however many peers were on the other end.
            key=get_source
            if args.PER_SOURCE and event.get('per_source', True)
            else None,
        )
        rate_counts[event['key']] = window['count']
        if not hits:
            continue
        rate_state = lib.base.get_state(
            window['count'],
            getattr(args, event['warning']) or None,
            getattr(args, event['critical']) or None,
        )
        state = lib.base.get_worst(state, rate_state)
        fact = f'{window["count"]} {lib.txt.pluralize(event["label"], window["count"])}'
        if window['busiest']:
            fact += f' from {window["busiest"]}'
        fact += lib.base.state2str(rate_state, prefix=' ')
        extras = []
        if window['total'] != window['count']:
            extras.append(
                f'{window["total"]} in total from {window["sources"]}'
                f' {lib.txt.pluralize("address", window["sources"], "es")}'
            )
        if len(hits) != window['total']:
            extras.append(f'{len(hits)} in the whole window')
        if extras:
            fact += ' (' + ', '.join(extras) + ')'
        if window['undated']:
            fact += (
                f', {window["undated"]} of them without a timestamp and therefore'
                f' not counted'
            )
        rate_facts.append(fact)
        if rate_state != STATE_OK:
            recommendations.append(event['recommendation'])

    counts = {}
    for item in LEVELS:
        lines = levels[item['level']]
        counts[item['level']] = len(lines)
        if not lines:
            continue
        level_state = level_states[item['level']]
        state = lib.base.get_worst(state, level_state)
        facts.append(
            f'{len(lines)} {item["level"]} '
            f'{lib.txt.pluralize("line", len(lines))} found'
            f'{lib.base.state2str(level_state, prefix=" ")}'
            f' (last: {lines[-1]})'
        )
    if not any(counts.values()) and not rate_facts:
        # Naming all five levels one by one would bury the one sentence a healthy
        # host is supposed to be.
        facts.append('No errors or warnings found')

    # Name the events the level counts above already carry, so the summary says
    # what happened and not only how bad it was, and raise the state for those
    # Apache logs below `warn`. Quiet events stay out of the line, which keeps a
    # healthy host down to a single sentence.
    named = []
    for event in EVENTS:
        hits = found[event['key']]
        if not hits:
            continue
        state = lib.base.get_worst(state, event['state'])
        named.append(
            f'{len(hits)} '
            f'{lib.txt.pluralize(event["label"], len(hits), event.get("suffix", "s"))}'
            f'{lib.base.state2str(event["state"], prefix=" ")}'
        )
        recommendations.append(event['recommendation'])
    if named:
        # Not "among them": a named event is counted here and nowhere else, so
        # these are lines the per-level counts above do not hold.
        facts.append('Found ' + ', '.join(named))

    for item in LIFECYCLE:
        lines = found[item['key']]
        # A counter at zero says nothing the performance data does not, and a
        # sentence about it would only push the verdict further right.
        if lines:
            facts.append(
                f'{len(lines)} {lib.txt.pluralize(item["label"], len(lines))}'
                f' {item["verb"]} (last: {lines[-1]})'
            )

    # A few lines without a level and without a message code are normal where
    # something else logs into the same file; a majority of them means the source
    # is mostly something else.
    if unrecognized_cnt:
        facts.append(
            f'{unrecognized_cnt} {lib.txt.pluralize("line", unrecognized_cnt)}'
            f' not written by Apache'
        )
        # Only worth saying for a file, which holds Apache's lines and nothing
        # else. A unit and a container mix in what systemd and the engine say
        # about the service, so a majority of foreign lines is normal there.
        if unrecognized_cnt > recognized_cnt and kind == lib.logsource.KIND_FILE:
            recommendations.append(
                'Most lines carry neither an Apache log level nor a message code;'
                ' check that the source is the error log and not the access log'
            )

    if main_log_notice:
        facts.append(main_log_notice)

    if result['notice']:
        facts.append(result['notice'].rstrip('.'))

    # A source that could not be read at all leaves a hole in the window: with a
    # second source still delivering, the run goes on, but reporting the state of
    # what happened to work would hide that nothing is watching the rest.
    if result['failed']:
        state = lib.base.get_worst(state, STATE_WARN)
        facts.append(
            '; '.join(item.rstrip('.') for item in result['failed'])
            + lib.base.state2str(STATE_WARN, prefix=' ')
        )
        recommendations.append(
            'Check the log sources this check was told to read; '
            'one of them could not be read at all'
        )

    # The window the summary reports on comes first, because every count in it
    # is a count within that window and reads differently against an hour than
    # against a week.
    # The two groups are labelled rather than run together, because they answer
    # about different spans and the numbers are not comparable: an event counted
    # over the whole window read and one counted over the rate window sat next to
    # each other with nothing but the wording of each to tell them apart, and
    # "12 stapling failures" next to "0 request errors in the last 10m" reads as
    # if both were about the last ten minutes.
    # Only the rate window is labelled. Everything before the semicolon is about
    # the window the line opens with, which is already named there, so saying so
    # again would be noise; what needed saying is that the counts after it are
    # not, because "12 stapling failures" next to "0 request errors" otherwise
    # reads as if both were about the same span. The separator is a semicolon and
    # not a `|`: `lib.base.oao()` replaces every `|` in the message with `!`,
    # because that is what divides the message from the performance data.
    summary = []
    if facts:
        summary.append('. '.join(facts))
    if rate_facts:
        summary.append(
            f'Last {lib.human.seconds2human(args.LOOKBACK)}: ' + '. '.join(rate_facts)
        )
    sections.append(covered + '; '.join(summary) + '.')

    for item in LEVELS:
        lines = levels[item['level']]
        if lines:
            sections.append(
                f'{item["level"].capitalize()} lines:\n'
                + '\n'.join(f'* {line}' for line in lib.txt.shorten_list(lines))
            )

    # What was read, last of the sections: the verdict and the lines behind it
    # come first, the provenance after them.
    sections.append(
        source_heading + '\n' + '\n'.join(f'* {item}' for item in described)
    )

    # And how every count above was arrived at. A number in a check is only worth
    # something if the administrator can go and look at the lines behind it, and
    # with a dozen sites logging into a dozen files, finding them means knowing
    # which file and which pattern. Only counts that actually fired are listed,
    # so a quiet host does not carry a block of commands nobody needs to run.
    origins = get_origins(result['sources'])
    matching = []
    for event in list(EVENTS) + list(RATE_EVENTS):
        hits = found[event['key']]
        if hits:
            matching.extend(describe_matching(event, hits, origins))
    if matching:
        sections.append(
            'Where the numbers come from:\n'
            + '\n'.join(
                f'* {item}' if not item.startswith('  ') else item for item in matching
            )
        )

    if recommendations:
        sections.append(
            'Recommendations:\n' + '\n'.join(f'* {item}' for item in recommendations)
        )

    msg = '\n\n'.join(sections)

    perfdata = ''
    if sizes:
        perfdata += lib.base.get_perfdata(
            'apache_httpd_logfile_size',
            sum(sizes.values()),
            uom='B',
            _min=0,
        )
    # The problem counters alert from their first hit on, hence a threshold of
    # `'0'` ("outside 0..0") rather than `1`. A level counter carries the
    # threshold of the state it actually raises, so the graph shows the line the
    # check follows and a level left quiet by `--critical-level` and
    # `--warning-level` draws none.
    for item in LEVELS:
        level_state = level_states[item['level']]
        perfdata += lib.base.get_perfdata(
            item['perfdata'],
            counts[item['level']],
            uom=None,
            warn='0' if level_state == STATE_WARN else None,
            crit='0' if level_state == STATE_CRIT else None,
            _min=0,
        )
    for event in EVENTS:
        perfdata += lib.base.get_perfdata(
            event['perfdata'],
            len(found[event['key']]),
            uom=None,
            warn='0' if event['state'] == STATE_WARN else None,
            crit='0' if event['state'] == STATE_CRIT else None,
            _min=0,
        )
    # The rate counters trend what the state follows: how many the busiest
    # single client produced within the window, not how many the log holds.
    for event in RATE_EVENTS:
        perfdata += lib.base.get_perfdata(
            event['perfdata'],
            rate_counts[event['key']],
            uom=None,
            warn=getattr(args, event['warning']) or None,
            crit=getattr(args, event['critical']) or None,
            _min=0,
        )
    for item in LIFECYCLE:
        perfdata += lib.base.get_perfdata(
            item['perfdata'], len(found[item['key']]), uom=None, _min=0
        )

    # Ask the monitoring server about the acknowledgement. Where the service is
    # acknowledged, persist the lines currently being reported so an unchanged
    # error log does not raise them again on the following runs.
    if args.ICINGA_CALLBACK and state != STATE_OK:
        acknowledged, note = lib.base.coe(
            lib.logmatch.service_acknowledged(
                args.ICINGA_URL,
                args.ICINGA_USERNAME,
                args.ICINGA_PASSWORD,
                args.ICINGA_SERVICE_NAME,
                insecure=args.INSECURE,
                no_proxy=args.NO_PROXY,
                proxy=args.PROXY,
                timeout=args.TIMEOUT,
            )
        )
        if acknowledged:
            # Both the lines a level flagged and the lines an event flagged, and
            # the same line can be both, so they are deduplicated on the way in.
            # The counted events stay out: what alerts there is the arrival rate
            # and not one line, so silencing the lines already read would not
            # keep the next run from alerting on the ones arriving meanwhile.
            reported = {}
            for item in LEVELS:
                reported.update(dict.fromkeys(levels[item['level']]))
            for event in EVENTS:
                reported.update(dict.fromkeys(found[event['key']]))
            lib.base.coe(
                lib.logmatch.acknowledge(
                    ack_conn,
                    [
                        {'key': lib.logmatch.key(line), 'line': line}
                        for line in reported
                    ],
                )
            )
            state = STATE_OK
        if note:
            msg += f'\n\n{note}'
    if suppressed_cnt:
        msg += (
            f'\n\n{suppressed_cnt} acknowledged'
            f' {lib.txt.pluralize("line", suppressed_cnt)} suppressed.'
        )
    if ack_conn is not None:
        lib.db_sqlite.close(ack_conn)

    # over and out
    lib.base.oao(
        msg, state, perfdata, always_ok=args.ALWAYS_OK, no_perfdata=args.NO_PERFDATA
    )


if __name__ == '__main__':
    try:
        main()
    except Exception:
        lib.base.cu()
