#!/usr/bin/env python3
# -*- coding: utf-8; py-indent-offset: 4 -*-
#
# Author:  Linuxfabrik GmbH, Zurich, Switzerland
# Contact: info (at) linuxfabrik (dot) ch
#          https://www.linuxfabrik.ch/
# License: The Unlicense, see LICENSE file.

# https://github.com/Linuxfabrik/monitoring-plugins/blob/main/CONTRIBUTING.md

"""See the check's README for more details."""

import argparse
import datetime
import os
import re
import stat
import sys

import lib.args
import lib.base
import lib.db_sqlite
import lib.disk
import lib.human
import lib.logmatch
import lib.logsource
import lib.txt
from lib.globals import STATE_CRIT, STATE_OK, STATE_UNKNOWN, STATE_WARN

__author__ = 'Linuxfabrik GmbH, Zurich/Switzerland'
__version__ = '2026090301'

DESCRIPTION = """Scans the PHP-FPM error log for the events an administrator has to act
on: rejected configurations, worker crashes, requests that ran into
`request_terminate_timeout` or `request_slowlog_timeout`, pools that hit
`pm.max_children`, and the emergency reload PHP-FPM performs after repeated worker
failures. Startups, reloads and shutdowns are counted alongside them, so a pool that
keeps restarting is visible.
Alerts when one of those events shows up, when the lines a slow site provokes cross
the rates the thresholds set, and when a line arrives at a level `--critical-level`
or `--warning-level` covers.
Requests that ran long and a pool spawning workers in bursts are counted within
`--lookback` and judged by how many of them arrived, not by the fact that they did: one
slow page is nobody's night, dozens within ten minutes say the pool is mis-sized. Those
lines are counted there and nowhere else, so a site with one heavy report page does not
leave the check permanently yellow.
What is left is counted by the level PHP-FPM writes at the head of the line: `ALERT` and
`ERROR` return CRITICAL and `WARNING` returns WARNING, which `--critical-level` and
`--warning-level` move. The events named above carry their own state and are counted there
and nowhere else, because the level says nothing about what happened. A
message that merely contains the word "error" never counts.
The log is read either from a file, from a systemd unit (`systemd:`) or from a
container (`docker:`/`podman:`/`kubectl:`). `--server-log` may be given several times,
and everything named is then read as one window. Without it the file path is taken from
the `error_log` directive of the PHP-FPM configuration, with the common locations of the
distributions probed when that yields nothing, and the journal of the PHP-FPM unit is read
along with it, because a master that fails to start writes why to its standard error and
never reaches the error log. What both hold is counted once. The most recent
rotated file is read along with the live one, so the window does not end where
logrotate last ran.
Note that PHP-FPM discards everything its workers write unless
`catch_workers_output = yes` is set, so an error log that only ever shows master
events is the default behaviour rather than a quiet application.
Requires root or sudo."""

# What `--critical-level` and `--warning-level` accept on top of the levels
# themselves: no level at all raises that state, which leaves the named events as
# the only way to reach it.
LEVEL_NONE = 'none'

DEFAULT_CRITICAL_LEVEL = 'ERROR'
DEFAULT_ICINGA_CALLBACK = False
DEFAULT_INSECURE = True
DEFAULT_LOOKBACK = 600  # seconds
DEFAULT_NO_MATCH_SEVERITY = 'ok'
DEFAULT_NO_PROXY = False
DEFAULT_REQUEST_TIMEOUTS_CRITICAL = 50
DEFAULT_REQUEST_TIMEOUTS_WARNING = 5
DEFAULT_SLOW_REQUESTS_CRITICAL = 200
DEFAULT_SLOW_REQUESTS_WARNING = 20
DEFAULT_SPAWN_PRESSURE_CRITICAL = 100
DEFAULT_SPAWN_PRESSURE_WARNING = 10
DEFAULT_TIMEOUT = 8
DEFAULT_WARNING_LEVEL = 'WARNING'

ACK_RETENTION_DAYS = 30
MAXLINES = 30000  # maximum log lines to consider from the end of the source
# Rotated predecessors of the log file to read along with it. The distributions
# rotate this log daily, so without one the window would end at the last
# rotation and an event from before it would stop being reported the moment
# logrotate runs. One covers a daily rotation with room to spare.
ROTATED_FILES = 1

# The unit the distributions run PHP-FPM as. Only the fixed locations systemd
# reads unit files from are looked at, and only for the names the distributions
# use, so nothing a log or a configuration file holds can point the check at
# another unit.
UNIT_CANDIDATES = ('php-fpm.service', 'php*-fpm.service')
UNIT_DIRECTORIES = (
    '/etc/systemd/system',
    '/usr/lib/systemd/system',
    '/lib/systemd/system',
)

# How long a path may be in the summary before it is abbreviated for display.
# The paths the distributions use fit within it untouched; a host serving a
# dozen sites names a dozen logs in one directory, and there the directory is
# the part that says nothing.
SOURCE_PATH_MAX_LEN = 32

# PHP-FPM configuration files that carry the global `error_log` directive. The
# directive is global-only, so the pool files in `php-fpm.d/` never override it
# and are not read. The Debian family installs one tree per PHP version, hence
# the glob; the last match wins, which is the highest version installed.
CONFIG_FILES = (
    '/etc/php-fpm.conf',
    '/etc/php/*/fpm/php-fpm.conf',
    '/usr/local/etc/php-fpm.conf',
)
ERROR_LOG_REGEX = re.compile(r'^\s*error_log\s*=\s*(\S+)', re.MULTILINE)

# What PHP-FPM accepts instead of a path, matched case-insensitively the way
# PHP-FPM itself compares it.
SYSLOG_TARGET = 'syslog'

# Where the distributions keep the error log when the configuration does not say.
# The last entry is the path a source build compiles in (`log/php-fpm.log`
# relative to the local state directory).
LOG_FILE_CANDIDATES = (
    '/var/log/php-fpm/error.log',
    '/var/log/php-fpm.log',
    '/var/log/php*-fpm.log',
    '/usr/local/var/log/php-fpm.log',
)

# The master writes `[28-Aug-2026 15:08:23] NOTICE: message`. A worker that logs
# without going through the master writes the same line without the timestamp,
# because PHP-FPM prints the time only in the master process. Both shapes carry
# the level, so both are recognized (verified against php-fpm 8.0.30 on Rocky 9).
# What the journal calls PHP-FPM, for telling its own lines from the ones
# something else logged into the same place. The Debian family carries the
# version in the name (`php8.2-fpm`), the RHEL family does not.
IDENTIFIER_REGEX = re.compile(r'^php[\d.]*-?fpm$', re.IGNORECASE)

LEVEL_REGEX = re.compile(
    r'\b(ALERT|ERROR|WARNING|NOTICE|DEBUG):\s*(.*)$',
)

# The timestamp PHP-FPM writes in front of a master line, `[28-Aug-2026
# 15:21:20]`, in the local time of the host and in English regardless of the
# locale, because PHP-FPM formats it in the C locale. Python parses it in the C
# locale as well, as nothing here calls `setlocale()`. A worker line and a line
# a transport prefixed carry none; `lib.logsource.timestamp()` falls back to the
# transport's.
TIMESTAMP_REGEX = re.compile(r'\[(\d{2}-\w{3}-\d{4} \d{2}:\d{2}:\d{2})\]')
TIMESTAMP_FORMAT = '%d-%b-%Y %H:%M:%S'

# The levels that say something is wrong, worst first. Which of them raise which
# state is left to `--critical-level` and `--warning-level`: a pool that logs one
# `WARNING` about a single slow request is not a reason to get anybody out of
# bed, and what genuinely is arrives as one of the named events below.
LEVELS = (
    {'level': 'ALERT', 'perfdata': 'php_fpm_alert_lines'},
    {'level': 'ERROR', 'perfdata': 'php_fpm_error_lines'},
    {'level': 'WARNING', 'perfdata': 'php_fpm_warning_lines'},
)
LEVEL_CHOICES = [item['level'] for item in LEVELS] + [LEVEL_NONE]

# The events worth naming on their own, because the level alone does not say what
# happened. Everything else PHP-FPM logs is covered by the per-level counts. The
# message texts are taken from php-src `sapi/fpm/fpm/` and reproduced on
# php-fpm 8.0.30; a pattern therefore matches the whole message, not a keyword.
EVENTS = (
    {
        'key': 'worker_crashes',
        'label': 'worker crash',
        'suffix': 'es',
        'perfdata': 'php_fpm_worker_crashes',
        # `SIGQUIT` is how PHP-FPM ends a worker gracefully and `SIGTERM` is how
        # it terminates one that ran over `request_terminate_timeout` (which logs
        # a line of its own), so both are ordinary. Every other signal means the
        # worker died on a fault or was killed from outside, the OOM killer for
        # example. The lookahead also lets the line through when PHP-FPM has no
        # name for the signal and prints an empty pair of brackets.
        'regex': re.compile(r'exited on signal \d+ \((?!SIGQUIT|SIGTERM)'),
        'state': STATE_WARN,
        'recommendation': (
            'Workers died on a signal PHP-FPM did not send them; look for a core'
            ' dump, a faulty PHP extension, or the OOM killer in the kernel log'
        ),
    },
    {
        'key': 'pool_saturations',
        'label': 'pool saturation',
        'perfdata': 'php_fpm_pool_saturations',
        'regex': re.compile(
            r'server reached (?:pm\.)?max_children setting'
            r'|the maximum number of processes has been reached',
            re.IGNORECASE,
        ),
        'state': STATE_CRIT,
        # Raising the cap is the obvious lever and the wrong one to reach for
        # first: a scanner walking a few hundred 404s fills a pool exactly the
        # way real traffic does, and where the app renders its own 404 through
        # PHP every one of those costs a worker. A higher cap then multiplies
        # the memory the pool can claim and hands the next burst more of it, so
        # the access log decides which case this is before the pool is resized.
        'recommendation': (
            'A pool ran out of workers and clients waited in the listen queue;'
            ' check the web server access log around the timestamp above before'
            ' raising `pm.max_children` (or `process.max`), because a scanner'
            ' walking 404s fills a pool the same way real traffic does and a'
            ' higher cap only hands it more workers'
        ),
    },
    {
        'key': 'emergency_restarts',
        'label': 'emergency restart',
        'perfdata': 'php_fpm_emergency_restarts',
        'regex': re.compile(
            r'failed processes threshold \(\d+ in \d+ sec\) is reached,'
            r' initiating reload',
        ),
        'state': STATE_CRIT,
        'recommendation': (
            'PHP-FPM reloaded itself because too many workers failed in a row;'
            ' treat this as a crashing application until proven otherwise'
        ),
    },
)

# What PHP-FPM writes once per request or once per maintenance tick rather than
# once per problem. One of these is a traffic peak or a single slow page and no
# reason to get anybody out of bed; dozens of them within ten minutes say the
# pool is mis-sized. They are therefore counted against a rate over `--lookback`,
# and they are kept out of the per-level counts above, so a site with one slow
# report page does not leave the check permanently yellow.
RATE_EVENTS = (
    {
        'key': 'request_timeouts',
        'label': 'request timeout',
        'perfdata': 'php_fpm_request_timeouts',
        # PHP-FPM kills the worker and the client gets nothing, so the threshold
        # is the lowest of the three: a handful of these is already a bad
        # afternoon for somebody.
        'regex': re.compile(r'execution timed out \([\d.]+ sec\), terminating'),
        'critical': 'REQUEST_TIMEOUTS_CRITICAL',
        'warning': 'REQUEST_TIMEOUTS_WARNING',
        'recommendation': (
            'Requests were terminated after `request_terminate_timeout`; profile'
            ' the scripts named above or raise the timeout for that pool'
        ),
    },
    {
        'key': 'slow_requests',
        'label': 'slow request',
        'perfdata': 'php_fpm_slow_requests',
        # The request still finishes; PHP-FPM only writes a backtrace. A site
        # with one heavy report page produces these all day.
        'regex': re.compile(r'executing too slow \([\d.]+ sec\), logging'),
        'critical': 'SLOW_REQUESTS_CRITICAL',
        'warning': 'SLOW_REQUESTS_WARNING',
        'recommendation': (
            'Requests ran longer than `request_slowlog_timeout`; the backtrace of'
            ' each one is in the pool `slowlog`'
        ),
    },
    {
        'key': 'spawn_pressure',
        'label': 'spawn pressure warning',
        'perfdata': 'php_fpm_spawn_pressure',
        # Only `pm = dynamic` writes this, and only once the spawn rate has
        # doubled its way to 8, which takes several seconds of sustained
        # pressure. Unlike `server reached pm.max_children`, which PHP-FPM guards
        # with a flag and logs once, this one has no guard at all: it is written
        # on every maintenance tick, so roughly once a second, for as long as the
        # pressure lasts. A traffic peak therefore leaves a burst of them behind
        # and then silence, which is a rate and not an event (verified against
        # `fpm_pctl_perform_idle_server_maintenance()` in php-src).
        'regex': re.compile(r'seems busy \(you may need to increase'),
        'critical': 'SPAWN_PRESSURE_CRITICAL',
        'warning': 'SPAWN_PRESSURE_WARNING',
        'recommendation': (
            'A pool had to spawn workers in bursts; raise `pm.start_servers` and'
            ' `pm.min_spare_servers` so requests do not wait for a fork'
        ),
    },
)

# PHP-FPM writes this once into the fresh file when logrotate has moved the old
# one away and signalled it with SIGUSR1, so the window of a just-rotated log
# holds exactly this line. Counted, and named next to the window size, which
# turns "no startup, no reload, no shutdown" from a puzzle into a sentence.
ROTATION_REGEX = re.compile(r'^error log file re-opened')

# Lifecycle markers, counted but never alerting on. A reload re-executes the
# master, so it shows up as a reload and as a startup. The reload is recognized
# by the line the exec itself writes, because the `Reloading in progress ...`
# that a `SIGUSR2` triggers is missing when PHP-FPM reloads itself after too many
# worker failures.
LIFECYCLE = (
    {
        'key': 'startups',
        'label': 'startup',
        'perfdata': 'php_fpm_startups',
        'regex': re.compile(r'^fpm is running, pid \d+'),
        'verb': 'detected',
    },
    {
        'key': 'reloads',
        'label': 'reload',
        'perfdata': 'php_fpm_reloads',
        'regex': re.compile(r'^reloading: execvp\('),
        'verb': 'detected',
    },
    {
        'key': 'shutdowns',
        'label': 'shutdown',
        'perfdata': 'php_fpm_shutdowns',
        'regex': re.compile(r'^exiting, bye-bye!'),
        'verb': 'detected',
    },
)


def parse_args():
    """Parse command line arguments using argparse."""
    parser = argparse.ArgumentParser(
        description=DESCRIPTION,
        epilog=lib.args.epilog(__file__),
        formatter_class=lib.args.HelpFormatter,
    )

    parser.add_argument(
        '-V',
        '--version',
        action='version',
        version=f'%(prog)s: v{__version__} by {__author__}',
    )

    parser.add_argument(
        '--always-ok',
        help=lib.args.help('--always-ok'),
        dest='ALWAYS_OK',
        action='store_true',
        default=False,
    )

    # Developer-only hook, like `--test` on the plugins that inject command
    # output: it prefixes every path the check opens, so the fixture tests can
    # stand a directory tree in for the host's file system. Deliberately not
    # named `--test-...`, because argparse would then resolve a bare `--test`
    # into it.
    parser.add_argument(
        '--config-root',
        help=argparse.SUPPRESS,
        dest='CONFIG_ROOT',
        default='',
    )

    parser.add_argument(
        '--critical-level',
        help='Least severe PHP-FPM log level that returns CRITICAL. '
        'Each level includes everything more severe than itself, so `WARNING` '
        'covers `ERROR` and `ALERT` as well. '
        'Case-sensitive. '
        '`none` lets no level return CRITICAL, which leaves the events this '
        'check names as the only way to reach it. '
        'Example: `--critical-level=WARNING`. '
        'Default: %(default)s',
        dest='CRITICAL_LEVEL',
        choices=LEVEL_CHOICES,
        default=DEFAULT_CRITICAL_LEVEL,
    )

    parser.add_argument(
        '--icinga-callback',
        help=lib.args.help('--icinga-callback'),
        dest='ICINGA_CALLBACK',
        action='store_true',
        default=DEFAULT_ICINGA_CALLBACK,
    )

    parser.add_argument(
        '--icinga-password',
        help=lib.args.help('--icinga-password'),
        dest='ICINGA_PASSWORD',
    )

    parser.add_argument(
        '--icinga-service-name',
        help=lib.args.help('--icinga-service-name'),
        dest='ICINGA_SERVICE_NAME',
    )

    parser.add_argument(
        '--icinga-url',
        help=lib.args.help('--icinga-url'),
        dest='ICINGA_URL',
    )

    parser.add_argument(
        '--icinga-username',
        help=lib.args.help('--icinga-username'),
        dest='ICINGA_USERNAME',
    )

    parser.add_argument(
        '--ignore',
        help='Ignore a log line matching this Python regular expression. '
        'The log line is lowercased before matching, so write the pattern in '
        'lowercase (or use the `(?i)` flag). '
        'Can be specified multiple times. '
        "Example: `--ignore='(?i)linuxfabrik'`.",
        action='append',
        default=None,
        dest='IGNORE',
    )

    parser.add_argument(
        '--insecure',
        help='Applies to the connection to the monitoring server that `--icinga-callback` makes, which is the only network connection this check opens. '
        + lib.args.help('--insecure'),
        dest='INSECURE',
        action='store_true',
        default=DEFAULT_INSECURE,
    )

    parser.add_argument(
        '--lookback',
        help='Request timeouts, slow requests and spawn pressure are counted '
        'within this window rather than reported one by one. '
        + lib.args.help('--lookback')
        + ' Example: `--lookback=3600`. '
        'Default: %(default)s (seconds)',
        dest='LOOKBACK',
        type=int,
        default=DEFAULT_LOOKBACK,
    )

    parser.add_argument(
        '--match',
        help='Only consider a log line matching this Python regular expression. '
        'The log line is lowercased before matching, so write the pattern in '
        'lowercase (or use the `(?i)` flag). '
        'Can be specified multiple times. '
        + lib.args.MATCH_IGNORE_PRECEDENCE
        + " Example: `--match='\\[pool www\\]'`.",
        action='append',
        default=None,
        dest='MATCH',
    )

    parser.add_argument(
        '--no-insecure',
        help='Applies to the connection to the monitoring server that `--icinga-callback` makes, which is the only network connection this check opens. '
        + lib.args.help('--no-insecure'),
        dest='INSECURE',
        action='store_false',
        default=DEFAULT_INSECURE,
    )

    parser.add_argument(
        '--no-match-severity',
        help=lib.args.help('--no-match-severity') + ' Default: %(default)s',
        dest='NO_MATCH_SEVERITY',
        choices=['ok', 'warn', 'crit', 'unknown'],
        default=DEFAULT_NO_MATCH_SEVERITY,
    )

    parser.add_argument(
        '--no-perfdata',
        help=lib.args.help('--no-perfdata'),
        dest='NO_PERFDATA',
        action='store_true',
        default=False,
    )

    parser.add_argument(
        '--no-proxy',
        help='Applies to the connection to the monitoring server that `--icinga-callback` makes, which is the only network connection this check opens. '
        + lib.args.help('--no-proxy'),
        dest='NO_PROXY',
        action='store_true',
        default=DEFAULT_NO_PROXY,
    )

    parser.add_argument(
        '--proxy',
        help='Applies to the connection to the monitoring server that `--icinga-callback` makes, which is the only network connection this check opens. '
        + lib.args.help('--proxy'),
        dest='PROXY',
        default=None,
    )

    parser.add_argument(
        '--request-timeouts-critical',
        help='Number of terminated requests within `--lookback` that returns '
        'CRITICAL. '
        '0 turns the threshold off. '
        'Example: `--request-timeouts-critical=20`. '
        'Default: %(default)s',
        dest='REQUEST_TIMEOUTS_CRITICAL',
        type=int,
        default=DEFAULT_REQUEST_TIMEOUTS_CRITICAL,
    )

    parser.add_argument(
        '--request-timeouts-warning',
        help='Number of terminated requests within `--lookback` that returns '
        'WARNING. '
        '0 turns the threshold off. '
        'Example: `--request-timeouts-warning=1`. '
        'Default: %(default)s',
        dest='REQUEST_TIMEOUTS_WARNING',
        type=int,
        default=DEFAULT_REQUEST_TIMEOUTS_WARNING,
    )

    parser.add_argument(
        '--server-log',
        help='Log source to read from. '
        'Accepts a file path, `docker:CONTAINER`, `podman:CONTAINER`, '
        '`kubectl:CONTAINER` or `systemd:UNITNAME`. '
        'Can be specified multiple times, and everything named is then read as '
        'one window; a source named twice is read once. '
        'If omitted, the check reads the `error_log` directive of the PHP-FPM '
        'configuration, falls back to the common locations of the '
        'distributions, and reads the journal of the PHP-FPM unit along with it; '
        'what the two share is counted once. '
        'Example: `--server-log=systemd:php-fpm.service`.',
        action='append',
        default=None,
        dest='SERVER_LOG',
    )

    parser.add_argument(
        '--slow-requests-critical',
        help='Number of slow requests within `--lookback` that returns CRITICAL. '
        '0 turns the threshold off. '
        'Example: `--slow-requests-critical=100`. '
        'Default: %(default)s',
        dest='SLOW_REQUESTS_CRITICAL',
        type=int,
        default=DEFAULT_SLOW_REQUESTS_CRITICAL,
    )

    parser.add_argument(
        '--slow-requests-warning',
        help='Number of slow requests within `--lookback` that returns WARNING. '
        '0 turns the threshold off. '
        'Example: `--slow-requests-warning=5`. '
        'Default: %(default)s',
        dest='SLOW_REQUESTS_WARNING',
        type=int,
        default=DEFAULT_SLOW_REQUESTS_WARNING,
    )

    parser.add_argument(
        '--spawn-pressure-critical',
        help='Number of spawn pressure warnings within `--lookback` that returns '
        'CRITICAL. '
        '0 turns the threshold off. '
        'Example: `--spawn-pressure-critical=50`. '
        'Default: %(default)s',
        dest='SPAWN_PRESSURE_CRITICAL',
        type=int,
        default=DEFAULT_SPAWN_PRESSURE_CRITICAL,
    )

    parser.add_argument(
        '--spawn-pressure-warning',
        help='Number of spawn pressure warnings within `--lookback` that returns '
        'WARNING. '
        'A pool logs one of these per second while it is spawning in bursts, so '
        'this counts how many arrived rather than that any did. '
        '0 turns the threshold off. '
        'Example: `--spawn-pressure-warning=30`. '
        'Default: %(default)s',
        dest='SPAWN_PRESSURE_WARNING',
        type=int,
        default=DEFAULT_SPAWN_PRESSURE_WARNING,
    )

    parser.add_argument(
        '--timeout',
        help=lib.args.help('--timeout') + ' Default: %(default)s (seconds)',
        dest='TIMEOUT',
        type=int,
        default=DEFAULT_TIMEOUT,
    )

    parser.add_argument(
        '--warning-level',
        help='Least severe PHP-FPM log level that returns WARNING. '
        'Each level includes everything more severe than itself, and a level '
        'that `--critical-level` already covers returns CRITICAL instead. '
        'Case-sensitive. '
        '`none` lets no level return WARNING, which leaves the events this '
        'check names as the only way to reach it. '
        'Example: `--warning-level=none`. '
        'Default: %(default)s',
        dest='WARNING_LEVEL',
        choices=LEVEL_CHOICES,
        default=DEFAULT_WARNING_LEVEL,
    )

    args, _ = parser.parse_known_args()
    return args


def get_configured_log_file(config_root=''):
    """Return the `error_log` the PHP-FPM configuration names, or None.

    The value `syslog` is handed back lowercased, because that is a value PHP-FPM
    understands (case-insensitively) and the caller has to report rather than
    open. Only the fixed configuration locations of the distributions are read,
    so nothing the log itself contains can steer the check to another file.
    """
    error_log = None
    for candidate in CONFIG_FILES:
        for filename in lib.disk.glob(f'{config_root}{candidate}'):
            success, content = lib.disk.read_file(filename)
            if not success or not content:
                continue
            match = ERROR_LOG_REGEX.search(content)
            if match:
                error_log = match.group(1)
    if error_log is None:
        return None
    if error_log.lower() == SYSLOG_TARGET:
        return SYSLOG_TARGET
    if not error_log.startswith('/'):
        # A relative path is resolved against a prefix that is compiled into
        # PHP-FPM and not readable from here, so probing the known locations
        # gets further than guessing that prefix.
        return None
    return f'{config_root}{error_log}'


def get_log_file_real_path(config_root=''):
    """Probe the locations the distributions keep the PHP-FPM error log in."""
    for candidate in LOG_FILE_CANDIDATES:
        for filename in lib.disk.glob(f'{config_root}{candidate}'):
            if lib.disk.file_exists(filename, allow_empty=True):
                return filename
    return None


def parse_timestamp(log_line):
    """Return the moment PHP-FPM stamped a line with, or None when it carries none.

    Only PHP-FPM's own timestamp; `lib.logsource.timestamp()` falls back to the
    one the transport prefixed. A line without either is not an error: PHP-FPM
    prints the time only in the master process, so a line a worker wrote on its
    own carries none.
    """
    match = TIMESTAMP_REGEX.search(log_line)
    if not match:
        return None
    try:
        return datetime.datetime.strptime(match.group(1), TIMESTAMP_FORMAT)
    except ValueError:
        return None


def get_level_states(critical_level, warning_level):
    """Map every counted level to the state it raises.

    A level raises a state when it is at least as severe as the level the
    parameter names, and CRITICAL wins wherever both would apply.
    """
    ranks = {item['level']: rank for rank, item in enumerate(LEVELS)}
    states = {}
    for level, rank in ranks.items():
        if critical_level != LEVEL_NONE and rank <= ranks[critical_level]:
            states[level] = STATE_CRIT
        elif warning_level != LEVEL_NONE and rank <= ranks[warning_level]:
            states[level] = STATE_WARN
        else:
            states[level] = STATE_OK
    return states


def get_units(config_root=''):
    """Return the PHP-FPM units this host has a unit file for.

    A unit file two names point at is one unit: the MariaDB package ships
    `mysqld.service` and `mysql.service` as symlinks to `mariadb.service`, and
    reading that journal three times would count every line three times.
    """
    found = []
    seen = set()
    for candidate in UNIT_CANDIDATES:
        for directory in UNIT_DIRECTORIES:
            for filename in lib.disk.glob(f'{config_root}{directory}/{candidate}'):
                if not lib.disk.file_exists(filename, allow_empty=True):
                    continue
                identity = os.path.realpath(filename)
                if identity in seen:
                    continue
                seen.add(identity)
                found.append(os.path.basename(filename))
    return found


def dedup_key(line):
    """Identify the event in a line, whichever transport delivered it.

    The error log and the journal of the unit are both read, and PHP-FPM writes
    its own timestamp into either, so what follows the syslog prefix the journal
    adds is the line the file holds.
    """
    written_at = lib.logsource.timestamp(line, parse_timestamp)
    return (
        written_at.replace(microsecond=0) if written_at else None,
        lib.logsource.strip_syslog_prefix(line),
    )


def main():
    """The main function. This is where the magic happens."""

    # parse the command line
    try:
        args = parse_args()
    except SystemExit:
        sys.exit(STATE_UNKNOWN)

    # set default values for append parameters that were not specified
    if args.IGNORE is None:
        args.IGNORE = []
    if args.MATCH is None:
        args.MATCH = []
    if args.SERVER_LOG is None:
        args.SERVER_LOG = []

    if args.ICINGA_CALLBACK and not all(
        (
            args.ICINGA_URL,
            args.ICINGA_PASSWORD,
            args.ICINGA_USERNAME,
            args.ICINGA_SERVICE_NAME,
        )
    ):
        lib.base.cu(
            '`--icinga-callback` requires `--icinga-url`, `--icinga-password`, '
            '`--icinga-username` and `--icinga-service-name`.'
        )

    # fetch data
    server_logs = args.SERVER_LOG
    if not server_logs:
        # The unit alongside the file, not instead of it: a master that failed to
        # start wrote why to its standard error and never reached the error log,
        # and a pool logging to syslog is in the journal only. What both carry is
        # counted once.
        found_log = get_configured_log_file(args.CONFIG_ROOT) or get_log_file_real_path(
            args.CONFIG_ROOT
        )
        server_logs = [found_log] if found_log else []
        server_logs += [f'systemd:{unit}' for unit in get_units(args.CONFIG_ROOT)]
    if not server_logs:
        lib.base.cu(
            'Found no PHP-FPM error log. Set `error_log` in the PHP-FPM '
            "configuration, or name the log with the check's `--server-log` "
            'parameter.'
        )
    if SYSLOG_TARGET in server_logs:
        lib.base.cu(
            '`error_log` is set to syslog, so the log is not a file this check '
            'can open. Point `--server-log` at the systemd unit '
            '(`--server-log=systemd:php-fpm.service`) or at the file the syslog '
            'daemon writes.'
        )

    # A size is known for plain on-disk files only, so the size fact and its
    # perfdata series stay out of the output for a unit or a container. Where
    # several files are read, the series is what they add up to.
    sizes = {}
    for server_log in server_logs:
        kind, _, target = lib.base.coe(lib.logsource.parse(server_log))
        if kind != lib.logsource.KIND_FILE:
            continue
        # Report a path that is not a regular file before reading it, so the
        # admin hears about a directory or a device rather than about an I/O
        # error further down. With one source that is the whole answer; with
        # several it is one source of many and the run goes on.
        log_stat = lib.disk.stat(target)
        if log_stat is None or not stat.S_ISREG(log_stat.st_mode):
            if len(server_logs) == 1:
                lib.base.oao(
                    f'Logging seems to be configured, but `{target}` does not'
                    f' seem to be an existing regular file. Check the path and'
                    f' file permissions, or provide the `--server-log`'
                    f' parameter.',
                    STATE_WARN,
                )
            continue
        sizes[server_log] = log_stat.st_size
    if len(server_logs) == 1 and sizes.get(server_logs[0]) == 0:
        # An empty log file is a deterministic "no events observed" state,
        # not an unknown one - typical right after logrotate fires.
        lib.base.oao(
            f'Log file `{server_logs[0]}` is empty. Assuming log-rotation.',
            STATE_OK,
        )

    # The plugin runs as root via sudo, so restrict an on-disk log to the system
    # log directory. The path the PHP-FPM configuration reports is deliberately
    # not trusted beyond that: a pool file an unprivileged user may edit could
    # otherwise point the check at any file on the host. The plugin's own
    # unit-test/ directory is allowed so the fixture tests work; it does not
    # exist next to the flat, root-owned plugin in production. Bind-mount a
    # custom log directory under /var/log to include it (see the README).
    allowed_roots = [
        '/var/log',
        os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), 'unit-test'),
    ]
    # No position is kept on purpose: this check reports on a window of the log,
    # counting startups, reloads and shutdowns next to the problems, so every run
    # has to see the whole window rather than only what is new.
    result = lib.base.coe(
        lib.logsource.read_many(
            server_logs,
            allowed_roots=allowed_roots,
            dedup_key=dedup_key,
            max_lines=MAXLINES,
            rotated=ROTATED_FILES,
            timeout=args.TIMEOUT,
        )
    )
    # What to call the whole of what was read, for the messages that have to
    # name it before the summary below spells every source out.
    # Abbreviated here, because this one shares a line with a sentence; the
    # listing further down gives the path in full.
    source_label = (
        '`'
        + lib.disk.shorten_path(
            result['sources'][0]['label'], max_len=SOURCE_PATH_MAX_LEN, truncate=False
        )
        + '`'
        if len(result['sources']) == 1
        else f'the {len(result["sources"])} sources read'
    )

    # init some vars
    state = STATE_OK
    sections = []
    facts = []
    # All recommendations from all WARN/CRIT paths land here and render once at
    # the end as a `Recommendations:\n* ...` bulleted block, regardless of which
    # combinations of paths fire.
    recommendations = []
    level_states = get_level_states(args.CRITICAL_LEVEL, args.WARNING_LEVEL)
    levels = {item['level']: [] for item in LEVELS}
    found = {item['key']: [] for item in EVENTS}
    found.update({item['key']: [] for item in RATE_EVENTS})
    found.update({item['key']: [] for item in LIFECYCLE})
    rate_since = datetime.datetime.now() - datetime.timedelta(seconds=args.LOOKBACK)
    rotations = []
    considered_cnt = 0
    recognized_cnt = 0
    unrecognized_cnt = 0
    suppressed_cnt = 0
    compiled_ignore = [
        lib.base.coe(item) for item in lib.txt.compile_regex(args.IGNORE, '--ignore')
    ]
    compiled_match = [
        lib.base.coe(item) for item in lib.txt.compile_regex(args.MATCH, '--match')
    ]

    # Persisted acknowledgements are only needed when the callback is in use.
    # Each unique combination of filters gets its own state database, so two
    # services watching the same error log for different things do not share what
    # has been acknowledged. Only the filters actually set take part in the
    # identifier, so adding a filter later does not orphan the state of a service
    # that does not use it.
    acked_fingerprints = set()
    ack_conn = None
    if args.ICINGA_CALLBACK:
        # Only what the operator set takes part in the identifier, so a host
        # that gains a second source - a unit next to the file - keeps the
        # acknowledgements it already has. `server_log` is in there when it was
        # named, because two services then deliberately watch different logs.
        instance_payload = {}
        for name, value in (
            ('ignore_regex', args.IGNORE),
            ('match', args.MATCH),
            ('server_log', args.SERVER_LOG),
        ):
            if value:
                instance_payload[name] = value
        ack_conn = lib.base.coe(
            lib.logmatch.connect(
                'php-fpm-logfile', lib.logmatch.instance_id(instance_payload)
            )
        )
        # Drop acknowledgements older than the retention: by that age the line has
        # left the window this check reads and can no longer re-appear.
        lib.base.coe(lib.logmatch.prune(ack_conn, retention=ACK_RETENTION_DAYS))
        acked_fingerprints = lib.base.coe(lib.logmatch.suppressed(ack_conn))

    # analyze data
    for log_line in result['lines']:
        haystack = log_line.lower()
        if any(item.search(haystack) for item in compiled_ignore):
            continue
        # `--match` (include) is applied first, then `--ignore` (exclude), so a
        # line hit by `--ignore` is dropped even if it also matches `--match`.
        if compiled_match and not any(item.search(haystack) for item in compiled_match):
            continue
        considered_cnt += 1
        # Drop a line an operator has already taken on. The whole log line is
        # keyed, timestamp included, so the very same message logged again later
        # is a new event and alerts again.
        if acked_fingerprints and lib.logmatch.key(log_line) in acked_fingerprints:
            suppressed_cnt += 1
            continue
        identifier = lib.logsource.syslog_identifier(log_line)
        if identifier is not None and not IDENTIFIER_REGEX.match(identifier):
            # Somebody else writing into the same place: systemd's own
            # bookkeeping about the unit ("Starting ...", "Started ..."), and
            # the application's own messages, which PHP-FPM passes on under the
            # application's identifier - the journal of a PHP-FPM unit serving
            # Icinga Web is mostly `icingaweb2[1234]: PDOException ...`. Neither
            # is PHP-FPM's line, and counting them as lines without a level
            # would report the wrong source on a host where the source is right.
            continue
        level_match = LEVEL_REGEX.search(log_line)
        if not level_match:
            # Not a line PHP-FPM wrote. Counted rather than dropped silently,
            # because a log full of them means the check is pointed at the wrong
            # file.
            unrecognized_cnt += 1
            continue
        recognized_cnt += 1
        level, message = level_match.group(1), level_match.group(2)
        rate_hits = [event for event in RATE_EVENTS if event['regex'].search(message)]
        for event in rate_hits:
            found[event['key']].append(log_line)
        hits = [event for event in EVENTS if event['regex'].search(message)]
        for event in hits:
            found[event['key']].append(log_line)
        # A line one of the catalogs owns is left out of the per-level counts.
        # A rate event is judged by how often it arrives, and counting it by its
        # level too would leave a site with one slow report page permanently
        # yellow. A named event carries its own state and says what happened,
        # which the level cannot; reporting the same lines once by level and
        # once by name says the same thing twice and doubles them in the graphs.
        if not rate_hits and not hits and level in levels:
            levels[level].append(log_line)
        for item in LIFECYCLE:
            if item['regex'].search(message):
                found[item['key']].append(log_line)
        if ROTATION_REGEX.search(message):
            rotations.append(log_line)

    # Everything collected is put back into the order it was written in. The
    # sources are read one after the other, so a line from the file sits before
    # a newer one from the journal however late it was written, and "the last
    # one" would name something days old.
    for key, lines in found.items():
        found[key] = lib.logsource.sort_by_time(lines, parse_timestamp)
    for level, lines in levels.items():
        levels[level] = lib.logsource.sort_by_time(lines, parse_timestamp)

    # `--match` narrowed the run down to nothing, so this run looked at no line at
    # all rather than at a quiet log. Only reachable when the operator set a
    # filter, so a log that simply has nothing to report stays OK.
    if compiled_match and result['lines'] and not considered_cnt:
        if ack_conn is not None:
            lib.db_sqlite.close(ack_conn)
        lib.base.oao(
            f'Nothing checked: `--match` dropped all {len(result["lines"])} '
            f'{lib.txt.pluralize("line", len(result["lines"]))} of '
            f'{source_label}.',
            lib.base.str2state(args.NO_MATCH_SEVERITY),
            always_ok=args.ALWAYS_OK,
        )

    # Every line PHP-FPM writes carries its level, so a window without a single
    # one is not a quiet PHP-FPM but a source that holds something else, the pool
    # `slowlog` or the access log for example. Saying so beats reporting the
    # green state a check would otherwise keep on the wrong file forever.
    if unrecognized_cnt and not recognized_cnt:
        if ack_conn is not None:
            lib.db_sqlite.close(ack_conn)
        lib.base.oao(
            f'None of the {unrecognized_cnt} '
            f'{lib.txt.pluralize("line", unrecognized_cnt)} read from '
            f'{source_label} carries a PHP-FPM log level, so this does not '
            f'look like a PHP-FPM error log. Check `--server-log` and the '
            f'`error_log` directive of the PHP-FPM configuration.',
            STATE_UNKNOWN,
        )

    # build the message
    # What was read, and where from. The size stands next to the live file
    # rather than next to the line count, because it describes that one file and
    # nothing else - right after logrotate a 34 KiB file said to hold 30000
    # lines reads like a miscount, when almost all of them came from the rotated
    # predecessor. Naming the predecessor says where they did come from.
    # The count matters because everything below is counted within it: a run
    # that reports no startup at all is telling the truth about the window rather
    # than about the day.
    # And where the window stopped at the cap this check reads rather than at
    # the start of the log, it says so, because a busy host then reports on the
    # last few hours rather than on the day.
    # Which stretch of time those lines cover, because the count alone does not
    # say whether the window is an hour or a week: on a busy host the cap is
    # reached within hours, on a quiet one the same 30000 lines reach back
    # months, and every count below has to be read against that. Taken from the
    # ends of each source rather than from the ends of everything read: the
    # sources arrive one after the other, so the newest line of the first one
    # sits in the middle. A source that stamps no line simply leaves it out.
    covered = ''
    window_from, window_to = lib.logsource.covered_window(
        [item['lines'] for item in result['sources']], parse_timestamp
    )
    if window_from is not None and window_to is not None:
        span = int((window_to - window_from).total_seconds())
        covered = (
            f'{window_from:%Y-%m-%d %H:%M} .. {window_to:%Y-%m-%d %H:%M}'
            f' ({lib.human.seconds2human(span)}): '
        )
    line_cnt = len(result['lines'])
    lines_read = (
        f'{lib.human.number2human(line_cnt)} {lib.txt.pluralize("line", line_cnt)}'
    )
    if result['truncated']:
        lines_read = f'the most recent {lines_read}'
    # Every source by name, with its own size and its own rotated predecessor,
    # because a run that read three files and found nothing in two of them has
    # said something different from a run that read one.
    # The sources get a section of their own, the way the lines behind every
    # count do: a reverse proxy serving a dozen sites reads a dozen logs, and
    # naming them in the summary would bury the verdict under a paragraph of
    # paths - and the summary line is what a monitoring server shows in a list.
    # The paths are not abbreviated there: a bullet has the room, and that is
    # where an administrator copies them from.
    described = [
        lib.logsource.describe(item, sizes.get(item['label']))
        for item in result['sources']
    ]
    sources = f'{len(described)} {lib.txt.pluralize("source", len(described))}'

    if result['duplicates']:
        # Named rather than silently dropped: an administrator who sees two
        # sources read and one count has to be able to tell that the check knows
        # they hold the same events.
        shared = result['duplicates']
        source_fact_tail = (
            f', {lib.human.number2human(shared)} '
            f'{lib.txt.pluralize("line", shared)} they share counted once'
        )
    else:
        source_fact_tail = ''
    source_heading = f'Read {lines_read} from {sources}{source_fact_tail}:'

    # The counted events. Their state comes from how many of them arrived within
    # the rate window, not from the fact that they arrived at all. Their facts
    # are collected before the level counts are rendered, because a run that has
    # something to say about them is not the quiet one the literal below claims.
    rate_counts = {}
    rate_facts = []
    for event in RATE_EVENTS:
        hits = found[event['key']]
        # No `key` here: PHP-FPM names the pool and the script of a request in
        # its error log but never the client, so there is no source to count per
        # and the number to judge by is everything that arrived.
        window = lib.logsource.count_within(
            hits, rate_since, parse_line=parse_timestamp
        )
        rate_counts[event['key']] = window['count']
        if not hits:
            continue
        rate_state = lib.base.get_state(
            window['count'],
            getattr(args, event['warning']) or None,
            getattr(args, event['critical']) or None,
        )
        state = lib.base.get_worst(state, rate_state)
        fact = (
            f'{window["count"]} '
            f'{lib.txt.pluralize(event["label"], window["count"])} in the last'
            f' {lib.human.seconds2human(args.LOOKBACK)}'
            f'{lib.base.state2str(rate_state, prefix=" ")}'
        )
        if len(hits) != window['count']:
            fact += f' ({len(hits)} in the window read)'
        if window['undated']:
            fact += (
                f', {window["undated"]} of them without a timestamp and therefore'
                f' not counted'
            )
        rate_facts.append(fact)
        if rate_state != STATE_OK:
            recommendations.append(event['recommendation'])

    counts = {}
    for item in LEVELS:
        lines = levels[item['level']]
        counts[item['level']] = len(lines)
        if not lines:
            continue
        level_state = level_states[item['level']]
        state = lib.base.get_worst(state, level_state)
        facts.append(
            f'{len(lines)} {item["level"]} '
            f'{lib.txt.pluralize("line", len(lines))} found'
            f'{lib.base.state2str(level_state, prefix=" ")}'
            f' (last: {lines[-1]})'
        )
    if not any(counts.values()) and not rate_facts:
        # Naming all three levels one by one would bury the one sentence a
        # healthy host is supposed to be.
        facts.append('No errors or warnings found')

    # Name the events, so the summary says what happened and not only how bad it
    # was, and raise the state for those PHP-FPM logs too quietly. Quiet events
    # stay out of the line, which keeps a healthy host down to a single sentence.
    named = []
    for event in EVENTS:
        hits = found[event['key']]
        if not hits:
            continue
        state = lib.base.get_worst(state, event['state'])
        named.append(
            f'{len(hits)} '
            f'{lib.txt.pluralize(event["label"], len(hits), event.get("suffix", "s"))}'
            f'{lib.base.state2str(event["state"], prefix=" ")}'
        )
        recommendations.append(event['recommendation'])
    if named:
        facts.append('Found ' + ', '.join(named))

    facts.extend(rate_facts)

    for item in LIFECYCLE:
        lines = found[item['key']]
        # A counter at zero says nothing the performance data does not, and a
        # sentence about it would only push the verdict further right.
        if lines:
            facts.append(
                f'{len(lines)} {lib.txt.pluralize(item["label"], len(lines))}'
                f' {item["verb"]} (last: {lines[-1]})'
            )

    # A few lines without a level are normal where the application logs into the
    # same file; a majority of them means the source is mostly something else.
    if unrecognized_cnt:
        facts.append(
            f'{unrecognized_cnt} {lib.txt.pluralize("line", unrecognized_cnt)}'
            f' without a PHP-FPM log level'
        )
        if unrecognized_cnt > recognized_cnt:
            recommendations.append(
                'Most lines carry no PHP-FPM log level; check that the source is'
                ' the PHP-FPM error log and not the pool `slowlog`, the access'
                ' log or the log of the application itself'
            )

    if result['notice']:
        facts.append(result['notice'].rstrip('.'))

    # A source that could not be read at all leaves a hole in the window: with a
    # second source still delivering, the run goes on, but reporting the state of
    # what happened to work would hide that nothing is watching the rest.
    if result['failed']:
        state = lib.base.get_worst(state, STATE_WARN)
        facts.append(
            '; '.join(item.rstrip('.') for item in result['failed'])
            + lib.base.state2str(STATE_WARN, prefix=' ')
        )
        recommendations.append(
            'Check the log sources this check was told to read; '
            'one of them could not be read at all'
        )

    # The window the summary reports on comes first, because every count in it
    # is a count within that window and reads differently against an hour than
    # against a week.
    sections.append(covered + '. '.join(facts) + '.')

    for item in LEVELS:
        lines = levels[item['level']]
        if lines:
            sections.append(
                f'{item["level"].capitalize()} lines:\n'
                + '\n'.join(f'* {line}' for line in lib.txt.shorten_list(lines))
            )

    # The lines behind the named events. They are kept out of the per-level
    # counts above, so without this they are the one thing the check alerts on
    # that it never shows: "1 pool saturation" says that a pool ran dry, not
    # when, not which one, and not whether it was five minutes ago or a burst
    # from yesterday still sitting in the rotated file. The timestamp is what
    # lets the access log be read for the same minute, which is where the cause
    # of a saturation is.
    for event in EVENTS:
        lines = found[event['key']]
        if lines:
            label = lib.txt.pluralize(
                event['label'], len(lines), event.get('suffix', 's')
            )
            sections.append(
                f'{label.capitalize()}:\n'
                + '\n'.join(f'* {line}' for line in lib.txt.shorten_list(lines))
            )

    # What was read, last of the sections: the verdict and the lines behind it
    # come first, the provenance after them.
    sections.append(
        source_heading + '\n' + '\n'.join(f'* {item}' for item in described)
    )

    if recommendations:
        sections.append(
            'Recommendations:\n' + '\n'.join(f'* {item}' for item in recommendations)
        )

    msg = '\n\n'.join(sections)

    perfdata = ''
    if sizes:
        perfdata += lib.base.get_perfdata(
            'php_fpm_logfile_size',
            sum(sizes.values()),
            uom='B',
            _min=0,
        )
    # The problem counters alert from their first hit on, hence `warn='0'`
    # ("outside 0..0") rather than `warn=1`.
    # A level counter carries the threshold of the state it actually raises, so
    # the graph shows the line the check follows and a level left quiet by
    # `--critical-level` and `--warning-level` draws none.
    for item in LEVELS:
        level_state = level_states[item['level']]
        perfdata += lib.base.get_perfdata(
            item['perfdata'],
            counts[item['level']],
            uom=None,
            warn='0' if level_state == STATE_WARN else None,
            crit='0' if level_state == STATE_CRIT else None,
            _min=0,
        )
    for event in EVENTS:
        perfdata += lib.base.get_perfdata(
            event['perfdata'],
            len(found[event['key']]),
            uom=None,
            warn='0' if event['state'] == STATE_WARN else None,
            crit='0' if event['state'] == STATE_CRIT else None,
            _min=0,
        )
    # The rate counters trend what the state follows: how many arrived within the
    # window, not how many the log holds.
    for event in RATE_EVENTS:
        perfdata += lib.base.get_perfdata(
            event['perfdata'],
            rate_counts[event['key']],
            uom=None,
            warn=getattr(args, event['warning']) or None,
            crit=getattr(args, event['critical']) or None,
            _min=0,
        )
    for item in LIFECYCLE:
        perfdata += lib.base.get_perfdata(
            item['perfdata'], len(found[item['key']]), uom=None, _min=0
        )
    perfdata += lib.base.get_perfdata(
        'php_fpm_log_rotations', len(rotations), uom=None, _min=0
    )

    # Ask the monitoring server about the acknowledgement. Where the service is
    # acknowledged, persist the lines currently being reported so an unchanged
    # error log does not raise them again on the following runs.
    if args.ICINGA_CALLBACK and state != STATE_OK:
        acknowledged, note = lib.base.coe(
            lib.logmatch.service_acknowledged(
                args.ICINGA_URL,
                args.ICINGA_USERNAME,
                args.ICINGA_PASSWORD,
                args.ICINGA_SERVICE_NAME,
                insecure=args.INSECURE,
                no_proxy=args.NO_PROXY,
                proxy=args.PROXY,
                timeout=args.TIMEOUT,
            )
        )
        if acknowledged:
            # Both the lines a level flagged and the lines an event flagged,
            # and the same line can be both, so they are deduplicated on the way
            # in. The counted events stay out: what alerts there is the arrival
            # rate and not one line, so silencing the lines already read would
            # not keep the next run from alerting on the ones arriving meanwhile.
            reported = {}
            for item in LEVELS:
                reported.update(dict.fromkeys(levels[item['level']]))
            for event in EVENTS:
                reported.update(dict.fromkeys(found[event['key']]))
            lib.base.coe(
                lib.logmatch.acknowledge(
                    ack_conn,
                    [
                        {'key': lib.logmatch.key(line), 'line': line}
                        for line in reported
                    ],
                )
            )
            state = STATE_OK
        if note:
            msg += f'\n\n{note}'
    if suppressed_cnt:
        msg += (
            f'\n\n{suppressed_cnt} acknowledged'
            f' {lib.txt.pluralize("line", suppressed_cnt)} suppressed.'
        )
    if ack_conn is not None:
        lib.db_sqlite.close(ack_conn)

    # over and out
    lib.base.oao(
        msg, state, perfdata, always_ok=args.ALWAYS_OK, no_perfdata=args.NO_PERFDATA
    )


if __name__ == '__main__':
    try:
        main()
    except Exception:
        lib.base.cu()
