#!/usr/bin/env python3
# -*- coding: utf-8; py-indent-offset: 4 -*-
#
# Author:  Linuxfabrik GmbH, Zurich, Switzerland
# Contact: info (at) linuxfabrik (dot) ch
#          https://www.linuxfabrik.ch/
# License: The Unlicense, see LICENSE file.

# https://github.com/Linuxfabrik/monitoring-plugins/blob/main/CONTRIBUTING.md

"""See the check's README for more details."""

import argparse
import os
import re
import sys
import uuid

import lib.args
import lib.base
import lib.db_sqlite
import lib.logmatch
import lib.logsource
import lib.time
import lib.txt
from lib.globals import STATE_CRIT, STATE_OK, STATE_UNKNOWN, STATE_WARN

__author__ = 'Linuxfabrik GmbH, Zurich/Switzerland'
__version__ = '2026092101'

DESCRIPTION = """Scans a logfile for matching patterns or regular expressions and alerts based on the
number of matches found. Only the lines added since the previous run are scanned, and the
whole file is rescanned whenever it was rotated, truncated or rewritten in place.
Optionally asks the monitoring server whether the service running this check is
acknowledged, and suppresses repeated alerts for known issues where it is.
Configurable alarm duration limits how long matches trigger alerts.

`--filename` accepts time macros, so logfiles whose name contains the current date
(`20260422.log`, `app-2026-04-22.log`, etc.) can be monitored directly. `{today}` /
`{yesterday}` resolve tolerantly: compact (`YYYYMMDD`) first, ISO 8601 (`YYYY-MM-DD`)
as fallback if the compact file does not exist. Read offset and pending matches carry
over when the filename changes on the next day, no wrapper script needed.

Prints the lines around each match on request, like `grep --context`, including lines
the logfile only receives after the run that found the match.

Requires root or sudo."""

# The plugin runs as root via sudo on many hosts, so it only opens files that
# resolve inside the system log directory. To monitor logs stored elsewhere,
# bind-mount that location under /var/log (see the README); a symlink is
# rejected because the confinement resolves symlinks.
ALLOWED_LOG_ROOTS = ['/var/log']

# Context lines are printed with every pending match on every run until it ages
# out or is acknowledged, so the number of them has to stay small.
MAX_CONTEXT = 10
CONTEXT_INDENT = '  '
CONTEXT_MATCH_MARKER = '>>> '

DEFAULT_ALARM_DURATION = 60  # minutes (1 hour)
DEFAULT_CONTEXT = 0
DEFAULT_CRIT = 1
DEFAULT_ICINGA_CALLBACK = False
DEFAULT_INSECURE = True
DEFAULT_NO_PROXY = False
DEFAULT_NO_MATCH_SEVERITY = 'ok'
DEFAULT_NO_SUMMARY = False
DEFAULT_TIMEOUT = 5
DEFAULT_WARN = 1


def add_context(lines, matches, waiting, remaining, before, after):
    """Attach the lines around each match to it, the way `grep --context` does.

    The check reads its logfile incrementally, so the lines around a match often
    belong to another run: the lines in front of it were read by the previous run,
    and the lines after it may not even be written yet. This is the same problem
    grep solves at the boundary between two read buffers, and it is solved the same
    way (see `grep()` and `prtext()` in grep's `src/grep.c`): the previous run keeps
    its last lines back as leading context, and a match at the end of a run waits
    for its trailing context on the following runs.

    As in grep, no line is printed twice. A matching line is never context, and a
    line between two matches is trailing context of the first one rather than
    leading context of the second.

    Parameters
    ----------
    lines : list of str
        The lines the previous run kept back, followed by the lines this run read.
    matches : list of dict
        The matches of this run, each with `index` pointing into `lines`. Gets
        `before` and `after` set.
    waiting : list of dict
        Matches of earlier runs that still wait for trailing context. Gets `after`
        extended.
    remaining : int
        How many lines of trailing context `waiting` still miss.
    before : int
        Lines of leading context.
    after : int
        Lines of trailing context.

    Returns
    -------
    tuple
          - tuple[0] (**list**): The lines to keep back for the next run.
          - tuple[1] (**list**): The matches that still wait for trailing context.
          - tuple[2] (**int**): How many lines of trailing context they still miss.
    """
    printed = 0  # index of the first line not printed yet
    for index in sorted({match['index'] for match in matches}):
        # the trailing context of the previous match ends where this match begins
        take = min(remaining, index - printed)
        for match in waiting:
            match['after'] += lines[printed : printed + take]
        printed += take
        group = [match for match in matches if match['index'] == index]
        for match in group:
            match['before'] = lines[max(printed, index - before) : index]
            match['after'] = []
        printed = index + 1
        waiting, remaining = group, after
    take = min(remaining, len(lines) - printed)
    for match in waiting:
        match['after'] += lines[printed : printed + take]
    printed += take
    remaining -= take
    if remaining:
        # every line after the match went into its trailing context, so there is
        # nothing left that could serve as leading context
        return [], waiting, remaining
    return lines[max(printed, len(lines) - before) :], [], 0


def render_match(match):
    """Render a match as it is printed and stored.

    Without context, that is the matching line. With context, it is the lines in
    the order the logfile holds them, the matching line marked among them. The
    first line carries the bullet of the list the match is printed in, the others
    are indented below it.
    """
    if not match['before'] and not match['after']:
        return match['line']
    lines = [
        *match['before'],
        f'{CONTEXT_MATCH_MARKER}{match["line"]}',
        *match['after'],
    ]
    return '\n'.join(lines[:1] + [f'{CONTEXT_INDENT}{line}' for line in lines[1:]])


def migrate_state(conn, filename):
    """Carry a state database written by an earlier version over to the current layout.

    Returns the read position recovered from it, or None where there was nothing to carry
    over. Without this, the first run after an update would start at offset 0 and report
    every match the logfile still holds, on every host at once.

    The migration works inside the state database that is already open, and never constructs,
    moves or opens a path of its own. Keep it that way: the previous attempt at carrying old
    state over moved a file from a predictable location in the shared temp directory and
    followed a symlink planted there (GHSA-w2gg-hx6w-24w3).
    """
    tables = lib.base.coe(lib.db_sqlite.get_tables(conn))
    position = None
    if 'file_stats' in tables:
        row = lib.base.coe(
            lib.db_sqlite.select(
                conn,
                'SELECT * FROM file_stats WHERE filename = :filename',
                {'filename': filename},
                fetchone=True,
            )
        )
        if row:
            position = {
                # A database from before the fingerprint was stored simply has
                # none, and a length of 0 is what tells the rewrite check that
                # there is nothing to compare against yet.
                'fingerprint': '',
                'inode': str(row.get('inode')),
                'kind': lib.logsource.KIND_FILE,
                'length': 0,
                'offset': row.get('offset', 0),
            }
    if position and 'file_fingerprints' in tables:
        row = lib.base.coe(
            lib.db_sqlite.select(
                conn,
                'SELECT * FROM file_fingerprints WHERE filename = :filename',
                {'filename': filename},
                fetchone=True,
            )
        )
        if row:
            position['fingerprint'] = row.get('fingerprint', '')
            position['length'] = row.get('length', 0)
    if 'matching_lines' in tables:
        rows = lib.base.coe(
            lib.db_sqlite.select(
                conn,
                'SELECT * FROM matching_lines WHERE filename = :filename',
                {'filename': filename},
                fetchone=False,
            )
        )
        if rows:
            # The alarm duration of a carried-over match restarts here, because
            # it is recorded as seen now. That costs one extra alarm duration
            # once, where dropping the matches would let a real pending problem
            # disappear silently.
            lib.base.coe(
                lib.logmatch.record(
                    conn,
                    [{'line': row['line'], 'state': row['state']} for row in rows],
                )
            )
    for table in ('file_fingerprints', 'file_stats', 'matching_lines'):
        if table in tables:
            lib.base.coe(lib.db_sqlite.drop_table(conn, table=table))
    return position


def parse_args():
    """Parse command line arguments using argparse."""
    parser = argparse.ArgumentParser(
        description=DESCRIPTION,
        epilog=lib.args.epilog(__file__),
        formatter_class=lib.args.HelpFormatter,
    )

    parser.add_argument(
        '-V',
        '--version',
        action='version',
        version=f'%(prog)s: v{__version__} by {__author__}',
    )

    # `--after-context` and `--before-context` default to None rather than to a
    # number, so that main() can tell an explicit 0 apart from "not given" and let
    # an explicit value take precedence over `--context`, as in grep.
    parser.add_argument(
        '--after-context',
        help='Print this many lines that follow a matching line along with it, '
        'like `grep --after-context`. '
        'Lines the logfile does not hold yet are added on the following runs. '
        'Applies to matches found from then on. '
        'Takes precedence over `--context`. '
        f'Takes 0 to {MAX_CONTEXT}. '
        'Default: the value of `--context`',
        dest='AFTER_CONTEXT',
        type=int,
        default=None,
    )

    parser.add_argument(
        '--alarm-duration',
        help='Duration in minutes for how long new matches trigger an alert. '
        'Overwritten by `--icinga-callback`. '
        'Default: %(default)s',
        dest='ALARM_DURATION',
        type=int,
        default=DEFAULT_ALARM_DURATION,
    )

    parser.add_argument(
        '--always-ok',
        help=lib.args.help('--always-ok'),
        dest='ALWAYS_OK',
        action='store_true',
        default=False,
    )

    parser.add_argument(
        '--before-context',
        help='Print this many lines that precede a matching line along with it, '
        'like `grep --before-context`. '
        'Lines read by the previous run count as well. '
        'Applies to matches found from then on. '
        'Takes precedence over `--context`. '
        f'Takes 0 to {MAX_CONTEXT}. '
        'Default: the value of `--context`',
        dest='BEFORE_CONTEXT',
        type=int,
        default=None,
    )

    parser.add_argument(
        '--context',
        help='Print this many lines before and after a matching line along with '
        'it, like `grep --context`. '
        'Sets `--before-context` and `--after-context` where they are not given. '
        'Applies to matches found from then on. '
        f'Takes 0 to {MAX_CONTEXT}. '
        'Default: %(default)s',
        dest='CONTEXT',
        type=int,
        default=DEFAULT_CONTEXT,
    )

    parser.add_argument(
        '-c',
        '--critical',
        help='CRIT threshold for the number of found critical matches. '
        'Default: %(default)s',
        dest='CRIT',
        default=DEFAULT_CRIT,
    )

    parser.add_argument(
        '--critical-pattern',
        help='Any line containing this pattern will count as a critical. '
        'Can be specified multiple times.',
        action='append',
        default=None,
        dest='CRIT_PATTERN',
    )

    parser.add_argument(
        '--critical-regex',
        help='Any line matching this Python regex will count as a critical. '
        'Can be specified multiple times.',
        action='append',
        dest='CRIT_REGEX',
        default=None,
    )

    parser.add_argument(
        '--filename',
        help='Path to the logfile. '
        'Supports time macros that are expanded on every run: '
        '`{today}` / `{yesterday}` first try the compact form `YYYYMMDD`, '
        'then fall back to `YYYY-MM-DD` if that file does not exist. '
        '`{%%Y}`, `{%%y}`, `{%%m}`, `{%%d}`, `{%%H}`, `{%%M}`, `{%%S}` '
        'render the matching strftime component of the current time. '
        'Example: `/var/log/app/{today}.log`. '
        'Example: `/var/log/app/app-{today}.log`. '
        'Example: `/var/log/app/{%%Y}{%%m}{%%d}.log`.',
        dest='FILENAME',
        required=True,
        type=str,
    )

    parser.add_argument(
        '--icinga-callback',
        help=lib.args.help('--icinga-callback'),
        dest='ICINGA_CALLBACK',
        action='store_true',
        default=DEFAULT_ICINGA_CALLBACK,
    )

    parser.add_argument(
        '--icinga-password',
        help=lib.args.help('--icinga-password'),
        dest='ICINGA_PASSWORD',
    )

    parser.add_argument(
        '--icinga-service-name',
        help=lib.args.help('--icinga-service-name'),
        dest='ICINGA_SERVICE_NAME',
    )

    parser.add_argument(
        '--icinga-url',
        help=lib.args.help('--icinga-url'),
        dest='ICINGA_URL',
    )

    parser.add_argument(
        '--icinga-username',
        help=lib.args.help('--icinga-username'),
        dest='ICINGA_USERNAME',
    )

    parser.add_argument(
        '--ignore',
        help='Ignore a line matching this Python regular expression, whichever '
        'warning or critical pattern it also matches. '
        'Case-sensitive by default; use `(?i)` for case-insensitive matching. '
        'Can be specified multiple times. '
        "Example: `--ignore='(?i)linuxfabrik'`.",
        action='append',
        default=None,
        dest='IGNORE',
    )

    parser.add_argument(
        '--ignore-pattern',
        help=argparse.SUPPRESS,
        action='append',
        default=None,
        dest='IGNORE_PATTERN',
    )

    parser.add_argument(
        '--ignore-regex',
        help=argparse.SUPPRESS,
        action='append',
        default=None,
        dest='IGNORE_REGEX',
    )

    parser.add_argument(
        '--insecure',
        help='Applies to the connection to the monitoring server that `--icinga-callback` makes, which is the only network connection this check opens. '
        + lib.args.help('--insecure'),
        dest='INSECURE',
        action='store_true',
        default=DEFAULT_INSECURE,
    )

    parser.add_argument(
        '--match',
        help='Only consider a line matching this Python regular expression. '
        'Applied before the warning and critical patterns decide the severity, '
        'so it narrows what is looked at rather than what counts as a problem. '
        'Case-sensitive by default; use `(?i)` for case-insensitive matching. '
        'Can be specified multiple times. '
        + lib.args.MATCH_IGNORE_PRECEDENCE
        + " Example: `--match='^\\[prod\\]'`.",
        action='append',
        default=None,
        dest='MATCH',
    )

    parser.add_argument(
        '--no-insecure',
        help='Applies to the connection to the monitoring server that `--icinga-callback` makes, which is the only network connection this check opens. '
        + lib.args.help('--no-insecure'),
        dest='INSECURE',
        action='store_false',
        default=DEFAULT_INSECURE,
    )

    parser.add_argument(
        '--no-match-severity',
        help=lib.args.help('--no-match-severity') + ' Default: %(default)s',
        dest='NO_MATCH_SEVERITY',
        choices=['ok', 'warn', 'crit', 'unknown'],
        default=DEFAULT_NO_MATCH_SEVERITY,
    )

    parser.add_argument(
        '--no-perfdata',
        help=lib.args.help('--no-perfdata'),
        dest='NO_PERFDATA',
        action='store_true',
        default=False,
    )

    parser.add_argument(
        '--no-proxy',
        help='Applies to the connection to the monitoring server that `--icinga-callback` makes, which is the only network connection this check opens. '
        + lib.args.help('--no-proxy'),
        dest='NO_PROXY',
        action='store_true',
        default=DEFAULT_NO_PROXY,
    )

    parser.add_argument(
        '--proxy',
        help='Applies to the connection to the monitoring server that `--icinga-callback` makes, which is the only network connection this check opens. '
        + lib.args.help('--proxy'),
        dest='PROXY',
        default=None,
    )

    parser.add_argument(
        '--suppress-lines',
        help='Suppress the found lines in the output and only report the number of findings.',
        dest='SUPPRESS_OUTPUT',
        action='store_true',
        default=False,
    )

    parser.add_argument(
        '--timeout',
        help=lib.args.help('--timeout') + ' Default: %(default)s (seconds)',
        dest='TIMEOUT',
        type=int,
        default=DEFAULT_TIMEOUT,
    )

    parser.add_argument(
        '-w',
        '--warning',
        help='WARN threshold for the number of found warning matches. '
        'Default: %(default)s',
        dest='WARN',
        default=DEFAULT_WARN,
    )

    parser.add_argument(
        '--warning-pattern',
        help='Any line containing this pattern will count as a warning. '
        'Can be specified multiple times.',
        action='append',
        default=None,
        dest='WARN_PATTERN',
    )

    parser.add_argument(
        '--warning-regex',
        help='Any line matching this Python regex will count as a warning. '
        'Can be specified multiple times.',
        action='append',
        dest='WARN_REGEX',
        default=None,
    )

    args, _ = parser.parse_known_args()
    return args


def main():
    """The main function. This is where the magic happens."""

    # parse the command line
    try:
        args = parse_args()
    except SystemExit:
        sys.exit(STATE_UNKNOWN)

    # set default values for append parameters that were not specified
    if args.CRIT_PATTERN is None:
        args.CRIT_PATTERN = []
    if args.CRIT_REGEX is None:
        args.CRIT_REGEX = []
    if args.IGNORE is None:
        args.IGNORE = []
    if args.IGNORE_PATTERN is None:
        args.IGNORE_PATTERN = []
    if args.MATCH is None:
        args.MATCH = []
    # `--ignore-regex` is the former name of `--ignore` and means the same
    # thing, so its values simply join the list. `--ignore-pattern` stays a
    # plain substring filter of its own, because folding a substring into a
    # regex list would change what it matches as soon as it carries a
    # metacharacter.
    if args.IGNORE_REGEX:
        args.IGNORE += args.IGNORE_REGEX
    if args.WARN_PATTERN is None:
        args.WARN_PATTERN = []
    if args.WARN_REGEX is None:
        args.WARN_REGEX = []

    if not any(
        (args.WARN_PATTERN, args.WARN_REGEX, args.CRIT_PATTERN, args.CRIT_REGEX)
    ):
        lib.base.cu('At least one pattern or regex is required.')

    # Checked each on its own, like grep does, so an out-of-range `--context` is
    # reported even where both explicit values override it.
    for name, value in (
        ('--after-context', args.AFTER_CONTEXT),
        ('--before-context', args.BEFORE_CONTEXT),
        ('--context', args.CONTEXT),
    ):
        if value is not None and not 0 <= value <= MAX_CONTEXT:
            lib.base.cu(f'`{name}` takes a number of lines from 0 to {MAX_CONTEXT}.')
    if args.AFTER_CONTEXT is None:
        args.AFTER_CONTEXT = args.CONTEXT
    if args.BEFORE_CONTEXT is None:
        args.BEFORE_CONTEXT = args.CONTEXT

    if args.ICINGA_CALLBACK and not all(
        (
            args.ICINGA_URL,
            args.ICINGA_PASSWORD,
            args.ICINGA_USERNAME,
            args.ICINGA_SERVICE_NAME,
        )
    ):
        lib.base.cu(
            '`--icinga-callback` requires `--icinga-url`, `--icinga-password`, '
            '`--icinga-username` and `--icinga-service-name`.'
        )

    # fetch data
    # Expand time macros for filesystem access only. args.FILENAME stays the
    # state key so offset and pending matches survive the daily filename change
    # (issue #678). For {today} / {yesterday}, try the compact form
    # (`20260422`) first because that is what most rotated logfiles use, and
    # fall back to ISO 8601 (`2026-04-22`) if no compact file exists.
    scan_path = lib.time.macro2timestr(args.FILENAME, format='%Y%m%d')
    if not os.path.exists(scan_path):
        iso_path = lib.time.macro2timestr(args.FILENAME, format='%Y-%m-%d')
        if iso_path != scan_path and os.path.exists(iso_path):
            scan_path = iso_path

    # This check reads a file, and only a file. The library also resolves
    # `systemd:` and `docker:` style sources, but neither the time macros above
    # nor the path confinement below mean anything for those, so accepting one
    # here would look like support that is not there.
    kind, _, _ = lib.base.coe(lib.logsource.parse(scan_path))
    if kind != lib.logsource.KIND_FILE:
        lib.base.cu('`--filename` takes the path to a logfile.')

    # Give each unique combination of patterns and regexes its own state
    # database, so two services watching the same logfile for different things
    # no longer trample each other's read position or match history (issue
    # #698). The basename keeps the file recognizable among the databases.
    # The payload names the filters actually in effect, and an unused filter
    # contributes nothing to it. That keeps the identifier of a service
    # configured before `--match` existed byte-identical, so its state database,
    # and with it the read position and every pending match, survives the
    # update. `--ignore` sits in the slot of `--ignore-regex`, which it replaces
    # and whose semantics it shares.
    instance_payload = {
        'crit_pattern': args.CRIT_PATTERN,
        'crit_regex': args.CRIT_REGEX,
        'ignore_pattern': args.IGNORE_PATTERN,
        'ignore_regex': args.IGNORE,
        'warn_pattern': args.WARN_PATTERN,
        'warn_regex': args.WARN_REGEX,
    }
    if args.MATCH:
        instance_payload['match'] = args.MATCH
    # The context parameters stay out of the payload on purpose: they change what
    # is printed along with a match, not which lines match, so changing them must
    # not start a new database that reports every match of the logfile again.
    instance = lib.logmatch.instance_id(instance_payload)
    basename = os.path.basename(args.FILENAME)
    conn = lib.base.coe(lib.logmatch.connect('logfile', f'{basename}-{instance}'))

    position = lib.base.coe(lib.logmatch.get_position(conn, args.FILENAME))
    if position is None:
        position = migrate_state(conn, args.FILENAME)

    # The plugin runs as root via sudo, so it only opens files that resolve
    # inside the system log directory. To monitor logs stored elsewhere,
    # bind-mount that location under /var/log (see the README); a symlink is
    # rejected because the confinement resolves symlinks. The plugin's own
    # unit-test/ directory is allowed too so the fixture-based tests work; on a
    # deployed host that directory does not exist next to the flat, root-owned
    # plugin, so it grants nothing (and needs no environment flag that a
    # misconfigured sudoers could leak).
    allowed_roots = [
        *ALLOWED_LOG_ROOTS,
        os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), 'unit-test'),
    ]
    success, result = lib.logsource.read(
        scan_path,
        position=position,
        allowed_roots=allowed_roots,
    )
    if not success:
        # no traceback wanted: a logfile that was rotated away between two runs
        # is an everyday event, not a defect in the plugin
        lib.db_sqlite.close(conn)
        lib.base.oao(result, STATE_UNKNOWN)

    # init some vars
    compiled_warn_regex = [re.compile(item) for item in args.WARN_REGEX]
    compiled_crit_regex = [re.compile(item) for item in args.CRIT_REGEX]
    compiled_ignore = [
        lib.base.coe(item) for item in lib.txt.compile_regex(args.IGNORE, '--ignore')
    ]
    compiled_match = [
        lib.base.coe(item) for item in lib.txt.compile_regex(args.MATCH, '--match')
    ]

    # per-pattern counters, for the verbose "matched N lines" summary.
    # These are purely for reporting; the scan semantics (which lines are
    # classified as warn/crit) remain unchanged.
    warn_pattern_hits = dict.fromkeys(args.WARN_PATTERN, 0)
    warn_regex_hits = dict.fromkeys(args.WARN_REGEX, 0)
    crit_pattern_hits = dict.fromkeys(args.CRIT_PATTERN, 0)
    crit_regex_hits = dict.fromkeys(args.CRIT_REGEX, 0)

    warn_matches = []
    crit_matches = []
    line_counter = len(result['lines'])

    # What the previous run left for the context: the lines it kept back as
    # leading context, and its matches that still wait for trailing context. It
    # travels with the read position, because both describe the same point in the
    # logfile: once the logfile was rotated, truncated or rewritten, neither means
    # anything any more. A change of the context parameters cuts it to size.
    previous = (position or {}).get('context') or {}
    if result['restarted']:
        previous = {}
    carry = previous.get('carry', [])
    carry = carry[-args.BEFORE_CONTEXT :] if args.BEFORE_CONTEXT else []
    remaining = min(previous.get('remaining', 0), args.AFTER_CONTEXT)
    waiting = previous.get('waiting', []) if remaining else []
    waiting_lengths = [len(match['after']) for match in waiting]
    lines = carry + result['lines']

    # analyze data
    matched_any = False
    # the lines kept back from the previous run were already scanned by it
    for index in range(len(carry), len(lines)):
        line = lines[index]
        # `--match` (include) is applied first, then `--ignore` (exclude), so a
        # line hit by `--ignore` is dropped even if it also matches `--match`.
        if compiled_match and not any(item.search(line) for item in compiled_match):
            continue
        matched_any = True
        is_ignored = any(
            ignore_pattern in line for ignore_pattern in args.IGNORE_PATTERN
        ) or any(item.search(line) for item in compiled_ignore)

        # due to lazy evaluation, the regex will only be executed
        # if the pattern does not match
        # see https://docs.python.org/3/reference/expressions.html#boolean-operations
        if any(warn_pattern in line for warn_pattern in args.WARN_PATTERN) or any(
            item.search(line) for item in compiled_warn_regex
        ):
            if not is_ignored:
                warn_matches.append(
                    {
                        'index': index,
                        'key': uuid.uuid4().hex,
                        'line': line.strip(),
                        'state': STATE_WARN,
                    }
                )
                for pat in args.WARN_PATTERN:
                    if pat in line:
                        warn_pattern_hits[pat] += 1
                for idx, rgx in enumerate(compiled_warn_regex):
                    if rgx.search(line):
                        warn_regex_hits[args.WARN_REGEX[idx]] += 1

        if any(crit_pattern in line for crit_pattern in args.CRIT_PATTERN) or any(
            item.search(line) for item in compiled_crit_regex
        ):
            if not is_ignored:
                crit_matches.append(
                    {
                        'index': index,
                        'key': uuid.uuid4().hex,
                        'line': line.strip(),
                        'state': STATE_CRIT,
                    }
                )
                for pat in args.CRIT_PATTERN:
                    if pat in line:
                        crit_pattern_hits[pat] += 1
                for idx, rgx in enumerate(compiled_crit_regex):
                    if rgx.search(line):
                        crit_regex_hits[args.CRIT_REGEX[idx]] += 1

    previous_waiting = waiting
    carry, waiting, remaining = add_context(
        lines,
        warn_matches + crit_matches,
        waiting,
        remaining,
        args.BEFORE_CONTEXT,
        args.AFTER_CONTEXT,
    )
    for match in warn_matches + crit_matches:
        match['text'] = render_match(match)
    # A match of an earlier run that received trailing context from this run is
    # stored again under its own key, which keeps when it was first seen and
    # whether it was acknowledged.
    lib.base.coe(
        lib.logmatch.record(
            conn,
            [
                {
                    'key': match['key'],
                    'line': render_match(match),
                    'state': match['state'],
                }
                for match, length in zip(previous_waiting, waiting_lengths)
                if len(match['after']) > length
            ],
        )
    )
    position = result['position']
    if carry or waiting:
        position['context'] = {
            'carry': carry,
            'remaining': remaining,
            'waiting': [
                {key: match[key] for key in ('after', 'before', 'key', 'line', 'state')}
                for match in waiting
            ],
        }
    # Stored only now, after the patterns were compiled and the lines scanned: a
    # run that ends in UNKNOWN on the way leaves the lines for the next run.
    lib.base.coe(lib.logmatch.set_position(conn, args.FILENAME, position))

    # `--match` narrowed the run down to nothing, so this run looked at no line
    # at all rather than at a quiet logfile. Only reachable when the operator
    # set a filter, so a logfile that simply has nothing to report stays OK.
    if compiled_match and line_counter and not matched_any:
        lib.db_sqlite.close(conn)
        lib.base.oao(
            f'Nothing checked: `--match` dropped all {line_counter} new '
            f'{lib.txt.pluralize("line", line_counter)} of {scan_path}.',
            lib.base.str2state(args.NO_MATCH_SEVERITY),
            always_ok=args.ALWAYS_OK,
        )

    # Read what earlier runs found before recording this run's matches, so the
    # two can be told apart in the output. Both drive the state: with no new
    # lines we still alarm the old ones, until they age out or the service is
    # acknowledged.
    max_age = None if args.ICINGA_CALLBACK else args.ALARM_DURATION
    old_matches = lib.base.coe(lib.logmatch.pending(conn, max_age=max_age))
    old_warn_matches = [item for item in old_matches if item['state'] == STATE_WARN]
    old_crit_matches = [item for item in old_matches if item['state'] == STATE_CRIT]

    state = lib.base.get_worst(
        lib.base.get_state(len(warn_matches) + len(old_warn_matches), args.WARN, None),
        lib.base.get_state(len(crit_matches) + len(old_crit_matches), None, args.CRIT),
    )

    msg_addendum = ''
    if args.ICINGA_CALLBACK and state != STATE_OK:
        acknowledged, msg_addendum = lib.base.coe(
            lib.logmatch.service_acknowledged(
                args.ICINGA_URL,
                args.ICINGA_USERNAME,
                args.ICINGA_PASSWORD,
                args.ICINGA_SERVICE_NAME,
                insecure=args.INSECURE,
                no_proxy=args.NO_PROXY,
                proxy=args.PROXY,
                timeout=args.TIMEOUT,
            )
        )
        if acknowledged:
            state = STATE_OK
            lib.base.coe(lib.logmatch.acknowledge(conn, old_matches))
            old_warn_matches = []
            old_crit_matches = []

    # Record this run's matches, so they keep alarming on the following runs
    # until they age out or the service is acknowledged. Every occurrence gets
    # its own key, because the file is read incrementally: a line that turns up
    # again after an acknowledgement is a new event and has to alarm again.
    lib.base.coe(
        lib.logmatch.record(
            conn,
            [
                {'key': match['key'], 'line': match['text'], 'state': match['state']}
                for match in warn_matches + crit_matches
            ],
        )
    )
    lib.base.coe(lib.logmatch.prune(conn))
    lib.db_sqlite.close(conn)

    # build the message: name the scanned file, every configured pattern /
    # regex with its per-pattern match count, the severity label if the
    # pattern produced hits, and the ignore patterns. See issue #547.
    def _fmt_pattern(pattern, count, severity):
        label = f' [{severity}]' if count > 0 else ''
        return (
            f"'{pattern}' (matched {count} {lib.txt.pluralize('line', count)}){label}"
        )

    def _join_and(items):
        if len(items) <= 1:
            return ''.join(items)
        if len(items) == 2:
            return f'{items[0]} and {items[1]}'
        return f'{", ".join(items[:-1])} and {items[-1]}'

    pattern_parts = []
    for pat in sorted(args.WARN_PATTERN):
        pattern_parts.append(_fmt_pattern(pat, warn_pattern_hits[pat], 'WARNING'))
    for rgx in sorted(args.WARN_REGEX):
        pattern_parts.append(_fmt_pattern(rgx, warn_regex_hits[rgx], 'WARNING'))
    for pat in sorted(args.CRIT_PATTERN):
        pattern_parts.append(_fmt_pattern(pat, crit_pattern_hits[pat], 'CRITICAL'))
    for rgx in sorted(args.CRIT_REGEX):
        pattern_parts.append(_fmt_pattern(rgx, crit_regex_hits[rgx], 'CRITICAL'))

    ignore_parts = [f"'{item}'" for item in sorted(args.IGNORE_PATTERN + args.IGNORE)]

    msg = (
        f'Scanned {scan_path} ({line_counter} '
        f'{lib.txt.pluralize("line", line_counter)}) '
        f'using patterns {_join_and(pattern_parts)}'
    )
    if ignore_parts:
        msg += f', ignoring {_join_and(ignore_parts)}'
    msg += '.'

    if old_warn_matches:
        msg += (
            f' {len(old_warn_matches)} unacknowledged warning'
            f' {lib.txt.pluralize("match", len(old_warn_matches), "es")}'
            f' from previous runs.'
        )

    if old_crit_matches:
        msg += (
            f' {len(old_crit_matches)} unacknowledged critical'
            f' {lib.txt.pluralize("match", len(old_crit_matches), "es")}'
            f' from previous runs.'
        )

    if not args.SUPPRESS_OUTPUT:
        if warn_matches:
            msg += '\n\nWarning matches:\n* ' + '\n* '.join(
                match['text'] for match in warn_matches
            )

        if crit_matches:
            msg += '\n\nCritical matches:\n* ' + '\n* '.join(
                match['text'] for match in crit_matches
            )

        if old_warn_matches:
            msg += (
                '\n\nUnacknowledged warning matches from previous runs:\n* '
                + '\n* '.join(match['line'] for match in old_warn_matches)
            )

        if old_crit_matches:
            msg += (
                '\n\nUnacknowledged critical matches from previous runs:\n* '
                + '\n* '.join(match['line'] for match in old_crit_matches)
            )

    perfdata = lib.base.get_perfdata(
        'scanned_lines',
        line_counter,
    )
    perfdata += lib.base.get_perfdata(
        'warn_matches',
        len(warn_matches),
        warn=args.WARN,
    )
    perfdata += lib.base.get_perfdata(
        'crit_matches',
        len(crit_matches),
        warn=args.WARN,
    )

    # over and out
    lib.base.oao(
        msg + '\n\n' + msg_addendum,
        state,
        perfdata,
        always_ok=args.ALWAYS_OK,
        no_perfdata=args.NO_PERFDATA,
    )


if __name__ == '__main__':
    try:
        main()
    except Exception:
        lib.base.cu()
