#!/usr/bin/env python3
# -*- coding: utf-8; py-indent-offset: 4 -*-
#
# Author:  Linuxfabrik GmbH, Zurich, Switzerland
# Contact: info (at) linuxfabrik (dot) ch
#          https://www.linuxfabrik.ch/
# License: The Unlicense, see LICENSE file.

# https://github.com/Linuxfabrik/monitoring-plugins/blob/main/CONTRIBUTING.md

"""See the check's README for more details."""

import argparse
import json
import sys

import lib.args
import lib.base
import lib.db_sqlite
import lib.disk
import lib.human
import lib.lftest
import lib.logmatch
import lib.shell
import lib.time
import lib.txt
from lib.globals import STATE_OK, STATE_UNKNOWN, STATE_WARN

__author__ = 'Linuxfabrik GmbH, Zurich/Switzerland'
__version__ = '2026083001'

DESCRIPTION = """Queries the systemd journal using journalctl and alerts when matching entries are found.
Supports all journalctl filtering options such as --unit, --priority, --facility,
--identifier, and --grep. Useful for monitoring specific log patterns in real time.
Optionally asks the monitoring server whether the service running this check is
acknowledged: where it is, the matching events are suppressed on following runs so
they don't re-alert.
Requires root or sudo."""

ACK_RETENTION_DAYS = 30

DEFAULT_FACILITY = None
DEFAULT_ICINGA_CALLBACK = False
DEFAULT_IDENTIFIER = None
DEFAULT_NO_MATCH_SEVERITY = 'ok'
DEFAULT_INSECURE = True
DEFAULT_NO_PROXY = False
DEFAULT_PRIORITY = 'emerg..err'
DEFAULT_SERVERITY = 'warn'
DEFAULT_SINCE = '-8h'
DEFAULT_TIMEOUT = 5
DEFAULT_UNIT = None
DEFAULT_USER_UNIT = None

# don't sort JOURNALD_PRIOS alphabetically, we need the indexes (0 = emerg etc.)
JOURNALD_PRIOS = [
    'emerg',
    'alert',
    'crit',
    'err',
    'warning',
    'notice',
    'info',
    'debug',
]


def parse_args():
    """Parse command line arguments using argparse."""
    parser = argparse.ArgumentParser(
        description=DESCRIPTION,
        epilog=lib.args.epilog(__file__),
        formatter_class=lib.args.HelpFormatter,
    )

    parser.add_argument(
        '-V',
        '--version',
        action='version',
        version=f'%(prog)s: v{__version__} by {__author__}',
    )

    parser.add_argument(
        '--always-ok',
        help=lib.args.help('--always-ok'),
        dest='ALWAYS_OK',
        action='store_true',
        default=False,
    )

    parser.add_argument(
        '--facility',
        help='Filter output by syslog facility (passed to journalctl). '
        'Takes a comma-separated list of numbers or facility names. '
        'Default: %(default)s',
        dest='FACILITY',
        default=DEFAULT_FACILITY,
    )

    parser.add_argument(
        '--icinga-callback',
        help=lib.args.help('--icinga-callback'),
        dest='ICINGA_CALLBACK',
        action='store_true',
        default=DEFAULT_ICINGA_CALLBACK,
    )

    parser.add_argument(
        '--icinga-password',
        help=lib.args.help('--icinga-password'),
        dest='ICINGA_PASSWORD',
    )

    parser.add_argument(
        '--icinga-service-name',
        help=lib.args.help('--icinga-service-name'),
        dest='ICINGA_SERVICE_NAME',
    )

    parser.add_argument(
        '--icinga-url',
        help=lib.args.help('--icinga-url'),
        dest='ICINGA_URL',
    )

    parser.add_argument(
        '--icinga-username',
        help=lib.args.help('--icinga-username'),
        dest='ICINGA_USERNAME',
    )

    parser.add_argument(
        '--identifier',
        help='Show messages for the specified syslog identifier (passed to journalctl). '
        'Default: %(default)s',
        dest='IDENTIFIER',
        default=DEFAULT_IDENTIFIER,
    )

    parser.add_argument(
        '--ignore',
        help='Ignore an event whose MESSAGE field matches this Python regular expression. '
        'Case-sensitive by default; use `(?i)` for case-insensitive matching. '
        'Can be specified multiple times. '
        "Example: `--ignore='(?i)linuxfabrik'`.",
        action='append',
        default=None,
        dest='IGNORE',
    )

    parser.add_argument(
        '--ignore-pattern',
        help=argparse.SUPPRESS,
        action='append',
        default=None,
        dest='IGNORE_PATTERN',
    )

    parser.add_argument(
        '--ignore-regex',
        help=argparse.SUPPRESS,
        action='append',
        default=None,
        dest='IGNORE_REGEX',
    )

    parser.add_argument(
        '--insecure',
        help='Applies to the connection to the monitoring server that `--icinga-callback` makes, which is the only network connection this check opens. '
        + lib.args.help('--insecure'),
        dest='INSECURE',
        action='store_true',
        default=DEFAULT_INSECURE,
    )

    parser.add_argument(
        '--match',
        help='Only report an event whose MESSAGE field matches this Python regular '
        'expression. '
        'Case-sensitive by default; use `(?i)` for case-insensitive matching. '
        'Can be specified multiple times. '
        + lib.args.MATCH_IGNORE_PRECEDENCE
        + " Example: `--match='(?i)out of memory'`.",
        action='append',
        default=None,
        dest='MATCH',
    )

    parser.add_argument(
        '--no-insecure',
        help='Applies to the connection to the monitoring server that `--icinga-callback` makes, which is the only network connection this check opens. '
        + lib.args.help('--no-insecure'),
        dest='INSECURE',
        action='store_false',
        default=DEFAULT_INSECURE,
    )

    parser.add_argument(
        '--no-match-severity',
        help=lib.args.help('--no-match-severity') + ' Default: %(default)s',
        dest='NO_MATCH_SEVERITY',
        choices=['ok', 'warn', 'crit', 'unknown'],
        default=DEFAULT_NO_MATCH_SEVERITY,
    )

    parser.add_argument(
        '--no-perfdata',
        help=lib.args.help('--no-perfdata'),
        dest='NO_PERFDATA',
        action='store_true',
        default=False,
    )

    parser.add_argument(
        '--no-proxy',
        help='Applies to the connection to the monitoring server that `--icinga-callback` makes, which is the only network connection this check opens. '
        + lib.args.help('--no-proxy'),
        dest='NO_PROXY',
        action='store_true',
        default=DEFAULT_NO_PROXY,
    )

    parser.add_argument(
        '--priority',
        help='Filter output by message priorities or priority ranges (passed to journalctl). '
        'Default: %(default)s',
        dest='PRIORITY',
        default=DEFAULT_PRIORITY,
    )

    parser.add_argument(
        '--proxy',
        help='Applies to the connection to the monitoring server that `--icinga-callback` makes, which is the only network connection this check opens. '
        + lib.args.help('--proxy'),
        dest='PROXY',
        default=None,
    )

    parser.add_argument(
        '--severity',
        help='Severity for alerts when journalctl returns results. '
        'Default: %(default)s',
        dest='SEVERITY',
        default=DEFAULT_SERVERITY,
        choices=['warn', 'crit'],
    )

    parser.add_argument(
        '--since',
        help='Show entries on or newer than the specified date (passed to journalctl). '
        'Default: %(default)s',
        dest='SINCE',
        default=DEFAULT_SINCE,
    )

    parser.add_argument(
        '--test',
        help=lib.args.help('--test'),
        dest='TEST',
        type=lib.args.csv,
    )

    parser.add_argument(
        '--timeout',
        help=lib.args.help('--timeout') + ' Default: %(default)s (seconds)',
        dest='TIMEOUT',
        type=int,
        default=DEFAULT_TIMEOUT,
    )

    parser.add_argument(
        '--unit',
        help='Show messages for the specified systemd unit UNIT|PATTERN (passed to journalctl). '
        'Can be specified multiple times. '
        'Default: %(default)s',
        dest='UNIT',
        default=DEFAULT_UNIT,
        action='append',
    )

    parser.add_argument(
        '--user-unit',
        help='Show messages for the specified user session unit (passed to journalctl). '
        'Can be specified multiple times. '
        'Default: %(default)s',
        dest='USER_UNIT',
        default=DEFAULT_USER_UNIT,
        action='append',
    )

    args, _ = parser.parse_known_args()
    return args


def migrate_ack_state(conn):
    """Carry acknowledgements written by an earlier version over to the current layout.

    Without this, the first run after an update finds no acknowledgement at all and raises
    every event an operator had already taken on, on every host at once.

    The migration works inside the state database that is already open, and never constructs,
    moves or opens a path of its own. Keep it that way: the previous attempt at carrying old
    state over moved a file from a predictable location in the shared temp directory and
    followed a symlink planted there (GHSA-w2gg-hx6w-24w3).
    """
    tables = lib.base.coe(lib.db_sqlite.get_tables(conn))
    if 'acknowledged_events' not in tables:
        return
    rows = lib.base.coe(
        lib.db_sqlite.select(
            conn,
            'SELECT event_hash FROM acknowledged_events',
            fetchone=False,
        )
    )
    if rows:
        # The message itself was never stored in the old layout, only its hash,
        # so the text stays empty here. It is not read back anywhere: the hash
        # is what an acknowledgement is matched on.
        lib.base.coe(
            lib.logmatch.acknowledge(
                conn,
                [{'key': row['event_hash'], 'line': ''} for row in rows],
            )
        )
    lib.base.coe(lib.db_sqlite.drop_table(conn, table='acknowledged_events'))


def main():
    """The main function. This is where the magic happens."""

    # parse the command line
    try:
        args = parse_args()
    except SystemExit:
        sys.exit(STATE_UNKNOWN)

    # Icinga Director escapes a leading dash as "\-" so a value like "-8h"
    # survives argparse on RHEL 8/9, where a value passed as a separate argv
    # token is otherwise read as an option (#789). Threshold parameters get
    # the backslash removed by the lib.base range parser; --since has no such
    # parser, so undo the escape here before the value reaches journalctl. A
    # journalctl time spec never legitimately starts with a backslash.
    args.SINCE = args.SINCE.lstrip('\\')

    # set default values for append parameters that were not specified
    if args.IGNORE is None:
        args.IGNORE = []
    if args.IGNORE_PATTERN is None:
        args.IGNORE_PATTERN = []
    if args.MATCH is None:
        args.MATCH = []
    # `--ignore-regex` is the former name of `--ignore` and means the same
    # thing, so its values simply join the list. `--ignore-pattern` stays a
    # plain substring filter of its own, because folding a substring into a
    # regex list would change what it matches as soon as it carries a
    # metacharacter.
    if args.IGNORE_REGEX:
        args.IGNORE += args.IGNORE_REGEX

    if args.ICINGA_CALLBACK and not all(
        (
            args.ICINGA_URL,
            args.ICINGA_PASSWORD,
            args.ICINGA_USERNAME,
            args.ICINGA_SERVICE_NAME,
        )
    ):
        lib.base.cu(
            '`--icinga-callback` requires `--icinga-url`, `--icinga-password`, `--icinga-username` and `--icinga-service-name`.'
        )

    # Persisted ack state is only needed when the Icinga callback is in use.
    # When it is, each unique combination of filter arguments gets its own
    # state DB so two Icinga services watching the journal with different
    # filters do not share ack state.
    acked_fingerprints = set()
    ack_conn = None
    if args.ICINGA_CALLBACK:
        # The payload names the filters actually in effect, and an unused filter
        # contributes nothing to it. That keeps the identifier of a service
        # configured before `--match` existed byte-identical, so its state
        # database, and with it every acknowledgement in there, survives the
        # update. `--ignore` sits in the slot of `--ignore-regex`, which it
        # replaces and whose semantics it shares.
        instance_payload = {
            'facility': args.FACILITY,
            'identifier': args.IDENTIFIER,
            'ignore_pattern': args.IGNORE_PATTERN,
            'ignore_regex': args.IGNORE,
            'priority': args.PRIORITY,
            'since': args.SINCE,
            'unit': args.UNIT or [],
            'user_unit': args.USER_UNIT or [],
        }
        if args.MATCH:
            instance_payload['match'] = args.MATCH
        instance = lib.logmatch.instance_id(instance_payload)
        ack_conn = lib.base.coe(lib.logmatch.connect('journald-query', instance))
        migrate_ack_state(ack_conn)
        # Drop ack records older than the retention: by that age the event has
        # rotated out of the journal and can no longer re-appear anyway.
        lib.base.coe(lib.logmatch.prune(ack_conn, retention=ACK_RETENTION_DAYS))
        acked_fingerprints = lib.base.coe(lib.logmatch.suppressed(ack_conn))

    # fetch data
    if args.TEST is None:
        cmd = [
            'journalctl',
            # '--boot',  # logs for the current boot will be shown
            '--reverse',
            '--quiet',
            '--output=json',
            f'--priority={args.PRIORITY}',
            f'--since={args.SINCE}',
        ]
        if args.FACILITY:
            cmd.append(f'--facility={args.FACILITY}')
        if args.IDENTIFIER:
            cmd.append(f'--identifier={args.IDENTIFIER}')
        # unfortunately only for newer journalctl commands:
        # cmd.append('--output-fields=UNIT,_SYSTEMD_UNIT,_SYSTEMD_SLICE,PRIORITY,MESSAGE')
        if args.UNIT is None and args.USER_UNIT is None:
            # Pre-define a standard set on basic system services we want to warn about,
            # found on fresh rhel 7+, ubuntu 16+ and debian 9+ systems altogether, if no unit
            # is provided. And yes, if called without any --unit parameter(s), we therefore ignore
            # errors on any specific application services like httpd etc. To check for application
            # services, call this check separately using --unit=httpd, for example.
            # Attention: '*' is the only wildcard that works.
            units = [
                '--unit=accounts-daemon.service',
                '--unit=acpid.service',
                '--unit=apparmor.service',
                '--unit=apport.service',
                '--unit=auditd.service',
                '--unit=cron.service',
                '--unit=crond.service',
                '--unit=dbus.service',
                '--unit=dracut-*.service',
                '--unit=haveged.service',
                '--unit=ifplugd.service',
                '--unit=ifup@*.service',
                '--unit=init.scope',
                '--unit=irqbalance.service',
                '--unit=iscsid.service',
                '--unit=lvm2-*.service',
                '--unit=lxcfs.service',
                '--unit=mdadm.service',
                '--unit=network.service',
                '--unit=NetworkManager*.service',
                '--unit=open-iscsi.service',
                '--unit=polkit.service',
                '--unit=polkitd.service',
                '--unit=qemu-guest-agent.service',
                '--unit=rsyslog.service',
                '--unit=session-*.scope',
                '--unit=snapd*.service',
                '--unit=ssh.service',
                '--unit=sshd*.service',
                '--unit=sssd.service',
                '--unit=sysstat.service',
                '--unit=systemd-*.service',
                '--unit=user@*.service',
            ]
            cmd += units
        if args.UNIT is not None:
            for unit in args.UNIT:
                cmd.append(f'--unit={unit}')
        if args.USER_UNIT is not None:
            for unit in args.USER_UNIT:
                cmd.append(f'--user-unit={unit}')
        stdout, stderr, retc = lib.base.coe(lib.shell.shell_exec(cmd))
        if retc != 0:
            lib.base.oao(
                f'`{" ".join(cmd)}` failed (exit {retc}): {stderr.strip()}',
                STATE_WARN,
                always_ok=args.ALWAYS_OK,
            )
    else:
        # do not call the command, put in test data
        cmd = ['no-real-command-used']
        stdout, stderr, _retc = lib.lftest.test(args.TEST)

    # init some vars
    cnt = 0
    shortened = False
    state = STATE_OK
    table_data = []

    # analyze data
    if stdout:
        # found something, so nothing good
        state = lib.base.str2state(args.SEVERITY)
        result = stdout.splitlines()

        compiled_ignore = [
            lib.base.coe(item)
            for item in lib.txt.compile_regex(args.IGNORE, '--ignore')
        ]
        compiled_match = [
            lib.base.coe(item) for item in lib.txt.compile_regex(args.MATCH, '--match')
        ]
        for item in result:
            try:
                event = json.loads(item)
            except Exception:
                lib.base.cu(f'Unable to interpret journald event: {item}')

            if event['MESSAGE'] is None:
                continue
            if any(
                ignore_pattern in event['MESSAGE']
                for ignore_pattern in args.IGNORE_PATTERN
            ) or any(item.search(event['MESSAGE']) for item in compiled_ignore):
                continue
            # `--match` (include) is applied first, then `--ignore` (exclude),
            # so an event hit by `--ignore` is dropped even if it also matches
            # `--match`.
            if compiled_match and not any(
                item.search(event['MESSAGE']) for item in compiled_match
            ):
                continue
            # Stable per-event fingerprint for ack persistence. Kept
            # deliberately simple (timestamp + message) so the same hash is
            # derived the next time journalctl returns the same entry.
            event_fingerprint = lib.logmatch.key(
                f'{event.get("__REALTIME_TIMESTAMP", "")}|{event.get("MESSAGE", "")}'
            )
            if event_fingerprint in acked_fingerprints:
                continue
            event['_fingerprint'] = event_fingerprint
            # shorten message if necessary
            if len(event['MESSAGE']) > 80:
                event['MESSAGE'] = event['MESSAGE'][0:77] + '...'

            try:
                event['unit'] = event['UNIT'].replace('.service', '')
            except Exception:
                try:
                    event['unit'] = event['_SYSTEMD_UNIT'].replace('.service', '')
                except Exception:
                    event['unit'] = event['_SYSTEMD_SLICE'].replace('.service', '')
            event['priority'] = JOURNALD_PRIOS[int(event['PRIORITY'])]
            event['timestamp'] = lib.time.epoch2iso(
                int(event['__REALTIME_TIMESTAMP']) / 1000000
            )

            table_data.append(event)

        cnt = len(table_data)
        if cnt == 0:
            # A journal that reported nothing is the normal quiet case and stays
            # OK. Only where the operator's own filters dropped everything the
            # journal did report does `--no-match-severity` get a say, because
            # then the check looked at nothing rather than at a quiet host.
            state = STATE_OK
            if result and (args.MATCH or args.IGNORE or args.IGNORE_PATTERN):
                state = lib.base.str2state(args.NO_MATCH_SEVERITY)
        if cnt > 10:
            # shorten the message
            table_data = table_data[0:5] + table_data[-5:]
            shortened = True
        else:
            shortened = False

    # Ask Icinga about the service acknowledgement. If acknowledged, persist
    # the fingerprints of the events that are currently being reported so
    # they do not re-alert on following runs, and return OK to Icinga. See
    # issue #649.
    msg_addendum = ''
    if args.ICINGA_CALLBACK and state != STATE_OK:
        acknowledged, msg_addendum = lib.base.coe(
            lib.logmatch.service_acknowledged(
                args.ICINGA_URL,
                args.ICINGA_USERNAME,
                args.ICINGA_PASSWORD,
                args.ICINGA_SERVICE_NAME,
                insecure=args.INSECURE,
                no_proxy=args.NO_PROXY,
                proxy=args.PROXY,
                timeout=args.TIMEOUT,
            )
        )
        if acknowledged:
            # Persist the events currently being reported, so they do not
            # re-alert on the following runs (issue #649).
            lib.base.coe(
                lib.logmatch.acknowledge(
                    ack_conn,
                    [
                        {'key': event['_fingerprint'], 'line': event['MESSAGE']}
                        for event in table_data
                    ],
                )
            )
            state = STATE_OK

    if ack_conn is not None:
        lib.db_sqlite.close(ack_conn)

    # build the message
    if table_data:
        sev_str = lib.base.state2str(
            lib.base.str2state(args.SEVERITY),
            prefix=' ',
        )
        # A journal entry may carry newlines, and one of them in the first line breaks
        # it in two, which is where a monitoring server cuts the summary from the long
        # output. Fold the whole entry onto one line instead.
        latest_message = ' '.join(str(table_data[0]['MESSAGE']).split())
        msg = (
            f'{cnt}'
            f' {lib.txt.pluralize("event", cnt)}.'
            f' Latest event at {table_data[0]["timestamp"]}'
            f' from {table_data[0]["unit"]},'
            f' level {table_data[0]["priority"]}:'
            f' `{latest_message}`{sev_str}'
        )
        if shortened:
            msg += (
                '\nAttention: Table below is truncated, showing the 5 newest and '
                'the 5 oldest messages.'
            )
        msg += '\n\n' + lib.base.get_table(
            table_data,
            [
                'timestamp',
                'unit',
                'priority',
                'MESSAGE',
            ],
            header=[
                'Timestamp',
                'Unit',
                'Prio',
                'Message',
            ],
        )
        if args.UNIT is None:
            msg += (
                f'\nUse `journalctl --reverse'
                f' --priority={args.PRIORITY}'
                f' --since={args.SINCE}`'
                f' as a starting point for debugging.'
                f' Be aware of the fact that you may'
                f' see even more messages then, as we'
                f' use a lot of unit filters to get'
                f' only messages from basic system'
                f' services.'
            )
    else:
        # Status line analog to the logfile check: name what was queried,
        # the hit count, the filters the query ran with and any ignores.
        def _join_and(items):
            if len(items) <= 1:
                return ''.join(items)
            if len(items) == 2:
                return f'{items[0]} and {items[1]}'
            return f'{", ".join(items[:-1])} and {items[-1]}'

        filter_parts = [
            f"priority='{args.PRIORITY}'",
            f"since='{args.SINCE}'",
        ]
        if args.UNIT:
            filter_parts.append(
                f'units {_join_and([f"{u!r}" for u in sorted(args.UNIT)])}'
            )
        if args.USER_UNIT:
            filter_parts.append(
                f'user-units {_join_and([f"{u!r}" for u in sorted(args.USER_UNIT)])}'
            )
        if args.FACILITY:
            filter_parts.append(f"facility='{args.FACILITY}'")
        if args.IDENTIFIER:
            filter_parts.append(f"identifier='{args.IDENTIFIER}'")

        ignore_parts = [
            f"'{item}'" for item in sorted(args.IGNORE_PATTERN + args.IGNORE)
        ]

        msg = f'Queried the systemd journal (0 events) using {_join_and(filter_parts)}'
        if ignore_parts:
            msg += f', ignoring {_join_and(ignore_parts)}'
        msg += '.'
        state = STATE_OK

    full_cmd = ' '.join(
        token for token in cmd if token not in ('--quiet', '--output=json')
    )
    msg += f'\nThe full command used was:\n`{full_cmd}`'
    perfdata = lib.base.get_perfdata(
        'journald-query',
        cnt,
        _min=0,
    )

    if msg_addendum:
        msg += '\n\n' + msg_addendum

    # over and out
    lib.base.oao(
        msg, state, perfdata, always_ok=args.ALWAYS_OK, no_perfdata=args.NO_PERFDATA
    )


if __name__ == '__main__':
    try:
        main()
    except Exception:
        lib.base.cu()
