#!/usr/bin/env python3
# -*- coding: utf-8; py-indent-offset: 4 -*-
#
# Author:  Linuxfabrik GmbH, Zurich, Switzerland
# Contact: info (at) linuxfabrik (dot) ch
#          https://www.linuxfabrik.ch/
# License: The Unlicense, see LICENSE file.

# https://github.com/Linuxfabrik/monitoring-plugins/blob/main/CONTRIBUTING.md

"""See the check's README for more details."""

import argparse
import hashlib
import os
import re
import sys
from collections import defaultdict

import lib.args
import lib.base
import lib.db_sqlite
import lib.disk
import lib.human
import lib.time
import lib.txt
from lib.globals import STATE_OK, STATE_UNKNOWN

try:
    import psutil
except ImportError:
    print('Python module "psutil" is not installed.')
    sys.exit(STATE_UNKNOWN)


__author__ = 'Linuxfabrik GmbH, Zurich/Switzerland'
__version__ = '2026082601'

DESCRIPTION = """Monitors running processes and alerts on process count, aggregated memory usage, or
aggregated CPU usage. Processes can be filtered by name, command-line arguments, and
user name using regular expressions. Optionally lists the top processes by CPU time
and memory usage.

On Linux, also reports how many processes the whole system creates per second. A
headcount stays flat while a retry loop, a runaway cron job or a fork bomb creates and
reaps thousands of processes a second, and this is the number that shows it. It covers
the whole system and is not narrowed down by the filters above.

Supports extended reporting via --lengthy."""

# The file the Linux kernel counts every process it has created in. No other system
# keeps it, and neither does a container started without a `/proc` of its own.
PROC_STAT = '/proc/stat'

DEFAULT_CRIT = None
DEFAULT_CRIT_AGE = None
DEFAULT_CRIT_CPU_PERCENT = None
DEFAULT_CRIT_MEM = None
DEFAULT_CRIT_MEM_PERCENT = None
DEFAULT_LENGTHY = False
DEFAULT_NO_KTHREADS = False
DEFAULT_STATUS = None
DEFAULT_TOP = 5
DEFAULT_WARN = None
DEFAULT_WARN_AGE = None
DEFAULT_WARN_CPU_PERCENT = None
DEFAULT_WARN_MEM = None
DEFAULT_WARN_MEM_PERCENT = None


def parse_args():
    """Parse command line arguments using argparse."""
    parser = argparse.ArgumentParser(
        description=DESCRIPTION,
        epilog=lib.args.epilog(__file__),
        formatter_class=lib.args.HelpFormatter,
    )

    parser.add_argument(
        '-V',
        '--version',
        action='version',
        version=f'%(prog)s: v{__version__} by {__author__}',
    )

    parser.add_argument(
        '--always-ok',
        help=lib.args.help('--always-ok'),
        dest='ALWAYS_OK',
        action='store_true',
        default=False,
    )

    parser.add_argument(
        '--argument',
        help='Filter by command line arguments using a regular expression (case-insensitive). '
        'Example: `--argument="--config.*production"`',
        dest='ARGUMENT',
    )

    parser.add_argument(
        '--command',
        help='Filter by process name using a regular expression (case-insensitive). '
        'Example: `--command="^(apache|httpd)"`',
        dest='COMMAND',
    )

    parser.add_argument(
        '-c',
        '--critical',
        help='CRIT threshold for the number of matching processes. '
        ''
        'Default: %(default)s',
        default=DEFAULT_CRIT,
        dest='CRIT',
    )

    parser.add_argument(
        '--critical-age',
        help='CRIT threshold for age of the oldest matching process, in seconds. '
        ''
        'Default: %(default)s',
        default=DEFAULT_CRIT_AGE,
        dest='CRIT_AGE',
    )

    parser.add_argument(
        '--critical-cpu-percent',
        help='CRIT threshold for aggregated CPU usage of all matching processes, in percent. '
        'Requires two consecutive check runs to calculate. '
        '100%% equals one fully utilized CPU core. '
        ''
        'Default: %(default)s',
        default=DEFAULT_CRIT_CPU_PERCENT,
        dest='CRIT_CPU_PERCENT',
    )

    parser.add_argument(
        '--critical-mem',
        help='CRIT threshold for aggregated memory usage, in bytes. '
        ''
        'Default: %(default)s',
        default=DEFAULT_CRIT_MEM,
        dest='CRIT_MEM',
    )

    parser.add_argument(
        '--critical-mem-percent',
        help='CRIT threshold for aggregated memory usage, in percent. '
        ''
        'Default: %(default)s',
        default=DEFAULT_CRIT_MEM_PERCENT,
        dest='CRIT_MEM_PERCENT',
    )

    parser.add_argument(
        '--lengthy',
        help=lib.args.help('--lengthy'),
        dest='LENGTHY',
        action='store_true',
        default=DEFAULT_LENGTHY,
    )

    parser.add_argument(
        '--no-kthreads',
        help='Exclude kernel threads from the scan (Linux only).',
        dest='NO_KTHREADS',
        default=DEFAULT_NO_KTHREADS,
        action='store_true',
    )

    parser.add_argument(
        '--no-perfdata',
        help=lib.args.help('--no-perfdata'),
        dest='NO_PERFDATA',
        action='store_true',
        default=False,
    )

    parser.add_argument(
        '--status',
        help='Filter by process status. Default: %(default)s',
        dest='STATUS',
        default=DEFAULT_STATUS,
        choices=[
            'dead',
            'disk-sleep',
            'idle',  # Linux, macOS, FreeBSD
            'locked',  # FreeBSD
            'parked',  # Linux
            'running',
            'sleeping',
            'stopped',
            'suspended',  # NetBSD
            'tracing-stop',
            'waiting',  # FreeBSD
            'wake-kill',
            'waking',
            'zombie',
        ],
    )

    parser.add_argument(
        '--top',
        help='Number of top processes by CPU time to display. '
        'Processes with zero CPU time are excluded. '
        'Use `--top=0` to disable. '
        'Default: %(default)s',
        dest='TOP',
        type=int,
        default=DEFAULT_TOP,
    )

    parser.add_argument(
        '--username',
        help='Filter by user name using a regular expression (case-insensitive). '
        'Example: `--username="^(apache|www-data)$"`',
        dest='USERNAME',
    )

    parser.add_argument(
        '-w',
        '--warning',
        help='WARN threshold for the number of matching processes. '
        ''
        'Default: %(default)s',
        dest='WARN',
        default=DEFAULT_WARN,
    )

    parser.add_argument(
        '--warning-age',
        help='WARN threshold for age of the oldest matching process, in seconds. '
        ''
        'Default: %(default)s',
        default=DEFAULT_WARN_AGE,
        dest='WARN_AGE',
    )

    parser.add_argument(
        '--warning-cpu-percent',
        help='WARN threshold for aggregated CPU usage of all matching processes, in percent. '
        'Requires two consecutive check runs to calculate. '
        '100%% equals one fully utilized CPU core. '
        ''
        'Default: %(default)s',
        default=DEFAULT_WARN_CPU_PERCENT,
        dest='WARN_CPU_PERCENT',
    )

    parser.add_argument(
        '--warning-mem',
        help='WARN threshold for aggregated memory usage, in bytes. '
        ''
        'Default: %(default)s',
        default=DEFAULT_WARN_MEM,
        dest='WARN_MEM',
    )

    parser.add_argument(
        '--warning-mem-percent',
        help='WARN threshold for aggregated memory usage, in percent. '
        ''
        'Default: %(default)s',
        default=DEFAULT_WARN_MEM_PERCENT,
        dest='WARN_MEM_PERCENT',
    )

    args, _ = parser.parse_known_args()
    return args


def get_forks_since_boot():
    """Return how many processes the kernel has created since boot, or None where
    that number is not available.

    The counter sits in the `processes` line of `/proc/stat` and covers every
    `fork()` and `clone()` the whole system has carried out. It is cumulative, so it
    is only meaningful as a delta against a previous reading.

    None is a state and not an error: no system except Linux keeps this file, and
    neither does a container without a `/proc` of its own. A malformed line yields
    None as well, because a rate against a counter this function could not read would
    be worse than no rate at all.

    Verified against `fs/proc/stat.c` of Linux 7.1.
    """
    # Files below /proc report a size of zero, so their presence has to be probed
    # allowing an empty file.
    if not lib.disk.file_exists(PROC_STAT, allow_empty=True):
        return None
    success, content = lib.disk.read_file(PROC_STAT)
    if not success:
        return None
    for line in content.splitlines():
        fields = line.split()
        # Read by line name, never by position. The kernel has added lines to this
        # file before, and a reader going by position picks up the wrong one as soon
        # as it adds the next. Only the first value of the line is read, so a line
        # the kernel one day appends to keeps working, the way `intr` and `softirq`
        # already print their total ahead of one number per interrupt.
        if len(fields) >= 2 and fields[0] == 'processes':
            try:
                return int(fields[1])
            except ValueError:
                return None
    return None


def main():
    """The main function. This is where the magic happens."""

    # parse the command line
    try:
        args = parse_args()
    except SystemExit:
        sys.exit(STATE_UNKNOWN)

    # init some vars
    msg = ''
    state = STATE_OK
    perfdata = ''

    # init more vars
    kthreadd_pid = 0
    my_pid = os.getpid()
    procs_count, procs_mem, procs_mem_percent, procs_age = 0, 0, 0, 0
    procs_cpu_percent = 0
    proc_states = defaultdict(lambda: 0)
    uninterruptibles, running, zombies = {}, {}, {}

    need_age = bool(args.WARN_AGE or args.CRIT_AGE)
    need_argument = bool(args.ARGUMENT)
    need_command = bool(args.COMMAND)
    need_cpu = bool(args.TOP or args.WARN_CPU_PERCENT or args.CRIT_CPU_PERCENT)
    need_cpu_percent = bool(args.WARN_CPU_PERCENT or args.CRIT_CPU_PERCENT)
    need_ppid = bool(args.NO_KTHREADS)
    need_status_filter = bool(args.STATUS)
    need_username = bool(args.USERNAME)
    show_top_table = bool(args.TOP)

    # platform-specific memory_info fields for --top table
    # (attr_name, header_label, format: 'bytes' or 'number')
    # mem_fields: all fields for aggregation and --lengthy display
    # mem_fields_short: key fields for compact display (without --lengthy)
    if lib.base.WINDOWS:
        # rss=wset, vms=pagefile on Windows, so skip wset and pagefile to avoid duplicates;
        # psutil 8.0 renamed peak_wset→peak_rss, peak_pagefile→peak_vms and moved
        # pool fields to memory_info_ex, so include both old and new names
        mem_fields = [
            ('rss', 'RSS', 'bytes'),
            ('vms', 'VMS', 'bytes'),
            ('num_page_faults', 'PageFaults', 'number'),
            ('peak_rss', 'Peak RSS', 'bytes'),
            ('peak_wset', 'Peak RSS', 'bytes'),
            ('peak_paged_pool', 'Peak Paged', 'bytes'),
            ('paged_pool', 'Paged', 'bytes'),
            ('peak_nonpaged_pool', 'Peak NonPaged', 'bytes'),
            ('nonpaged_pool', 'NonPaged', 'bytes'),
            ('peak_vms', 'Peak VMS', 'bytes'),
            ('peak_pagefile', 'Peak VMS', 'bytes'),
            ('private', 'Private', 'bytes'),
        ]
        mem_fields_short = [
            ('rss', 'RSS', 'bytes'),
            ('private', 'Private', 'bytes'),
            ('peak_rss', 'Peak RSS', 'bytes'),
            ('peak_wset', 'Peak RSS', 'bytes'),
        ]
    elif sys.platform.startswith('linux'):
        mem_fields = [
            ('rss', 'RSS', 'bytes'),
            ('vms', 'VMS', 'bytes'),
            ('shared', 'Shared', 'bytes'),
            ('text', 'Text', 'bytes'),
            ('lib', 'Lib', 'bytes'),
            ('data', 'Data', 'bytes'),
            ('dirty', 'Dirty', 'bytes'),
        ]
        mem_fields_short = [
            ('rss', 'RSS', 'bytes'),
        ]
    else:
        mem_fields = [
            ('rss', 'RSS', 'bytes'),
            ('vms', 'VMS', 'bytes'),
            ('shared', 'Shared', 'bytes'),
            ('text', 'Text', 'bytes'),
            ('data', 'Data', 'bytes'),
            ('stack', 'Stack', 'bytes'),
            ('peak_rss', 'Peak RSS', 'bytes'),
        ]
        mem_fields_short = [
            ('rss', 'RSS', 'bytes'),
        ]

    # filter to only include fields actually available in the installed psutil version
    try:
        available_mem_fields = set(psutil.Process(my_pid).memory_info()._fields)
    except (psutil.NoSuchProcess, psutil.AccessDenied):
        available_mem_fields = {'rss', 'vms'}
    mem_fields = [f for f in mem_fields if f[0] in available_mem_fields]
    mem_fields_short = [f for f in mem_fields_short if f[0] in available_mem_fields]

    # We always print a per-status / per-name breakdown later,
    # so we will collect 'status' and 'name' (even if not used for filters).
    attrs = ['name', 'status', 'memory_info']
    if need_cpu:
        attrs.append('cpu_times')
    if need_age:
        attrs.append('create_time')
    if need_argument:
        attrs.append('cmdline')
    if need_command:
        attrs.append('cmdline')
    if need_ppid:
        attrs.append('ppid')
    if need_username:
        attrs.append('username')

    # Precompile regex filters once
    try:
        re_username = (
            re.compile(args.USERNAME, re.IGNORECASE) if need_username else None
        )
        re_command = re.compile(args.COMMAND, re.IGNORECASE) if need_command else None
        re_argument = (
            re.compile(args.ARGUMENT, re.IGNORECASE) if need_argument else None
        )
    except re.error as e:
        lib.base.cu(f'Invalid regular expression: {e}')

    # per-process-name aggregation for --top table and CPU% calculation
    top_procs = {}

    total_ram = psutil.virtual_memory().total or 0

    # fetch data
    # Using ad_value avoids slow exception paths for
    # AccessDenied / racing processes.
    # https://psutil.readthedocs.io/en/latest/#psutil.process_iter
    for process in psutil.process_iter(attrs=attrs, ad_value=''):
        info = process.info

        # always ignore myself
        if process.pid == my_pid:
            continue

        # Filter: username regex
        if need_username:
            uname = info.get('username') or ''
            if not re_username.search(uname):
                continue

        # Filter: command name regex
        if need_command:
            pname = info.get('name') or ''
            if not re_command.search(pname):
                continue

        # Filter: command line argument regex
        if need_argument:
            cmdline = info.get('cmdline') or []
            cmdline_str = ' '.join(part or '' for part in cmdline)
            if not re_argument.search(cmdline_str):
                continue

        # Filter kernel threads (childs of KTHREAD_PARENT)
        if need_ppid:
            pname = info.get('name') or ''
            if pname == 'kthreadd':
                kthreadd_pid = process.pid
            else:
                if kthreadd_pid and info.get('ppid') == kthreadd_pid:
                    continue

        # Search only for processes in a specific state (if set)
        if need_status_filter:
            if info.get('status') != args.STATUS:
                continue

        # Start counting
        procs_count += 1

        rss = getattr(info.get('memory_info'), 'rss', 0) or 0
        procs_mem += rss

        # compute the max age (seconds);
        # on Windows, the "System Idle Process" always returns 0 - do not calculate the age
        # of such processes, otherwise we get `max(0, Seconds since 1970)`, which is 50Y+.
        if need_age:
            ctime = info.get('create_time') or 0
            if ctime:
                procs_age = max(procs_age, lib.time.now() - ctime)

        # count processes per status ("83x sleeping, 5x running")
        # plus group and summarize some states
        # handle far most status' first
        try:
            st = info.get('status')
            pname = info.get('name') or ''
            if st == psutil.STATUS_SLEEPING:
                proc_states[psutil.STATUS_SLEEPING] += 1
            elif st == psutil.STATUS_RUNNING:
                proc_states[psutil.STATUS_RUNNING] += 1
                running[pname] = running.get(pname, 0) + 1
            elif st == psutil.STATUS_DISK_SLEEP:  # == uninterruptible
                proc_states[psutil.STATUS_DISK_SLEEP] += 1
                uninterruptibles[pname] = uninterruptibles.get(pname, 0) + 1
            elif st == psutil.STATUS_ZOMBIE:
                proc_states[psutil.STATUS_ZOMBIE] += 1
                zname = pname.replace(' ', '')
                zombies[zname] = zombies.get(zname, 0) + 1
            elif st == psutil.STATUS_DEAD:
                proc_states[psutil.STATUS_DEAD] += 1
            elif st == psutil.STATUS_IDLE:
                proc_states[psutil.STATUS_SLEEPING] += 1
            elif st == psutil.STATUS_LOCKED:
                proc_states[psutil.STATUS_LOCKED] += 1
            elif st == psutil.STATUS_PARKED:
                proc_states[psutil.STATUS_PARKED] += 1
            elif st == psutil.STATUS_STOPPED or st == psutil.STATUS_TRACING_STOP:
                proc_states[psutil.STATUS_STOPPED] += 1
            elif st == psutil.STATUS_WAITING:
                proc_states[psutil.STATUS_WAITING] += 1
            elif st == psutil.STATUS_WAKING:
                proc_states[psutil.STATUS_WAKING] += 1
            else:
                # psutil.STATUS_SUSPENDED, psutil.STATUS_WAKE_KILL
                proc_states['other'] += 1

            # aggregate cpu_times, memory and statuses per process name for --top table / CPU%
            if need_cpu:
                pname = info.get('name') or ''
                if not (lib.base.WINDOWS and pname == 'System Idle Process'):
                    if pname not in top_procs:
                        init = {
                            'user': 0.0,
                            'system': 0.0,
                            'statuses': defaultdict(int),
                        }
                        for attr, _, _ in mem_fields:
                            init[attr] = 0
                        top_procs[pname] = init
                    cput = info.get('cpu_times')
                    if cput:
                        top_procs[pname]['user'] += cput.user or 0
                        top_procs[pname]['system'] += getattr(cput, 'system', 0) or 0
                    meminfo = info.get('memory_info')
                    if meminfo:
                        for attr, _, _ in mem_fields:
                            top_procs[pname][attr] += getattr(meminfo, attr, 0) or 0
                    top_procs[pname]['statuses'][st] += 1

        except Exception:
            # specific psutil version does not have such an attribute from above
            continue

    # analyze data

    # compute memory percent once (mathematically equals sum of per-proc percentages)
    # Rounded here rather than where it is printed, so the message, the graph and
    # the value the threshold is compared against are all the same number. Perfdata
    # carrying thirteen decimals is neither readable nor of any use to a graph.
    procs_mem_percent = round(procs_mem / total_ram * 100.0, 1) if total_ram else 0

    # compute CPU% via SQLite delta if thresholds are set
    if need_cpu_percent:
        total_cpu_time = sum(d['user'] + d['system'] for d in top_procs.values())
        now = lib.time.now()

        # use a hash of the filter parameters to separate data for different
        # service definitions sharing the same DB file
        filter_key = (
            f'{args.COMMAND or ""}'
            f':{args.ARGUMENT or ""}'
            f':{args.USERNAME or ""}'
            f':{args.STATUS or ""}'
        )
        filter_hash = hashlib.sha256(filter_key.encode()).hexdigest()

        conn = lib.base.coe(
            lib.db_sqlite.connect(
                filename='linuxfabrik-monitoring-plugins-procs.db',
            )
        )
        definition = """
            filter_hash TEXT PRIMARY KEY,
            cpu_time REAL NOT NULL,
            ts REAL NOT NULL
        """
        lib.base.coe(lib.db_sqlite.create_table(conn, definition, table='cpu_snapshot'))

        old = lib.base.coe(
            lib.db_sqlite.select(
                conn,
                'SELECT cpu_time, ts FROM cpu_snapshot WHERE filter_hash = :filter_hash',
                {'filter_hash': filter_hash},
                fetchone=True,
            )
        )

        if old:
            delta_time = now - old['ts']
            delta_cpu = total_cpu_time - old['cpu_time']
            if delta_time > 0 and delta_cpu >= 0:
                procs_cpu_percent = (delta_cpu / delta_time) * 100.0

        # store current snapshot
        lib.base.coe(
            lib.db_sqlite.delete(
                conn,
                'DELETE FROM cpu_snapshot WHERE filter_hash = :filter_hash',
                {'filter_hash': filter_hash},
            )
        )
        lib.base.coe(
            lib.db_sqlite.insert(
                conn,
                {'filter_hash': filter_hash, 'cpu_time': total_cpu_time, 'ts': now},
                table='cpu_snapshot',
            )
        )
        lib.base.coe(lib.db_sqlite.commit(conn))
        lib.db_sqlite.close(conn)

    # How many processes the system created per second, which is the one thing about
    # process activity a headcount cannot show: a runaway cron job, a retry loop or a
    # fork bomb keeps the number of processes roughly constant while creating and
    # reaping thousands of them a second. The kernel counts them for the whole system,
    # so the filters above do not narrow it down, and no system except Linux keeps that
    # counter at all. Read and turned into a rate here, after the connection above is
    # closed: the helper opens its own, and an uncommitted write would lock it out and
    # cost the rate without saying so.
    forks_per_second = None
    forks = get_forks_since_boot()
    if forks is not None:
        rates = lib.db_sqlite.per_second_deltas(
            'linuxfabrik-monitoring-plugins-procs.db',
            'forks',
            {'processes': forks},
        )
        if rates is not None:
            forks_per_second = round(rates['processes'], 1)

    # list used filters
    msg_filter = ''
    if need_username:
        msg_filter += f'user "{args.USERNAME}", '
    if need_command:
        msg_filter += f'cmd "{args.COMMAND}", '
    if need_argument:
        msg_filter += f'args "{args.ARGUMENT}", '
    if need_ppid:
        msg_filter += 'w/o kthreads, '
    if need_status_filter:
        msg_filter += f'status "{args.STATUS}", '
    if msg_filter:
        msg_filter = f' (filtered by {msg_filter[:-2]})'

    # system uptime for context (helps interpret cumulative CPU times)
    uptime = lib.time.now() - psutil.boot_time()

    # build the message

    # number of processes
    local_state = lib.base.get_state(procs_count, args.WARN, args.CRIT, 'range')
    state = lib.base.get_worst(state, local_state)
    msg = (
        f'{procs_count} {lib.txt.pluralize("proc", procs_count)}'
        f'{msg_filter}{lib.base.state2str(local_state, prefix=" ")}'
    )

    # memory usage (bytes)
    local_state = lib.base.get_state(procs_mem, args.WARN_MEM, args.CRIT_MEM, 'range')
    state = lib.base.get_worst(state, local_state)
    msg += (
        f' using {lib.human.bytes2human(procs_mem)} RAM'
        f'{lib.base.state2str(local_state, prefix=" ")}, '
    )

    # memory usage (percent)
    local_state = lib.base.get_state(
        procs_mem_percent,
        args.WARN_MEM_PERCENT,
        args.CRIT_MEM_PERCENT,
        'range',
    )
    state = lib.base.get_worst(state, local_state)
    msg = (
        msg[:-2] + f' ({procs_mem_percent}%)'
        f'{lib.base.state2str(local_state, prefix=" ")}, '
    )

    # CPU usage (percent)
    if need_cpu_percent:
        local_state = lib.base.get_state(
            procs_cpu_percent,
            args.WARN_CPU_PERCENT,
            args.CRIT_CPU_PERCENT,
            'range',
        )
        state = lib.base.get_worst(state, local_state)
        msg += (
            f'{round(procs_cpu_percent, 1)}% CPU'
            f'{lib.base.state2str(local_state, prefix=" ")}, '
        )

    # age (seconds)
    local_state = lib.base.get_state(procs_age, args.WARN_AGE, args.CRIT_AGE, 'range')
    state = lib.base.get_worst(state, local_state)
    if need_age and procs_count > 1:
        msg += 'oldest proc '
        msg += (
            f'created {lib.human.seconds2human(procs_age)} ago'
            f'{lib.base.state2str(local_state, prefix=" ")}, '
        )

    if proc_states[psutil.STATUS_DEAD]:
        msg += f'{proc_states[psutil.STATUS_DEAD]} dead, '

    if proc_states[psutil.STATUS_DISK_SLEEP]:
        msg += (
            f'{proc_states[psutil.STATUS_DISK_SLEEP]} '
            f'{lib.txt.pluralize("uninterruptible", proc_states[psutil.STATUS_DISK_SLEEP])} ('
        )
        for name, cnt in uninterruptibles.items():
            msg += f'{cnt}x {name}, '
        msg = msg[:-2] + '), '

    if proc_states[psutil.STATUS_RUNNING]:
        msg += f'{proc_states[psutil.STATUS_RUNNING]} running ('
        for name, cnt in running.items():
            msg += f'{cnt}x {name}, '
        msg = msg[:-2] + '), '

    if proc_states[psutil.STATUS_SLEEPING]:
        msg += f'{proc_states[psutil.STATUS_SLEEPING]} sleeping, '

    if proc_states[psutil.STATUS_STOPPED]:
        msg += f'{proc_states[psutil.STATUS_STOPPED]} stopped, '

    if proc_states[psutil.STATUS_ZOMBIE]:
        msg += (
            f'{proc_states[psutil.STATUS_ZOMBIE]} '
            f'{lib.txt.pluralize("zombie", proc_states[psutil.STATUS_ZOMBIE])} ('
        )
        for name, cnt in zombies.items():
            msg += f'{cnt}x {name}, '
        msg = msg[:-2] + '), '

    # strip trailing ", " from the summary line, append the fork rate and the uptime
    msg = msg.rstrip(', ')
    if forks_per_second is not None:
        msg += f', {lib.human.number2human(forks_per_second)} forks/s system-wide'
    msg += f', up {lib.human.seconds2human(uptime)}'

    # build --top table
    if show_top_table and top_procs:
        # filter out processes with zero CPU time,
        # sort by total cpu time descending, take top N
        table_data = []
        for pname, data in top_procs.items():
            statuses = data['statuses']
            total = data['user'] + data['system']
            if total == 0:
                continue
            status_parts = []
            for s, cnt in sorted(statuses.items()):
                status_parts.append(f'{cnt}x {s}')
            row = {
                'name': pname,
                'total': lib.human.seconds2human(total),
                'status': ', '.join(status_parts),
                '_total_raw': total,
            }
            if args.LENGTHY:
                row['user'] = lib.human.seconds2human(data['user'])
                row['system'] = lib.human.seconds2human(data['system'])
                display_fields = mem_fields
            else:
                display_fields = mem_fields_short
            for attr, _, fmt in display_fields:
                if fmt == 'bytes':
                    row[attr] = lib.human.bytes2human(data[attr])
                else:
                    row[attr] = data[attr]
            table_data.append(row)
        table_data.sort(key=lambda x: x['_total_raw'], reverse=True)
        table_data = table_data[: args.TOP]
        for row in table_data:
            del row['_total_raw']
        if table_data:
            if args.LENGTHY:
                keys = ['name', 'user', 'system', 'total']
                headers = ['Name', 'CPU User', 'CPU System', 'CPU Total']
                display_fields = mem_fields
            else:
                keys = ['name', 'total']
                headers = ['Name', 'CPU Total']
                display_fields = mem_fields_short
            for attr, header, _ in display_fields:
                keys.append(attr)
                headers.append(header)
            keys.append('status')
            headers.append('Status')
            msg += '\n\n'
            msg += lib.base.get_table(table_data, keys, header=headers)

    # build perfdata
    perfdata += lib.base.get_perfdata(
        'procs',
        procs_count,
        uom=None,
        warn=args.WARN,
        crit=args.CRIT,
        _min=0,
        _max=None,
    )
    perfdata += lib.base.get_perfdata(
        'procs_age',
        procs_age,
        uom='s',
        warn=args.WARN_AGE,
        crit=args.CRIT_AGE,
        _min=0,
        _max=None,
    )
    if need_cpu_percent:
        perfdata += lib.base.get_perfdata(
            'procs_cpu_percent',
            procs_cpu_percent,
            uom='%',
            warn=args.WARN_CPU_PERCENT,
            crit=args.CRIT_CPU_PERCENT,
            _min=0,
            _max=None,
        )
    perfdata += lib.base.get_perfdata(
        'procs_dead',
        proc_states[psutil.STATUS_DEAD],
        uom=None,
        warn=None,
        crit=None,
        _min=0,
        _max=None,
    )
    if forks_per_second is not None:
        perfdata += lib.base.get_perfdata(
            'procs_forks_per_second',
            forks_per_second,
            uom=None,
            warn=None,
            crit=None,
            _min=0,
            _max=None,
        )
    perfdata += lib.base.get_perfdata(
        'procs_mem',
        procs_mem,
        uom='B',
        warn=args.WARN_MEM,
        crit=args.CRIT_MEM,
        _min=0,
        _max=None,
    )
    perfdata += lib.base.get_perfdata(
        'procs_mem_percent',
        procs_mem_percent,
        uom='%',
        warn=args.WARN_MEM_PERCENT,
        crit=args.CRIT_MEM_PERCENT,
        _min=0,
        _max=None,
    )
    perfdata += lib.base.get_perfdata(
        'procs_running',
        proc_states[psutil.STATUS_RUNNING],
        uom=None,
        warn=None,
        crit=None,
        _min=0,
        _max=None,
    )
    perfdata += lib.base.get_perfdata(
        'procs_sleeping',
        proc_states[psutil.STATUS_SLEEPING],
        uom=None,
        warn=None,
        crit=None,
        _min=0,
        _max=None,
    )
    perfdata += lib.base.get_perfdata(
        'procs_stopped',
        proc_states[psutil.STATUS_STOPPED],
        uom=None,
        warn=None,
        crit=None,
        _min=0,
        _max=None,
    )
    perfdata += lib.base.get_perfdata(
        'procs_uninterruptible',
        proc_states[psutil.STATUS_DISK_SLEEP],
        uom=None,
        warn=None,
        crit=None,
        _min=0,
        _max=None,
    )
    perfdata += lib.base.get_perfdata(
        'procs_zombies',
        proc_states[psutil.STATUS_ZOMBIE],
        uom=None,
        warn=None,
        crit=None,
        _min=0,
        _max=None,
    )

    # over and out
    lib.base.oao(
        msg, state, perfdata, always_ok=args.ALWAYS_OK, no_perfdata=args.NO_PERFDATA
    )


if __name__ == '__main__':
    try:
        main()
    except Exception:
        lib.base.cu()
