snapshot of https://github.com/zulip/zulip.git @ 48c69911c7dd4e03b20552eb3bd77ad8151c1357 (2026-09-15)

puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_cron_file

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+"""
+Nagios plugin paired with a cron job.  This just verifies that the
+file output by the cron job is correct.
+"""
+
+import sys
+import time
+
+
+def nagios_from_file(results_file: str, max_time_diff: int = 60 * 2) -> tuple[int, str]:
+    """Returns a nagios-appropriate string and return code obtained by
+    parsing the desired file on disk. The file on disk should be of format
+
+    %s|%s % (timestamp, nagios_string)
+
+    This file is created by various nagios checking cron jobs such as
+    check-rabbitmq-queues and check-rabbitmq-consumers"""
+
+    try:
+        with open(results_file) as f:
+            data = f.read().strip()
+    except FileNotFoundError:
+        state = "UNKNOWN"
+        ret = 3
+        data = "Results file is missing"
+    else:
+        pieces = data.split("|")
+
+        if len(pieces) != 4:
+            state = "UNKNOWN"
+            ret = 3
+            data = "Results file malformed"
+        else:
+            timestamp = int(pieces[0])
+
+            time_diff = int(time.time() - timestamp)
+            if time_diff > max_time_diff:
+                ret = 3
+                state = "UNKNOWN"
+                data = "Results file is stale"
+            else:
+                ret = int(pieces[1])
+                state = pieces[2]
+                data = pieces[3]
+
+    return (ret, f"{state}: {data}")
+
+
+if __name__ == "__main__":
+    RESULTS_FILE = sys.argv[1]
+    extra_args = {}
+    if len(sys.argv) > 2:
+        extra_args["max_time_diff"] = int(sys.argv[2])
+    ret, result = nagios_from_file(RESULTS_FILE, **extra_args)
+
+    print(result)
+    sys.exit(ret)

puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_queue_worker_errors

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+
+"""
+Nagios plugin to check that none of our queue workers have reported errors.
+"""
+
+import glob
+import os
+import sys
+
+# settings.QUEUE_ERROR_DIR; not importing Django so that this can run
+# as the nagios user.
+wildcard = os.path.join("/var/log/zulip/queue_error", "*.errors")
+clean = True
+for fn in glob.glob(wildcard):
+    print(f"WARNING: Queue errors logged in {fn}")
+    clean = False
+
+if not clean:
+    sys.exit(1)
+
+sys.exit(0)

puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_send_receive_time

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+
+"""
+Script to provide information about send-receive times.
+
+It must be run on a machine that is using the live database for the
+Django ORM.
+"""
+
+import argparse
+import os
+import random
+import sys
+import time
+import traceback
+from typing import Any, Literal, NoReturn
+
+sys.path.append(".")
+sys.path.append("/home/zulip/deployments/current")
+from scripts.lib.setup_path import setup_path
+from scripts.lib.zulip_tools import atomic_nagios_write
+
+setup_path()
+
+import django
+import zulip
+
+sys.path.append("/home/zulip/deployments/current")
+os.environ["DJANGO_SETTINGS_MODULE"] = "zproject.settings"
+
+django.setup()
+
+from django.conf import settings
+
+from zerver.models.realms import get_realm
+from zerver.models.users import get_system_bot
+from zproject.config import get_config
+
+usage = """Usage: send-receive.py [options] [config]
+
+       'config' is optional, if present will return config info.
+        Otherwise, returns the output data."""
+
+parser = argparse.ArgumentParser(usage=usage)
+parser.add_argument("--site", default=f"https://{settings.NAGIOS_BOT_HOST}")
+parser.add_argument("--insecure", action="store_true")
+
+options = parser.parse_args()
+
+
+def report(
+    state: Literal["ok", "warning", "critical", "unknown"],
+    timestamp: float | None = None,
+    msg: str | None = None,
+) -> NoReturn:
+    if msg is None:
+        msg = f"send time was {timestamp}"
+    sys.exit(atomic_nagios_write("check_send_receive_state", state, msg))
+
+
+def send_zulip(sender: zulip.Client, message: dict[str, Any]) -> None:
+    result = sender.send_message(message)
+    if result["result"] != "success":
+        report("critical", msg=f"Error sending Zulip, args were: {message}, {result}")
+
+
+def get_zulips() -> list[dict[str, Any]]:
+    global last_event_id
+    res = zulip_recipient.get_events(queue_id=queue_id, last_event_id=last_event_id)
+    if "error" in res.get("result", {}):
+        report("critical", msg="Error receiving Zulips, error was: {}".format(res["msg"]))
+    for event in res["events"]:
+        last_event_id = max(last_event_id, int(event["id"]))
+    # If we get a heartbeat event, that means we've been hanging for
+    # 40s, and we should bail.
+    if "heartbeat" in (event["type"] for event in res["events"]):
+        report("critical", msg="Got heartbeat waiting for Zulip, which means get_events is hanging")
+    return [event["message"] for event in res["events"]]
+
+
+internal_realm_id = get_realm(settings.SYSTEM_BOT_REALM).id
+if (
+    get_config("machine", "deploy_type") == "staging"
+    and settings.NAGIOS_STAGING_SEND_BOT is not None
+    and settings.NAGIOS_STAGING_RECEIVE_BOT is not None
+):
+    sender = get_system_bot(settings.NAGIOS_STAGING_SEND_BOT, internal_realm_id)
+    recipient = get_system_bot(settings.NAGIOS_STAGING_RECEIVE_BOT, internal_realm_id)
+else:
+    sender = get_system_bot(settings.NAGIOS_SEND_BOT, internal_realm_id)
+    recipient = get_system_bot(settings.NAGIOS_RECEIVE_BOT, internal_realm_id)
+
+zulip_sender = zulip.Client(
+    email=sender.email,
+    api_key=sender.api_key,
+    verbose=True,
+    insecure=options.insecure,
+    client="ZulipMonitoring/0.1",
+    site=options.site,
+)
+
+zulip_recipient = zulip.Client(
+    email=recipient.email,
+    api_key=recipient.api_key,
+    verbose=True,
+    insecure=options.insecure,
+    client="ZulipMonitoring/0.1",
+    site=options.site,
+)
+
+try:
+    res = zulip_recipient.register(event_types=["message"])
+    if "error" in res.get("result", {}):
+        report("critical", msg="Error subscribing to Zulips: {}".format(res["msg"]))
+    queue_id, last_event_id = (res["queue_id"], res["last_event_id"])
+except Exception:
+    report("critical", msg=f"Error subscribing to Zulips:\n{traceback.format_exc()}")
+msg_to_send = str(random.getrandbits(64))
+time_start = time.perf_counter()
+
+send_zulip(
+    zulip_sender,
+    {
+        "type": "private",
+        "content": msg_to_send,
+        "subject": "time to send",
+        "to": recipient.email,
+    },
+)
+
+complete = False
+while not complete:
+    messages = get_zulips()
+    seconds_diff = time.perf_counter() - time_start
+    for m in messages:
+        if msg_to_send == m["content"]:
+            zulip_sender.delete_message(m["id"])
+            complete = True
+            break
+
+zulip_recipient.deregister(queue_id)
+
+if seconds_diff > 12:
+    report("critical", timestamp=seconds_diff)
+if seconds_diff > 3:
+    report("warning", timestamp=seconds_diff)
+else:
+    report("ok", timestamp=seconds_diff)

puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_worker_memory

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# Checks for any Zulip queue workers that are leaking memory and thus have a high vsize
+datafile=$(mktemp)
+
+# We expect other Nagios checks to monitor for whether no queue
+# workers are running, so we give that condition a pass
+processes=$(pgrep -xf 'python.* /home/zulip/deployments/current/manage.py process_queue .*')
+if [ -z "$processes" ]; then
+    echo "No workers running"
+    exit 0
+fi
+mapfile -t processes <<<"$processes"
+ps -o vsize,size,pid,user,command --sort -vsize "${processes[@]}" >"$datafile"
+cat "$datafile"
+top_worker=$(head -n2 "$datafile" | tail -n1)
+top_worker_memory_usage=$(echo "$top_worker" | cut -f1 -d" ")
+rm -f "$datafile"
+if [ "$top_worker_memory_usage" -gt 800000 ]; then
+    exit 2
+elif [ "$top_worker_memory_usage" -gt 600000 ]; then
+    exit 1
+else
+    exit 0
+fi

puppet/zulip/files/nagios_plugins/zulip_base/check_debian_packages

@@ -0,0 +1,146 @@
+#!/usr/bin/perl -w
+#
+# check_debian_packages - nagios plugin
+#
+#
+# Copyright (C) 2005 Francesc Guasch
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+# Report bugs to: frankie@etsetb.upc.edu
+#
+use strict;
+use lib '/usr/lib/nagios/plugins';
+use utils qw(%ERRORS &print_revision &support &usage);
+use Getopt::Long;
+
+my $VERSION = '0.06';
+
+my $RET = 'OK';
+my $LOCK_FILE = "/var/lib/dpkg/lock";
+my $CMD_APT = "/usr/bin/apt-get -s upgrade";
+my $TIMEOUT = 60;
+my $DEBUG	= 0;
+
+#####################################################################
+#
+# Command line arguments
+#
+
+sub print_usage ();
+
+my ($help,$version);
+GetOptions(    help => \$help,
+			  debug => \$DEBUG,
+		    version => \$version,
+		'timeout=s' => \$TIMEOUT
+);
+my ($PROGNAME) = $0 =~ m#.*/(.*)#;
+
+if ($help) {
+	print_revision($PROGNAME,"\$Revision: $VERSION \$");
+	print "Copyright (c) 2005 Francesc Guasch - Ortiz
+
+	Perl Check debian packages plugin for Nagios
+
+";
+	print_usage();
+	exit($ERRORS{OK});
+}
+
+if ($version) {
+	print_revision($PROGNAME,"\$Revision: $VERSION \$");
+	exit($ERRORS{OK});
+}
+
+#
+# unlikely but compliant
+#
+$SIG{'ALRM'} = sub {
+	print ("ERROR: Timeout\n");
+	exit $ERRORS{"UNKNOWN"};
+};
+alarm($TIMEOUT);
+
+
+######################################################################
+#
+# subs
+#
+
+sub print_usage () {
+	print "Usage: $PROGNAME [--debug] [--version] [--help]"
+			." [--timeout=$TIMEOUT]\n";
+}
+
+sub add_info {
+	my ($info,$type,$pkg) = @_;
+	$$info .= scalar(keys %$pkg)." new pkgs in $type: ";
+	if (keys %$pkg< 5 ) {
+		$$info .= join " ",keys %$pkg;
+	} else {
+		my $alguns = join " ",keys %$pkg;
+		$alguns = substr($alguns,0,80);
+		$alguns .= "...";
+		$$info .= $alguns;
+	}
+}
+
+sub exit_unknown {
+	my ($info) = @_;
+	chomp $info;
+    $RET='UNKNOWN';
+    print "$RET: $info\n";
+    exit $ERRORS{$RET};
+};
+
+sub run_apt {
+	my ($pkg,$ver,$type,$release);
+	open APT,"$CMD_APT 2>&1|" or exit_unknown($!);
+	my (%stable,%security,%other);
+	while (<APT>) {
+		print "APT: $_" if $DEBUG;
+		exit_unknown($_) if /(Could not open lock file)|(Could not get lock)/;
+		next unless /^Inst/;
+		($pkg,$ver,$release) = /Inst (.*?) .*\((.*?) (.*?)\)/;
+		print "$_\npkg=$pkg ver=$ver release=$release\n" if $DEBUG;
+		die "$_\n" unless defined $release;
+		$release = 'stable'  
+				if $release =~ /stable$/ && $release !~/security/i;
+		$release = 'security' 
+				if $release =~ /security/i;
+		if ($release eq 'stable') {
+			$stable{$pkg} = $ver;
+		} elsif ($release eq 'security') {
+			$security{$pkg} = $ver;
+		} else {
+			$other{$pkg}=$ver;
+		}
+	}
+	close APT;
+	my $info = '';
+	if (keys (%security)) {
+		$RET = 'CRITICAL';
+		add_info(\$info,'security',\%security);
+	} elsif (keys (%other) or keys(%stable)) {
+    	$RET = 'WARNING';
+		add_info(\$info,'stable',\%stable);
+		add_info(\$info,'other',\%other) if keys %other;
+	}
+	print "$RET: $info\n";
+}
+
+run_apt();
+exit $ERRORS{$RET};

puppet/zulip/files/nagios_plugins/zulip_postgresql/check_postgresql_replication_lag

@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+
+"""Nagios plugin to check the difference between the primary and
+replica PostgreSQL servers' xlog location.  Requires that the user this
+connects to PostgreSQL as has been granted the `pg_monitor` role.
+
+This can only use stdlib modules from python.
+"""
+
+import configparser
+import re
+import subprocess
+import sys
+
+
+def get_config(
+    config_file: configparser.RawConfigParser,
+    section: str,
+    key: str,
+    default_value: str = "",
+) -> str:
+    if config_file.has_option(section, key):
+        return config_file.get(section, key)
+    return default_value
+
+
+config_file = configparser.RawConfigParser()
+config_file.read("/etc/zulip/zulip.conf")
+
+states = {
+    "OK": 0,
+    "WARNING": 1,
+    "CRITICAL": 2,
+    "UNKNOWN": 3,
+}
+
+MAXSTATE = 0
+
+
+def report(state: str, msg: str) -> None:
+    global MAXSTATE
+    print(f"{state}: {msg}")
+    MAXSTATE = max(MAXSTATE, states[state])
+
+
+def run_sql_query(query: str) -> list[list[str]]:
+    command = [
+        "psql",
+        "-t",  # Omit header line
+        "-A",  # Don't pad with spaces
+        "-z",  # Separate columns with nulls
+        "-v",
+        "ON_ERROR_STOP=1",
+        "-d",
+        get_config(config_file, "postgresql", "database_name", "zulip"),
+        # No -U; nagios connects as itself
+        "-c",
+        f"SELECT {query}",
+    ]
+    try:
+        output = subprocess.check_output(command, stderr=subprocess.STDOUT, text=True).strip()
+        if not output:
+            return []
+        return [x.split("\0") for x in output.split("\n")]
+    except subprocess.CalledProcessError as e:
+        report("CRITICAL", f"psql failed: {e}: {e.output}")
+        sys.exit(MAXSTATE)
+
+
+def loc_to_abs_offset(loc_str: str) -> int:
+    m = re.match(r"^\s*([0-9a-fA-F]+)/([0-9a-fA-F]+)\s*$", loc_str)
+    if not m:
+        raise ValueError("Unknown xlog location format: " + loc_str)
+    (xlog_file, file_offset) = (m.group(1), m.group(2))
+
+    # From PostgreSQL 9.2's pg_xlog_location_diff:
+    #   result = XLogFileSize * (xlogid1 - xlogid2) + xrecoff1 - xrecoff2
+    # Taking xlogid2 and xrecoff2 to be zero to get the absolute offset:
+    #   result = XLogFileSize * xlogid1 + xrecoff1
+    #
+    # xlog_internal.h says:
+    #   #define XLogSegSize ((uint32) XLOG_SEG_SIZE)
+    #   #define XLogSegsPerFile (((uint32) 0xffffffff) / XLogSegSize)
+    #   #define XLogFileSize (XLogSegsPerFile * XLogSegSize)
+    #
+    # Since XLOG_SEG_SIZE is normally 16MB, XLogFileSize comes out to 0xFF000000
+    return 0xFF000000 * int(xlog_file, 16) + int(file_offset, 16)
+
+
+is_in_recovery = run_sql_query("pg_is_in_recovery()")
+
+if is_in_recovery[0][0] == "t":
+    replication_info = run_sql_query(
+        "sender_host, status, pg_last_wal_replay_lsn(), pg_last_wal_receive_lsn()"
+        " from pg_stat_wal_receiver"
+    )
+    if not replication_info:
+        report("CRITICAL", "Replaying WAL logs from backups")
+    else:
+        (primary_server, state, replay_loc, recv_loc) = replication_info[0]
+        recv_offset = loc_to_abs_offset(recv_loc)
+        replay_lag = recv_offset - loc_to_abs_offset(replay_loc)
+
+        if state != "streaming":
+            report("CRITICAL", f"replica is in state {state}, not streaming")
+
+        msg = f"replica is {replay_lag} bytes behind in replay of WAL logs from {primary_server}"
+        if replay_lag > 5 * 16 * 1024**2:
+            report("CRITICAL", msg)
+        elif replay_lag > 16 * 1024**2:
+            report("WARNING", msg)
+        else:
+            report("OK", msg)
+
+else:
+    replication_info = run_sql_query(
+        "client_addr, state, sent_lsn, write_lsn, flush_lsn, replay_lsn from pg_stat_replication"
+    )
+    if not replication_info:
+        report("CRITICAL", "No replicas!")
+    elif len(replication_info) == 1:
+        report("WARNING", "Only one replica!")
+    else:
+        report("OK", f"Found {len(replication_info)} replicas")
+
+    for replica in replication_info:
+        (client_addr, state, sent_lsn, write_lsn, flush_lsn, replay_lsn) = replica
+        if state != "streaming":
+            report("CRITICAL", f"replica {client_addr} is in state {state}, not streaming")
+
+        sent_offset = loc_to_abs_offset(sent_lsn)
+        lag: dict[str, int] = {}
+        lag["write"] = sent_offset - loc_to_abs_offset(write_lsn)
+        lag["flush"] = sent_offset - loc_to_abs_offset(flush_lsn)
+        lag["replay"] = sent_offset - loc_to_abs_offset(replay_lsn)
+        for lag_type in ("write", "flush", "replay"):
+            lag_bytes = lag[lag_type]
+            msg = f"replica {client_addr} is {lag_bytes} bytes behind in {lag_type} of WAL logs"
+            if lag_bytes > 5 * 16 * 1024**2:
+                report("CRITICAL", msg)
+            elif lag_bytes > 16 * 1024**2:
+                report("WARNING", msg)
+            else:
+                report("OK", msg)
+
+sys.exit(MAXSTATE)