snapshot of https://github.com/zulip/zulip.git @ 48c69911c7dd4e03b20552eb3bd77ad8151c1357 (2026-09-15)

puppet/deps.yaml

@@ -0,0 +1,6 @@
+puppetlabs-stdlib:
+  version: 8.6.0
+  sha256sum: 5bbaf59faf684726ca683ecddd5f62da34a725cdda6eabb6141db824431b807b
+puppetlabs-concat:
+  version: 7.3.3
+  sha256sum: 63d59db5fe908e851e4b92b5c3f8230c08f38d4fff000e15f92bc67c8a46a36b

puppet/kandra/files/apache/ports.conf

@@ -0,0 +1,9 @@
+# If you just change the port or add more ports here, you will likely also
+# have to change the VirtualHost statement in
+# /etc/apache2/sites-enabled/000-default
+# This is also true if you have upgraded from before 2.2.9-3 (i.e. from
+# Debian etch). See /usr/share/doc/apache2.2-common/NEWS.Debian.gz and
+# README.Debian.gz
+
+# Only serve Nagios on localhost
+Listen 127.0.0.1:3000

puppet/kandra/files/apt/apt.conf.d/02periodic

@@ -0,0 +1,7 @@
+# Every day, update package lists and install upgrades; see 50unattended-upgrades
+
+APT::Periodic::Enable "1";
+APT::Periodic::Update-Package-Lists "1";
+APT::Periodic::Download-Upgradeable-Packages "1";
+APT::Periodic::AutocleanInterval "7";
+APT::Periodic::Unattended-Upgrade "1";

puppet/kandra/files/apt/apt.conf.d/50unattended-upgrades

@@ -0,0 +1,59 @@
+// Automatically upgrade packages from these (origin:archive) pairs
+//
+// Note that in Ubuntu security updates may pull in new dependencies
+// from non-security sources (e.g. chromium). By allowing the release
+// pocket these get automatically pulled in.
+Unattended-Upgrade::Allowed-Origins {
+    "${distro_id}:${distro_codename}";
+    "${distro_id}:${distro_codename}-security";
+    "${distro_id}:${distro_codename}-updates";
+    // Extended Security Maintenance; doesn't necessarily exist for
+    // every release and this system may not have it installed, but if
+    // available, the policy for updates is such that unattended-upgrades
+    // should also install from here by default.
+    "${distro_id}ESMApps:${distro_codename}-apps-security";
+    "${distro_id}ESM:${distro_codename}-infra-security";
+};
+
+// Python regular expressions, matching packages to exclude from upgrading
+Unattended-Upgrade::Package-Blacklist {
+    "libc\d+";
+    "memcached$";
+    "nginx-full$";
+    "postgresql-\d+$";
+    "postgresql-\d+-pgdg-pgroonga$";
+    "rabbitmq-server$";
+    "erlang-base$";
+    "redis-server$";
+    "supervisor$";
+};
+
+// This option controls whether the development release of Ubuntu will be
+// upgraded automatically. Valid values are "true", "false", and "auto".
+Unattended-Upgrade::DevRelease "false";
+
+// This option allows you to control if on a unclean dpkg exit
+// unattended-upgrades will automatically run
+//   dpkg --force-confold --configure -a
+Unattended-Upgrade::AutoFixInterruptedDpkg "true";
+
+// Split the upgrade into the smallest possible chunks so that
+// they can be interrupted with SIGTERM. This makes the upgrade
+// a bit slower but it has the benefit that shutdown while a upgrade
+// is running is possible (with a small delay)
+Unattended-Upgrade::MinimalSteps "true";
+
+// Remove unused automatically installed kernel-related packages
+// (kernel images, kernel headers and kernel version locked tools).
+Unattended-Upgrade::Remove-Unused-Kernel-Packages "true";
+
+// Do automatic removal of newly unused dependencies after the upgrade
+Unattended-Upgrade::Remove-New-Unused-Dependencies "true";
+
+// Do automatic removal of unused packages after the upgrade
+// (equivalent to apt-get autoremove)
+//Unattended-Upgrade::Remove-Unused-Dependencies "false";
+
+// Automatically reboot *WITHOUT CONFIRMATION*
+//  if the file /var/run/reboot-required is found after the upgrade 
+Unattended-Upgrade::Automatic-Reboot "false";

puppet/kandra/files/certs/.gitignore

@@ -0,0 +1,2 @@
+# Don't check SSL private keys into git!
+*.key

puppet/kandra/files/chrony.conf

@@ -0,0 +1,49 @@
+# Welcome to the chrony configuration file. See chrony.conf(5) for more
+# information about usable directives.
+
+# Include configuration files found in /etc/chrony/conf.d.
+confdir /etc/chrony/conf.d
+
+# This will use the AWS local atomic clocks as a datasource; see
+# https://aws.amazon.com/blogs/aws/keeping-time-with-amazon-time-sync-service/
+server 169.254.169.123 prefer iburst minpoll 4 maxpoll 4
+
+# Use time sources from DHCP.
+sourcedir /run/chrony-dhcp
+
+# Use NTP sources found in /etc/chrony/sources.d.
+sourcedir /etc/chrony/sources.d
+
+# This directive specify the location of the file containing ID/key pairs for
+# NTP authentication.
+keyfile /etc/chrony/chrony.keys
+
+# This directive specify the file into which chronyd will store the rate
+# information.
+driftfile /var/lib/chrony/chrony.drift
+
+# Save NTS keys and cookies.
+ntsdumpdir /var/lib/chrony
+
+# Uncomment the following line to turn logging on.
+#log tracking measurements statistics
+
+# Log files location.
+logdir /var/log/chrony
+
+# Stop bad estimates upsetting machine clock.
+maxupdateskew 100.0
+
+# This directive enables kernel synchronisation (every 11 minutes) of the
+# real-time clock. Note that it can’t be used along with the 'rtcfile' directive.
+rtcsync
+
+# Step the system clock instead of slewing it if the adjustment is larger than
+# one second, but only in the first three clock updates.
+makestep 1 3
+
+# Get TAI-UTC offset and leap seconds from the system tz database.
+# This directive is commented out because AWS time servers do smear
+# leap seconds.
+#
+#leapsectz right/UTC

puppet/kandra/files/dotfiles/emacs.el

@@ -0,0 +1,3 @@
+; Move automatic backups into a directory off to the side
+(defvar backup-dir "~/.emacs-backups/")
+(setq backup-directory-alist (list (cons "." backup-dir)))

puppet/kandra/files/github.keys

@@ -0,0 +1,3 @@
+github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl
+github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg=
+github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=

puppet/kandra/files/hooks/post-deploy.d/restart_mirror_to_czo.hook

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -e
+set -u
+
+# This script uses `uv run`, and thus uses the production venv.  This
+# means that we need to make sure that we restart into the new venv,
+# so we don't start to error when the old venv gets garbage collected.
+supervisorctl restart mirror_to_czo:*

puppet/kandra/files/hooks/post-deploy.d/zulip_notify_schema_diff.hook

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+set -e
+set -u
+
+source "$(dirname "$0")/../common/zulip_notify.sh"
+
+if ! ref=$(crudini --get /etc/zulip/zulip.conf push_git_ref ref); then
+    exit 0
+fi
+
+diffstat=$(git diff --stat "$ZULIP_NEW_MERGE_BASE_COMMIT" ./*/migrations/)
+
+if [ -z "$diffstat" ]; then
+    exit 0
+fi
+
+echo "zulip_notify_schema_diff: Found schema differences from merge-base!"
+
+zulip_send "${deploy_environment} deploy on $(hostname) found differences in schema from merge-base:
+
+\`\`\`
+${diffstat}
+\`\`\`
+
+Please manually adjust the \`${ref}\` ref.
+"

puppet/kandra/files/install-aws-cli

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+set -eu
+
+ARCH=$(uname -m)
+
+AWS_CLI_VERSION="2.23.6"
+if [ "$ARCH" == "x86_64" ]; then
+    AWS_CLI_SHA="8d26ffa2e9427804a9f8928c692ee83e1c88456e0519a69e63bc992caf2b2339"
+elif [ "$ARCH" == "aarch64" ]; then
+    AWS_CLI_SHA="9c2b05dfc370cc99ec47ad374be52b828cd77d14702b2edae071d73224aacd6e"
+else
+    echo "Unsupported architecture: $ARCH"
+    exit 1
+fi
+
+# Check if we're already installed
+if [ -L "/srv/zulip-aws-tools/v2/current" ] \
+    && [ "$(readlink /srv/zulip-aws-tools/v2/current)" = "/srv/zulip-aws-tools/v2/$AWS_CLI_VERSION" ]; then
+    exit 0
+fi
+
+# If not running in the initial host bootstrap, and was called with
+# arguments, this is just a check.
+if [ -z "${RUNNING_IN_CLOUD_INIT+x}" ] && [[ $# -gt 0 ]]; then
+    exit 1
+fi
+
+mkdir -p /srv/zulip-aws-tools
+
+cd /srv/zulip-aws-tools || exit 1
+rm -rf awscli.zip awscli.zip.sha256 aws/
+curl -fL --retry 3 "https://awscli.amazonaws.com/awscli-exe-linux-$ARCH-$AWS_CLI_VERSION.zip" -o awscli.zip
+echo "$AWS_CLI_SHA  awscli.zip" >awscli.zip.sha256
+sha256sum -c awscli.zip.sha256
+unzip -q awscli.zip
+
+cd ./aws || exit 1
+./install -i /srv/zulip-aws-tools -b /srv/zulip-aws-tools/bin -u
+echo "$AWS_CLI_VERSION" >/srv/zulip-aws-tools/version
+
+(
+    cd /srv/zulip-aws-tools/
+    rm -rf awscli.zip awscli.zip.sha256 aws/
+)

puppet/kandra/files/install-ssh-authorized-keys

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+args="$(getopt -o '' --long check -- "$@")"
+eval "set -- $args"
+check=false
+while true; do
+    case "$1" in
+        --check)
+            check=true
+            shift
+            ;;
+        --)
+            shift
+            break
+            ;;
+    esac
+done
+
+username="$1"
+shift
+
+homedir="$(getent passwd "$username" | cut -d: -f6)"
+sshdir="$homedir/.ssh"
+
+workfile=$(mktemp)
+cleanup() { rm "$workfile"; }
+trap cleanup EXIT
+
+for ssh_secret_name in "$@"; do
+    /srv/zulip-aws-tools/bin/aws --output text \
+        secretsmanager get-secret-value \
+        --secret-id "$ssh_secret_name" \
+        --query SecretString \
+        | jq -r 'keys[] as $k | "\(.[$k]) \($k)"' \
+            >>"$workfile"
+done
+
+chmod 644 "$workfile"
+chown "$username:$username" "$workfile"
+
+if [ "$check" = "true" ]; then
+    diff -N "$workfile" "$sshdir/authorized_keys"
+    exit 0
+fi
+
+rsync -av "$workfile" "$sshdir/authorized_keys"

puppet/kandra/files/install-ssh-keys

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+args="$(getopt -o '' --long check -- "$@")"
+eval "set -- $args"
+check=false
+while true; do
+    case "$1" in
+        --check)
+            check=true
+            shift
+            ;;
+        --)
+            shift
+            break
+            ;;
+    esac
+done
+
+username="$1"
+shift
+
+homedir="$(getent passwd "$username" | cut -d: -f6)"
+sshdir="$homedir/.ssh"
+
+umask 077
+workdir=$(mktemp -d)
+chown "$username:$username" "$workdir"
+cleanup() { ls -al "$workdir" && rm -rf "$workdir"; }
+trap cleanup EXIT
+
+umask 033
+
+for ssh_secret_name in "$@"; do
+    keydata="$(/srv/zulip-aws-tools/bin/aws --output text \
+        secretsmanager get-secret-value \
+        --secret-id "$ssh_secret_name" \
+        --query SecretString)"
+    for keyfile in $(jq -r 'keys[]' <<<"$keydata"); do
+        touch "$workdir/$keyfile"
+        if [[ "$keyfile" != *".pub" ]]; then
+            chmod 600 "$workdir/$keyfile"
+        fi
+        jq -r ".[\"$keyfile\"]" <<<"$keydata" | base64 -d >"$workdir/$keyfile"
+        chown "$username:$username" "$workdir/$keyfile"
+    done
+done
+
+if [ "$check" = "true" ]; then
+    diff -rN -x config -x authorized_keys -x known_hosts \
+        "$workdir/" "$sshdir/"
+    exit 0
+fi
+
+rsync -av --delete \
+    --exclude config --exclude authorized_keys --exclude known_hosts \
+    "$workdir/" "$sshdir/"

puppet/kandra/files/iptables/header.v4

@@ -0,0 +1,30 @@
+# This file was auto-generated by Puppet.  Do not edit by hand.
+
+# The raw table is used to disable connection tracking for DNS
+# traffic, so it works even when the conntrack table fills.
+*raw
+:PREROUTING ACCEPT [0:0]
+:OUTPUT ACCEPT [0:0]
+-A PREROUTING -p udp -m udp --dport 53 -j CT --notrack
+-A PREROUTING -p udp -m udp --sport 53 -j CT --notrack
+-A OUTPUT -p udp -m udp --dport 53 -j CT --notrack
+-A OUTPUT -p udp -m udp --sport 53 -j CT --notrack
+COMMIT
+
+
+*filter
+# Allow all outbound traffic
+-A OUTPUT -j ACCEPT
+
+# Accept all loopback traffic
+-A INPUT -i lo -j ACCEPT
+
+# Drop all traffic to loopback IPs on other interfaces
+-A INPUT ! -i lo -d 127.0.0.0/8 -j DROP
+
+# Accept incoming traffic related to established connections, or the
+# untracked port-53-UDP set up above.  See iptables-extensions(8) for
+# the --state flag.  This drops INVALID and NEW states.
+-A INPUT -m state --state ESTABLISHED,RELATED,UNTRACKED -j ACCEPT
+
+# Host-specific rules follow:

puppet/kandra/files/iptables/header.v6

@@ -0,0 +1,19 @@
+# This file was auto-generated by Puppet.  Do not edit by hand.
+*filter
+
+# Allow all outbound traffic
+-A OUTPUT -j ACCEPT
+
+# Accept all loopback traffic
+-A INPUT -i lo -j ACCEPT
+
+# Drop all traffic to loopback IPs on other interfaces
+-A INPUT ! -i lo -d ::1/128 -j DROP
+
+# Allow ICMP; it is more fundamental to IPv6 functioning.
+-A INPUT -p icmpv6 -j ACCEPT
+
+# Accept incoming traffic related to established connections
+-A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT
+
+# Host-specific rules follow:

puppet/kandra/files/iptables/trailer.v4

@@ -0,0 +1,6 @@
+
+# Drop everything else
+-A INPUT   -j DROP
+-A FORWARD -j DROP
+
+COMMIT

puppet/kandra/files/iptables/trailer.v6

@@ -0,0 +1,6 @@
+
+# Drop everything else
+-A INPUT   -j DROP
+-A FORWARD -j DROP
+
+COMMIT

puppet/kandra/files/memcached_exporter

@@ -0,0 +1,398 @@
+#!/usr/bin/env python3
+
+import contextlib
+import sys
+import time
+from collections.abc import Iterable, Sequence
+from typing import Any
+
+sys.path.append("/home/zulip/deployments/current")
+from scripts.lib.setup_path import setup_path
+
+setup_path()
+
+import bmemcached
+from prometheus_client import start_http_server
+from prometheus_client.core import REGISTRY, CounterMetricFamily, GaugeMetricFamily
+from prometheus_client.metrics_core import Metric
+from prometheus_client.registry import Collector
+from prometheus_client.samples import Sample
+from typing_extensions import override
+
+from zproject import settings
+
+
+class MemcachedCollector(Collector):
+    @override
+    def collect(self) -> Iterable[Metric]:
+        def gauge(
+            name: str,
+            doc: str,
+            value: float | bytes | None = None,
+            labels: Sequence[str] | None = None,
+        ) -> GaugeMetricFamily:
+            return GaugeMetricFamily(
+                f"memcached_{name}", doc, float(value) if value else None, labels
+            )
+
+        def counter(
+            name: str,
+            doc: str,
+            labels: Iterable[str] | None = None,
+        ) -> CounterMetricFamily:
+            return CounterMetricFamily(
+                f"memcached_{name}", doc, labels=list(labels) if labels is not None else None
+            )
+
+        def counter_value(
+            name: str,
+            doc: str,
+            value: bytes | float,
+            labels: dict[str, str] | None = None,
+        ) -> CounterMetricFamily:
+            if labels is None:
+                labels = {}
+            metric = counter(name, doc, labels=labels.keys())
+            # CounterMetricFamily strips off a trailing "_total" from
+            # the metric's .name, and force-appends "_total" when you
+            # use .add_metric.  Since we have counters that don't end
+            # in _total, manually add samples, re-appending _total
+            # only if we originally ended with it.
+            append_total = name.endswith("_total")
+            metric.samples.append(
+                Sample(metric.name + ("_total" if append_total else ""), labels, float(value), None)
+            )
+            return metric
+
+        cache: dict[str, Any] = settings.CACHES["default"]
+        client = None
+        with contextlib.suppress(Exception):
+            client = bmemcached.Client((cache["LOCATION"],), **cache["OPTIONS"])
+        yield gauge("up", "If memcached is up", value=client is not None)
+
+        if client is None:
+            return
+
+        raw_stats = client.stats()
+        stats: dict[str, bytes] = next(iter(raw_stats.values()))
+
+        version_gauge = gauge(
+            "version", "The version of this memcached server.", labels=["version"]
+        )
+        version_gauge.add_metric(value=1, labels=[stats["version"].decode()])
+        yield version_gauge
+        yield counter_value(
+            "uptime_seconds", "Number of seconds since the server started.", value=stats["uptime"]
+        )
+
+        commands_counter = counter(
+            "commands_total",
+            "Total number of all requests broken down by command (get, set, etc.) and status.",
+            labels=["command", "status"],
+        )
+        for op in ("get", "delete", "incr", "decr", "cas", "touch"):
+            commands_counter.add_metric(value=float(stats[f"{op}_hits"]), labels=[op, "hit"])
+            commands_counter.add_metric(value=float(stats[f"{op}_misses"]), labels=[op, "miss"])
+        commands_counter.add_metric(value=float(stats["cas_badval"]), labels=["cas", "badval"])
+        commands_counter.add_metric(value=float(stats["cmd_flush"]), labels=["flush", "hit"])
+
+        # memcached includes cas operations again in cmd_set
+        commands_counter.add_metric(
+            value=int(stats["cmd_set"])
+            - (int(stats["cas_hits"]) + int(stats["cas_hits"]) + int(stats["cas_badval"])),
+            labels=["set", "hit"],
+        )
+        yield commands_counter
+
+        yield counter_value(
+            "process_user_cpu_seconds_total",
+            "Accumulated user time for this process.",
+            value=stats["rusage_user"],
+        )
+
+        yield counter_value(
+            "process_system_cpu_seconds_total",
+            "Accumulated system time for this process.",
+            value=stats["rusage_system"],
+        )
+
+        yield gauge(
+            "current_bytes", "Current number of bytes used to store items.", value=stats["bytes"]
+        )
+        yield gauge(
+            "limit_bytes",
+            "Number of bytes this server is allowed to use for storage.",
+            value=stats["limit_maxbytes"],
+        )
+        yield gauge(
+            "current_items",
+            "Current number of items stored by this instance.",
+            value=stats["curr_items"],
+        )
+        yield counter_value(
+            "items_total",
+            "Total number of items stored during the life of this instance.",
+            value=stats["total_items"],
+        )
+
+        yield counter_value(
+            "read_bytes_total",
+            "Total number of bytes read by this server from network.",
+            value=stats["bytes_read"],
+        )
+        yield counter_value(
+            "written_bytes_total",
+            "Total number of bytes sent by this server to network.",
+            value=stats["bytes_written"],
+        )
+
+        yield gauge(
+            "current_connections",
+            "Current number of open connections.",
+            value=stats["curr_connections"],
+        )
+        yield counter_value(
+            "connections_total",
+            "Total number of connections opened since the server started running.",
+            value=stats["total_connections"],
+        )
+        yield counter_value(
+            "connections_rejected_total",
+            "Total number of connections rejected due to hitting the memcached's -c limit in maxconns_fast mode.",
+            value=stats["rejected_connections"],
+        )
+        yield counter_value(
+            "connections_yielded_total",
+            "Total number of connections yielded running due to hitting the memcached's -R limit.",
+            value=stats["conn_yields"],
+        )
+        yield counter_value(
+            "connections_listener_disabled_total",
+            "Number of times that memcached has hit its connections limit and disabled its listener.",
+            value=stats["listen_disabled_num"],
+        )
+
+        yield counter_value(
+            "items_evicted_total",
+            "Total number of valid items removed from cache to free memory for new items.",
+            value=stats["evictions"],
+        )
+        yield counter_value(
+            "items_reclaimed_total",
+            "Total number of times an entry was stored using memory from an expired entry.",
+            value=stats["reclaimed"],
+        )
+        if "store_too_large" in stats:
+            yield counter_value(
+                "item_too_large_total",
+                "The number of times an item exceeded the max-item-size when being stored.",
+                value=stats["store_too_large"],
+            )
+        if "store_no_memory" in stats:
+            yield counter_value(
+                "item_no_memory_total",
+                "The number of times an item could not be stored due to no more memory.",
+                value=stats["store_no_memory"],
+            )
+
+        raw_stats = client.stats("settings")
+        settings_stats = next(iter(raw_stats.values()))
+        yield counter_value(
+            "max_connections",
+            "Maximum number of clients allowed.",
+            value=settings_stats["maxconns"],
+        )
+        yield counter_value(
+            "max_item_size_bytes",
+            "Maximum item size.",
+            value=settings_stats["item_size_max"],
+        )
+
+        raw_stats = client.stats("slabs")
+        slab_stats = next(iter(raw_stats.values()))
+        yield counter_value(
+            "malloced_bytes",
+            "Number of bytes of memory allocated to slab pages.",
+            value=slab_stats["total_malloced"],
+        )
+
+        slabs = {key.split(":", 1)[0] for key in slab_stats if ":" in key}
+        slab_commands = counter(
+            "slab_commands_total",
+            "Total number of all requests broken down by command (get, set, etc.) and status per slab.",
+            labels=["slab", "command", "status"],
+        )
+        for slab_no in slabs:
+            for op in ("get", "delete", "incr", "decr", "cas", "touch"):
+                slab_commands.add_metric(
+                    labels=[slab_no, op, "hit"], value=slab_stats[f"{slab_no}:{op}_hits"]
+                )
+            slab_commands.add_metric(
+                labels=[slab_no, "cas", "badval"], value=slab_stats[f"{slab_no}:cas_badval"]
+            )
+            slab_commands.add_metric(
+                labels=[slab_no, "set", "hit"],
+                value=float(slab_stats[f"{slab_no}:cmd_set"])
+                - (
+                    float(slab_stats[f"{slab_no}:cas_hits"])
+                    + float(slab_stats[f"{slab_no}:cas_badval"])
+                ),
+            )
+        yield slab_commands
+
+        def slab_counter(name: str, doc: str) -> CounterMetricFamily:
+            return counter(f"slab_{name}", doc, labels=["slab"])
+
+        def slab_gauge(name: str, doc: str) -> GaugeMetricFamily:
+            return gauge(f"slab_{name}", doc, labels=["slab"])
+
+        slab_metrics = {
+            "chunk_size": slab_gauge("chunk_size_bytes", "The amount of space each chunk uses."),
+            "chunks_per_page": slab_gauge(
+                "chunks_per_page", "How many chunks exist within one page."
+            ),
+            "total_pages": slab_gauge(
+                "current_pages", "Total number of pages allocated to the slab class."
+            ),
+            "total_chunks": slab_gauge(
+                "current_chunks", "Total number of chunks allocated to the slab class."
+            ),
+            "used_chunks": slab_gauge(
+                "chunks_used", "How many chunks have been allocated to items."
+            ),
+            "free_chunks": slab_gauge(
+                "chunks_free", "Chunks not yet allocated to items, or freed via delete."
+            ),
+            "free_chunks_end": slab_gauge(
+                "chunks_free_end", "Number of free chunks at the end of the last allocated page."
+            ),
+        }
+        for slab_no in slabs:
+            for key, slab_metric in slab_metrics.items():
+                slab_metric.samples.append(
+                    Sample(
+                        slab_metric.name,
+                        {"slab": slab_no},
+                        slab_stats.get(f"{slab_no}:{key}", b"0"),
+                    )
+                )
+        for slab_metric in slab_metrics.values():
+            yield slab_metric
+
+        raw_stats = client.stats("items")
+        item_stats = next(iter(raw_stats.values()))
+        item_hits_counter = counter(
+            "slab_lru_hits_total", "Number of get_hits to the LRU.", labels=["slab", "lru"]
+        )
+        for slab_no in slabs:
+            for lru in ("hot", "warm", "cold", "temp"):
+                item_hits_counter.add_metric(
+                    labels=[slab_no, lru],
+                    value=item_stats.get(f"items:{slab_no}:hits_to_{lru}", b"0"),
+                )
+        yield item_hits_counter
+
+        item_metrics = {
+            "number": slab_gauge(
+                "current_items", "Number of items presently stored in this class."
+            ),
+            "number_hot": slab_gauge(
+                "hot_items", "Number of items presently stored in the HOT LRU."
+            ),
+            "number_warm": slab_gauge(
+                "warm_items", "Number of items presently stored in the WARM LRU."
+            ),
+            "number_cold": slab_gauge(
+                "cold_items", "Number of items presently stored in the COLD LRU."
+            ),
+            "number_temp": slab_gauge(
+                "temporary_items", "Number of items presently stored in the TEMPORARY LRU."
+            ),
+            "age_hot": slab_gauge("hot_age_seconds", "Age of the oldest item in HOT LRU."),
+            "age_warm": slab_gauge("warm_age_seconds", "Age of the oldest item in WARM LRU."),
+            "age": slab_gauge("items_age_seconds", "Age of the oldest item in the LRU."),
+            "mem_requested": slab_gauge(
+                "mem_requested_bytes", "Number of bytes requested to be stored in this LRU."
+            ),
+            "evicted": slab_counter(
+                "items_evicted_total",
+                "Total number of times an item had to be evicted from the LRU before it expired.",
+            ),
+            "evicted_nonzero": slab_counter(
+                "items_evicted_nonzero_total",
+                "Number of times an item which had an explicit expire time set had to be evicted from the LRU before it expired.",
+            ),
+            "evicted_time": slab_gauge(
+                "items_evicted_time_seconds",
+                "Seconds since the last access for the most recent item evicted from this class.",
+            ),
+            "outofmemory": slab_counter(
+                "items_outofmemory_total",
+                " Number of times the underlying slab class was unable to store a new item.",
+            ),
+            "tailrepairs": slab_counter(
+                "items_tailrepairs_total",
+                "Number of times we self-healed a slab with a refcount leak.",
+            ),
+            "reclaimed": slab_counter(
+                "items_reclaimed_total",
+                "Number of times an entry was stored using memory from an expired entry.",
+            ),
+            "expired_unfetched": slab_counter(
+                "items_expired_unfetched_total",
+                "Number of expired items reclaimed from the LRU which were never touched after being set.",
+            ),
+            "evicted_unfetched": slab_counter(
+                "items_evicted_unfetched_total",
+                "Number of valid items evicted from the LRU which were never touched after being set.",
+            ),
+            "evicted_active": slab_counter(
+                "items_evicted_active_total",
+                "Number of valid items evicted from the LRU which were recently touched but were evicted before being moved to the top of the LRU again.",
+            ),
+            "crawler_reclaimed": slab_counter(
+                "items_crawler_reclaimed_total", "Number of items freed by the LRU Crawler."
+            ),
+            "lrutail_reflocked": slab_counter(
+                "items_lrutail_reflocked_total",
+                "Number of items found to be refcount locked in the LRU tail.",
+            ),
+            "moves_to_cold": slab_counter(
+                "moves_to_cold_total", "Number of items moved from HOT or WARM into COLD."
+            ),
+            "moves_to_warm": slab_counter(
+                "moves_to_warm_total", "Number of items moved from COLD to WARM."
+            ),
+            "moves_within_lru": slab_counter(
+                "moves_within_lru_total",
+                "Number of times active items were bumped within HOW or WARM.",
+            ),
+        }
+        for slab_no in slabs:
+            for key, item_metric in item_metrics.items():
+                item_metric.samples.append(
+                    Sample(
+                        item_metric.name,
+                        {"slab": slab_no},
+                        item_stats.get(f"items:{slab_no}:{key}", b"0"),
+                    )
+                )
+        for item_metric in item_metrics.values():
+            yield item_metric
+
+        raw_stats = client.stats("sizes")
+        sizes_stats = next(iter(raw_stats.values()))
+        if sizes_stats.get("sizes_status") != b"disabled" and sizes_stats != {}:
+            sizes = sorted([int(x) for x in sizes_stats])
+            yield gauge(
+                "item_max_bytes", "Largest item (rounded to 32 bytes) in bytes.", value=sizes[-1]
+            )
+
+        client.disconnect_all()
+
+
+if __name__ == "__main__":
+    REGISTRY.register(MemcachedCollector())
+    start_http_server(11212, addr="127.0.0.1")
+    while True:
+        time.sleep(60)

puppet/kandra/files/mirror_to_czo

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+
+import configparser
+import logging
+import time
+from contextlib import suppress
+from typing import Any
+
+# This import (and its dependencies) come from the production venv,
+# because we use `uv run` to execute this script.
+import zulip
+
+config_file = configparser.RawConfigParser()
+config_file.read("/etc/zulip/zulip.conf")
+send_to_channel = config_file.get("mirror_to_czo", "send_to_channel")
+
+reading = zulip.Client(config_file="/etc/zulip/mirror_to_czo.zulipcore.zuliprc")
+sending = zulip.Client(config_file="/etc/zulip/mirror_to_czo.czo.zuliprc")
+
+logging.Formatter.converter = time.gmtime
+logging.basicConfig(format="%(asctime)s mirror_to_czo: %(message)s", level=logging.INFO)
+
+
+def send_mirror(msg: dict[str, Any]) -> None:
+    logging.info(msg["message"]["content"])
+    sending.send_message(
+        {
+            "type": "stream",
+            "to": send_to_channel,
+            "topic": msg["message"]["subject"],
+            "content": msg["message"]["content"],
+        }
+    )
+
+
+with suppress(KeyboardInterrupt):
+    reading.call_on_each_event(
+        send_mirror,
+        event_types=["message"],
+        narrow=[["stream", "signups"]],
+    )

puppet/kandra/files/nagios4/commands.cfg

@@ -0,0 +1,167 @@
+###############################################################################
+# COMMANDS.CFG - SAMPLE COMMAND DEFINITIONS FOR NAGIOS
+###############################################################################
+
+
+################################################################################
+# NOTIFICATION COMMANDS
+################################################################################
+
+# 'notify-host-by-email' command definition
+define command{
+      command_name notify-host-by-email
+      command_line /usr/bin/printf "%b" "Subject: $NOTIFICATIONTYPE$ Host Alert: $HOSTNAME$ is $HOSTSTATE$\n\n***** Nagios *****\n\nNotification Type: $NOTIFICATIONTYPE$\nHost: $HOSTNAME$\nState: $HOSTSTATE$\nAddress: $HOSTADDRESS$\nInfo: $HOSTOUTPUT$\n\nDate/Time: $LONGDATETIME$\n" | /usr/bin/msmtp -C /var/lib/nagios/msmtprc -vt $CONTACTEMAIL$ 2>&1
+      }
+
+# 'notify-service-by-email' command definition
+define command{
+      command_name notify-service-by-email
+      command_line /usr/bin/printf "%b" "Subject: $NOTIFICATIONTYPE$ Service Alert: $HOSTALIAS$/$SERVICEDESC$ is $SERVICESTATE$\n\n***** Nagios *****\n\nNotification Type: $NOTIFICATIONTYPE$\n\nService: $SERVICEDESC$\nHost: $HOSTALIAS$\nAddress: $HOSTADDRESS$\nState: $SERVICESTATE$\n\nDate/Time: $LONGDATETIME$\n\nAdditional Info:\n\n$SERVICEOUTPUT$\n$LONGSERVICEOUTPUT$\n" | /usr/bin/msmtp -C /var/lib/nagios/msmtprc -vt $CONTACTEMAIL$
+      }
+
+# Zulip commands
+define command {
+    command_name    notify-host-by-zulip
+    command_line    /usr/local/share/zulip/integrations/nagios/nagios-notify-zulip --stream="kandra ops" --type="$NOTIFICATIONTYPE$" --host="$HOSTADDRESS$" --state="$HOSTSTATE$" --output="$HOSTOUTPUT$" --long-output="$LONGHOSTOUTPUT$"
+}
+
+define command {
+    command_name    notify-service-by-zulip
+    command_line    /usr/local/share/zulip/integrations/nagios/nagios-notify-zulip --stream="kandra ops" --type="$NOTIFICATIONTYPE$" --host="$HOSTADDRESS$" --service="$SERVICEDESC$" --state="$SERVICESTATE$" --output="$SERVICEOUTPUT$" --long-output="$LONGSERVICEOUTPUT$"
+}
+
+################################################################################
+# HOST CHECK COMMANDS
+################################################################################
+
+# On Debian, check-host-alive is being defined from within the
+# monitoring-plugins-basic package
+
+################################################################################
+# PERFORMANCE DATA COMMANDS
+################################################################################
+
+
+# 'process-host-perfdata' command definition
+define command{
+        command_name    process-host-perfdata
+        command_line    /usr/bin/printf "%b" "$LASTHOSTCHECK$\t$HOSTNAME$\t$HOSTSTATE$\t$HOSTATTEMPT$\t$HOSTSTATETYPE$\t$HOSTEXECUTIONTIME$\t$HOSTOUTPUT$\t$HOSTPERFDATA$\n" >> /var/lib/nagios4/host-perfdata.out
+        }
+
+
+# 'process-service-perfdata' command definition
+define command{
+        command_name    process-service-perfdata
+        command_line    /usr/bin/printf "%b" "$LASTSERVICECHECK$\t$HOSTNAME$\t$SERVICEDESC$\t$SERVICESTATE$\t$SERVICEATTEMPT$\t$SERVICESTATETYPE$\t$SERVICEEXECUTIONTIME$\t$SERVICELATENCY$\t$SERVICEOUTPUT$\t$SERVICEPERFDATA$\n" >> /var/lib/nagios4/service-perfdata.out
+        }
+
+define command{
+        command_name    check_remote_disk
+        command_line    /usr/lib/nagios/plugins/check_by_ssh -p 22 -l nagios -t 30 -i /var/lib/nagios/.ssh/id_ed25519 -H $HOSTADDRESS$ -C '/usr/lib/nagios/plugins/check_disk -W$ARG1$ -K$ARG2$ -w $ARG1$ -c $ARG2$ -A -l -X cgroup -X tmpfs -X devtmpfs -X squashfs -X configfs -X tracefs'
+}
+
+# Copy of check_remote_disk without the SSH
+define command{
+        command_name    check_local_disk
+        command_line    /usr/lib/nagios/plugins/check_disk -W$ARG1$ -K$ARG2$ -w $ARG1$ -c $ARG2$ -A -l -X cgroup -X tmpfs -X devtmpfs -X squashfs -X configfs -X tracefs
+}
+
+define command{
+        command_name    check_debian_packages
+        command_line    /usr/lib/nagios/plugins/check_by_ssh -p $ARG1$ -l nagios -t 30 -i /var/lib/nagios/.ssh/id_ed25519 -H $HOSTADDRESS$ -C '/usr/lib/nagios/plugins/zulip_base/check_debian_packages'
+}
+
+define command{
+        command_name    check_ntp_time
+        command_line    /usr/lib/nagios/plugins/check_by_ssh -p $ARG1$ -l nagios -t 30 -i /var/lib/nagios/.ssh/id_ed25519 -H $HOSTADDRESS$ -C '/usr/lib/nagios/plugins/check_ntp_time -H $ARG2$ -w .5 -c 1'
+}
+
+define command{
+        command_name    check_queue_worker_errors
+        command_line    /usr/lib/nagios/plugins/check_by_ssh -p $ARG1$ -l nagios -t 30 -i /var/lib/nagios/.ssh/id_ed25519 -H $HOSTADDRESS$ -C '/usr/lib/nagios/plugins/zulip_app_frontend/check_queue_worker_errors'
+}
+
+define command{
+        command_name    check_postgres
+        command_line    /usr/lib/nagios/plugins/check_by_ssh -p 22 -l nagios -t 30 -i /var/lib/nagios/.ssh/id_ed25519 -H $HOSTADDRESS$ -C '/usr/bin/check_postgres --dbname=$ARG1$ --dbuser=$ARG2$ --action $ARG3$'
+}
+
+define command{
+        command_name    check_postgres_alert_args
+        command_line    /usr/lib/nagios/plugins/check_by_ssh -p 22 -l nagios -t 30 -i /var/lib/nagios/.ssh/id_ed25519 -H $HOSTADDRESS$ -C '/usr/bin/check_postgres --dbname=$ARG1$ --dbuser=$ARG2$ --action $ARG3$ --warning="$ARG4$" --critical="$ARG5$"'
+}
+
+define command{
+        command_name    check_redis_ssh
+        command_line    /usr/lib/nagios/plugins/check_by_ssh -p $ARG1$ -l nagios -t 30 -i /var/lib/nagios/.ssh/id_ed25519 -H $HOSTADDRESS$ -C '/usr/lib/nagios/plugins/check_redis -H 127.0.0.1 -C /var/lib/nagios/redis_password'
+}
+
+define command{
+        command_name    check_rabbitmq_queues
+        command_line    /usr/lib/nagios/plugins/check_by_ssh -p $ARG1$ -l nagios -t 30 -i /var/lib/nagios/.ssh/id_ed25519 -H $HOSTADDRESS$ -C '/usr/lib/nagios/plugins/zulip_app_frontend/check_cron_file /var/lib/nagios_state/check-rabbitmq-results'
+}
+
+define command{
+        command_name    check_send_receive_time
+        command_line    /usr/lib/nagios/plugins/check_by_ssh -p $ARG1$ -l nagios -t 30 -i /var/lib/nagios/.ssh/id_ed25519 -H $HOSTADDRESS$ -C '/usr/lib/nagios/plugins/zulip_app_frontend/check_cron_file /var/lib/nagios_state/check_send_receive_state'
+}
+
+define command{
+        command_name    check_rabbitmq_consumers
+        command_line    /usr/lib/nagios/plugins/check_by_ssh -p 22 -l nagios -t 30 -i /var/lib/nagios/.ssh/id_ed25519 -H $HOSTADDRESS$ -C '/usr/lib/nagios/plugins/zulip_app_frontend/check_cron_file /var/lib/nagios_state/check-rabbitmq-consumers-$ARG1$'
+}
+
+define command{
+        command_name    check_analytics_state
+        command_line    /usr/lib/nagios/plugins/check_by_ssh -p 22 -l nagios -t 30 -i /var/lib/nagios/.ssh/id_ed25519 -H $HOSTADDRESS$ -C '/usr/lib/nagios/plugins/zulip_app_frontend/check_cron_file /var/lib/nagios_state/check-analytics-state 4000'  # Last argument is acceptable delay in seconds.
+}
+
+define command {
+        command_name    check_named_procs
+        command_line    /usr/lib/nagios/plugins/check_procs -C $ARG1$ -w $ARG2$ -c $ARG3$
+}
+
+define command{
+        command_name    check_procs_nokthreads
+        command_line    /usr/lib/nagios/plugins/check_procs -w '$ARG1$' -c '$ARG2$' -k
+}
+
+define command {
+        command_name    check_fts_update_log
+        command_line    /usr/lib/nagios/plugins/check_by_ssh -l nagios -t 30 -i /var/lib/nagios/.ssh/id_ed25519 -H $HOSTADDRESS$ -C '/usr/local/bin/process_fts_updates --nagios-check'
+}
+
+define command {
+        command_name    check_postgresql_replication_lag
+        command_line    /usr/lib/nagios/plugins/check_by_ssh -l nagios -t 30 -i /var/lib/nagios/.ssh/id_ed25519 -H $HOSTADDRESS$ -C '/usr/lib/nagios/plugins/zulip_postgresql/check_postgresql_replication_lag'
+}
+
+define command{
+       command_name     check_worker_memory
+       command_line    /usr/lib/nagios/plugins/check_by_ssh -l nagios -t 30 -i /var/lib/nagios/.ssh/id_ed25519 -H $HOSTADDRESS$ -C '/usr/lib/nagios/plugins/zulip_app_frontend/check_worker_memory'
+}
+
+define command{
+       command_name     check_https_status
+       command_line     /usr/lib/nagios/plugins/check_http --ssl -H '$HOSTADDRESS$' -I '$HOSTADDRESS$' --expect=200,302,401
+}
+
+define command{
+       command_name     check_ssl_certificate
+       command_line     /usr/lib/nagios/plugins/check_http --ssl -H '$ARG1$' -I '$ARG1$' --sni --expect=200,302,401 -C30,14
+}
+
+define command{
+       command_name     check_proxy_status
+       command_line     /usr/lib/nagios/plugins/check_http --ssl -H 'www.google.com' -u 'https://www.google.com/' -j CONNECT -I '$HOSTADDRESS$' -p 4750 --expect=200,302,401 -s Search
+}
+
+define command{
+       command_name     check_apt_repo_status
+       command_line     /usr/lib/nagios/plugins/check_http --sni --ssl -H '$ARG1$' -u 'https://$ARG1$$ARG2$/dists/stable/Release' --expect=200 -s Contents-amd64
+}
+
+define command{
+       command_name     check_camo
+       command_line     /usr/lib/nagios/plugins/check_http --sni --ssl -H '$ARG1$' -u '$ARG2$' -k 'Accept-Encoding: identity' -w '$ARG3$' -c '$ARG4$'
+}

puppet/kandra/files/nagios4/conf.d/generic-host_nagios2.cfg

@@ -0,0 +1,19 @@
+# Generic host definition template - This is NOT a real host, just a template!
+
+define host{
+        name                            generic-host    ; The name of this host template
+        notifications_enabled           1       ; Host notifications are enabled
+        event_handler_enabled           1       ; Host event handler is enabled
+        flap_detection_enabled          0
+        failure_prediction_enabled      1       ; Failure prediction is enabled
+        process_perf_data               1       ; Process performance data
+        retain_status_information       1       ; Retain status information across program restarts
+        retain_nonstatus_information    1       ; Retain non-status information across program restarts
+                check_command           check_tcp!22
+                max_check_attempts              10
+                notification_interval           0
+                notification_period             24x7
+                notification_options            d,u,r
+                contact_groups                  admins
+        register                        0       ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL HOST, JUST A TEMPLATE!
+        }

puppet/kandra/files/nagios4/conf.d/generic-service_nagios2.cfg

@@ -0,0 +1,26 @@
+# generic service template definition
+define service{
+        name                            generic-service ; The 'name' of this service template
+        active_checks_enabled           1       ; Active service checks are enabled
+        passive_checks_enabled          1       ; Passive service checks are enabled/accepted
+        parallelize_check               1       ; Active service checks should be parallelized (disabling this can lead to major performance problems)
+        obsess_over_service             1       ; We should obsess over this service (if necessary)
+        check_freshness                 0       ; Default is to NOT check service 'freshness'
+        notifications_enabled           1       ; Service notifications are enabled
+        event_handler_enabled           1       ; Service event handler is enabled
+        flap_detection_enabled          0
+        failure_prediction_enabled      1       ; Failure prediction is enabled
+        process_perf_data               1       ; Process performance data
+        retain_status_information       1       ; Retain status information across program restarts
+        retain_nonstatus_information    1       ; Retain non-status information across program restarts
+                notification_interval           0               ; Only send notifications on status change by default.
+                is_volatile                     0
+                check_period                    24x7
+                check_interval                  1
+                retry_interval                  1
+                max_check_attempts              2
+                notification_period             24x7
+                notification_options            w,u,c,r
+                contact_groups                  admins
+        register                        0       ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL SERVICE, JUST A TEMPLATE!
+        }

puppet/kandra/files/nagios4/conf.d/hostgroups.cfg

@@ -0,0 +1,82 @@
+define hostgroup {
+        hostgroup_name          all
+        alias                   All servers
+        members                 *
+}
+
+define hostgroup {
+        hostgroup_name          aws_host
+        alias                   Hosts in AWS
+}
+
+define hostgroup {
+        hostgroup_name          non_aws_host
+        alias                   Hosts not in AWS
+}
+
+define hostgroup {
+        hostgroup_name          pageable_servers
+        alias                   Servers for whom we want to page
+}
+
+define hostgroup {
+        hostgroup_name          not_pageable_servers
+        alias                   Servers for whom we do not want to page
+}
+
+define hostgroup {
+        hostgroup_name          flaky_servers
+        alias                   Servers for whom we do not page, and have flakier net
+}
+
+define hostgroup {
+        hostgroup_name          fullstack
+        alias                   Fullstack hosts
+}
+
+define hostgroup {
+        hostgroup_name          frontends
+        alias                   Frontend web servers
+        hostgroup_members       staging_frontends, prod_frontends
+}
+
+define hostgroup {
+        hostgroup_name          staging_frontends
+        alias                   Staging frontend web servers
+}
+
+define hostgroup {
+        hostgroup_name          prod_frontends
+        alias                   Production frontend web servers
+}
+
+define hostgroup {
+        hostgroup_name          redis
+        alias                   Redis servers
+}
+
+define hostgroup {
+        hostgroup_name          postgresql
+        alias                   PostgreSQL app servers
+        hostgroup_members       postgresql_primary, postgresql_replica
+}
+
+define hostgroup {
+        hostgroup_name          postgresql_primary
+        alias                   Primary PostgreSQL app servers
+}
+
+define hostgroup {
+        hostgroup_name          postgresql_replica
+        alias                   Replica PostgreSQL servers
+}
+
+define hostgroup {
+        hostgroup_name          smokescreen
+        alias                   Servers that run the Smokescreen HTTP proxy
+}
+
+define hostgroup {
+        hostgroup_name          other
+        alias                   Other servers
+}

puppet/kandra/files/nagios4/conf.d/services.cfg

@@ -0,0 +1,436 @@
+### SSH
+
+define service {
+        use                             generic-service
+        service_description             SSH - pageable
+        hostgroup_name                  pageable_servers
+        check_command                   check_ssh
+        contact_groups                  page_admins
+}
+
+define service {
+        use                             generic-service
+        service_description             SSH
+        hostgroup_name                  not_pageable_servers
+        check_command                   check_ssh
+        contact_groups                  admins
+}
+
+define service {
+        use                             generic-service
+        service_description             SSH for flaky machines
+        hostgroup_name                  flaky_servers
+        check_command                   check_ssh
+        normal_check_interval           2
+        retry_check_interval            2
+        max_check_attempts              5
+        contact_groups                  admins
+}
+
+
+### Disk usage
+
+define service {
+        use                             generic-service
+        service_description             Disk usage - pageable
+        hostgroup_name                  pageable_servers
+        check_command                   check_remote_disk!20%!10%
+        contact_groups                  ops_message
+}
+
+define service {
+        use                             generic-service
+        service_description             Disk usage
+        hostgroup_name                  not_pageable_servers
+        check_command                   check_remote_disk!20%!10%
+        contact_groups                  ops_message
+}
+
+define service {
+        use                             generic-service
+        service_description             Disk usage for flaky machines
+        hostgroup_name                  flaky_servers
+        check_command                   check_remote_disk!20%!10%
+        normal_check_interval           2
+        retry_check_interval            2
+        max_check_attempts              5
+        contact_groups                  admins
+}
+
+
+### System updates
+
+define service {
+        use                             generic-service
+        service_description             Debian update availability
+        hostgroup_name                  all
+        check_command                   check_debian_packages!22
+        contact_groups                  admins
+}
+
+
+### NTP
+
+define service {
+        use                             generic-service
+        service_description             Check AWS NTP time
+        hostgroup_name                  aws_host
+        check_command                   check_ntp_time!22!169.254.169.123
+        max_check_attempts              3
+        contact_groups                  admins
+}
+
+define service {
+        use                             generic-service
+        service_description             Check non-AWS NTP time
+        hostgroup_name                  non_aws_host
+        check_command                   check_ntp_time!22!pool.ntp.org
+        max_check_attempts              5
+        contact_groups                  admins
+}
+
+
+
+
+
+### Service groups
+
+#### Application frontends
+
+define service {
+        use                             generic-service
+        service_description             HTTPS
+        hostgroup_name                  prod_frontends, fullstack
+        check_command                   check_https_status
+        contact_groups                  page_admins
+}
+
+define service {
+        use                             generic-service
+        service_description             HTTPS
+        hostgroup_name                  staging_frontends
+        check_command                   check_https_status
+        contact_groups                  ops_message
+}
+
+define service {
+        use                             generic-service
+        service_description             Check send receive time
+        hostgroup_name                  prod_frontends, fullstack
+        check_command                   check_send_receive_time!22
+        max_check_attempts              2
+        contact_groups                  page_admins
+}
+
+define service {
+        use                             generic-service
+        service_description             Check send receive time
+        hostgroup_name                  staging_frontends
+        check_command                   check_send_receive_time!22
+        max_check_attempts              2
+        contact_groups                  ops_message
+}
+
+define service {
+        use                             generic-service
+        service_description             Check analytics state
+        hostgroup_name                  prod_frontends
+        check_command                   check_analytics_state!22
+        max_check_attempts              2
+        contact_groups                  admins
+}
+
+
+#### PostgreSQL
+
+define service {
+        use                             generic-service
+        service_description             Check PostgreSQL connection
+        hostgroup_name                  postgresql
+        check_command                   check_postgres!zulip!nagios!connection
+        contact_groups                  page_admins
+}
+
+define service {
+        use                             generic-service
+        service_description             Check PostgreSQL autovac_freeze
+        hostgroup_name                  postgresql
+        check_command                   check_postgres_alert_args!zulip!nagios!autovac_freeze!101%!105%
+        contact_groups                  admins
+}
+
+define service {
+        use                             generic-service
+        service_description             Check PostgreSQL backends
+        hostgroup_name                  postgresql
+        check_command                   check_postgres!zulip!nagios!backends
+        contact_groups                  admins
+}
+
+define service {
+        use                             generic-service
+        service_description             Check PostgreSQL disabled triggers
+        hostgroup_name                  postgresql
+        check_command                   check_postgres!zulip!nagios!disabled_triggers
+        contact_groups                  admins
+}
+
+define service {
+        use                             generic-service
+        service_description             Check PostgreSQL hitratio
+        hostgroup_name                  postgresql
+        check_command                   check_postgres!zulip!nagios!hitratio
+        contact_groups                  admins
+}
+
+# define service {
+#         use                             generic-service
+#         service_description             Check PostgreSQL locks
+#         hostgroup_name                  postgresql
+#         check_command                   check_postgres_alert_args!zulip!nagios!locks!400!600
+#         contact_groups                  admins
+# }
+
+define service {
+        use                             generic-service
+        service_description             Check PostgreSQL query_time
+        hostgroup_name                  postgresql
+        check_command                   check_postgres_alert_args!zulip!nagios!query_time!20 seconds!40 seconds
+        contact_groups                  admins
+}
+
+define service {
+        use                             generic-service
+        service_description             Check PostgreSQL sequence
+        hostgroup_name                  postgresql
+        check_command                   check_postgres!zulip!nagios!sequence
+        contact_groups                  admins
+}
+
+define service {
+        use                             generic-service
+        service_description             Check PostgreSQL timesync
+        hostgroup_name                  postgresql
+        check_command                   check_postgres!zulip!nagios!timesync
+        contact_groups                  admins
+}
+
+# define service {
+#         use                             generic-service
+#         service_description             Check PostgreSQL txn_idle
+#         hostgroup_name                  postgresql
+#         check_command                   check_postgres_alert_args!zulip!nagios!txn_idle!20 seconds!40 seconds
+#         contact_groups                  admins
+# }
+
+define service {
+        use                             generic-service
+        service_description             Check PostgreSQL txn_time
+        hostgroup_name                  postgresql
+        check_command                   check_postgres_alert_args!zulip!nagios!txn_time!20 seconds!40 seconds
+        contact_groups                  admins
+}
+
+define service {
+        use                             generic-service
+        service_description             Check FTS update log length
+        hostgroup_name                  postgresql_primary
+        check_command                   check_fts_update_log
+        contact_groups                  admins
+}
+
+define service {
+        use                             generic-service
+        service_description             Check PostgreSQL replication lag
+        hostgroup_name                  postgresql_primary, postgresql_replica
+        check_command                   check_postgresql_replication_lag
+        contact_groups                  admins
+}
+
+
+#### Redis
+
+define service {
+        use                             generic-service
+        service_description             Check redis service
+        # Both redis and frontends hostgroups, since frontends SSH proxy redis to themselves
+        hostgroup_name                  prod_frontends, fullstack, redis
+        check_command                   check_redis_ssh!22
+        max_check_attempts              3
+        contact_groups                  page_admins
+}
+
+define service {
+        use                             generic-service
+        service_description             Check redis service
+        hostgroup_name                  staging_frontends
+        check_command                   check_redis_ssh!22
+        max_check_attempts              3
+        contact_groups                  ops_message
+}
+
+#### RabbitMQ / queue workers
+
+define service {
+        use                             generic-service
+        service_description             Check RabbitMQ queue sizes
+        hostgroup_name                  prod_frontends, fullstack
+        check_command                   check_rabbitmq_queues!22
+        # Workaround weird checks 40s after first error causing alerts
+        # from a single failure because cron hasn't run again yet
+        max_check_attempts              3
+        contact_groups                  page_admins_and_warn_czo
+}
+
+define service {
+        use                             generic-service
+        service_description             Check RabbitMQ queue sizes
+        hostgroup_name                  staging_frontends
+        check_command                   check_rabbitmq_queues!22
+        # Workaround weird checks 40s after first error causing alerts
+        # from a single failure because cron hasn't run again yet
+        max_check_attempts              3
+        contact_groups                  ops_message
+}
+
+define service {
+        name                            rabbitmq-consumer-service
+        use                             generic-service
+        service_description             RabbitMQ consumer check template
+        hostgroup_name                  frontends
+        # Workaround weird checks 40s after first error causing alerts
+        # from a single failure because cron hasn't run again yet
+        max_check_attempts              3
+        contact_groups                  admins
+        register                        0
+}
+
+define service {
+        use                             rabbitmq-consumer-service
+        service_description             Check RabbitMQ notify_tornado consumers
+        hostgroup_name                  prod_frontends, fullstack
+        check_command                   check_rabbitmq_consumers!notify_tornado
+        contact_groups                  page_admins
+}
+
+define service {
+        use                             rabbitmq-consumer-service
+        service_description             Check RabbitMQ notify_tornado consumers
+        hostgroup_name                  staging_frontends
+        check_command                   check_rabbitmq_consumers!notify_tornado
+        contact_groups                  admins
+}
+
+define service {
+        use                             rabbitmq-consumer-service
+        service_description             Check RabbitMQ deferred_work consumers
+        check_command                   check_rabbitmq_consumers!deferred_work
+}
+
+define service {
+        use                             rabbitmq-consumer-service
+        service_description             Check RabbitMQ digest digest_emails consumers
+        check_command                   check_rabbitmq_consumers!digest_emails
+}
+
+define service {
+        use                             rabbitmq-consumer-service
+        service_description             Check RabbitMQ email_mirror consumers
+        check_command                   check_rabbitmq_consumers!email_mirror
+}
+
+define service {
+        use                             rabbitmq-consumer-service
+        service_description             Check RabbitMQ email_senders consumers
+        check_command                   check_rabbitmq_consumers!email_senders
+}
+
+define service {
+        use                             rabbitmq-consumer-service
+        service_description             Check RabbitMQ deferred_email_senders consumers
+        check_command                   check_rabbitmq_consumers!deferred_email_senders
+}
+
+define service {
+        use                             rabbitmq-consumer-service
+        service_description             Check RabbitMQ embed_links consumers
+        check_command                   check_rabbitmq_consumers!embed_links
+
+}
+
+define service {
+        use                             rabbitmq-consumer-service
+        service_description             Check RabbitMQ embedded_bots consumers
+        check_command                   check_rabbitmq_consumers!embedded_bots
+}
+
+define service {
+        use                             rabbitmq-consumer-service
+        service_description             Check RabbitMQ missedmessage_emails consumers
+        check_command                   check_rabbitmq_consumers!missedmessage_emails
+}
+
+define service {
+        use                             rabbitmq-consumer-service
+        service_description             Check RabbitMQ missedmessage_mobile_notifications consumers
+        check_command                   check_rabbitmq_consumers!missedmessage_mobile_notifications
+}
+
+define service {
+        use                             rabbitmq-consumer-service
+        service_description             Check RabbitMQ outgoing_webhooks consumers
+        check_command                   check_rabbitmq_consumers!outgoing_webhooks
+}
+
+define service {
+        use                             rabbitmq-consumer-service
+        service_description             Check RabbitMQ soft_reactivation consumers
+        check_command                   check_rabbitmq_consumers!soft_reactivation
+}
+
+define service {
+        use                             rabbitmq-consumer-service
+        service_description             Check RabbitMQ thumbnail consumers
+        check_command                   check_rabbitmq_consumers!thumbnail
+
+}
+
+define service {
+        use                             rabbitmq-consumer-service
+        service_description             Check RabbitMQ user_activity consumers
+        check_command                   check_rabbitmq_consumers!user_activity
+}
+
+define service {
+        use                             rabbitmq-consumer-service
+        service_description             Check RabbitMQ user_activity_interval consumers
+        check_command                   check_rabbitmq_consumers!user_activity_interval
+}
+
+define service {
+        use                             generic-service
+        service_description             Check worker memory usage
+        hostgroup_name                  frontends
+        check_command                   check_worker_memory
+        contact_groups                  admins
+}
+
+define service {
+        use                             generic-service
+        service_description             Check for queue worker errors.
+        hostgroup_name                  frontends
+        check_command                   check_queue_worker_errors!22
+        contact_groups                  admins
+}
+
+
+
+### Smokescreen
+
+define service {
+        use                             generic-service
+        service_description             Check Smokescreen proxy
+        hostgroup_name                  smokescreen
+        check_command                   check_proxy_status
+        contact_groups                  page_admins
+}

puppet/kandra/files/nagios4/conf.d/timeperiods_nagios2.cfg

@@ -0,0 +1,50 @@
+###############################################################################
+# timeperiods.cfg
+###############################################################################
+
+# This defines a timeperiod where all times are valid for checks,
+# notifications, etc.  The classic "24x7" support nightmare. :-)
+
+define timeperiod{
+        timeperiod_name 24x7
+        alias           24 Hours A Day, 7 Days A Week
+        sunday          00:00-24:00
+        monday          00:00-24:00
+        tuesday         00:00-24:00
+        wednesday       00:00-24:00
+        thursday        00:00-24:00
+        friday          00:00-24:00
+        saturday        00:00-24:00
+        }
+
+# Here is a slightly friendlier period during work hours
+define timeperiod{
+        timeperiod_name workhours
+        alias           Standard Work Hours
+        monday          09:00-17:00
+        tuesday         09:00-17:00
+        wednesday       09:00-17:00
+        thursday        09:00-17:00
+        friday          09:00-17:00
+        }
+
+# The complement of workhours
+define timeperiod{
+        timeperiod_name nonworkhours
+        alias           Non-Work Hours
+        sunday          00:00-24:00
+        monday          00:00-09:00,17:00-24:00
+        tuesday         00:00-09:00,17:00-24:00
+        wednesday       00:00-09:00,17:00-24:00
+        thursday        00:00-09:00,17:00-24:00
+        friday          00:00-09:00,17:00-24:00
+        saturday        00:00-24:00
+        }
+
+# This one is a favorite: never :)
+define timeperiod{
+        timeperiod_name never
+        alias           Never
+        }
+
+# end of file

puppet/kandra/files/nagios4/resource.cfg

@@ -0,0 +1,31 @@
+###########################################################################
+#
+# RESOURCE.CFG - Resource File for Nagios
+#
+# You can define $USERx$ macros in this file, which can in turn be used
+# in command definitions in your host config file(s).  $USERx$ macros are
+# useful for storing sensitive information such as usernames, passwords,
+# etc.  They are also handy for specifying the path to plugins and
+# event handlers - if you decide to move the plugins or event handlers to
+# a different directory in the future, you can just update one or two
+# $USERx$ macros, instead of modifying a lot of command definitions.
+#
+# The CGIs will not attempt to read the contents of resource files, so
+# you can set restrictive permissions (600 or 660) on them.
+#
+# Nagios supports up to 32 $USERx$ macros ($USER1$ through $USER32$)
+#
+# Resource files may also be used to store configuration directives for
+# external data sources like MySQL...
+#
+###########################################################################
+
+# Sets $USER1$ to be the path to the plugins
+$USER1$=/usr/lib/nagios/plugins
+
+# Sets $USER2$ to be the path to event handlers
+#$USER2$=/usr/lib/nagios/plugins/eventhandlers
+
+# Store some usernames and passwords (hidden from the CGIs)
+#$USER3$=someuser
+#$USER4$=somepassword

puppet/kandra/files/nagios4/zuliprc

@@ -0,0 +1,7 @@
+# Zulip, Inc's internal nagios plugin configuration.
+# The plugin and example config are under api/integrations/
+
+[api]
+email = nagios-bot@zulip.com
+key = xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+site = https://staging.zulip.com

puppet/kandra/files/nagios_ssh_config

@@ -0,0 +1,6 @@
+Host *
+  ControlMaster no
+  ControlPath /tmp/ssh-%C
+
+  ServerAliveInterval 30
+  ServerAliveCountMax 3

puppet/kandra/files/needrestart/zulip.conf

@@ -0,0 +1,17 @@
+# -*-cperl-*-
+my @ignore = (
+    qr/^memcached\.service$/,
+    qr/^nginx\.service$/,
+    qr/^postgresql(@[0-9a-zA-Z_-]+)?.service$/,
+    qr/^epmd\.service$/,
+    qr/^rabbitmq-server\.service$/,
+    qr/^redis-server\.service$/,
+    qr/^supervisor\.service$/,
+    qr/^teleport(\w*)\.service$/,
+);
+
+$nrconf{override_rc}{$_} = 0 for @ignore;
+
+# ksplice keeps the kernel updated in a way that needrestart does not
+# understand; silence these hints
+$nrconf{kernelhints} = -1

puppet/kandra/files/nginx/sites-available/zulip

@@ -0,0 +1,35 @@
+server {
+    # If coming from localhost, we do allow access to internal
+    # APIs over HTTP.
+    listen 127.0.0.1:80;
+    listen [::1]:80;
+
+    location /api/internal/ {
+        include /etc/nginx/zulip-include/api_headers;
+        include uwsgi_params;
+    }
+}
+
+include /etc/nginx/zulip-include/trusted-ip;
+include /etc/nginx/zulip-include/trusted-proto;
+include /etc/nginx/zulip-include/s3-cache;
+include /etc/nginx/zulip-include/upstreams;
+include /etc/zulip/nginx_sharding_map.conf;
+
+server {
+    listen 443 ssl http2;
+    listen [::]:443 ssl http2;
+
+    # This server is behind an ALB, which does not check the
+    # certificate validity:
+    # https://kevin.burke.dev/kevin/aws-alb-validation-tls-reply/
+    #
+    # Snakeoil verts are good for 10 years after initial creation, but
+    # the ALBs don't even check expiration. ¯\_(ツ)_/¯
+    ssl_certificate /etc/ssl/certs/ssl-cert-snakeoil.pem;
+    ssl_certificate_key /etc/ssl/private/ssl-cert-snakeoil.key;
+
+    server_name zulipchat.com *.zulipchat.com;
+
+    include /etc/nginx/zulip-include/app;
+}

puppet/kandra/files/nginx/sites-available/zulip-staging

@@ -0,0 +1,35 @@
+server {
+    # If coming from localhost, we do allow access to internal
+    # APIs over HTTP.
+    listen 127.0.0.1:80;
+    listen [::1]:80;
+
+    location /api/internal/ {
+        include /etc/nginx/zulip-include/api_headers;
+        include uwsgi_params;
+    }
+}
+
+include /etc/nginx/zulip-include/trusted-ip;
+include /etc/nginx/zulip-include/trusted-proto;
+include /etc/nginx/zulip-include/s3-cache;
+include /etc/nginx/zulip-include/upstreams;
+include /etc/zulip/nginx_sharding_map.conf;
+
+server {
+    listen 443 ssl http2;
+    listen [::]:443 ssl http2;
+
+    # This server is behind an ALB, which does not check the
+    # certificate validity:
+    # https://kevin.burke.dev/kevin/aws-alb-validation-tls-reply/
+    #
+    # Snakeoil verts are good for 10 years after initial creation, but
+    # the ALBs don't even check expiration. ¯\_(ツ)_/¯
+    ssl_certificate /etc/ssl/certs/ssl-cert-snakeoil.pem;
+    ssl_certificate_key /etc/ssl/private/ssl-cert-snakeoil.key;
+
+    server_name zulipstaging.com;
+
+    include /etc/nginx/zulip-include/app;
+}

puppet/kandra/files/nginx/zulip-include-app.d/navigation-tour-video.conf

@@ -0,0 +1,9 @@
+location /static/navigation-tour-video/ {
+    alias /home/zulip/prod-static/navigation-tour-video/;
+    error_page 404 /django_static_404.html;
+    include /etc/nginx/zulip-include/immutable_headers;
+}
+
+location = /static/navigation-tour-video/zulip-10.mp4 {
+    return 302 fffe1bb0baefab1a448fd9082738eaf271feb7ec247901242cd7bccb2280914b.mp4;
+}

puppet/kandra/files/nginx/zulip-include-app.d/well-known.conf

@@ -0,0 +1,3 @@
+location /.well-known/ {
+    alias /etc/zulip/well-known/;
+}

puppet/kandra/files/postgresql/pg_hba.conf

@@ -0,0 +1,106 @@
+# PostgreSQL Client Authentication Configuration File
+# ===================================================
+#
+# Refer to the "Client Authentication" section in the PostgreSQL
+# documentation for a complete description of this file.  A short
+# synopsis follows.
+#
+# This file controls: which hosts are allowed to connect, how clients
+# are authenticated, which PostgreSQL user names they can use, which
+# databases they can access.  Records take one of these forms:
+#
+# local      DATABASE  USER  METHOD  [OPTIONS]
+# host       DATABASE  USER  ADDRESS  METHOD  [OPTIONS]
+# hostssl    DATABASE  USER  ADDRESS  METHOD  [OPTIONS]
+# hostnossl  DATABASE  USER  ADDRESS  METHOD  [OPTIONS]
+#
+# (The uppercase items must be replaced by actual values.)
+#
+# The first field is the connection type: "local" is a Unix-domain
+# socket, "host" is either a plain or SSL-encrypted TCP/IP socket,
+# "hostssl" is an SSL-encrypted TCP/IP socket, and "hostnossl" is a
+# plain TCP/IP socket.
+#
+# DATABASE can be "all", "sameuser", "samerole", "replication", a
+# database name, or a comma-separated list thereof. The "all"
+# keyword does not match "replication". Access to replication
+# must be enabled in a separate record (see example below).
+#
+# USER can be "all", a user name, a group name prefixed with "+", or a
+# comma-separated list thereof.  In both the DATABASE and USER fields
+# you can also write a file name prefixed with "@" to include names
+# from a separate file.
+#
+# ADDRESS specifies the set of hosts the record matches.  It can be a
+# host name, or it is made up of an IP address and a CIDR mask that is
+# an integer (between 0 and 32 (IPv4) or 128 (IPv6) inclusive) that
+# specifies the number of significant bits in the mask.  A host name
+# that starts with a dot (.) matches a suffix of the actual host name.
+# Alternatively, you can write an IP address and netmask in separate
+# columns to specify the set of hosts.  Instead of a CIDR-address, you
+# can write "samehost" to match any of the server's own IP addresses,
+# or "samenet" to match any address in any subnet that the server is
+# directly connected to.
+#
+# METHOD can be "trust", "reject", "md5", "password", "gss", "sspi",
+# "krb5", "ident", "peer", "pam", "ldap", "radius" or "cert".  Note that
+# "password" sends passwords in clear text; "md5" is preferred since
+# it sends encrypted passwords.
+#
+# OPTIONS are a set of options for the authentication in the format
+# NAME=VALUE.  The available options depend on the different
+# authentication methods -- refer to the "Client Authentication"
+# section in the documentation for a list of which options are
+# available for which authentication methods.
+#
+# Database and user names containing spaces, commas, quotes and other
+# special characters must be quoted.  Quoting one of the keywords
+# "all", "sameuser", "samerole" or "replication" makes the name lose
+# its special character, and just match a database or username with
+# that name.
+#
+# This file is read on server startup and when the postmaster receives
+# a SIGHUP signal.  If you edit the file on a running system, you have
+# to SIGHUP the postmaster for the changes to take effect.  You can
+# use "pg_ctl reload" to do that.
+
+# Put your actual configuration here
+# ----------------------------------
+#
+# If you want to allow non-local connections, you need to add more
+# "host" records.  In that case you will also need to make PostgreSQL
+# listen on a non-local interface via the listen_addresses
+# configuration parameter, or via the -i or -h command line switches.
+
+
+
+
+# DO NOT DISABLE!
+# If you change this first entry you will need to make sure that the
+# database superuser can access the database using some other method.
+# Noninteractive access to all databases is required during automatic
+# maintenance (custom daily cronjobs, replication, and similar tasks).
+#
+# Database administrative login by Unix domain socket
+local   all             postgres                                peer
+
+# TYPE  DATABASE        USER            ADDRESS                 METHOD
+
+# "local" is for Unix domain socket connections only
+local   all             all                                     peer
+# IPv4 local connections:
+#host    all             all             127.0.0.1/32            md5
+# IPv6 local connections:
+#host    all             all             ::1/128                 md5
+# Allow replication connections from localhost, by a user with the
+# replication privilege.
+#local   replication     postgres                                peer
+#host    replication     postgres        127.0.0.1/32            md5
+#host    replication     postgres        ::1/128                 md5
+
+# Local connections with certs from Teleport
+hostssl zulip           zulip           127.0.0.1/32            cert
+# Zulip app frontends access the zulip user
+hostssl zulip           zulip           172.31.0.0/16           cert
+# Zulip replication accesses the replicator user
+hostssl replication     replicator      172.31.0.0/16           cert

puppet/kandra/files/postgresql/setup_data.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+set -eux
+
+service postgresql stop
+
+cert_file="$(crudini --get /etc/zulip/zulip.conf postgresql ssl_cert_file)"
+if [ -z "$cert_file" ] || [ ! -f "$cert_file" ]; then
+    echo "Certificate file is not set or does not exist!"
+    exit 1
+fi
+
+key_file="$(crudini --get /etc/zulip/zulip.conf postgresql ssl_key_file)"
+if [ -z "$key_file" ] || [ ! -f "$key_file" ]; then
+    echo "Key file is not set or does not exist!"
+    exit 1
+fi
+
+cert_cn="$(openssl x509 -noout -subject -in "$cert_file" | sed -n '/^subject/s/^.*CN\s*=\s*//p')"
+
+if [ "$cert_cn" != "$(hostname)" ]; then
+    echo "Configured certificate does not match host!"
+    exit 1
+fi
+
+echo "Checking for S3 secrets..."
+crudini --get /etc/zulip/zulip-secrets.conf secrets s3_region >/dev/null
+crudini --get /etc/zulip/zulip-secrets.conf secrets s3_backups_bucket >/dev/null
+crudini --get /etc/zulip/zulip-secrets.conf secrets s3_backups_key >/dev/null
+crudini --get /etc/zulip/zulip-secrets.conf secrets s3_backups_secret_key >/dev/null
+
+if [ ! -f "/var/lib/postgresql/.postgresql/postgresql.crt" ]; then
+    echo "Replication certificate file is not set or does not exist!"
+    exit 1
+fi
+if [ ! -f "/var/lib/postgresql/.postgresql/postgresql.key" ]; then
+    echo "Replication key file is not set or does not exist!"
+    exit 1
+fi
+
+version="$(crudini --get /etc/zulip/zulip.conf postgresql version)"
+mkdir -p "/srv/data/postgresql/$version"
+chown postgres.postgres "/srv/data/postgresql/$version"
+chmod 700 "/srv/data/postgresql/$version"
+
+/usr/local/bin/env-wal-g backup-fetch "/var/lib/postgresql/$version/main" LATEST
+chown -R postgres.postgres "/var/lib/postgresql/$version/main"

puppet/kandra/files/postgresql/setup_disks.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+set -x
+set -e
+
+set -o pipefail
+
+LOCALDISK=$(
+    nvme list -o json \
+        | jq -r '.Devices[] | select(.ModelNumber | contains("Instance Storage")) | .DevicePath' \
+        | head -n1
+)
+
+if [ -z "$LOCALDISK" ]; then
+    echo "No instance storage found!"
+    nvme list
+    exit 1
+fi
+
+if ! grep -q "$LOCALDISK" /etc/fstab; then
+    echo "$LOCALDISK   /srv/data  xfs    nofail,noatime 1 1" >>/etc/fstab
+fi
+
+if [ ! -d /srv/data ]; then
+    mkdir /srv/data
+fi
+
+if ! mountpoint -q /srv/data; then
+    mkfs.xfs "$LOCALDISK"
+    mount /srv/data
+fi
+
+if [ ! -L /var/lib/postgresql ]; then
+    service postgresql stop
+    if [ -e /var/lib/postgresql ]; then
+        mv /var/lib/postgresql "/root/postgresql-data-$(date +'%m-%d-%Y-%T')"
+    fi
+    ln -s /srv/data/postgresql/ /var/lib
+fi
+
+if [ ! -e "/srv/data/postgresql" ]; then
+    service postgresql stop
+    mkdir "/srv/data/postgresql"
+    chown postgres:postgres /srv/data/postgresql
+fi

puppet/kandra/files/process_exporter.yaml

@@ -0,0 +1,31 @@
+process_names:
+  - name: "tornado-{{.Matches.Port}}"
+    comm:
+      - python3
+    cmdline:
+      - runtornado\s+127\.0\.0\.1:(?P<Port>\S+)
+  - name: "uwsgi-{{.Matches.Worker}}"
+    comm:
+      - uwsgi
+    cmdline:
+      - zulip-django uWSGI worker (?P<Worker>\d+)
+  - name: process_fts_updates
+    comm:
+      - python3
+    cmdline:
+      - /usr/local/bin/process_fts_updates
+  - name: email_server
+    comm:
+      - python3
+    cmdline:
+      - /home/zulip/deployments/current/manage.py email_server
+  - name: "worker-{{.Matches.Queue}}"
+    comm:
+      - python3
+    cmdline:
+      - /home/zulip/deployments/current/manage.py process_queue --queue_name=(?P<Queue>\S+)
+  - name: "deliver-scheduled-{{.Matches.What}}"
+    comm:
+      - python3
+    cmdline:
+      - /home/zulip/deployments/current/manage.py deliver_scheduled_(?P<What>\S+)

puppet/kandra/files/statuspage-pusher

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+
+import configparser
+import http.client
+import json
+import logging
+import time
+from urllib.parse import urlencode
+
+
+def fetch_metric(metric_id: str, query: str) -> float:
+    logging.info("Fetching %s", metric_id)
+    params = urlencode({"query": query})
+    conn = http.client.HTTPConnection("localhost:9090")
+    conn.request("GET", "/api/v1/query?" + params)
+    response = conn.getresponse()
+    if response.status != 200:
+        raise Exception(
+            f"Failed to request {metric_id} from Prometheus: {response.status} {response.reason}"
+        )
+    raw_bytes = response.read()
+    conn.close()
+    data = json.loads(raw_bytes)
+    if data["status"] != "success":
+        raise Exception(f"Failed to request {metric_id}: {raw_bytes.decode()}")
+    return float(data["data"]["result"][0]["value"][1])
+
+
+def push_metric(metric_id: str, metric_value: float, page_id: str, oauth_token: str) -> None:
+    conn = http.client.HTTPSConnection("api.statuspage.io")
+    params = json.dumps({"data": {"timestamp": time.time(), "value": metric_value}})
+    headers = {"Content-Type": "application/json", "Authorization": "OAuth " + oauth_token}
+    conn.request(
+        "POST", "/v1/pages/" + page_id + "/metrics/" + metric_id + "/data.json", params, headers
+    )
+    response = conn.getresponse()
+    if not (response.status >= 200 and response.status < 300):
+        raise Exception(
+            f"Failed to push {metric_id} to statuspage.io: {response.status} {response.reason}"
+        )
+    logging.info("Wrote %s: %s", metric_id, response.read().decode())
+    conn.close()
+
+
+def update_metric(metric_id: str, query: str, page_id: str, oauth_token: str) -> None:
+    metric_value = fetch_metric(metric_id, query)
+    push_metric(metric_id, metric_value, page_id, oauth_token)
+
+
+def main() -> None:
+    logging.basicConfig(format="%(asctime)s statuspage-pusher: %(message)s", level=logging.INFO)
+
+    secrets_file = configparser.RawConfigParser()
+    secrets_file.read("/etc/zulip/zulip-secrets.conf")
+
+    oauth_token = secrets_file.get("secrets", "statuspage_token")
+    if oauth_token is None:
+        raise RuntimeError("statuspage_token secret is required")
+
+    config_file = configparser.RawConfigParser()
+    config_file.read("/etc/zulip/zulip.conf")
+    page_id = config_file.get("statuspage", "page_id")
+    if page_id is None:
+        raise RuntimeError("statuspage.page_id in zulip.conf is required")
+
+    metrics_file = configparser.RawConfigParser()
+    metrics_file.read("/etc/zulip/statuspage.conf")
+    metrics = metrics_file["metrics"]
+
+    while True:
+        for metric_id, query in metrics.items():
+            try:
+                update_metric(metric_id, query, page_id, oauth_token)
+            except Exception:
+                logging.exception("Error while updating metric")
+        time.sleep(30)
+
+
+if __name__ == "__main__":
+    main()

puppet/kandra/files/tbot.service

@@ -0,0 +1,18 @@
+[Unit]
+Description=tbot - Teleport Machine & Workload Identity Service
+After=network.target
+
+[Service]
+Type=simple
+User=teleport
+Group=teleport
+Restart=always
+RestartSec=5
+Environment="TELEPORT_ANONYMOUS_TELEMETRY=0"
+ExecStart=/usr/local/bin/tbot start -c /etc/tbot.yaml --pid-file=/var/lib/teleport/bot/pid
+ExecReload=/bin/kill -HUP $MAINPID
+PIDFile=/var/lib/teleport/bot/pid
+LimitNOFILE=524288
+
+[Install]
+WantedBy=multi-user.target

puppet/kandra/files/tbot.yaml

@@ -0,0 +1,24 @@
+version: v2
+proxy_server: teleport.zulipchat.net:443
+onboarding:
+  join_method: iam
+  # This "token" is defined in Teleport, and links to the AWS role
+  # https://teleport.zulipchat.net/web/tokens
+  # https://goteleport.com/docs/reference/deployment/join-methods/#aws-iam-role-iam
+  # https://goteleport.com/docs/installation/agents/aws-iam/
+  token: prometheus-scrape-bot
+  ca_pins:
+    - "sha256:062db37249ea74c450579da8f02043b317cb8a174d653bb5090a89752d68efe7"
+storage:
+  type: directory
+  path: /var/lib/teleport/bot
+services:
+  # Prometheus proxies all exporter traffic through this proxy
+  - type: application-proxy
+    listen: tcp://127.0.0.1:8080
+outputs:
+  # This is consumed by teleport-sd to list exporters
+  - type: identity
+    destination:
+      type: directory
+      path: /var/lib/teleport/bot-identity

puppet/kandra/files/teleport-aws-credentials

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+# This uses AWS IAM Roles Anywhere[1] to acquire the NonAWSInstance
+# role for the whole server.  It does this by having configured the
+# Teleport CA to be a "trust anchor," allowing the Teleport host
+# certificate to be used for authentication; the NonAWSInstance role
+# itself has a trust relationship which lists the CA as sufficient to
+# authorize for the role.
+#
+# [1]: https://docs.aws.amazon.com/rolesanywhere/latest/userguide
+
+set -eu
+
+# Check if we have cached credentials; they're good for 60 minutes, so
+# check within 55min.
+cache="/etc/aws-credential-cache.json"
+if [ -s "$cache" ] && [ "$(find "$cache" -mmin -55)" ]; then
+    cat "$cache"
+    exit 0
+fi
+
+# Extract the current host cert and key from the Teleport database.
+# These are nominally static for the lifetime of the host, but there's
+# little reason to not refresh them on every run, in case they get
+# rotated.
+
+teleport_json=$(
+    sqlite3 /var/lib/teleport/proc/sqlite.db "select value from kv where key = '/ids/node/current'"
+)
+
+# $teleport_json is a secret (it has the host key) so extract it
+# carefully (i.e. not using `echo $teleport_json"`")
+jq -r .spec.tls_cert <<<"$teleport_json" | base64 -d >/var/lib/teleport/host.crt
+jq -r .spec.key <<<"$teleport_json" | base64 -d >/var/lib/teleport/host.key
+
+# Write the cache out
+/srv/zulip-aws-tools/bin/aws_signing_helper credential-process \
+    --certificate /var/lib/teleport/host.crt \
+    --private-key /var/lib/teleport/host.key \
+    "$@" >"$cache"
+
+cat "$cache"

puppet/kandra/files/teleport_app.yaml

@@ -0,0 +1,3 @@
+app_service:
+  enabled: yes
+  apps:

puppet/kandra/files/teleport_server.yaml

@@ -0,0 +1,32 @@
+# See https://goteleport.com/docs/config-reference/ and
+# https://goteleport.com/docs/admin-guide/#configuration
+
+version: v3
+teleport:
+  ca_pin: "sha256:062db37249ea74c450579da8f02043b317cb8a174d653bb5090a89752d68efe7"
+  data_dir: /var/lib/teleport-server
+
+auth_service:
+  enabled: "yes"
+  proxy_listener_mode: multiplex
+  listen_addr: 127.0.0.1:3025
+  cluster_name: teleport.zulipchat.net
+  authentication:
+    type: local
+    second_factor: on
+    signature_algorithm_suite: balanced-v1
+    u2f:
+      app_id: https://teleport.zulipchat.net
+    webauthn:
+      rp_id: teleport.zulipchat.net
+
+proxy_service:
+  enabled: "yes"
+  web_listen_addr: "[::]:443"
+  public_addr: teleport.zulipchat.net:443
+  acme:
+    enabled: "yes"
+    email: zulip-ops@zulip.com
+
+ssh_service:
+  enabled: no

puppet/kandra/files/weblate_exporter

@@ -0,0 +1,275 @@
+#!/usr/bin/env python3
+
+import argparse
+import configparser
+import json
+import logging
+import os
+import sys
+import time
+from collections.abc import Callable
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from typing import Any
+from urllib.request import Request, urlopen
+
+DEFAULT_PORT = 9189
+PROJECT = "zulip"
+COMPONENTS = ["frontend", "django", "desktop", "zulip-flutter"]
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
+logger = logging.getLogger(__name__)
+
+
+class WeblateMetricsCollector:
+    def __init__(self, token: str) -> None:
+        self.token = token
+
+    def make_request(self, endpoint: str) -> dict[str, Any]:
+        request = Request(f"https://hosted.weblate.org/{endpoint}")
+        request.add_header("Authorization", f"Token {self.token}")
+        request.add_header("Accept", "application/json")
+        request.add_header("User-Agent", "WeblatePrometheusExporter/1.0")
+        with urlopen(request, timeout=30) as response:
+            return json.loads(response.read().decode("utf-8"))
+
+    def fetch_component_languages(self, project: str, component: str) -> list[str]:
+        endpoint = f"api/components/{project}/{component}/translations/"
+        try:
+            response = self.make_request(endpoint)
+            logger.debug("Fetched translations for %s/%s: %s", project, component, response)
+
+            # Extract language codes from the paginated response
+            languages = []
+            if response.get("results"):
+                languages.extend(
+                    [
+                        translation["language"]["code"]
+                        for translation in response["results"]
+                        if "language" in translation and "code" in translation["language"]
+                    ]
+                )
+
+                logger.debug("Found languages for %s/%s: %s", project, component, languages)
+                return languages
+            else:
+                logger.warning("No translations found for %s/%s", project, component)
+                return []
+        except Exception as e:
+            logger.error("Failed to fetch languages for %s/%s: %s", project, component, e)
+            return []
+
+    def fetch_translation_statistics(
+        self, project: str, component: str, language: str
+    ) -> dict[str, Any] | None:
+        endpoint = f"api/translations/{project}/{component}/{language}/statistics/"
+        try:
+            stats = self.make_request(endpoint)
+            logger.debug("Fetched stats for %s/%s/%s: %s", project, component, language, stats)
+            return stats
+        except Exception as e:
+            logger.error(
+                "Failed to fetch statistics for %s/%s/%s: %s", project, component, language, e
+            )
+            return None
+
+    def collect_all_metrics(self) -> dict[str, dict[str, dict[str, Any]]]:
+        metrics = {}
+
+        for component in COMPONENTS:
+            logger.info("Fetching statistics for %s/%s", PROJECT, component)
+
+            languages = self.fetch_component_languages(PROJECT, component)
+            if not languages:
+                logger.warning("No languages found for %s/%s", PROJECT, component)
+                continue
+
+            component_metrics = {}
+            for language in languages:
+                logger.info("Fetching statistics for %s/%s/%s", PROJECT, component, language)
+                stats = self.fetch_translation_statistics(PROJECT, component, language)
+
+                if stats:
+                    component_metrics[language] = stats
+                else:
+                    logger.warning(
+                        "No statistics available for %s/%s/%s", PROJECT, component, language
+                    )
+
+            if component_metrics:
+                metrics[component] = component_metrics
+            else:
+                logger.warning("No translation statistics available for component %s", component)
+
+        logger.info("Collected metrics for %d components", len(metrics))
+        return metrics
+
+    def format_prometheus_metrics(self, metrics_data: dict[str, dict[str, dict[str, Any]]]) -> str:
+        if not metrics_data:
+            return "# No metrics data available\n"
+
+        lines = [
+            "# HELP weblate_translation_info Translation information",
+            "# TYPE weblate_translation_info gauge",
+        ]
+
+        for component, languages in metrics_data.items():
+            for language, stats in languages.items():
+                lines.append(
+                    f'weblate_translation_info{{component="{component}",language="{language}",name="{stats.get("name", f"{component}-{language}")}"}} 1'
+                )
+
+        metric_definitions = [
+            ("translated", "Number of translated strings"),
+            ("translated_words", "Number of translated words"),
+            ("translated_chars", "Number of translated characters"),
+            ("total", "Total number of strings"),
+            ("total_words", "Total number of words"),
+            ("total_chars", "Total number of characters"),
+            ("fuzzy", "Number of fuzzy strings"),
+            ("fuzzy_words", "Number of fuzzy words"),
+            ("fuzzy_chars", "Number of fuzzy characters"),
+            ("failing", "Number of failing checks"),
+            ("failing_words", "Number of words with failing checks"),
+            ("failing_chars", "Number of characters with failing checks"),
+            ("approved", "Number of approved strings"),
+            ("approved_words", "Number of approved words"),
+            ("approved_chars", "Number of approved characters"),
+            ("suggestions", "Number of suggestions"),
+            ("comments", "Number of comments"),
+            ("translated_percent", "Percentage of translated strings"),
+            ("translated_words_percent", "Percentage of translated words"),
+            ("translated_chars_percent", "Percentage of translated characters"),
+            ("approved_percent", "Percentage of approved strings"),
+            ("approved_words_percent", "Percentage of approved words"),
+            ("approved_chars_percent", "Percentage of approved characters"),
+        ]
+
+        for metric_key, description in metric_definitions:
+            lines.extend(
+                [
+                    f"# HELP weblate_{metric_key} {description}",
+                    f"# TYPE weblate_{metric_key} gauge",
+                ]
+            )
+            for component, languages in metrics_data.items():
+                for language, stats in languages.items():
+                    value = stats.get(metric_key, 0)
+                    lines.append(
+                        f'weblate_{metric_key}{{component="{component}",language="{language}"}} {value}'
+                    )
+
+        lines.extend(
+            [
+                "# HELP weblate_last_update_timestamp Unix timestamp of last metrics update",
+                "# TYPE weblate_last_update_timestamp gauge",
+                f"weblate_last_update_timestamp {time.time()}",
+            ]
+        )
+
+        return "\n".join(lines) + "\n"
+
+
+class PrometheusHandler(BaseHTTPRequestHandler):
+    def __init__(self, collector: WeblateMetricsCollector, *args: Any, **kwargs: Any) -> None:
+        self.collector = collector
+        super().__init__(*args, **kwargs)
+
+    def do_GET(self) -> None:
+        if self.path == "/metrics":
+            self.send_response(200)
+            self.send_header("Content-Type", "text/plain; charset=utf-8")
+            self.end_headers()
+
+            try:
+                metrics_data = self.collector.collect_all_metrics()
+                metrics = self.collector.format_prometheus_metrics(metrics_data)
+            except Exception as e:
+                logger.error("Error collecting metrics: %s", e)
+                metrics = f"# Error collecting metrics: {e}\n"
+
+            self.wfile.write(metrics.encode("utf-8"))
+            return
+
+        elif self.path == "/health":
+            self.send_response(200)
+            self.send_header("Content-Type", "text/plain")
+            self.end_headers()
+            self.wfile.write(b"OK\n")
+            return
+
+        elif self.path == "/":
+            self.send_response(200)
+            self.send_header("Content-Type", "text/html")
+            self.end_headers()
+            html = """<!DOCTYPE html>
+<html>
+<head><title>Weblate Prometheus Exporter</title></head>
+<body>
+<h1>Weblate Prometheus Exporter</h1>
+<p><a href="/metrics">Metrics</a></p>
+<p><a href="/health">Health Check</a></p>
+</body>
+</html>"""
+            self.wfile.write(html.encode("utf-8"))
+            return
+
+        else:
+            self.send_response(404)
+            self.end_headers()
+            self.wfile.write(b"Not Found\n")
+            return
+
+    def log_message(self, format: str, *args: Any) -> None:  # type: ignore[explicit-override]  # @override is available in typing_extensions, which is not core Python
+        logger.info("%s - %s", self.address_string(), format % args)
+
+
+def create_handler(
+    collector: WeblateMetricsCollector,
+) -> Callable[..., PrometheusHandler]:
+    def handler(*args: Any, **kwargs: Any) -> PrometheusHandler:
+        return PrometheusHandler(collector, *args, **kwargs)
+
+    return handler
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Weblate Prometheus Exporter")
+    parser.add_argument(
+        "--port",
+        type=int,
+        default=DEFAULT_PORT,
+        help=f"Port to listen on (default: {DEFAULT_PORT})",
+    )
+    parser.add_argument(
+        "--token",
+        default=os.getenv("WEBLATE_TOKEN"),
+        help="Weblate API token (can also be set via WEBLATE_TOKEN env var)",
+    )
+    parser.add_argument("--verbose", "-v", action="store_true", help="Enable verbose logging")
+
+    args = parser.parse_args()
+
+    if args.verbose:
+        logging.getLogger().setLevel(logging.DEBUG)
+
+    token = args.token
+    if args.token is None:
+        secrets_file = configparser.RawConfigParser()
+        secrets_file.read("/etc/zulip/zulip-secrets.conf")
+        token = secrets_file["secrets"]["weblate_api_key"]
+
+    handler = create_handler(WeblateMetricsCollector(token=token))
+    server = HTTPServer(("127.0.0.1", args.port), handler)
+
+    logger.info("Metrics available at http://localhost:%d/metrics", args.port)
+
+    try:
+        server.serve_forever()
+    except KeyboardInterrupt:
+        logger.info("Shutting down server...")
+        server.shutdown()
+        sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()

puppet/kandra/manifests/apache.pp

@@ -0,0 +1,40 @@
+class kandra::apache {
+  package { 'apache2':
+    ensure => installed,
+  }
+  service { 'apache2':
+    require => Package['apache2'],
+  }
+
+  apache2mod { [ 'headers', 'proxy', 'proxy_http', 'rewrite', 'auth_digest', 'ssl' ]:
+    ensure  => present,
+    require => Package['apache2'],
+    notify  => Service['apache2'],
+  }
+
+  file { '/etc/apache2/certs/':
+    ensure  => directory,
+    require => Package['apache2'],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0755',
+  }
+
+  file { '/etc/apache2/ports.conf':
+    ensure  => file,
+    require => Package[apache2],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0640',
+    source  => 'puppet:///modules/kandra/apache/ports.conf',
+    notify  => Service['apache2'],
+  }
+
+  file { '/etc/apache2/sites-available/':
+    ensure  => directory,
+    require => Package[apache2],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0750',
+  }
+}

puppet/kandra/manifests/app_frontend.pp

@@ -0,0 +1,82 @@
+class kandra::app_frontend {
+  include zulip::snakeoil
+  include zulip::app_frontend_base
+  include zulip::profile::memcached
+  include zulip::profile::rabbitmq
+  include zulip::local_mailserver
+  include zulip::hooks::sentry
+  include kandra::app_frontend_monitoring
+  include kandra::app_frontend_tmpfs
+
+  kandra::firewall_allow{ 'smtp': }
+  kandra::firewall_allow{ 'http': }
+  kandra::firewall_allow{ 'https': }
+
+  $redis_hostname = zulipconf('redis', 'hostname', undef)
+  group { 'redistunnel':
+    ensure => present,
+    gid    => '1080',
+  }
+  user { 'redistunnel':
+    ensure     => present,
+    uid        => '1080',
+    gid        => '1080',
+    groups     => ['zulip'],
+    shell      => '/bin/true',
+    home       => '/home/redistunnel',
+    managehome => true,
+  }
+  kandra::user_dotfiles { 'redistunnel':
+    keys        => true,
+    known_hosts => [$redis_hostname],
+  }
+  package { 'autossh': ensure => installed }
+  file { "${zulip::common::supervisor_conf_dir}/redis_tunnel.conf":
+    ensure  => file,
+    require => [
+      Package['supervisor', 'autossh'],
+      Kandra::User_Dotfiles['redistunnel'],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/supervisor/conf.d/redis_tunnel.conf.template.erb'),
+    notify  => Service['supervisor'],
+  }
+  # Need redis_password in its own file for Nagios
+  file { '/var/lib/nagios/redis_password':
+    ensure  => file,
+    mode    => '0600',
+    owner   => 'nagios',
+    group   => 'nagios',
+    content => zulipsecret('secrets', 'redis_password', ''),
+  }
+
+  # Mount /etc/zulip/well-known/ as /.well-known/
+  file { '/etc/nginx/zulip-include/app.d/well-known.conf':
+    require => File['/etc/nginx/zulip-include/app.d'],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    source  => 'puppet:///modules/kandra/nginx/zulip-include-app.d/well-known.conf',
+    notify  => Service['nginx'],
+  }
+
+  # Serve /static/navigation-tour-video/
+  file { '/etc/nginx/zulip-include/app.d/navigation-tour-video.conf':
+    require => File['/etc/nginx/zulip-include/app.d'],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    source  => 'puppet:///modules/kandra/nginx/zulip-include-app.d/navigation-tour-video.conf',
+    notify  => Service['nginx'],
+  }
+
+  # Each server does its own fetching of contributor data, since
+  # we don't have a way to synchronize that among several servers.
+  zulip::cron { 'fetch-contributor-data':
+    hour    => '8',
+    minute  => '0',
+    command => '/home/zulip/deployments/current/tools/fetch-contributor-data',
+  }
+}

puppet/kandra/manifests/app_frontend_monitoring.pp

@@ -0,0 +1,25 @@
+# @summary Prometheus monitoring of a Django frontend and RabbitMQ server.
+#
+class kandra::app_frontend_monitoring {
+  include kandra::prometheus::memcached
+  include kandra::prometheus::rabbitmq
+  include kandra::prometheus::uwsgi
+  include kandra::prometheus::process
+  include kandra::prometheus::grok
+  kandra::teleport::prometheus_app { 'tusd': port => '9900' }
+  kandra::teleport::prometheus_app { 'katex': port => '9700' }
+
+  file { '/etc/cron.d/rabbitmq-monitoring':
+    ensure => absent,
+  }
+  zulip::cron { 'check-rabbitmq-queue':
+    minute  => '*',
+    user    => 'root',
+    command => '/home/zulip/deployments/current/scripts/nagios/check-rabbitmq-queue',
+  }
+  zulip::cron { 'check-rabbitmq-consumers':
+    minute  => '*',
+    user    => 'root',
+    command => '/home/zulip/deployments/current/scripts/nagios/check-rabbitmq-consumers',
+  }
+}

puppet/kandra/manifests/app_frontend_tmpfs.pp

@@ -0,0 +1,43 @@
+class kandra::app_frontend_tmpfs {
+  $disk = '/dev/nvme1n1'
+  $partition = "${disk}p1"
+
+  exec { 'create partition table':
+    command => "/usr/sbin/parted -s ${disk} mklabel gpt",
+    unless  => "/usr/sbin/blkid -s PTUUID ${disk}",
+  }
+
+  exec { 'create partition':
+    command => "/usr/sbin/parted -a optimal -s ${disk} mkpart slack-tmp ext4 0% 100%",
+    unless  => "/usr/sbin/blkid ${partition}",
+    require => Exec['create partition table'],
+  }
+
+  exec { 'format partition':
+    command => "/sbin/mkfs.ext4 -L slack-tmp ${partition}",
+    unless  => "/sbin/blkid -t TYPE=ext4 ${partition}",
+    require => Exec['create partition'],
+  }
+
+  file { '/tmp/slack-conversions':
+    ensure => 'directory',
+    mode   => '0755',
+  }
+  mount { '/tmp/slack-conversions':
+    ensure  => 'mounted',
+    device  => $partition,
+    fstype  => 'ext4',
+    atboot  => true,
+    options => 'defaults,noatime,discard,commit=60,errors=remount-ro',
+    require => Exec['format partition'],
+  }
+
+  # We want ownership to be enforced after the mount, so the File
+  # directive above doesn't have user/group, and we manage it
+  # explicitly via chown.
+  exec { 'chown-slack-conversions':
+    command => '/bin/chown zulip:zulip /tmp/slack-conversions',
+    unless  => '/usr/bin/test "$(stat -c %U:%G /tmp/slack-conversions)" = "zulip:zulip"',
+    require => Mount['/tmp/slack-conversions'],
+  }
+}

puppet/kandra/manifests/aws_tools.pp

@@ -0,0 +1,90 @@
+# @summary Installs the AWS CLI
+#
+class kandra::aws_tools {
+  $is_ec2 = zulipconf('machine', 'hosting_provider', 'ec2') == 'ec2'
+
+  file { '/usr/local/bin/install-aws-cli':
+    ensure => file,
+    mode   => '0755',
+    source => 'puppet:///modules/kandra/install-aws-cli',
+  }
+  exec { 'install-aws-cli':
+    require => File['/usr/local/bin/install-aws-cli'],
+    command => '/usr/local/bin/install-aws-cli',
+    # When puppet is initially determining which resources need to be
+    # applied, it will call the unless -- but install-aws-cli may not
+    # exist yet.  Count this as needing to run.
+    unless  => '[ -f /usr/local/bin/install-aws-cli ] && /usr/local/bin/install-aws-cli check',
+  }
+
+  if ! $is_ec2 {
+    if $facts['os']['architecture'] == 'amd64' {
+      $archname = 'X86_64'
+    } else {
+      $archname = 'Aarch64'
+    }
+    $helper_version = $zulip::common::versions['aws_signing_helper']['version']
+    zulip::external_dep { 'aws_signing_helper':
+      version => $helper_version,
+      url     => "https://rolesanywhere.amazonaws.com/releases/${helper_version}/${archname}/Linux/Amzn2023/aws_signing_helper",
+      before  => File['/root/.aws/config'],
+    }
+    file { '/srv/zulip-aws-tools/bin/aws_signing_helper':
+      ensure  => link,
+      target  => "/srv/zulip-aws_signing_helper-${helper_version}",
+      require => [
+        File["/srv/zulip-aws_signing_helper-${helper_version}"],
+        Exec['install-aws-cli'],
+      ],
+      before  => Exec['Cleanup aws_signing_helper'],
+    }
+    file { '/usr/local/bin/teleport-aws-credentials':
+      ensure  => file,
+      require => [
+        Package['sqlite3'],
+        Service['teleport_node'],
+      ],
+      before  => [
+        File['/root/.aws/config'],
+      ],
+      mode    => '0755',
+      owner   => 'root',
+      group   => 'root',
+      source  => 'puppet:///modules/kandra/teleport-aws-credentials',
+    }
+  }
+  file { '/root/.aws':
+    ensure => directory,
+    mode   => '0755',
+    owner  => 'root',
+    group  => 'root',
+  }
+  $aws_trust_arn = zulipsecret('secrets','aws_trust_arn','')
+  $aws_profile_arn = zulipsecret('secrets','aws_profile_arn','')
+  $aws_role_arn = zulipsecret('secrets','aws_role_arn','')
+  file { '/root/.aws/config':
+    ensure  => file,
+    mode    => '0644',
+    owner   => 'root',
+    group   => 'root',
+    content => template('kandra/dotfiles/aws_config.erb'),
+  }
+
+  # Pull keys and authorized_keys from AWS secretsmanager
+  file { '/usr/local/bin/install-ssh-keys':
+    ensure  => file,
+    require => File['/root/.aws/config'],
+    mode    => '0755',
+    owner   => 'root',
+    group   => 'root',
+    source  => 'puppet:///modules/kandra/install-ssh-keys',
+  }
+  file { '/usr/local/bin/install-ssh-authorized-keys':
+    ensure  => file,
+    require => File['/root/.aws/config'],
+    mode    => '0755',
+    owner   => 'root',
+    group   => 'root',
+    source  => 'puppet:///modules/kandra/install-ssh-authorized-keys',
+  }
+}

puppet/kandra/manifests/camo.pp

@@ -0,0 +1,8 @@
+class kandra::camo {
+  class { 'zulip::camo':
+    listen_address => '0.0.0.0',
+  }
+
+  kandra::firewall_allow { 'camo': port => '9292' }
+  kandra::teleport::prometheus_app { 'camo': port => '9292' }
+}

puppet/kandra/manifests/firewall.pp

@@ -0,0 +1,56 @@
+class kandra::firewall {
+  package { 'iptables-persistent': }
+  concat { '/etc/iptables/rules.v4':
+    ensure  => present,
+    mode    => '0600',
+    require => Package['iptables-persistent'],
+  }
+  concat::fragment { 'iptables-header.v4':
+    target => '/etc/iptables/rules.v4',
+    source => 'puppet:///modules/kandra/iptables/header.v4',
+    order  => '01',
+  }
+  concat::fragment { 'iptables-trailer.v4':
+    target => '/etc/iptables/rules.v4',
+    source => 'puppet:///modules/kandra/iptables/trailer.v4',
+    order  => '99',
+  }
+
+  concat { '/etc/iptables/rules.v6':
+    ensure  => present,
+    mode    => '0600',
+    require => Package['iptables-persistent'],
+  }
+  concat::fragment { 'iptables-header.v6':
+    target => '/etc/iptables/rules.v6',
+    source => 'puppet:///modules/kandra/iptables/header.v6',
+    order  => '01',
+  }
+  concat::fragment { 'iptables-trailer.v6':
+    target => '/etc/iptables/rules.v6',
+    source => 'puppet:///modules/kandra/iptables/trailer.v6',
+    order  => '99',
+  }
+
+  service { 'netfilter-persistent':
+    ensure     => running,
+
+    # Because there is no running process for this service, the normal status
+    # checks fail.  Because Puppet then thinks the service has been manually
+    # stopped, it won't restart it.  This fake status command will trick Puppet
+    # into thinking the service is *always* running (which in a way it is, as
+    # iptables is part of the kernel.)
+    hasstatus  => true,
+    status     => '/bin/true',
+
+    # Under Debian, the "restart" parameter does not reload the rules, so tell
+    # Puppet to fall back to stop/start, which does work.
+    hasrestart => false,
+
+    require    => Package['iptables-persistent'],
+    subscribe  => [
+      Concat['/etc/iptables/rules.v4'],
+      Concat['/etc/iptables/rules.v6'],
+    ],
+  }
+}

puppet/kandra/manifests/firewall_allow.pp

@@ -0,0 +1,27 @@
+# @summary Adds an iptables "allow" rule for the host for a port.
+#
+# Rules with the same ordering are ordered by the rule name.
+#
+define kandra::firewall_allow (
+  $port = '',
+  $proto = 'tcp',
+  $order = '50',
+) {
+  if $port == '' {
+    $portname = $name
+  } else {
+    $portname = $port
+  }
+
+  concat::fragment { "iptables_v4_${portname}":
+    target  => '/etc/iptables/rules.v4',
+    order   => $order,
+    content => "-A INPUT -p ${proto} --dport ${portname} -j ACCEPT -m comment --comment \"${name}\"\n",
+  }
+
+  concat::fragment { "iptables_v6_${portname}":
+    target  => '/etc/iptables/rules.v6',
+    order   => $order,
+    content => "-A INPUT -p ${proto} --dport ${portname} -j ACCEPT -m comment --comment \"${name}\"\n",
+  }
+}

puppet/kandra/manifests/hooks/file.pp

@@ -0,0 +1,14 @@
+# @summary Install a static hook file
+#
+define kandra::hooks::file() {
+  include zulip::hooks::base
+
+  file { "/etc/zulip/hooks/${title}":
+    ensure => file,
+    mode   => '0755',
+    owner  => 'zulip',
+    group  => 'zulip',
+    source => "puppet:///modules/kandra/hooks/${title}",
+    tag    => ['hooks'],
+  }
+}

puppet/kandra/manifests/hooks/zulip_notify_schema_diff.pp

@@ -0,0 +1,8 @@
+# @summary Install hook that checks for schema drift from published ref
+#
+class kandra::hooks::zulip_notify_schema_diff {
+  include zulip::hooks::base
+  include zulip::hooks::zulip_common
+
+  kandra::hooks::file { 'post-deploy.d/zulip_notify_schema_diff.hook': }
+}

puppet/kandra/manifests/ksplice_uptrack.pp

@@ -0,0 +1,31 @@
+class kandra::ksplice_uptrack {
+  $ksplice_access_key = zulipsecret('secrets', 'ksplice_access_key', '')
+  if $ksplice_access_key != '' {
+    file { '/etc/uptrack':
+      ensure => directory,
+      owner  => 'root',
+      group  => 'adm',
+      mode   => '0750',
+    }
+    file { '/etc/uptrack/uptrack.conf':
+      ensure  => file,
+      owner   => 'root',
+      group   => 'adm',
+      mode    => '0640',
+      content => template('kandra/uptrack/uptrack.conf.erb'),
+    }
+    $setup_apt_repo_file = "${facts['zulip_scripts_path']}/lib/setup-apt-repo"
+    exec{ 'setup-apt-repo-ksplice':
+      command => "${setup_apt_repo_file} --list ksplice",
+      unless  => "${setup_apt_repo_file} --list ksplice --verify",
+    }
+    Package { 'uptrack':
+      require => [
+        Exec['setup-apt-repo-ksplice'],
+        File['/etc/uptrack/uptrack.conf'],
+      ],
+    }
+  } else {
+    warning('No ksplice uptrack key is configured; ksplice is not installed!')
+  }
+}

puppet/kandra/manifests/mirror_to_czo.pp

@@ -0,0 +1,34 @@
+class kandra::mirror_to_czo {
+  include zulip::hooks::base
+  include zulip::supervisor
+
+  # We embed the hash of the contents into the name of the process, so
+  # that `supervisorctl reread` knows that it has updated.
+  $full_script_hash = sha256(file('kandra/mirror_to_czo'))
+  $script_hash = $full_script_hash[0,8]
+
+  $bin = '/usr/local/bin/mirror_to_czo'
+  file { $bin:
+    ensure => file,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0755',
+    source => 'puppet:///modules/kandra/mirror_to_czo',
+  }
+
+  file { "${zulip::common::supervisor_conf_dir}/mirror_to_czo.conf":
+    ensure  => file,
+    require => [
+      User[zulip],
+      Package[supervisor],
+      File[$bin],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/supervisor/conf.d/mirror_to_czo.conf.template.erb'),
+    notify  => Service[supervisor],
+  }
+
+  kandra::hooks::file { 'post-deploy.d/restart_mirror_to_czo.hook': }
+}

puppet/kandra/manifests/prod_app_frontend_once.pp

@@ -0,0 +1,29 @@
+class kandra::prod_app_frontend_once {
+  include zulip::app_frontend_once
+  include zulip::hooks::push_git_ref
+  include zulip::hooks::zulip_notify
+  include kandra::hooks::zulip_notify_schema_diff
+  include kandra::mirror_to_czo
+
+  zulip::cron { 'update-first-visible-message-id':
+    hour   => '7',
+    minute => '0',
+    manage => 'calculate_first_visible_message_id --lookback-hours 30',
+  }
+
+  zulip::cron { 'invoice-plans':
+    hour   => '22',
+    minute => '0',
+  }
+  zulip::cron { 'downgrade-small-realms-behind-on-payments':
+    hour   => '17',
+    minute => '0',
+  }
+
+  zulip::cron { 'check_send_receive_time':
+    hour      => '*',
+    minute    => '*',
+    command   => '/usr/lib/nagios/plugins/zulip_app_frontend/check_send_receive_time',
+    use_proxy => false,
+  }
+}

puppet/kandra/manifests/profile/base.pp

@@ -0,0 +1,126 @@
+class kandra::profile::base {
+  include zulip::profile::base
+  include kandra::ksplice_uptrack
+  include kandra::firewall
+  include kandra::teleport::node
+  include kandra::prometheus::node
+
+  kandra::firewall_allow { 'ssh': order => '10'}
+  $is_ec2 = zulipconf('machine', 'hosting_provider', 'ec2') == 'ec2'
+
+  $org_base_packages = [
+    # Standard kernel, not AWS', so ksplice works
+    'linux-image-virtual',
+    # Management for our systems
+    'openssh-server',
+    # package management
+    'aptitude',
+    # SSL certificates
+    'certbot',
+    # For managing our current Debian packages
+    'debian-goodies',
+    # Popular editors
+    'vim',
+    'emacs-nox',
+    # Prevent accidental reboots
+    'molly-guard',
+    # For extracting Teleport certs
+    'sqlite3',
+    # Useful tools in a production environment
+    'screen',
+    'strace',
+    'bind9-host',
+    'git',
+    'nagios-plugins-contrib',
+    'ripgrep',
+    'bat',
+  ]
+  zulip::safepackage { $org_base_packages: ensure => installed }
+  file { '/usr/local/bin/bat':
+    ensure => link,
+    target => '/usr/bin/batcat',
+  }
+
+  # Uninstall the AWS kernel, but only after we install the usual one
+  package { ['linux-image-aws', 'linux-headers-aws', 'linux-aws-*', 'linux-image-*-aws', 'linux-modules-*-aws']:
+    ensure  => absent,
+    require => Package['linux-image-virtual'],
+  }
+
+  file { '/etc/apt/apt.conf.d/02periodic':
+    ensure => file,
+    mode   => '0644',
+    source => 'puppet:///modules/kandra/apt/apt.conf.d/02periodic',
+  }
+
+  file { '/etc/apt/apt.conf.d/50unattended-upgrades':
+    ensure => file,
+    mode   => '0644',
+    source => 'puppet:///modules/kandra/apt/apt.conf.d/50unattended-upgrades',
+  }
+  file { '/etc/needrestart/conf.d/zulip.conf':
+    ensure => file,
+    mode   => '0644',
+    source => 'puppet:///modules/kandra/needrestart/zulip.conf',
+  }
+
+  user { 'root': }
+  kandra::user_dotfiles { 'root':
+    home            => '/root',
+    keys            => 'internal-read-only-deploy-key',
+    authorized_keys => 'common',
+    known_hosts     => ['github.com'],
+  }
+
+  kandra::user_dotfiles { 'zulip':
+    keys            => 'internal-read-only-deploy-key',
+    authorized_keys => 'common',
+    known_hosts     => ['github.com'],
+  }
+
+  service { 'ssh':
+    ensure => running,
+  }
+
+  include kandra::aws_tools
+
+  if $is_ec2 {
+    # EC2 hosts can use the in-VPC timeserver
+    file { '/etc/chrony/chrony.conf':
+      ensure  => file,
+      mode    => '0644',
+      source  => 'puppet:///modules/kandra/chrony.conf',
+      require => Package['chrony'],
+      notify  => Service['chrony'],
+    }
+  }
+
+  group { 'nagios':
+    ensure => present,
+    gid    => '1050',
+  }
+  user { 'nagios':
+    ensure     => present,
+    uid        => '1050',
+    gid        => '1050',
+    shell      => '/bin/bash',
+    home       => '/var/lib/nagios',
+    managehome => true,
+  }
+  file { '/var/lib/nagios':
+    ensure  => directory,
+    require => User['nagios'],
+    owner   => 'nagios',
+    group   => 'nagios',
+    mode    => '0700',
+  }
+  kandra::user_dotfiles { 'nagios':
+    home            => '/var/lib/nagios',
+    authorized_keys => true,
+  }
+  file { '/home/nagios':
+    ensure  => absent,
+    force   => true,
+    recurse => true,
+  }
+}

puppet/kandra/manifests/profile/chat_zulip_org.pp

@@ -0,0 +1,23 @@
+class kandra::profile::chat_zulip_org inherits kandra::profile::base {
+  include zulip::profile::standalone
+  include zulip::local_mailserver
+  include zulip::hooks::sentry
+  include zulip::hooks::zulip_notify
+
+  include kandra::app_frontend_monitoring
+  include kandra::prometheus::redis
+  include kandra::prometheus::postgresql
+  kandra::teleport::prometheus_app { 'smokescreen': port => '4760' }
+  kandra::teleport::prometheus_app { 'camo': port => '9292' }
+
+  kandra::firewall_allow { 'http': }
+  kandra::firewall_allow { 'https': }
+  kandra::firewall_allow { 'smtp': }
+
+  Kandra::User_Dotfiles['root'] {
+    keys => false,
+  }
+  Kandra::User_Dotfiles['zulip'] {
+    keys => false,
+  }
+}

puppet/kandra/manifests/profile/grafana.pp

@@ -0,0 +1,78 @@
+# @summary Observability using Grafana
+#
+class kandra::profile::grafana inherits kandra::profile::base {
+
+  include zulip::supervisor
+
+  $version = $zulip::common::versions['grafana']['version']
+  $dir = "/srv/zulip-grafana-${version}"
+  $bin = "${dir}/bin/grafana-server"
+  $data_dir = '/var/lib/grafana'
+
+  zulip::external_dep { 'grafana':
+    version        => $version,
+    url            => "https://dl.grafana.com/oss/release/grafana-${version}.linux-${zulip::common::goarch}.tar.gz",
+    tarball_prefix => "grafana-${version}",
+    bin            => [$bin],
+    cleanup_after  => [Service[supervisor]],
+  }
+
+  group { 'grafana':
+    ensure => present,
+    gid    => '1070',
+  }
+  user { 'grafana':
+    ensure     => present,
+    uid        => '1070',
+    gid        => '1070',
+    shell      => '/bin/bash',
+    home       => $data_dir,
+    managehome => false,
+  }
+  file { $data_dir:
+    ensure  => directory,
+    owner   => 'grafana',
+    group   => 'grafana',
+    require => [ User[grafana], Group[grafana] ],
+  }
+  file { '/var/log/grafana':
+    ensure => directory,
+    owner  => 'grafana',
+    group  => 'grafana',
+  }
+
+  kandra::teleport::application { 'monitoring': port => '3000' }
+  file { "${zulip::common::supervisor_conf_dir}/grafana.conf":
+    ensure  => file,
+    require => [
+      Package[supervisor],
+      File[$bin],
+      File[$data_dir],
+      File['/var/log/grafana'],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/supervisor/conf.d/grafana.conf.erb'),
+    notify  => Service[supervisor],
+  }
+
+  $email_host = zulipconf('grafana', 'email_host', '')
+  $email_from = zulipconf('grafana', 'email_from', '')
+  $email_user = zulipsecret('secrets', 'grafana_email_user', '')
+  $email_password = zulipsecret('secrets', 'grafana_email_password', '')
+  file { '/etc/grafana':
+    ensure => directory,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0755',
+  }
+  file { '/etc/grafana/grafana.ini':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/grafana.ini.template.erb'),
+    notify  => Service[supervisor],
+  }
+}

puppet/kandra/manifests/profile/nagios.pp

@@ -0,0 +1,178 @@
+class kandra::profile::nagios inherits kandra::profile::base {
+
+  include kandra::apache
+
+  package { ['nagios4', 'msmtp', 'autossh']: ensure => installed }
+  $nagios_alert_email = zulipconf('nagios', 'alert_email', undef)
+  $nagios_test_email = zulipconf('nagios', 'test_email', undef)
+  $nagios_pager_email = zulipconf('nagios', 'pager_email', undef)
+
+  $nagios_mail_domain = zulipconf('nagios', 'mail_domain', undef)
+  $nagios_mail_host = zulipconf('nagios', 'mail_host', undef)
+  $nagios_mail_password = zulipsecret('secrets', 'nagios_mail_password', '')
+  if zulipconf('nagios', 'camo_check_url', undef) =~ /^https:\/\/([^\/]*)(\/.*)$/ {
+    $nagios_camo_check_host = $1
+    $nagios_camo_check_path = $2
+  }
+
+  $default_host_domain = zulipconf('nagios', 'default_host_domain', undef)
+  $hosts_app_prod = split(zulipconf('nagios', 'hosts_app_prod', undef), ',')
+  $hosts_app_staging = split(zulipconf('nagios', 'hosts_app_staging', undef), ',')
+  $hosts_postgresql_primary = split(zulipconf('nagios', 'hosts_postgresql_primary', undef), ',')
+  $hosts_postgresql_replica = split(zulipconf('nagios', 'hosts_postgresql_replica', undef), ',')
+  $hosts_redis = split(zulipconf('nagios', 'hosts_redis', undef), ',')
+  $hosts_fullstack = split(zulipconf('nagios', 'hosts_fullstack', undef), ',')
+  $hosts_smokescreen = split(zulipconf('nagios', 'hosts_smokescreen', undef), ',')
+  $hosts_other = split(zulipconf('nagios', 'hosts_other', undef), ',')
+
+  $hosts = zulipconf_nagios_hosts()
+  $qualified_hosts = $hosts.map |$h| { if '.' in $h { $h } else { "${h}.${default_host_domain}" }}
+  Kandra::User_Dotfiles['nagios'] {
+    keys        => 'nagios',
+    known_hosts => $qualified_hosts,
+  }
+
+  file { '/etc/nagios4/':
+    recurse => true,
+    purge   => false,
+    require => Package[nagios4],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    source  => 'puppet:///modules/kandra/nagios4/',
+    notify  => Service['nagios4'],
+  }
+
+  file { '/etc/apache2/sites-available/nagios.conf':
+    purge   => false,
+    require => Package[apache2],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0640',
+    content => template('kandra/nagios_apache_site.conf.template.erb'),
+  }
+  apache2site { 'nagios':
+    ensure  => present,
+    require => [
+      File['/etc/apache2/sites-available/nagios.conf'],
+      Apache2mod['headers'], Apache2mod['ssl'],
+    ],
+    notify  => Service['apache2'],
+  }
+  kandra::teleport::application{ 'nagios':
+    description => 'Monitoring: nagios',
+    port        => '3000',
+  }
+
+  file { '/etc/nagios4/conf.d/contacts.cfg':
+    require => Package[nagios4],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/nagios4/contacts.cfg.template.erb'),
+    notify  => Service['nagios4'],
+  }
+  file { '/etc/nagios4/conf.d/hosts.cfg':
+    require => Package[nagios4],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/nagios4/hosts.cfg.template.erb'),
+    notify  => Service['nagios4'],
+  }
+  file { '/etc/nagios4/conf.d/localhost.cfg':
+    require => Package[nagios4],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/nagios4/localhost.cfg.template.erb'),
+    notify  => Service['nagios4'],
+  }
+
+  file { '/etc/nagios4/cgi.cfg':
+    require => Package[nagios4],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/nagios4/cgi.cfg.template.erb'),
+    notify  => Service['nagios4'],
+  }
+
+  service { 'nagios4':
+    ensure  => running,
+    require => Kandra::User_Dotfiles['nagios'],
+  }
+
+  file { [
+    '/etc/nagios4/conf.d/extinfo_nagios2.cfg',
+    '/etc/nagios4/conf.d/services_nagios2.cfg',
+    '/etc/nagios4/conf.d/contacts_nagios2.cfg',
+    '/etc/nagios4/conf.d/hostgroups_nagios2.cfg',
+    '/etc/nagios4/conf.d/localhost_nagios2.cfg',
+    '/etc/nagios4/conf.d/zulip_nagios.cfg',
+  ]:
+    ensure => absent,
+  }
+
+  file { "${zulip::common::supervisor_conf_dir}/autossh_tunnels.conf":
+    ensure  => file,
+    require => [
+      Package['supervisor', 'autossh'],
+      Kandra::User_Dotfiles['nagios'],
+    ],
+    mode    => '0644',
+    owner   => 'root',
+    group   => 'root',
+    content => template('kandra/supervisor/conf.d/autossh_tunnels.conf.erb'),
+    notify  => Service['supervisor'],
+  }
+  file { '/etc/nagios4/conf.d/zulip_autossh.cfg':
+    ensure  => file,
+    mode    => '0644',
+    owner   => 'root',
+    group   => 'root',
+    content => template('kandra/nagios_autossh.template.erb'),
+    notify  => Service['nagios4'],
+  }
+
+  file { '/var/lib/nagios/msmtprc':
+    ensure  => file,
+    mode    => '0600',
+    owner   => 'nagios',
+    group   => 'nagios',
+    content => template('kandra/msmtprc_nagios.template.erb'),
+    require => File['/var/lib/nagios'],
+  }
+
+  file { '/var/lib/nagios/.ssh/config':
+    ensure => file,
+    mode   => '0644',
+    owner  => 'nagios',
+    group  => 'nagios',
+    source => 'puppet:///modules/kandra/nagios_ssh_config',
+  }
+
+  # Disable apparmor for msmtp so it can read the above config file
+  file { '/etc/apparmor.d/disable/usr.bin.msmtp':
+    ensure => link,
+    target => '/etc/apparmor.d/usr.bin.msmtp',
+    notify => Exec['reload apparmor'],
+  }
+  exec {'reload apparmor':
+    command     => '/sbin/apparmor_parser -R /etc/apparmor.d/usr.bin.msmtp',
+    refreshonly => true,
+  }
+
+  exec { 'fix_nagios_permissions':
+    command => 'dpkg-statoverride --update --add nagios www-data 2710 /var/lib/nagios4/rw',
+    unless  => 'bash -c "ls -ld /var/lib/nagios4/rw | grep ^drwx--s--- -q"',
+    notify  => Service['nagios4'],
+  }
+  exec { 'fix_nagios_permissions2':
+    command => 'dpkg-statoverride --update --add nagios nagios 751 /var/lib/nagios4',
+    unless  => 'bash -c "ls -ld /var/lib/nagios4 | grep ^drwxr-x--x -q"',
+    notify  => Service['nagios4'],
+  }
+
+  # TODO: Install our API
+}

puppet/kandra/manifests/profile/postgresql.pp

@@ -0,0 +1,74 @@
+class kandra::profile::postgresql inherits kandra::profile::base {
+  include kandra::teleport::db
+  include kandra::prometheus::postgresql
+
+  # We key off of `listen_addresses` being set to know if we should be starting PostgreSQL.
+  $listen_addresses = zulipconf('postgresql', 'listen_addresses', undef)
+  $is_stage1 = ($listen_addresses == undef)
+  class { 'zulip::profile::postgresql':
+    start => ! $is_stage1,
+  }
+
+  package { ['xfsprogs', 'nvme-cli']: ensure => installed }
+
+  kandra::firewall_allow{ 'postgresql': }
+
+  zulip::sysctl { 'postgresql-swappiness':
+    key   => 'vm.swappiness',
+    value => '0',
+  }
+  zulip::sysctl { 'postgresql-overcommit':
+    key   => 'vm.overcommit_memory',
+    value => '2',
+  }
+
+  file { '/root/setup_disks.sh':
+    ensure => file,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0744',
+    source => 'puppet:///modules/kandra/postgresql/setup_disks.sh',
+  }
+  exec { 'setup_disks':
+    command => '/root/setup_disks.sh',
+    # We need to not have started installing the non-AWS kernel, so
+    # the xfs module gets installed for the running kernel, and we can
+    # mount it.
+    before  => Package['linux-image-virtual'],
+    require => Package["postgresql-${zulip::postgresql_common::version}", 'xfsprogs', 'nvme-cli'],
+    unless  => 'test /var/lib/postgresql/ -ef /srv/data/postgresql/',
+  }
+
+  # This is the second stage, after secrets are configured
+  if (! $is_stage1) {
+    $replication_primary = zulipconf('postgresql', 'replication_primary', undef)
+    $replication_user = zulipconf('postgresql', 'replication_user', undef)
+    if $replication_primary != undef and $replication_user != undef {
+      file { '/root/setup_data.sh':
+        ensure => file,
+        owner  => 'root',
+        group  => 'root',
+        mode   => '0744',
+        source => 'puppet:///modules/kandra/postgresql/setup_data.sh',
+      }
+      exec { 'setup_data':
+        command => '/root/setup_data.sh',
+        require => [File['/usr/local/bin/env-wal-g'], Exec['setup_disks']],
+        unless  => "test -d /srv/data/postgresql/${zulip::postgresql_common::version}/main",
+        timeout => 0,
+        before  => File["${zulip::postgresql_base::postgresql_datadir}/standby.signal"],
+        notify  => Service['postgresql'],
+      }
+    }
+  }
+
+  file { "${zulip::postgresql_base::postgresql_confdir}/pg_hba.conf":
+    ensure  => file,
+    require => Package["postgresql-${zulip::postgresql_common::version}"],
+    notify  => Service['postgresql'],
+    owner   => 'postgres',
+    group   => 'postgres',
+    mode    => '0640',
+    source  => 'puppet:///modules/kandra/postgresql/pg_hba.conf',
+  }
+}

puppet/kandra/manifests/profile/prod_app_frontend.pp

@@ -0,0 +1,37 @@
+class kandra::profile::prod_app_frontend inherits kandra::profile::base {
+  include kandra::app_frontend
+  include zulip::hooks::zulip_notify
+
+  Kandra::User_Dotfiles['root'] {
+    keys => 'internal-limited-write-deploy-key',
+  }
+  Kandra::User_Dotfiles['zulip'] {
+    keys => 'internal-limited-write-deploy-key',
+  }
+
+  zulip::sysctl { 'conntrack':
+    comment => 'Increase conntrack kernel table size',
+    key     => 'net.nf_conntrack_max',
+    value   => zulipconf('application_server', 'conntrack_max', 262144),
+  }
+
+  file { '/etc/nginx/sites-available/zulip':
+    ensure  => file,
+    require => [Package['nginx-full'], Exec['generate-default-snakeoil']],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    source  => 'puppet:///modules/kandra/nginx/sites-available/zulip',
+    notify  => Service['nginx'],
+  }
+
+  file { '/etc/nginx/sites-enabled/zulip':
+    ensure  => link,
+    require => Package['nginx-full'],
+    target  => '/etc/nginx/sites-available/zulip',
+    notify  => Service['nginx'],
+  }
+
+  # Prod has our Apple Push Notifications Service private key at
+  # /etc/ssl/django-private/apns-dist.pem
+}

puppet/kandra/manifests/profile/prometheus_server.pp

@@ -0,0 +1,87 @@
+# @summary Gathers Prometheus statistics from all nodes.
+#
+# Only one instance is necessary.
+#
+class kandra::profile::prometheus_server inherits kandra::profile::base {
+
+  include kandra::prometheus::base
+
+  # Service discovery via Teleport
+  include kandra::teleport::tbot
+  include kandra::teleport::sd
+
+  # This blackbox monitoring of the backup system runs locally
+  include kandra::prometheus::wal_g
+
+  # Ditto the Akamai logs
+  include kandra::prometheus::akamai
+
+  # Ditto Weblate
+  include kandra::prometheus::weblate
+
+  # The SES log ETL (writing to S3) runs on vector
+  include kandra::ses_logs
+
+  # Export prometheus stats to status.zulip.com
+  include kandra::statuspage
+
+  $version = $zulip::common::versions['prometheus']['version']
+  $dir = "/srv/zulip-prometheus-${version}"
+  $bin = "${dir}/prometheus"
+  $data_dir = '/var/lib/prometheus'
+
+  zulip::external_dep { 'prometheus':
+    version        => $version,
+    url            => "https://github.com/prometheus/prometheus/releases/download/v${version}/prometheus-${version}.linux-${zulip::common::goarch}.tar.gz",
+    tarball_prefix => "prometheus-${version}.linux-${zulip::common::goarch}",
+    bin            => [$bin, "${dir}/promtool"],
+    cleanup_after  => [Service[supervisor]],
+  }
+  file { '/usr/local/bin/promtool':
+    ensure  => link,
+    target  => "${dir}/promtool",
+    require => File["${dir}/promtool"],
+    before  => Exec['Cleanup prometheus'],
+  }
+
+  file { $data_dir:
+    ensure  => directory,
+    owner   => 'prometheus',
+    group   => 'prometheus',
+    require => [ User[prometheus], Group[prometheus] ],
+  }
+  file { '/etc/prometheus':
+    ensure => directory,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0755',
+  }
+
+  $czo = zulipconf('prometheus', 'czo', '')
+  $other_hosts = split(zulipconf('prometheus', 'other_hosts', ''), ',')
+  $backup_buckets = split(zulipconf('prometheus', 'walg_buckets', ''), ',')
+  file { '/etc/prometheus/prometheus.yaml':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/prometheus/prometheus.yaml.template.erb'),
+    notify  => Service[supervisor],
+  }
+
+  file { "${zulip::common::supervisor_conf_dir}/prometheus.conf":
+    ensure  => file,
+    require => [
+      Package[supervisor],
+      File[$bin],
+      File[$data_dir],
+      File['/etc/prometheus/prometheus.yaml'],
+      Service['tbot'],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/supervisor/conf.d/prometheus.conf.template.erb'),
+    notify  => Service[supervisor],
+  }
+}

puppet/kandra/manifests/profile/redis.pp

@@ -0,0 +1,35 @@
+class kandra::profile::redis inherits kandra::profile::base {
+  include zulip::profile::redis
+  include kandra::prometheus::redis
+
+  zulip::sysctl { 'redis-somaxconn':
+    key   => 'net.core.somaxconn',
+    value => '65535',
+  }
+
+  # Need redis_password in its own file for Nagios
+  file { '/var/lib/nagios/redis_password':
+    ensure  => file,
+    mode    => '0600',
+    owner   => 'nagios',
+    group   => 'nagios',
+    content => "${zulip::profile::redis::redis_password}\n",
+  }
+
+  group { 'redistunnel':
+    ensure => present,
+    gid    => '1080',
+  }
+  user { 'redistunnel':
+    ensure     => present,
+    uid        => '1080',
+    gid        => '1080',
+    groups     => ['zulip'],
+    shell      => '/bin/true',
+    home       => '/home/redistunnel',
+    managehome => true,
+  }
+  kandra::user_dotfiles { 'redistunnel':
+    authorized_keys => true,
+  }
+}

puppet/kandra/manifests/profile/smokescreen.pp

@@ -0,0 +1,9 @@
+class kandra::profile::smokescreen inherits kandra::profile::base {
+
+
+  include zulip::profile::smokescreen
+  kandra::firewall_allow { 'smokescreen': port => '4750' }
+  kandra::teleport::prometheus_app { 'smokescreen': port => '4760' }
+
+  include kandra::camo
+}

puppet/kandra/manifests/profile/staging_app_frontend.pp

@@ -0,0 +1,28 @@
+class kandra::profile::staging_app_frontend inherits kandra::profile::base {
+
+  include kandra::app_frontend
+
+  file { '/etc/nginx/sites-available/zulip-staging':
+    ensure  => file,
+    require => [Package['nginx-full'], Exec['generate-default-snakeoil']],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    source  => 'puppet:///modules/kandra/nginx/sites-available/zulip-staging',
+    notify  => Service['nginx'],
+  }
+  file { '/etc/nginx/sites-enabled/zulip-staging':
+    ensure  => link,
+    require => Package['nginx-full'],
+    target  => '/etc/nginx/sites-available/zulip-staging',
+    notify  => Service['nginx'],
+  }
+
+  # Eventually, this will go in a staging_app_frontend_once.pp
+  zulip::cron { 'check_send_receive_time':
+    hour      => '*',
+    minute    => '*',
+    command   => '/usr/lib/nagios/plugins/zulip_app_frontend/check_send_receive_time',
+    use_proxy => false,
+  }
+}

puppet/kandra/manifests/profile/teleport.pp

@@ -0,0 +1,23 @@
+class kandra::profile::teleport inherits kandra::profile::base {
+
+
+  file { '/etc/teleport_server.yaml':
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0644',
+    source => 'puppet:///modules/kandra/teleport_server.yaml',
+    before => Kandra::Teleport::Part['server'],
+    notify => Exec['reload teleport_server'],
+  }
+  file { '/var/lib/teleport-server':
+    ensure => directory,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0700',
+    before => Kandra::Teleport::Part['server'],
+  }
+  kandra::teleport::part { 'server': }
+
+  # https://goteleport.com/docs/reference/deployment/networking/#ports-with-tls-routing
+  kandra::firewall_allow { 'teleport_server_ui': port => 443 }
+}

puppet/kandra/manifests/profile/zulipbot_zulip_org.pp

@@ -0,0 +1,8 @@
+class kandra::profile::zulipbot_zulip_org inherits kandra::profile::base {
+
+  kandra::firewall_allow { 'http': }
+  kandra::firewall_allow { 'https': }
+
+  # TODO: This does not do any configuration of zulipbot itself, or of
+  # caddy.
+}

puppet/kandra/manifests/prometheus/akamai.pp

@@ -0,0 +1,23 @@
+# @summary Prometheus monitoring of Akamai access logs
+#
+class kandra::prometheus::akamai {
+  include kandra::prometheus::base
+  include kandra::vector
+  include zulip::supervisor
+
+  $pipelines = {
+    'static' => zulipsecret('secrets', 'akamai_static_sqs_url', ''),
+    'realm'  => zulipsecret('secrets', 'akamai_realm_sqs_url', ''),
+  }
+
+  concat::fragment { 'vector_akamai':
+    target  => $kandra::vector::conf,
+    order   => '50',
+    content => template('kandra/vector_akamai.toml.template.erb'),
+  }
+  concat::fragment { 'vector_akamai_export':
+    target  => $kandra::vector::conf,
+    content => ',"akamai_logs2metrics*"',
+    order   => '90',
+  }
+}

puppet/kandra/manifests/prometheus/base.pp

@@ -0,0 +1,16 @@
+# @summary Configures a node for monitoring with Prometheus
+#
+class kandra::prometheus::base {
+  group { 'prometheus':
+    ensure => present,
+    gid    => '1060',
+  }
+  user { 'prometheus':
+    ensure     => present,
+    uid        => '1060',
+    gid        => '1060',
+    shell      => '/bin/bash',
+    home       => '/nonexistent',
+    managehome => false,
+  }
+}

puppet/kandra/manifests/prometheus/grok.pp

@@ -0,0 +1,45 @@
+# @summary Parses nginx access_log files
+#
+class kandra::prometheus::grok {
+  include kandra::prometheus::base
+  include zulip::supervisor
+
+  $version = $zulip::common::versions['grok_exporter']['version']
+  $dir = "/srv/zulip-grok_exporter-${version}"
+  $bin = "${dir}/grok_exporter"
+
+  zulip::external_dep { 'grok_exporter':
+    version        => $version,
+    url            => "https://github.com/fstab/grok_exporter/releases/download/v${version}/grok_exporter-${version}.linux-${zulip::common::goarch}.zip",
+    tarball_prefix => "grok_exporter-${version}.linux-${zulip::common::goarch}",
+    bin            => [$bin],
+    cleanup_after  => [Service[supervisor]],
+  }
+
+  $realm_names_regex = zulipconf('grok_exporter', 'realm_names_regex', '__impossible__')
+  $include_dir = "${dir}/patterns"
+  file { '/etc/grok_exporter.yaml':
+    ensure  => file,
+    owner   => zulip,
+    group   => zulip,
+    mode    => '0644',
+    content => template('kandra/prometheus/grok_exporter.yaml.template.erb'),
+    notify  => Service[supervisor],
+  }
+
+  kandra::teleport::prometheus_app { 'grok_exporter': port => '9144' }
+  file { "${zulip::common::supervisor_conf_dir}/prometheus_grok_exporter.conf":
+    ensure  => file,
+    require => [
+      User[zulip],
+      Package[supervisor],
+      File[$bin],
+      File['/etc/grok_exporter.yaml'],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/supervisor/conf.d/prometheus_grok_exporter.conf.template.erb'),
+    notify  => Service[supervisor],
+  }
+}

puppet/kandra/manifests/prometheus/memcached.pp

@@ -0,0 +1,39 @@
+# @summary Export memcached stats, with SASL auth
+#
+# We cannot use the stock
+# https://github.com/prometheus/memcached_exporter because it does not
+# support SASL auth, which we require.  Re-implement it in Python,
+# using bmemcached.
+class kandra::prometheus::memcached {
+  include kandra::prometheus::base
+  include zulip::supervisor
+
+  # We embed the hash of the contents into the name of the process, so
+  # that `supervisorctl reread` knows that it has updated.
+  $full_exporter_hash = sha256(file('kandra/memcached_exporter'))
+  $exporter_hash = $full_exporter_hash[0,8]
+
+  $bin = '/usr/local/bin/memcached_exporter'
+  file { $bin:
+    ensure => file,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0755',
+    source => 'puppet:///modules/kandra/memcached_exporter',
+  }
+
+  kandra::teleport::prometheus_app { 'memcached_exporter': port => '11212' }
+  file { "${zulip::common::supervisor_conf_dir}/memcached_exporter.conf":
+    ensure  => file,
+    require => [
+      User[zulip],
+      Package[supervisor],
+      File[$bin],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/supervisor/conf.d/memcached_exporter.conf.template.erb'),
+    notify  => Service[supervisor],
+  }
+}

puppet/kandra/manifests/prometheus/node.pp

@@ -0,0 +1,33 @@
+# @summary Configures a node for monitoring with Prometheus
+#
+class kandra::prometheus::node {
+  include kandra::prometheus::base
+  include zulip::supervisor
+
+  $version = $zulip::common::versions['node_exporter']['version']
+  $dir = "/srv/zulip-node_exporter-${version}"
+  $bin = "${dir}/node_exporter"
+
+  zulip::external_dep { 'node_exporter':
+    version        => $version,
+    url            => "https://github.com/prometheus/node_exporter/releases/download/v${version}/node_exporter-${version}.linux-${zulip::common::goarch}.tar.gz",
+    tarball_prefix => "node_exporter-${version}.linux-${zulip::common::goarch}",
+    bin            => [$bin],
+    cleanup_after  => [Service[supervisor]],
+  }
+
+  kandra::teleport::prometheus_app { 'node_exporter': port => '9100' }
+  file { "${zulip::common::supervisor_conf_dir}/prometheus_node_exporter.conf":
+    ensure  => file,
+    require => [
+      User[zulip],
+      Package[supervisor],
+      File[$bin],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/supervisor/conf.d/prometheus_node_exporter.conf.template.erb'),
+    notify  => Service[supervisor],
+  }
+}

puppet/kandra/manifests/prometheus/postgresql.pp

@@ -0,0 +1,83 @@
+# @summary Prometheus monitoring of postgresql servers
+#
+class kandra::prometheus::postgresql {
+  include kandra::prometheus::base
+  include zulip::supervisor
+  include zulip::golang
+
+  $version = $zulip::common::versions['postgres_exporter-src']['version']
+  $dir = "/srv/zulip-postgres_exporter-src-${version}"
+  $bin = "/usr/local/bin/postgres_exporter-${version}-go-${zulip::golang::version}"
+
+  # Binary builds: https://github.com/prometheus-community/postgres_exporter/releases/download/v${version}/postgres_exporter-${version}.linux-${zulip::common::goarch}.tar.gz
+
+  zulip::external_dep { 'postgres_exporter-src':
+    version        => $version,
+    url            => "https://github.com/alexmv/postgres_exporter/archive/${version}.tar.gz",
+    tarball_prefix => "postgres_exporter-${version}",
+  }
+
+  exec { 'compile postgres_exporter':
+    command     => "make build && cp ./postgres_exporter ${bin}",
+    cwd         => $dir,
+    # GOCACHE is required; nothing is written to GOPATH, but it is required to be set
+    environment => ['GOCACHE=/tmp/gocache', 'GOPATH=/root/go'],
+    path        => [
+      "${zulip::golang::dir}/bin",
+      '/usr/local/bin',
+      '/usr/bin',
+      '/bin',
+    ],
+    creates     => $bin,
+    require     => [
+      Zulip::External_Dep['golang'],
+      Zulip::External_Dep['postgres_exporter-src'],
+    ],
+    notify      => Exec['Cleanup postgres_exporter'],
+  }
+  # This resource is created by the 'compile postgres_exporter' step.
+  file { $bin:
+    ensure  => file,
+    require => Exec['compile postgres_exporter'],
+  }
+  exec { 'Cleanup postgres_exporter':
+    refreshonly => true,
+    provider    => shell,
+    onlyif      => "ls /usr/local/bin/postgres_exporter-* | grep -xv '${bin}'",
+    command     => "ls /usr/local/bin/postgres_exporter-* | grep -xv '${bin}' | xargs rm -r",
+    require     => [File[$bin], Service[supervisor]],
+  }
+
+  if false {
+    # This is left commented out, since it only makes sense to run
+    # against a server where the database exists and is writable --
+    # the former of which happens outside th scope of puppet right
+    # now, and the latter of which can only be determined after the
+    # database is in place.  Given that it has been run once, we do
+    # not expect to ever need it to run again; it is left here for
+    # completeness.
+    include zulip::postgresql_client
+    exec { 'create prometheus postgres user':
+      require => Class['zulip::postgresql_client'],
+      command => '/usr/bin/createuser -g pg_monitor prometheus',
+      unless  => 'test -f /usr/bin/psql && /usr/bin/psql -tAc "select usename from pg_user" | /bin/grep -xq prometheus)',
+      user    => 'postgres',
+      before  => File["${zulip::common::supervisor_conf_dir}/prometheus_postgres_exporter.conf"],
+    }
+  }
+
+  kandra::teleport::prometheus_app { 'postgres_exporter': port => '9187' }
+  file { "${zulip::common::supervisor_conf_dir}/prometheus_postgres_exporter.conf":
+    ensure  => file,
+    require => [
+      User[prometheus],
+      Package[supervisor],
+      File[$bin],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/supervisor/conf.d/prometheus_postgres_exporter.conf.template.erb'),
+    notify  => Service[supervisor],
+  }
+}

puppet/kandra/manifests/prometheus/process.pp

@@ -0,0 +1,43 @@
+# @summary Prometheus monitoring of Zulip server processes
+#
+class kandra::prometheus::process {
+  include kandra::prometheus::base
+  include zulip::supervisor
+
+  $version = $zulip::common::versions['process_exporter']['version']
+  $dir = "/srv/zulip-process_exporter-${version}"
+  $bin = "${dir}/process-exporter"
+  $conf = '/etc/zulip/process_exporter.yaml'
+
+  zulip::external_dep { 'process_exporter':
+    version        => $version,
+    url            => "https://github.com/ncabatoff/process-exporter/releases/download/v${version}/process-exporter-${version}.linux-${zulip::common::goarch}.tar.gz",
+    tarball_prefix => "process-exporter-${version}.linux-${zulip::common::goarch}",
+    bin            => [$bin],
+    cleanup_after  => [Service[supervisor]],
+  }
+
+  kandra::teleport::prometheus_app { 'process_exporter': port => '9256' }
+  file { $conf:
+    ensure  => file,
+    require => User[zulip],
+    owner   => 'zulip',
+    group   => 'zulip',
+    mode    => '0644',
+    source  => 'puppet:///modules/kandra/process_exporter.yaml',
+  }
+  file { "${zulip::common::supervisor_conf_dir}/prometheus_process_exporter.conf":
+    ensure  => file,
+    require => [
+      User[zulip],
+      Package[supervisor],
+      File[$bin],
+      File[$conf],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/supervisor/conf.d/prometheus_process_exporter.conf.template.erb'),
+    notify  => Service[supervisor],
+  }
+}

puppet/kandra/manifests/prometheus/pushgateway.pp

@@ -0,0 +1,32 @@
+# @summary Pushgateway for cron jobs
+#
+class kandra::prometheus::pushgateway {
+  include kandra::prometheus::base
+  include zulip::supervisor
+
+  $version = $zulip::common::versions['pushgateway']['version']
+  $dir = "/srv/zulip-pushgateway-${version}"
+  $bin = "${dir}/pushgateway"
+
+  zulip::external_dep { 'pushgateway':
+    version        => $version,
+    url            => "https://github.com/prometheus/pushgateway/releases/download/v${version}/pushgateway-${version}.linux-${zulip::common::goarch}.tar.gz",
+    tarball_prefix => "pushgateway-${version}.linux-${zulip::common::goarch}",
+    bin            => [$bin],
+    cleanup_after  => [Service[supervisor]],
+  }
+
+  file { "${zulip::common::supervisor_conf_dir}/prometheus_pushgateway.conf":
+    ensure  => file,
+    require => [
+      User[zulip],
+      Package[supervisor],
+      File[$bin],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/supervisor/conf.d/prometheus_pushgateway.conf.template.erb'),
+    notify  => Service[supervisor],
+  }
+}

puppet/kandra/manifests/prometheus/rabbitmq.pp

@@ -0,0 +1,14 @@
+# @summary Prometheus monitoring of rabbitmq server.  This is done via
+# the built-in prometheus plugin which serves on port 15692:
+# https://www.rabbitmq.com/prometheus.html
+#
+class kandra::prometheus::rabbitmq {
+  include kandra::prometheus::base
+
+  exec { 'enable rabbitmq-prometheus':
+    command => 'rabbitmq-plugins enable rabbitmq_prometheus',
+    unless  => 'grep -q rabbitmq_prometheus /etc/rabbitmq/enabled_plugins',
+    require => Service['rabbitmq-server'],
+  }
+  kandra::teleport::prometheus_app { 'rabbitmq': port => '15692' }
+}

puppet/kandra/manifests/prometheus/redis.pp

@@ -0,0 +1,33 @@
+# @summary Prometheus monitoring of redis servers
+#
+class kandra::prometheus::redis {
+  include kandra::prometheus::base
+  include zulip::supervisor
+
+  $version = $zulip::common::versions['redis_exporter']['version']
+  $dir = "/srv/zulip-redis_exporter-${version}"
+  $bin = "${dir}/redis_exporter"
+
+  zulip::external_dep { 'redis_exporter':
+    version        => $version,
+    url            => "https://github.com/oliver006/redis_exporter/releases/download/v${version}/redis_exporter-v${version}.linux-${zulip::common::goarch}.tar.gz",
+    tarball_prefix => "redis_exporter-v${version}.linux-${zulip::common::goarch}",
+    bin            => [$bin],
+    cleanup_after  => [Service[supervisor]],
+  }
+
+  kandra::teleport::prometheus_app { 'redis_exporter': port => '9121' }
+  file { "${zulip::common::supervisor_conf_dir}/prometheus_redis_exporter.conf":
+    ensure  => file,
+    require => [
+      User[zulip],
+      Package[supervisor],
+      File[$bin],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/supervisor/conf.d/prometheus_redis_exporter.conf.template.erb'),
+    notify  => Service[supervisor],
+  }
+}

puppet/kandra/manifests/prometheus/uwsgi.pp

@@ -0,0 +1,33 @@
+# @summary Prometheus monitoring of uwsgi servers
+#
+class kandra::prometheus::uwsgi {
+  include kandra::prometheus::base
+  include zulip::supervisor
+
+  $version = $zulip::common::versions['uwsgi_exporter']['version']
+  $dir = "/srv/zulip-uwsgi_exporter-${version}"
+  $bin = "${dir}/uwsgi_exporter"
+
+  zulip::external_dep { 'uwsgi_exporter':
+    version        => $version,
+    url            => "https://github.com/timonwong/uwsgi_exporter/releases/download/v${version}/uwsgi_exporter-${version}.linux-${zulip::common::goarch}.tar.gz",
+    tarball_prefix => "uwsgi_exporter-${version}.linux-${zulip::common::goarch}",
+    bin            => [$bin],
+    cleanup_after  => [Service[supervisor]],
+  }
+
+  kandra::teleport::prometheus_app { 'uwsgi_exporter': port => '9238' }
+  file { "${zulip::common::supervisor_conf_dir}/prometheus_uwsgi_exporter.conf":
+    ensure  => file,
+    require => [
+      User[zulip],
+      Package[supervisor],
+      File[$bin],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/supervisor/conf.d/prometheus_uwsgi_exporter.conf.template.erb'),
+    notify  => Service[supervisor],
+  }
+}

puppet/kandra/manifests/prometheus/wal_g.pp

@@ -0,0 +1,34 @@
+# @summary Prometheus monitoring of wal-g backups
+#
+class kandra::prometheus::wal_g {
+  include kandra::prometheus::base
+  include zulip::supervisor
+  include zulip::wal_g
+
+  file { '/usr/local/bin/wal-g-exporter':
+    ensure  => file,
+    require => User[zulip],
+    owner   => 'zulip',
+    group   => 'zulip',
+    mode    => '0755',
+    source  => 'puppet:///modules/zulip/postgresql/wal-g-exporter',
+  }
+
+  # We embed the hash of the contents into the name of the process, so
+  # that `supervisorctl reread` knows that it has updated.
+  $full_exporter_hash = sha256(file('zulip/postgresql/wal-g-exporter'))
+  $exporter_hash = $full_exporter_hash[0,8]
+  file { "${zulip::common::supervisor_conf_dir}/prometheus_wal_g_exporter.conf":
+    ensure  => file,
+    require => [
+      User[zulip],
+      Package[supervisor],
+      File['/usr/local/bin/wal-g-exporter'],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/supervisor/conf.d/prometheus_wal_g_exporter.conf.template.erb'),
+    notify  => Service[supervisor],
+  }
+}

puppet/kandra/manifests/prometheus/weblate.pp

@@ -0,0 +1,40 @@
+# @summary Export Weblate translation stats
+class kandra::prometheus::weblate {
+  include kandra::prometheus::base
+  include zulip::supervisor
+
+  # We embed the hash of the contents into the name of the process, so
+  # that `supervisorctl reread` knows that it has updated.
+  $full_exporter_hash = sha256(file('kandra/weblate_exporter'))
+  $exporter_hash = $full_exporter_hash[0,8]
+
+  $bin = '/usr/local/bin/weblate_exporter'
+  file { $bin:
+    ensure => file,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0755',
+    source => 'puppet:///modules/kandra/weblate_exporter',
+  }
+
+  file { "${zulip::common::supervisor_conf_dir}/weblate_exporter.conf":
+    ensure  => file,
+    require => [
+      User[zulip],
+      Package[supervisor],
+      File[$bin],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/supervisor/conf.d/weblate_exporter.conf.template.erb'),
+    notify  => Service[supervisor],
+  }
+
+  include kandra::prometheus::pushgateway
+  zulip::cron { 'weblate-to-pushgateway':
+    minute    => '*/15',
+    command   => 'curl http://localhost:9189/metrics | curl --data-binary @- http://localhost:9091/metrics/job/weblate',
+    use_proxy => false,
+  }
+}

puppet/kandra/manifests/ses_logs.pp

@@ -0,0 +1,12 @@
+class kandra::ses_logs {
+  include kandra::vector
+
+  $ses_logs_sqs_url = zulipsecret('secrets', 'ses_logs_sqs_url', '')
+  $ses_logs_s3_bucket = zulipsecret('secrets', 'ses_logs_s3_bucket', '')
+
+  concat::fragment { 'vector_ses_logs':
+    target  => $kandra::vector::conf,
+    order   => '50',
+    content => template('kandra/vector_ses.toml.template.erb'),
+  }
+}

puppet/kandra/manifests/ssh_authorized_keys.pp

@@ -0,0 +1,17 @@
+define kandra::ssh_authorized_keys(
+  $keys = true,
+) {
+  $user = $name
+  if $keys == true {
+    $keypath = "prod/ssh/authorized_keys/${user}"
+  } elsif $keys.is_a(Array) {
+    $keypath = join($keys.map |$k| {"prod/ssh/authorized_keys/${k}"}, ' ')
+  } else {
+    $keypath = "prod/ssh/authorized_keys/${keys}"
+  }
+  exec { "ssh_authorized_keys ${user}":
+    require => File['/usr/local/bin/install-ssh-authorized-keys'],
+    command => "/usr/local/bin/install-ssh-authorized-keys ${user} ${keypath}",
+    unless  => "[ -f /usr/local/bin/install-ssh-authorized-keys ] && /usr/local/bin/install-ssh-authorized-keys --check ${user} ${keypath}",
+  }
+}

puppet/kandra/manifests/ssh_keys.pp

@@ -0,0 +1,17 @@
+define kandra::ssh_keys(
+  $keys = true,
+) {
+  $user = $name
+  if $keys == true {
+    $keypath = "prod/ssh/keys/${user}"
+  } elsif $keys.is_a(Array) {
+    $keypath = join($keys.map |$k| {"prod/ssh/keys/${k}"}, ' ')
+  } else {
+    $keypath = "prod/ssh/keys/${keys}"
+  }
+  exec { "ssh_keys ${user}":
+    require => File['/usr/local/bin/install-ssh-keys'],
+    command => "/usr/local/bin/install-ssh-keys ${user} ${keypath}",
+    unless  => "[ -f /usr/local/bin/install-ssh-keys ] && /usr/local/bin/install-ssh-keys --check ${user} ${keypath}",
+  }
+}

puppet/kandra/manifests/statuspage.pp

@@ -0,0 +1,28 @@
+# @summary Plumb Prometheus stats into status.zulip.com
+#
+# Requires a /etc/zulip/statuspage.conf which maps statuspage.io
+# metric_ids to Prometheus queries.
+class kandra::statuspage {
+  $bin = '/usr/local/bin/statuspage-pusher'
+
+  file { $bin:
+    ensure => file,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0755',
+    source => 'puppet:///modules/kandra/statuspage-pusher',
+  }
+
+  file { "${zulip::common::supervisor_conf_dir}/statuspage-pusher.conf":
+    ensure  => file,
+    require => [
+      Package[supervisor],
+      File[$bin],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/supervisor/conf.d/statuspage-pusher.conf.template.erb'),
+    notify  => Service[supervisor],
+  }
+}

puppet/kandra/manifests/teleport/application.pp

@@ -0,0 +1,30 @@
+# @summary Adds an http "application" to the Teleport configuration for the host.
+#
+# See https://goteleport.com/docs/application-access/
+define kandra::teleport::application (
+  $port,
+  $description = '',
+  $order = '50',
+  $labels = {},
+) {
+  include kandra::teleport::application_top
+  $app_data = [
+    {
+      name        => $name,
+      description => $description,
+      uri         => "http://localhost:${port}",
+      labels      => {
+        name => $name,
+      } + $labels,
+    },
+  ]
+
+  # This is appended to puppet/kandra/files/teleport_node.yaml, so
+  # must be indented, and the leading document dashes stripped.
+  $app_yaml = to_yaml($app_data).regsubst(/\A---\s*\n/, '').regsubst(/^/, '    ', 'G')
+  concat::fragment { "teleport_app_${name}":
+    target  => '/etc/teleport_node.yaml',
+    order   => $order,
+    content => $app_yaml,
+  }
+}

puppet/kandra/manifests/teleport/application_top.pp

@@ -0,0 +1,10 @@
+# @summary Enables application support on the node; include once.
+#
+# See https://goteleport.com/docs/application-access/
+class kandra::teleport::application_top {
+  concat::fragment { 'teleport_app':
+    target => '/etc/teleport_node.yaml',
+    order  => '10',
+    source => 'puppet:///modules/kandra/teleport_app.yaml',
+  }
+}

puppet/kandra/manifests/teleport/base.pp

@@ -0,0 +1,18 @@
+class kandra::teleport::base {
+  include zulip::supervisor
+
+  $setup_apt_repo_file = "${facts['zulip_scripts_path']}/lib/setup-apt-repo"
+  exec{ 'setup-apt-repo-teleport':
+    command => "${setup_apt_repo_file} --list teleport",
+    unless  => "${setup_apt_repo_file} --list teleport --verify",
+  }
+  package { 'teleport':
+    ensure  => installed,
+    require => Exec['setup-apt-repo-teleport'],
+  }
+  service { 'teleport':
+    ensure  => stopped,
+    enable  => mask,
+    require => Package['teleport'],
+  }
+}

puppet/kandra/manifests/teleport/db.pp

@@ -0,0 +1,23 @@
+# @summary Provide Teleport SSH access to a node.
+#
+# https://goteleport.com/docs/admin-guide/#adding-nodes-to-the-cluster
+# details additional manual steps to allow a node to join the cluster.
+class kandra::teleport::db {
+  include kandra::teleport::base
+
+  $fqdn = $facts['networking']['fqdn']
+  $hostname = $facts['networking']['hostname']
+  $is_ec2 = zulipconf('machine', 'hosting_provider', 'ec2') == 'ec2'
+  $join_token = zulipsecret('secrets', 'teleport_join_token', '')
+  file { '/etc/teleport_db.yaml':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/teleport_db.yaml.template.erb'),
+    before  => Kandra::Teleport::Part['db'],
+    notify  => Exec['reload teleport_db'],
+  }
+
+  kandra::teleport::part { 'db': }
+}

puppet/kandra/manifests/teleport/node.pp

@@ -0,0 +1,39 @@
+# @summary Provide Teleport SSH access to a node.
+#
+# EC2 nodes will automatically join the cluster; non-EC2 hosts will
+# need to set a teleport_join_token secret.  See
+# https://goteleport.com/docs/agents/join-services-to-your-cluster/join-token/#generate-a-token
+class kandra::teleport::node {
+  include kandra::teleport::base
+
+  $is_ec2 = zulipconf('machine', 'hosting_provider', 'ec2') == 'ec2'
+  $join_token = zulipsecret('secrets', 'teleport_join_token', '')
+  concat { '/etc/teleport_node.yaml':
+    ensure => present,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0644',
+    before => Kandra::Teleport::Part['node'],
+    notify => Exec['reload teleport_node'],
+  }
+  concat::fragment { 'teleport_node_base':
+    target  => '/etc/teleport_node.yaml',
+    content => template('kandra/teleport_node.yaml.template.erb'),
+    order   => '01',
+  }
+
+  kandra::teleport::part { 'node': }
+
+  $host_ca_path = '/etc/ssl/certs/teleport-ca-host.cert'
+  $host_ca_extract = @("EOT")
+    sqlite3 /var/lib/teleport/proc/sqlite.db "select value from kv where key = '/ids/node/current'" \
+      | jq -r .spec.tls_ca_certs[] \
+      | base64 -d \
+      > ${host_ca_path}
+    | EOT
+  exec { 'teleport host CA':
+    require => [Service['teleport_node'], Package['sqlite3']],
+    creates => $host_ca_path,
+    command => $host_ca_extract,
+  }
+}

puppet/kandra/manifests/teleport/part.pp

@@ -0,0 +1,32 @@
+# @summary Adds a systemd service named teleport_$name
+#
+define kandra::teleport::part() {
+  $part = $name
+
+  include zulip::systemd_daemon_reload
+  file { "/etc/systemd/system/teleport_${part}.service":
+    require => [
+      Package[teleport],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/teleport.service.template.erb'),
+    notify  => [Exec['reload systemd'], Service["teleport_${part}"]],
+  }
+
+  service {"teleport_${part}":
+    ensure  => running,
+    enable  => true,
+    require => [Service['supervisor'], Service['teleport'], Exec['reload systemd']],
+  }
+
+  # Notify this from YAML config changes (rather than the service)
+  # to get a SIGHUP-based graceful restart -- Teleport forks a new
+  # daemon and drains the old -- instead of a hard stop+start.
+  exec { "reload teleport_${part}":
+    command     => "/bin/systemctl reload teleport_${part}",
+    refreshonly => true,
+    require     => Service["teleport_${part}"],
+  }
+}

puppet/kandra/manifests/teleport/prometheus_app.pp

@@ -0,0 +1,24 @@
+define kandra::teleport::prometheus_app (
+  $port,
+  $description = '',
+  $order = '50',
+) {
+  $app_name = regsubst("${facts['networking']['hostname']}-${name}", '_', '-', 'G')
+  # Handle non-AWS hosts which don't have ec2 metadata
+  $fallback_role = $facts['networking']['hostname'] ? {
+    /^([^.-]+)/ => $1,
+    default     => '',
+  }
+  $role = pick_default($facts.dig('ec2_metadata', 'tags', 'instance', 'role'), $fallback_role)
+  kandra::teleport::application { $app_name:
+    port        => $port,
+    order       => $order,
+    description => $description,
+    labels      => {
+      type     => 'prometheus',
+      instance => $facts['networking']['hostname'],
+      exporter => $name,
+      role     => $role,
+    }
+  }
+}

puppet/kandra/manifests/teleport/sd.pp

@@ -0,0 +1,31 @@
+# @summary Returns Prometheus scrape targets based on Teleport apps
+#
+# Only one instance is necessary.
+#
+class kandra::teleport::sd {
+  include zulip::supervisor
+  include kandra::teleport::tbot
+
+  $version = $zulip::common::versions['teleport-sd']['version']
+  $bin = "/srv/zulip-teleport-sd-${version}"
+
+  zulip::external_dep { 'teleport-sd':
+    version       => $version,
+    url           => "https://github.com/alexmv/teleport-sd/releases/download/v${version}/teleport-sd-linux-${zulip::common::goarch}",
+    cleanup_after => [Service[supervisor]],
+  }
+
+  file { "${zulip::common::supervisor_conf_dir}/teleport-sd.conf":
+    ensure  => file,
+    require => [
+      Package[teleport],
+      Package[supervisor],
+      File[$bin],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/supervisor/conf.d/teleport-sd.conf.erb'),
+    notify  => Service[supervisor],
+  }
+}

puppet/kandra/manifests/teleport/tbot.pp

@@ -0,0 +1,48 @@
+# @summary Provide a local application proxy
+# https://goteleport.com/docs/reference/machine-workload-identity/configuration/#application-proxy
+class kandra::teleport::tbot {
+  include kandra::teleport::base
+
+  file { '/etc/tbot.yaml':
+    ensure => file,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0644',
+    source => 'puppet:///modules/kandra/tbot.yaml',
+    before => Service['tbot'],
+    notify => Exec['reload tbot'],
+  }
+
+  file { '/etc/systemd/system/tbot.service':
+    require => [
+      Package[teleport],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    source  => 'puppet:///modules/kandra/tbot.service',
+    notify  => [Exec['reload systemd'], Service['tbot']],
+  }
+
+  file { ['/var/lib/teleport/bot', '/var/lib/teleport/bot-identity']:
+    ensure  => directory,
+    owner   => 'teleport',
+    group   => 'teleport',
+    mode    => '0644',
+    require => Service['teleport'],
+    before  => Service['tbot'],
+  }
+
+  service {'tbot':
+    ensure  => running,
+    enable  => true,
+    require => [Service['teleport'], Exec['reload systemd']],
+  }
+
+  # See the equivalent in kandra::teleport::part.
+  exec { 'reload tbot':
+    command     => '/bin/systemctl reload tbot',
+    refreshonly => true,
+    require     => Service['tbot'],
+  }
+}

puppet/kandra/manifests/user_dotfiles.pp

@@ -0,0 +1,85 @@
+define kandra::user_dotfiles (
+  $home = '',
+  $keys = false,
+  $authorized_keys = false,
+  $known_hosts = false,
+) {
+  $user = $name
+
+  if $home == '' {
+    $homedir = "/home/${user}"
+  } else {
+    $homedir = $home
+  }
+
+  file { "${homedir}/.ssh":
+    ensure  => directory,
+    require => User[$user],
+    owner   => $user,
+    group   => $user,
+    mode    => '0700',
+  }
+
+  file { "${homedir}/.emacs":
+    ensure  => file,
+    require => User[$user],
+    owner   => $user,
+    group   => $user,
+    mode    => '0644',
+    source  => 'puppet:///modules/kandra/dotfiles/emacs.el',
+  }
+
+  # Suppress MOTD printing, to fix load problems with Nagios caused by
+  # Ubuntu's default MOTD tools for things like "checking for the next
+  # release" being super slow.
+  file { "${homedir}/.hushlogin":
+    ensure  => file,
+    require => User[$user],
+    owner   => $user,
+    group   => $user,
+    mode    => '0644',
+    content => '',
+  }
+
+  if $keys != false {
+    kandra::ssh_keys{ $user:
+      keys    => $keys,
+      require => File["${homedir}/.ssh"],
+    }
+  }
+  if $authorized_keys != false {
+    kandra::ssh_authorized_keys{ $user:
+      keys    => $authorized_keys,
+      require => File["${homedir}/.ssh"],
+    }
+  }
+  if $known_hosts != false {
+    file { "${homedir}/.ssh/known_hosts":
+      # We mark this as "present" to ensure that it exists, but not to
+      # directly control its contents.
+      ensure  => present,
+      owner   => $user,
+      group   => $user,
+      mode    => '0644',
+      require => File["${homedir}/.ssh"],
+    }
+    $known_hosts.each |Optional[String] $hostname| {
+      if $hostname == undef {
+        # pass
+      } elsif $hostname == 'github.com' {
+        $github_keys = file('kandra/github.keys')
+        exec { "${user} ssh known_hosts ${hostname}":
+          command => "echo '${github_keys}' >> ${homedir}/.ssh/known_hosts",
+          unless  => "grep ${hostname} ${homedir}/.ssh/known_hosts",
+          require => File["${homedir}/.ssh/known_hosts"],
+        }
+      } else {
+        exec { "${user} ssh known_hosts ${hostname}":
+          command => "ssh-keyscan ${hostname} >> ${homedir}/.ssh/known_hosts",
+          unless  => "grep ${hostname} ${homedir}/.ssh/known_hosts",
+          require => File["${homedir}/.ssh/known_hosts"],
+        }
+      }
+    }
+  }
+}

puppet/kandra/manifests/vector.pp

@@ -0,0 +1,73 @@
+# @summary Installs Vector to transform Prometheus data
+#
+class kandra::vector {
+  $version = $zulip::common::versions['vector']['version']
+  $dir = "/srv/zulip-vector-${version}"
+  $bin = "${dir}/bin/vector"
+  $conf = '/etc/vector.toml'
+
+  $arch = $facts['os']['architecture'] ? {
+    'amd64'   => 'x86_64',
+    'aarch64' => 'aarch64',
+  }
+
+  zulip::external_dep { 'vector':
+    version        => $version,
+    url            => "https://packages.timber.io/vector/${version}/vector-${version}-${arch}-unknown-linux-gnu.tar.gz",
+    tarball_prefix => "vector-${arch}-unknown-linux-gnu",
+    bin            => [$bin],
+    cleanup_after  => [Service[supervisor]],
+  }
+  file { "${zulip::common::supervisor_conf_dir}/vector.conf":
+    ensure  => file,
+    require => [
+      User[zulip],
+      Package[supervisor],
+      Concat[$conf],
+      File[$bin],
+    ],
+    before  => Exec['Cleanup vector'],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/supervisor/conf.d/vector.conf.template.erb'),
+    notify  => Service[supervisor],
+  }
+
+  exec { 'reload vector':
+    command     => 'supervisorctl signal HUP vector',
+    require     => Service[supervisor],
+    refreshonly => true,
+  }
+  concat { $conf:
+    ensure => present,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0644',
+    notify => Exec['reload vector'],
+  }
+  # All of the pipelines need to be included in the Prometheus
+  # exporter; they insert their strings at order 90, with a leading
+  # comma, in the middle of the "inputs" block
+  $vector_export = @(VECTOR)
+    [sources.vector_metrics]
+      type = "internal_metrics"
+    [sinks.prometheus_exporter]
+      type = "prometheus_exporter"
+      address = "127.0.0.1:9081"
+      flush_period_secs = 120
+      suppress_timestamp = true
+      buckets = [0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5]
+      inputs = ["vector_metrics"
+    |-VECTOR
+  concat::fragment { 'vector_export_prefix':
+    target  => $conf,
+    content => $vector_export,
+    order   => '89',
+  }
+  concat::fragment { 'vector_export_suffix':
+    target  => $conf,
+    content => "]\n",
+    order   => '99',
+  }
+}

puppet/kandra/templates/dotfiles/aws_config.erb

@@ -0,0 +1,10 @@
+[default]
+region = us-east-1
+output = text
+<% if @is_ec2 -%>
+# Credentials are from the IAM role attached to the EC2 instance
+<% else -%>
+# We pull the Teleport host certificate and use that to auth to AWS
+# using IAM Roles Anywhere
+credential_process = /usr/local/bin/teleport-aws-credentials --trust-anchor-arn <%= @aws_trust_arn %> --profile-arn <%= @aws_profile_arn %> --role-arn <%= @aws_role_arn %>
+<% end %>

puppet/kandra/templates/grafana.ini.template.erb

@@ -0,0 +1,64 @@
+#################################### Paths ####################################
+[paths]
+# Path to where grafana can store temp files, sessions, and the sqlite3 db
+data = /var/lib/grafana
+
+# Directory where grafana can store logs
+logs = /var/log/grafana
+
+# Directory where grafana will automatically scan and look for plugins
+plugins = /var/lib/grafana/plugins
+
+
+#################################### Server ####################################
+[server]
+# Protocol (http, https, h2, socket)
+protocol = http
+
+# The ip address to bind to, empty will bind to all interfaces
+http_addr = 127.0.0.1
+
+# The http port to use
+http_port = 3000
+
+# The public facing domain name used to access grafana from a browser
+domain = monitoring.teleport.zulipchat.net
+
+# The full public facing url you use in browser, used for redirects and emails
+# If you use reverse proxy and sub path specify full url (with sub path)
+root_url = https://monitoring.teleport.zulipchat.net/
+
+
+#################################### SMTP ####################################
+[smtp]
+enabled = true
+from_name = Grafana
+from_address = <%= @email_from %>
+host = <%= @email_host %>
+user = <%= @email_user %>
+password = <%= @email_password %>
+startTLS_policy = MandatoryStartTLS
+
+
+#################################### Auth JWT ####################################
+[auth.jwt]
+# Grafana should trust the JWT that is provided in a header from Teleport; see
+# https://goteleport.com/docs/application-access/guides/jwt/ and
+# https://grafana.com/docs/grafana/latest/auth/jwt/
+enabled = true
+header_name = Teleport-Jwt-Assertion
+username_claim = sub
+jwk_set_url = https://teleport.zulipchat.net/.well-known/jwks.json
+cache_ttl = 24h
+expect_claims = {"iss": "teleport.zulipchat.net"}
+auto_sign_up = true
+skip_org_role_sync = true
+
+#################################### Alerting ####################################
+# Switch to the Grafana 8 unified alerting
+# https://grafana.com/docs/grafana/latest/alerting/unified-alerting/opt-in/#enable-grafana-alerting
+[unified_alerting]
+enabled = true
+
+[alerting]
+enabled = false