snapshot of https://github.com/zulip/zulip.git @ 48c69911c7dd4e03b20552eb3bd77ad8151c1357 (2026-09-15)

scripts/README.md

@@ -0,0 +1,11 @@
+This directory contains scripts that:
+
+- Generally do not require access to Django or the database (those are
+  "management commands"), and thus are suitable to run operationally.
+
+- Are useful for managing a production deployment of Zulip (many are
+  also used in a Zulip development environment, though
+  development-only scripts live in `tools/`).
+
+For more details, see
+https://zulip.readthedocs.io/en/latest/subsystems/directory-structure.html.

scripts/get-django-setting

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+import os
+import sys
+
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(BASE_DIR)
+from scripts.lib.setup_path import setup_path
+
+setup_path()
+
+os.environ["DJANGO_SETTINGS_MODULE"] = "zproject.settings"
+from django.conf import settings
+
+print(getattr(settings, sys.argv[1]))

scripts/lib/build-pgroonga

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+version="4.0.6"
+sha256=d0048944763c18f91bc67e043aafa64c2c53f6246547c9474311efbc05ccfe66
+
+tmpdir="$(mktemp -d)"
+trap 'rm -r "$tmpdir"' EXIT
+cd "$tmpdir"
+tarball="pgroonga-$version.tar.gz"
+curl -fLO --retry 3 "https://github.com/pgroonga/pgroonga/releases/download/$version/$tarball"
+sha256sum -c <<<"$sha256 $tarball"
+tar -xzf "$tarball"
+cd "pgroonga-$version"
+
+meson setup build .
+meson install -C build

scripts/lib/check-database-compatibility

@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+import logging
+import os
+import subprocess
+import sys
+
+ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.insert(0, ZULIP_PATH)
+
+from scripts.lib.setup_path import setup_path
+from scripts.lib.zulip_tools import (
+    DEPLOYMENTS_DIR,
+    assert_not_running_as_root,
+    get_config,
+    get_config_file,
+    parse_version_from,
+)
+from version import ZULIP_VERSION as NEW_VERSION
+
+assert_not_running_as_root()
+setup_path()
+os.environ["DJANGO_SETTINGS_MODULE"] = "zproject.settings"
+
+import django
+from django.db import connection
+from django.db.migrations.loader import MigrationLoader
+
+from zerver.lib.migration_status import STALE_MIGRATIONS
+
+django.setup()
+
+django_pg_version = connection.cursor().connection.server_version // 10000
+if os.path.exists("/etc/init.d/postgresql") and os.path.exists("/etc/zulip/zulip.conf"):
+    postgresql_version = int(get_config(get_config_file(), "postgresql", "version", "0"))
+    if postgresql_version == 0:
+        postgresql_version = django_pg_version
+        subprocess.check_call(
+            [
+                "crudini",
+                "--set",
+                "/etc/zulip/zulip.conf",
+                "postgresql",
+                "version",
+                str(postgresql_version),
+            ]
+        )
+    elif postgresql_version != django_pg_version:
+        logging.critical(
+            "PostgreSQL version mismatch: %d (running) vs %d (configured)",
+            django_pg_version,
+            postgresql_version,
+        )
+        logging.info(
+            "/etc/zulip/zulip.conf claims that Zulip is running PostgreSQL\n"
+            "%d, but the server is connected to a PostgreSQL running\n"
+            "version %d.  Check the output from pg_lsclusters to verify\n"
+            "which clusters are running, and update /etc/zulip/zulip.conf to match.\n"
+            "\n"
+            "In general, this results from manually upgrading PostgreSQL; you\n"
+            "should follow our instructions for using our tool to do so:\n"
+            "https://zulip.readthedocs.io/en/stable/production/upgrade.html#upgrading-postgresql",
+            postgresql_version,
+            django_pg_version,
+        )
+        sys.exit(1)
+
+if django_pg_version < 14:
+    logging.critical("Unsupported PostgreSQL version: %d", postgresql_version)
+    logging.info(
+        "Please upgrade to PostgreSQL 14 or newer first.\n"
+        "See https://zulip.readthedocs.io/en/stable/production/"
+        "upgrade.html#upgrading-postgresql"
+    )
+    sys.exit(1)
+
+loader = MigrationLoader(connection)
+missing = set(loader.applied_migrations)
+
+missing.difference_update(STALE_MIGRATIONS)
+
+for key, migration in loader.disk_migrations.items():
+    missing.discard(key)
+    missing.difference_update(migration.replaces)
+if not missing:
+    sys.exit(0)
+
+print("Migrations which are currently applied, but missing in the new version:")
+for app, migration_name in sorted(missing):
+    print(f"  {app} - {migration_name}")
+
+current_version = parse_version_from(os.path.join(DEPLOYMENTS_DIR, "current"))
+logging.error(
+    "This is not an upgrade -- the current deployment (version %s) "
+    "contains %s database migrations which %s (version %s) does not.",
+    current_version,
+    len(missing),
+    ZULIP_PATH,
+    NEW_VERSION,
+)
+sys.exit(1)

scripts/lib/check_rabbitmq_queue.py

@@ -0,0 +1,233 @@
+import json
+import os
+import re
+import subprocess
+import sys
+import time
+from collections import defaultdict
+from typing import Any
+
+ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+sys.path.append(ZULIP_PATH)
+from scripts.lib.zulip_tools import atomic_nagios_write, get_config, get_config_file
+
+normal_queues = [
+    "deferred_work",
+    "deferred_email_senders",
+    "digest_emails",
+    "email_mirror",
+    "email_senders",
+    "embed_links",
+    "embedded_bots",
+    "missedmessage_emails",
+    "missedmessage_mobile_notifications",
+    "outgoing_webhooks",
+    "thumbnail",
+    "user_activity",
+    "user_activity_interval",
+]
+
+# Soft reactivations share the deferred_work queue unless a server opts
+# into a dedicated queue; only then is there a consumer to monitor.
+if get_config(get_config_file(), "application_server", "dedicated_soft_reactivation_queue", False):
+    normal_queues.append("soft_reactivation")
+
+mobile_notification_shards = int(
+    get_config(get_config_file(), "application_server", "mobile_notification_shards", "1")
+)
+user_activity_shards = int(
+    get_config(get_config_file(), "application_server", "user_activity_shards", "1")
+)
+
+OK = 0
+WARNING = 1
+CRITICAL = 2
+UNKNOWN = 3
+
+states = {
+    0: "OK",
+    1: "WARNING",
+    2: "CRITICAL",
+    3: "UNKNOWN",
+}
+
+MAX_SECONDS_TO_CLEAR: defaultdict[str, int] = defaultdict(
+    lambda: 30,
+    deferred_work=600,
+    # Soft reactivations are the deferred_work backfills split into their
+    # own queue; keep deferred_work's generous clear-time budget so the slow
+    # backfills that motivated the split don't trip false alerts.
+    soft_reactivation=600,
+    digest_emails=1200,
+    missedmessage_mobile_notifications=120,
+    embed_links=60,
+    email_senders=90,
+    deferred_email_senders=3600,
+)
+CRITICAL_SECONDS_TO_CLEAR: defaultdict[str, int] = defaultdict(
+    lambda: 60,
+    deferred_work=900,
+    soft_reactivation=900,
+    missedmessage_mobile_notifications=180,
+    digest_emails=1800,
+    embed_links=90,
+    email_senders=300,
+    deferred_email_senders=4500,
+)
+
+
+def analyze_queue_stats(
+    queue_name: str, stats: dict[str, Any], queue_count_rabbitmqctl: int
+) -> dict[str, Any]:
+    now = int(time.time())
+    if stats == {}:
+        return dict(status=UNKNOWN, name=queue_name, message="invalid or no stats data")
+
+    if now - stats["update_time"] > 180 and queue_count_rabbitmqctl > 10:
+        # Queue isn't updating the stats file and has some events in
+        # the backlog, it's likely stuck.
+        #
+        # TODO: There's an unlikely race condition here - if the queue
+        # was fully emptied and was idle due to no new events coming
+        # for over 180 seconds, suddenly gets a burst of events and
+        # this code runs exactly in the very small time window between
+        # those events popping up and the queue beginning to process
+        # the first one (which will refresh the stats file at the very
+        # start), we'll incorrectly return the CRITICAL status. The
+        # chance of that happening should be negligible because the queue
+        # worker should wake up immediately and log statistics before
+        # starting to process the first event.
+        return dict(
+            status=CRITICAL,
+            name=queue_name,
+            message="queue appears to be stuck, last update {}, queue size {}".format(
+                stats["update_time"], queue_count_rabbitmqctl
+            ),
+        )
+
+    current_size = queue_count_rabbitmqctl
+    average_consume_time = stats["recent_average_consume_time"]
+    if average_consume_time is None:
+        # Queue just started; we can't effectively estimate anything.
+        #
+        # If the queue is stuck in this state and not processing
+        # anything, eventually the `update_time` rule above will fire.
+        return dict(status=OK, name=queue_name, message="")
+
+    expected_time_to_clear_backlog = current_size * average_consume_time
+    if expected_time_to_clear_backlog > MAX_SECONDS_TO_CLEAR[queue_name]:
+        if expected_time_to_clear_backlog > CRITICAL_SECONDS_TO_CLEAR[queue_name]:
+            status = CRITICAL
+        else:
+            status = WARNING
+
+        return dict(
+            status=status,
+            name=queue_name,
+            message=f"clearing the backlog will take too long: {expected_time_to_clear_backlog}s, size: {current_size}",
+        )
+
+    return dict(status=OK, name=queue_name, message="")
+
+
+WARN_COUNT_THRESHOLD_DEFAULT = 10
+CRITICAL_COUNT_THRESHOLD_DEFAULT = 50
+
+
+def check_other_queues(queue_counts_dict: dict[str, int]) -> list[dict[str, Any]]:
+    """Do a simple queue size check for queues whose workers don't publish stats files."""
+
+    results = []
+    for queue, count in queue_counts_dict.items():
+        if queue in normal_queues:
+            continue
+
+        if count > CRITICAL_COUNT_THRESHOLD_DEFAULT:
+            results.append(dict(status=CRITICAL, name=queue, message=f"count critical: {count}"))
+        elif count > WARN_COUNT_THRESHOLD_DEFAULT:
+            results.append(dict(status=WARNING, name=queue, message=f"count warning: {count}"))
+        else:
+            results.append(dict(status=OK, name=queue, message=""))
+
+    return results
+
+
+def check_rabbitmq_queues() -> None:
+    pattern = re.compile(r"(\w+)\t(\d+)\t(\d+)")
+    if "USER" in os.environ and os.environ["USER"] not in ["root", "rabbitmq"]:
+        print("This script must be run as the root or rabbitmq user")
+
+    list_queues_output = subprocess.check_output(
+        ["/usr/sbin/rabbitmqctl", "list_queues", "name", "messages", "consumers"],
+        text=True,
+    )
+    queue_counts_rabbitmqctl = {}
+    queues_with_consumers = []
+    for line in list_queues_output.split("\n"):
+        line = line.strip()
+        m = pattern.match(line)
+        if m:
+            queue = m.group(1)
+            count = int(m.group(2))
+            consumers = int(m.group(3))
+            queue_counts_rabbitmqctl[queue] = count
+            if consumers > 0 and not queue.startswith("notify_tornado"):
+                queues_with_consumers.append(queue)
+
+    queue_stats_dir = subprocess.check_output(
+        [os.path.join(ZULIP_PATH, "scripts/get-django-setting"), "QUEUE_STATS_DIR"],
+        text=True,
+    ).strip()
+    queue_stats: dict[str, dict[str, Any]] = {}
+
+    check_queues = normal_queues
+    if mobile_notification_shards > 1:
+        # For sharded queue workers, where there's a separate queue
+        # for each shard, we need to make sure none of those are
+        # backlogged.
+        check_queues += [
+            f"missedmessage_mobile_notifications_shard{d}"
+            for d in range(1, mobile_notification_shards + 1)
+        ]
+    if user_activity_shards > 1:
+        check_queues += [f"user_activity_shard{d}" for d in range(1, user_activity_shards + 1)]
+
+    queues_to_check = set(check_queues).intersection(set(queues_with_consumers))
+    for queue in queues_to_check:
+        fn = queue + ".stats"
+        file_path = os.path.join(queue_stats_dir, fn)
+        if not os.path.exists(file_path):
+            queue_stats[queue] = {}
+            continue
+
+        with open(file_path) as f:
+            try:
+                queue_stats[queue] = json.load(f)
+            except json.decoder.JSONDecodeError:
+                queue_stats[queue] = {}
+
+    results = []
+    for queue_name, stats in queue_stats.items():
+        results.append(analyze_queue_stats(queue_name, stats, queue_counts_rabbitmqctl[queue_name]))
+
+    results.extend(check_other_queues(queue_counts_rabbitmqctl))
+
+    status = max(result["status"] for result in results)
+
+    if status > 0:
+        queue_error_template = "queue {} problem: {}:{}"
+        error_message = "; ".join(
+            queue_error_template.format(result["name"], states[result["status"]], result["message"])
+            for result in results
+            if result["status"] > 0
+        )
+        sys.exit(
+            atomic_nagios_write(
+                "check-rabbitmq-results",
+                "critical" if status == CRITICAL else "warning",
+                error_message,
+            )
+        )
+    else:
+        atomic_nagios_write("check-rabbitmq-results", "ok", "queues normal")

scripts/lib/clean_emoji_cache.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+import argparse
+import os
+import sys
+
+ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.append(ZULIP_PATH)
+from scripts.lib.zulip_tools import (
+    get_environment,
+    get_recent_deployments,
+    parse_cache_script_args,
+    purge_unused_caches,
+)
+
+ENV = get_environment()
+EMOJI_CACHE_PATH = "/srv/zulip-emoji-cache"
+
+
+def get_caches_in_use(threshold_days: int) -> set[str]:
+    setups_to_check = {ZULIP_PATH}
+    caches_in_use = set()
+
+    if ENV == "prod":
+        setups_to_check |= get_recent_deployments(threshold_days)
+    if ENV == "dev":
+        CACHE_SYMLINK = os.path.join(ZULIP_PATH, "static", "generated", "emoji")
+        CURRENT_CACHE = os.path.dirname(os.path.realpath(CACHE_SYMLINK))
+        caches_in_use.add(CURRENT_CACHE)
+
+    for setup_dir in setups_to_check:
+        emoji_link_path = os.path.join(setup_dir, "static/generated/emoji")
+        if not os.path.islink(emoji_link_path):
+            # This happens for a deployment directory extracted from a
+            # tarball, which just has a copy of the emoji data, not a symlink.
+            continue
+        # The actual cache path doesn't include the /emoji
+        caches_in_use.add(os.path.dirname(os.readlink(emoji_link_path)))
+    return caches_in_use
+
+
+def main(args: argparse.Namespace) -> None:
+    caches_in_use = get_caches_in_use(args.threshold_days)
+    purge_unused_caches(EMOJI_CACHE_PATH, caches_in_use, "emoji cache", args)
+
+
+if __name__ == "__main__":
+    args = parse_cache_script_args("This script cleans unused Zulip emoji caches.")
+    main(args)

scripts/lib/clean_unused_caches.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+import argparse
+import os
+import shutil
+import sys
+
+ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.append(ZULIP_PATH)
+from scripts.lib import clean_emoji_cache
+from scripts.lib.zulip_tools import parse_cache_script_args
+
+
+def main(args: argparse.Namespace) -> None:
+    os.chdir(ZULIP_PATH)
+    shutil.rmtree("/srv/zulip-venv-cache", ignore_errors=True)  # Replaced as of 10.0
+    shutil.rmtree("/srv/zulip-npm-cache", ignore_errors=True)  # Replaced as of 7.0
+    clean_emoji_cache.main(args)
+
+
+if __name__ == "__main__":
+    args = parse_cache_script_args("This script cleans unused Zulip caches.")
+    main(args)

scripts/lib/create-production-venv

@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+import argparse
+import os
+import platform
+import sys
+
+ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+if ZULIP_PATH not in sys.path:
+    sys.path.append(ZULIP_PATH)
+
+from scripts.lib.setup_venv import get_venv_dependencies
+from scripts.lib.zulip_tools import os_families, run
+
+parser = argparse.ArgumentParser(description="Create a production virtualenv with caching")
+parser.add_argument("deploy_path")
+args = parser.parse_args()
+
+# install dependencies for setting up the virtualenv
+distro_info = platform.freedesktop_os_release()
+vendor = distro_info["ID"]
+os_version = distro_info["VERSION_ID"]
+VENV_DEPENDENCIES = get_venv_dependencies(vendor, os_version)
+if "debian" in os_families():
+    run(["apt-get", "-y", "install", "--no-install-recommends", *VENV_DEPENDENCIES])
+elif "fedora" in os_families():
+    run(["yum", "-y", "install", *VENV_DEPENDENCIES])
+else:
+    print("Unsupported platform: {}".format(distro_info["ID"]))
+    sys.exit(1)
+
+os.chdir(ZULIP_PATH)
+run(["scripts/lib/install-uv"])
+run(
+    ["uv", "sync", "--frozen", "--no-managed-python", "--only-group=prod"],
+    env={k: v for k, v in os.environ.items() if k not in {"PYTHONDEVMODE", "PYTHONWARNINGS"}},
+)

scripts/lib/install

@@ -0,0 +1,790 @@
+#!/usr/bin/env bash
+set -e
+
+usage() {
+    # A subset of this documentation also appears in docs/production/install.md
+    cat <<'EOF'
+Usage:
+  install --hostname=zulip.example.com --email=zulip-admin@example.com [options...]
+  install --help
+
+Options:
+  --hostname=zulip.example.com
+      The user-accessible domain name for this Zulip server, i.e.,
+      what users will type in their web browser.  Required, unless
+      --no-init-db or --puppet-classes is set, and --certbot is not.
+      Use Punycode for non-ASCII domains.
+
+  --email=zulip-admin@example.com
+      The email address of the person or team who should get support
+      and error emails from this Zulip server.  Required, unless
+      --no-init-db or --puppet-classes is set and --certbot is not.
+
+  --certbot
+      Obtains a free SSL certificate for the server using Certbot,
+      https://certbot.eff.org/  Recommended.  Conflicts with --self-signed-cert.
+  --self-signed-cert
+      Generate a self-signed SSL certificate for the server. This isn’t suitable for
+      production use, but may be convenient for testing.  Conflicts with --certbot.
+
+  --postgresql-database-name=zulip
+      Sets the PostgreSQL database name.
+  --postgresql-database-user=zulip
+      Sets the PostgreSQL database user.
+  --postgresql-version=18
+      Sets the version of PostgreSQL that will be installed.
+
+  --no-init-db
+      Does not do any database initialization; use when you already have a Zulip
+      database.
+  --puppet-classes
+      Comma-separated list of Puppet classes to install; defaults to
+      'zulip:::profile::standalone'.  Implies --no-init-db.
+
+  --no-overwrite-settings
+      Preserve existing `/etc/zulip` configuration files.
+  --no-dist-upgrade
+      Skip the initial `apt-get dist-upgrade`.
+
+  --push-notifications
+      With this option, the Zulip installer registers your server for the Mobile
+      Push Notification Service, and sets up the initial default configuration.
+      You will be immediately prompted to agree to the Terms of Service, and your
+      server will be registered at the end of the installation process.
+  --no-push-notifications
+      Disable push notifications registration (the default if neither
+      flag is provided).
+
+  --no-submit-usage-statistics
+      If you enable push notifications, by default your server
+      will submit basic metadata (required for billing and for
+      determining free plan eligibility), as well as aggregate usage
+      statistics. You can disable submitting usage statistics by passing
+      this flag.
+      If push notifications are not enabled, data won't be submitted, so
+      this flag is redundant.
+
+  --agree-to-terms-of-service
+      If you enable push notifications, you can pass this flag to indicate
+      that you have read and agree to the Zulip Terms of Service:
+      <https://zulip.com/policies/terms>.
+      This skips the Terms of Service prompt, allowing for running the installer
+      with --push-notifications in scripts without requiring user input.
+
+
+EOF
+}
+
+system_requirements_failure() {
+    set +x
+    echo >&2
+    cat >&2
+    cat <<EOF >&2
+
+For more information, see:
+  https://zulip.readthedocs.io/en/latest/production/requirements.html
+EOF
+    exit 1
+}
+
+# Shell option parsing.  Over time, we'll want to move some of the
+# environment variables below into this self-documenting system.
+args="$(getopt -o '' --long help,hostname:,email:,certbot,self-signed-cert,cacert:,postgresql-database-name:,postgresql-database-user:,postgresql-version:,no-init-db,puppet-classes:,no-overwrite-settings,no-dist-upgrade,push-notifications,no-push-notifications,no-submit-usage-statistics,agree-to-terms-of-service -n "$0" -- "$@")"
+
+eval "set -- $args"
+while true; do
+    case "$1" in
+        --help)
+            usage
+            exit 0
+            ;;
+
+        --hostname)
+            EXTERNAL_HOST="$2"
+            shift
+            shift
+            ;;
+        --email)
+            ZULIP_ADMINISTRATOR="$2"
+            shift
+            shift
+            ;;
+
+        --certbot)
+            USE_CERTBOT=1
+            shift
+            ;;
+        --self-signed-cert)
+            SELF_SIGNED_CERT=1
+            shift
+            ;;
+        --postgresql-database-name)
+            POSTGRESQL_DATABASE_NAME="$2"
+            shift
+            shift
+            ;;
+        --postgresql-database-user)
+            POSTGRESQL_DATABASE_USER="$2"
+            shift
+            shift
+            ;;
+        --postgresql-version)
+            POSTGRESQL_VERSION="$2"
+            shift
+            shift
+            ;;
+        --no-init-db)
+            NO_INIT_DB=1
+            shift
+            ;;
+        --puppet-classes)
+            PUPPET_CLASSES="$2"
+            NO_INIT_DB=1
+            shift
+            shift
+            ;;
+
+        --no-overwrite-settings)
+            NO_OVERWRITE_SETTINGS=1
+            shift
+            ;;
+        --no-dist-upgrade)
+            NO_DIST_UPGRADE=1
+            shift
+            ;;
+        --push-notifications)
+            if [ -n "$NO_PUSH_NOTIFICATIONS" ]; then
+                echo "error: --push-notifications and --no-push-notifications are incompatible." >&2
+                exit 1
+            fi
+            PUSH_NOTIFICATIONS=1
+            shift
+            ;;
+        --no-push-notifications)
+            if [ -n "$PUSH_NOTIFICATIONS" ]; then
+                echo "error: --push-notifications and --no-push-notifications are incompatible." >&2
+                exit 1
+            fi
+            NO_PUSH_NOTIFICATIONS=1
+            shift
+            ;;
+        --no-submit-usage-statistics)
+            NO_SUBMIT_USAGE_STATISTICS=1
+            shift
+            ;;
+        --agree-to-terms-of-service)
+            AGREE_TO_TERMS_OF_SERVICE_FLAG=1
+            shift
+            ;;
+        --)
+            shift
+            break
+            ;;
+    esac
+done
+
+if [ "$#" -gt 0 ]; then
+    usage >&2
+    exit 1
+fi
+
+## Options from environment variables.
+#
+# Specify options for apt.
+read -r -a APT_OPTIONS <<<"${APT_OPTIONS:-}"
+# Install additional packages.
+read -r -a ADDITIONAL_PACKAGES <<<"${ADDITIONAL_PACKAGES:-}"
+# Comma-separated list of Puppet manifests to install.  The default is
+# zulip::profile::standalone for an all-in-one system or
+# zulip::profile::docker for Docker.  Use
+# e.g. zulip::profile::app_frontend for a Zulip frontend server.
+PUPPET_CLASSES="${PUPPET_CLASSES:-zulip::profile::standalone}"
+POSTGRESQL_VERSION="${POSTGRESQL_VERSION:-18}"
+
+if [ -n "$SELF_SIGNED_CERT" ] && [ -n "$USE_CERTBOT" ]; then
+    set +x
+    echo "error: --self-signed-cert and --certbot are incompatible" >&2
+    echo >&2
+    usage >&2
+    exit 1
+fi
+
+if [ -z "$EXTERNAL_HOST" ] || [ -z "$ZULIP_ADMINISTRATOR" ]; then
+    if [ -n "$USE_CERTBOT" ] || [ -z "$NO_INIT_DB" ]; then
+        usage >&2
+        exit 1
+    fi
+fi
+
+if [ "$EXTERNAL_HOST" = zulip.example.com ] \
+    || [ "$ZULIP_ADMINISTRATOR" = zulip-admin@example.com ]; then
+    # These example values are specifically checked for and would fail
+    # later; see zerver/checks.py
+    echo 'error: The example hostname and email must be replaced with real values.' >&2
+    echo >&2
+    usage >&2
+    exit 1
+fi
+
+case "$EXTERNAL_HOST" in
+    *[![:ascii:]]*)
+        # Non-ASCII hostnames are specifically checked for and would
+        # fail later; see zerver/checks.py
+        echo "error: The hostname ($EXTERNAL_HOST) contains non-ASCII characters." >&2
+        echo 'Internationalized domain names must be specified in punycode ("xn--") form:' >&2
+        # python3 is not installed until later, so it may not be
+        # available to compute the suggested encoding.
+        if command -v python3 >/dev/null 2>&1; then
+            suggestion="$(python3 -c '
+import sys
+
+hostname, sep, port = sys.argv[1].partition(":")
+print(
+    ".".join(
+        "xn--" + label.encode("punycode").decode() if not label.isascii() else label
+        for label in hostname.split(".")
+    )
+    + sep
+    + port
+)' "$EXTERNAL_HOST")"
+            echo "    --hostname=$suggestion" >&2
+        fi
+        exit 1
+        ;;
+esac
+
+case "$POSTGRESQL_VERSION" in
+    [0-9] | [0-9].* | 1[0-3] | 1[0-3].*)
+        echo "error: PostgreSQL 14 or newer is required." >&2
+        exit 1
+        ;;
+esac
+
+if [ -z "$PUSH_NOTIFICATIONS" ] && [ -z "$NO_PUSH_NOTIFICATIONS" ]; then
+    # Unless specified, we default to --no-push-notifications
+    NO_PUSH_NOTIFICATIONS=1
+fi
+
+# We set these to Python-style True/False string values, for easy
+# insertion into the settings.py file.
+if [ -n "$PUSH_NOTIFICATIONS" ]; then
+    SERVICE_MOBILE_PUSH="True"
+
+    # --push-notifications also enables SUBMIT_USAGE_STATISTICS, unless the user
+    # explicitly opted out by passing --no-submit-usage-statistics.
+    if [ -n "$NO_SUBMIT_USAGE_STATISTICS" ]; then
+        SERVICE_SUBMIT_USAGE_STATISTICS="False"
+    else
+        SERVICE_SUBMIT_USAGE_STATISTICS="True"
+    fi
+else
+    SERVICE_MOBILE_PUSH="False"
+    # SUBMIT_USAGE_STATISTICS without PUSH_NOTIFICATIONS is an unusual
+    # configuration that we don't need to offer in the installer.
+    SERVICE_SUBMIT_USAGE_STATISTICS="False"
+fi
+
+# ToS acceptance needs to be ensured if --push-notifications is passed.
+if [ -n "$PUSH_NOTIFICATIONS" ]; then
+    # If the user provided the --agree-to-terms-of-service flag, we can just proceed.
+    if [ -n "$AGREE_TO_TERMS_OF_SERVICE_FLAG" ]; then
+        PUSH_NOTIFICATIONS_SERVICE_TOS_AGREED=1
+        echo "Push notifications will be enabled, as you agreed to the Zulip Terms of Service"
+        echo "by passing the --agree-to-terms-of-service flag."
+        sleep 2
+    else
+        # If user asked for push notifications, prompt for ToS acceptance.
+        echo
+        echo "You chose to register your server for the Mobile Push Notifications Service."
+        echo "Doing so will share basic metadata with the service's maintainers, including:"
+        echo
+        echo "* The server's configured hostname: $EXTERNAL_HOST"
+        echo "* The server's configured contact email address: $ZULIP_ADMINISTRATOR"
+        echo "* Basic metadata about each organization hosted by the server; see:"
+        echo "    <https://zulip.com/doc-permalinks/basic-metadata>"
+        if [ -z "$NO_SUBMIT_USAGE_STATISTICS" ]; then
+            echo "* The server's usage statistics; see:"
+            echo "    <https://zulip.com/doc-permalinks/usage-statistics>"
+        fi
+        echo
+        echo "For details on why a centralized push notification service is necessary, see:"
+        echo "    <https://zulip.com/doc-permalinks/why-service>"
+        echo
+        echo "Use of this service is governed by the Zulip Terms of Service:"
+        echo "    <https://zulip.com/policies/terms>"
+        echo
+        read -r -p "Do you want to agree to the Zulip Terms of Service and proceed? [Y/n] " tos_prompt
+        echo
+
+        # Normalize the user’s response to lowercase.
+        case "${tos_prompt,,}" in
+            "" | y | yes)
+                echo "Great! Push notifications will be enabled; continuing with installation..."
+                PUSH_NOTIFICATIONS_SERVICE_TOS_AGREED=1
+                sleep 2
+                ;;
+            *)
+                echo "In order to enable push notifications, you must agree to the Terms of Service."
+                echo "If you do not want to enable push notifications, run the command without the --push-notifications flag."
+                exit 1
+                ;;
+        esac
+    fi
+fi
+
+# Do set -x after option parsing is complete
+set -x
+
+ZULIP_PATH="$(readlink -f "$(dirname "$0")"/../..)"
+
+# Force a known locale.  Some packages on PyPI fail to install in some locales.
+export LC_ALL="C.UTF-8"
+export LANG="C.UTF-8"
+export LANGUAGE="C.UTF-8"
+
+# Force a known path; this fixes problems on Debian where `su` from
+# non-root may not adjust `$PATH` to root's.
+export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+
+# Check for a supported OS release.
+if [ -f /etc/os-release ]; then
+    os_info="$(
+        . /etc/os-release
+        printf '%s\n' "$ID" "$ID_LIKE" "$VERSION_ID" "$VERSION_CODENAME"
+    )"
+    {
+        read -r os_id
+        read -r os_id_like
+        read -r os_version_id
+        read -r os_version_codename || true
+    } <<<"$os_info"
+    case " $os_id $os_id_like " in
+        *' debian '*)
+            package_system="apt"
+            ;;
+        *' rhel '*)
+            package_system="yum"
+            ;;
+    esac
+fi
+
+if ! "$ZULIP_PATH/scripts/lib/supported-os"; then
+    system_requirements_failure <<EOF
+Unsupported OS release: $os_id $os_version_id
+
+Zulip in production is supported only on:
+ - Debian 12
+ - Debian 13
+ - Ubuntu 22.04 LTS
+ - Ubuntu 24.04 LTS
+ - Ubuntu 26.04 LTS
+EOF
+fi
+
+machine="$(uname -m)"
+if [ "$machine" != x86_64 ] && [ "$machine" != aarch64 ]; then
+    system_requirements_failure <<EOF
+Unsupported CPU architecture: $machine (expected x86_64 or aarch64).
+EOF
+fi
+
+dpkg_architecture="$(dpkg --print-architecture)"
+if [ "$dpkg_architecture" != amd64 ] && [ "$dpkg_architecture" != arm64 ]; then
+    system_requirements_failure <<EOF
+Unsupported OS architecture: $dpkg_architecture (expected amd64 or arm64).
+EOF
+fi
+
+has_universe() {
+    apt-cache policy \
+        | grep -q "^     release v=$os_version_id,o=Ubuntu,a=$os_version_codename,n=$os_version_codename,l=Ubuntu,c=universe"
+}
+
+if [ "$os_id" = ubuntu ] && ! has_universe && ! { apt-get update && has_universe; }; then
+    system_requirements_failure <<EOF
+You must enable the Ubuntu Universe repository before installing
+Zulip.  You can do this with:
+
+    sudo add-apt-repository universe
+    sudo apt update
+EOF
+fi
+
+case ",$PUPPET_CLASSES," in
+    *,zulip::profile::standalone,* | *,zulip::profile::postgresql,*)
+        if [ "$package_system" = apt ]; then
+            # We're going to install PostgreSQL from the PostgreSQL apt
+            # repository; this may conflict with the existing PostgreSQL.
+            OTHER_PG="$(dpkg --get-selections \
+                | grep -E '^postgresql-[0-9]+\s+install$' \
+                | grep -v "^postgresql-$POSTGRESQL_VERSION\b" \
+                | cut -f 1)" || true
+            if [ -n "$OTHER_PG" ]; then
+                INDENTED="${OTHER_PG//$'\n'/$'\n'    }"
+                SPACED="${OTHER_PG//$'\n'/ }"
+                cat <<EOF
+
+The following PostgreSQL servers were found to already be installed:
+
+    $INDENTED
+
+Zulip needs to install PostgreSQL $POSTGRESQL_VERSION, but does not wish
+to uninstall existing databases in order to do so.  Remove all other
+PostgreSQL servers manually before running the installer:
+
+    sudo apt-get remove $SPACED
+
+EOF
+                exit 1
+            fi
+        fi
+        ;;
+esac
+
+# Check for at least ~1.86GB of RAM before starting installation;
+# otherwise users will find out about insufficient RAM via weird
+# errors like a segfault running `pip install`.
+# Additionally, some AWS images that are advertised to be 2 GB
+# are actually 1880000B in size.
+mem_kb=$(head -n1 /proc/meminfo | awk '{print $2}')
+if [ "$mem_kb" -lt 1860000 ]; then
+    set +x
+    echo -e '\033[0;31m' >&2
+    echo "Insufficient RAM.  Zulip requires at least 2GB of RAM." >&2
+    echo >&2
+    echo -e '\033[0m' >&2
+    exit 1
+fi
+
+# Anything under 5GB, we recommended allocating 2GB of swap.  Error
+# out if there's no swap.
+swap_kb=$(grep SwapTotal /proc/meminfo | awk '{print $2}')
+if [ "$mem_kb" -lt 5000000 ] && [ "$swap_kb" -eq 0 ]; then
+    set +x
+    echo -e '\033[0;31m' >&2
+    echo "No swap allocated; when running with < 5GB of RAM, we recommend at least 2GB of swap." >&2
+    echo "https://www.digitalocean.com/community/tutorials/how-to-add-swap-space-on-ubuntu-22-04" >&2
+    echo >&2
+    echo -e '\033[0m' >&2
+    exit 1
+fi
+
+# Do package update, e.g. do `apt-get update` on Debian
+if [ "$package_system" = apt ]; then
+    # setup-apt-repo does an `apt-get update`
+    "$ZULIP_PATH"/scripts/lib/setup-apt-repo
+elif [ "$package_system" = yum ]; then
+    "$ZULIP_PATH"/scripts/lib/setup-yum-repo
+fi
+
+# Check early for missing SSL certificates
+if [ "$PUPPET_CLASSES" = "zulip::profile::standalone" ] && [ -z "$USE_CERTBOT""$SELF_SIGNED_CERT" ] && { ! [ -e "/etc/ssl/private/zulip.key" ] || ! [ -e "/etc/ssl/certs/zulip.combined-chain.crt" ]; }; then
+    set +x
+    cat <<EOF
+
+No SSL certificate found.  One or both required files is missing:
+    /etc/ssl/private/zulip.key
+    /etc/ssl/certs/zulip.combined-chain.crt
+
+Suggested solutions:
+ * For most sites, the --certbot option is recommended.
+ * If you have your own key and cert, see docs linked below
+   for how to install them.
+ * For non-production testing, try the --self-signed-cert option.
+
+For help and more details, see our SSL documentation:
+  https://zulip.readthedocs.io/en/latest/production/ssl-certificates.html
+
+Once fixed, just rerun scripts/setup/install; it'll pick up from here!
+
+EOF
+    exit 1
+fi
+
+# don't run dist-upgrade in one click apps to make the
+# installation process more seamless.
+if [ -z "$NO_DIST_UPGRADE" ]; then
+    if [ "$package_system" = apt ]; then
+        apt-get -y dist-upgrade "${APT_OPTIONS[@]}"
+    elif [ "$package_system" = yum ]; then
+        # On CentOS, there is no need to do `yum -y upgrade` because `yum -y
+        # update` already does the same thing.
+        :
+    fi
+fi
+
+if [ "$package_system" = apt ]; then
+    # Note that any additions to these lists must also be added to
+    # `zulip::profile::base` such that the new dependency is seen by
+    # upgrades, as well as new installs.
+    if ! apt-get install -y --no-install-recommends \
+        python3 python3-yaml puppet git curl jq crudini \
+        "${ADDITIONAL_PACKAGES[@]}"; then
+        set +x
+        echo -e '\033[0;31m' >&2
+        echo "Installing packages failed; is network working and (on Ubuntu) the universe repository enabled?" >&2
+        echo >&2
+        echo -e '\033[0m' >&2
+        exit 1
+    fi
+elif [ "$package_system" = yum ]; then
+    if ! yum install -y \
+        python3 python3-pyyaml puppet git curl jq crudini \
+        "${ADDITIONAL_PACKAGES[@]}"; then
+        set +x
+        echo -e '\033[0;31m' >&2
+        echo "Installing packages failed; is network working?" >&2
+        echo >&2
+        echo -e '\033[0m' >&2
+        exit 1
+    fi
+fi
+
+# We generate a self-signed cert even with certbot, so we can use the
+# webroot authenticator, which requires nginx be set up with a
+# certificate.
+if [ -n "$SELF_SIGNED_CERT" ] || [ -n "$USE_CERTBOT" ]; then
+    "$ZULIP_PATH"/scripts/setup/generate-self-signed-cert \
+        --exists-ok "${EXTERNAL_HOST:-$(hostname)}"
+fi
+
+# Generate /etc/zulip/zulip.conf .
+mkdir -p /etc/zulip
+has_class() {
+    grep -qx "$1" /var/lib/puppet/classes.txt
+}
+
+# puppet apply --noop fails unless the user that it _would_ chown
+# files to exists; https://tickets.puppetlabs.com/browse/PUP-3907
+#
+# The home directory here should match what's declared in base.pp.
+id -u zulip &>/dev/null || useradd -m zulip --home-dir /home/zulip
+if [ -n "$NO_OVERWRITE_SETTINGS" ] && [ -e "/etc/zulip/zulip.conf" ]; then
+    "$ZULIP_PATH"/scripts/zulip-puppet-apply --noop \
+        --write-catalog-summary \
+        --classfile=/var/lib/puppet/classes.txt
+else
+    # Write out more than we need, and remove sections that are not
+    # applicable to the classes that are actually necessary.
+    cat <<EOF >/etc/zulip/zulip.conf
+[machine]
+puppet_classes = $PUPPET_CLASSES
+deploy_type = production
+
+[postgresql]
+version = $POSTGRESQL_VERSION
+EOF
+
+    "$ZULIP_PATH"/scripts/zulip-puppet-apply --noop \
+        --write-catalog-summary \
+        --classfile=/var/lib/puppet/classes.txt
+
+    # We only need the PostgreSQL version setting on database hosts,
+    # or hosts which talk directly to the database (e.g. application
+    # hosts); but we don't know if this is a database host until we
+    # have the catalog summary.
+    if (! has_class "zulip::postgresql_common" && ! has_class "zulip::postgresql_client") || [ "$package_system" != apt ]; then
+        crudini --del /etc/zulip/zulip.conf postgresql
+    fi
+
+    if [ -n "$POSTGRESQL_DATABASE_NAME" ]; then
+        crudini --set /etc/zulip/zulip.conf postgresql database_name "$POSTGRESQL_DATABASE_NAME"
+    fi
+
+    if [ -n "$POSTGRESQL_DATABASE_USER" ]; then
+        crudini --set /etc/zulip/zulip.conf postgresql database_user "$POSTGRESQL_DATABASE_USER"
+    fi
+fi
+
+if has_class "zulip::app_frontend_base"; then
+    # Frontend deploys use /home/zulip/deployments; without this, the
+    # install directory is also only readable by root.
+    mkdir -p /home/zulip/deployments
+    deploy_path=$("$ZULIP_PATH"/scripts/lib/zulip_tools.py make_deploy_path)
+    mv "$ZULIP_PATH" "$deploy_path"
+    ln -nsf /home/zulip/deployments/next "$ZULIP_PATH"
+    ln -nsf "$deploy_path" /home/zulip/deployments/next
+
+    # Create and activate a virtualenv
+    "$deploy_path"/scripts/lib/create-production-venv "$deploy_path"
+
+    "$deploy_path"/scripts/lib/install-node
+
+    if [ -z "$NO_OVERWRITE_SETTINGS" ] || ! [ -e "/etc/zulip/settings.py" ]; then
+        cp -a "$deploy_path"/zproject/prod_settings_template.py /etc/zulip/settings.py
+        if [ -n "$EXTERNAL_HOST" ]; then
+            sed -i "s/^EXTERNAL_HOST =.*/EXTERNAL_HOST = '$EXTERNAL_HOST'/" /etc/zulip/settings.py
+        fi
+        if [ -n "$ZULIP_ADMINISTRATOR" ]; then
+            sed -i "s/^ZULIP_ADMINISTRATOR =.*/ZULIP_ADMINISTRATOR = '$ZULIP_ADMINISTRATOR'/" /etc/zulip/settings.py
+        fi
+
+        # Set SERVICE settings based on what the user provided.
+        if grep -q -E "^#?\s*ZULIP_SERVICE_PUSH_NOTIFICATIONS = " /etc/zulip/settings.py; then
+            sed -i -E "s/^#?\s*ZULIP_SERVICE_PUSH_NOTIFICATIONS = .*/ZULIP_SERVICE_PUSH_NOTIFICATIONS = $SERVICE_MOBILE_PUSH/" /etc/zulip/settings.py
+        else
+            echo "ZULIP_SERVICE_PUSH_NOTIFICATIONS = $SERVICE_MOBILE_PUSH" >>/etc/zulip/settings.py
+        fi
+
+        if grep -q -E "^#?\s*ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS = " /etc/zulip/settings.py; then
+            sed -i -E "s/^#?\s*ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS = .*/ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS = $SERVICE_SUBMIT_USAGE_STATISTICS/" /etc/zulip/settings.py
+        else
+            echo "ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS = $SERVICE_SUBMIT_USAGE_STATISTICS" >>/etc/zulip/settings.py
+        fi
+    fi
+    ln -nsf /etc/zulip/settings.py "$deploy_path"/zproject/prod_settings.py
+    "$deploy_path"/scripts/setup/generate_secrets.py --production
+else
+    deploy_path="$ZULIP_PATH"
+fi
+
+"$deploy_path"/scripts/zulip-puppet-apply -f
+
+if [ "$package_system" = apt ]; then
+    apt-get -y --with-new-pkgs upgrade
+elif [ "$package_system" = yum ]; then
+    # No action is required because `yum update` already does upgrade.
+    :
+fi
+
+if [ -n "$USE_CERTBOT" ]; then
+    "$deploy_path"/scripts/setup/setup-certbot \
+        "$EXTERNAL_HOST" --email "$ZULIP_ADMINISTRATOR"
+fi
+
+if has_class "zulip::nginx" && ! has_class "zulip::profile::docker"; then
+    # Check nginx was configured properly now that we've installed it.
+    # Most common failure mode is certs not having been installed.
+    if ! nginx -t; then
+        (
+            set +x
+            cat <<EOF
+
+Verifying the Zulip nginx configuration failed!
+
+This is almost always a problem with your SSL certificates.  See:
+  https://zulip.readthedocs.io/en/latest/production/ssl-certificates.html
+
+Once fixed, just rerun scripts/setup/install; it'll pick up from here!
+
+EOF
+            exit 1
+        )
+    fi
+fi
+
+if has_class "zulip::profile::rabbitmq"; then
+    if ! rabbitmqctl status >/dev/null; then
+        set +x
+        cat <<EOF
+
+RabbitMQ seems to not have started properly after the installation process.
+Often this is caused by misconfigured /etc/hosts in virtualized environments.
+For more information, see:
+  https://github.com/zulip/zulip/issues/53#issuecomment-143805121
+
+EOF
+        exit 1
+    fi
+fi
+
+# Set up a basic .gitconfig for the 'zulip' user
+if [ -n "$ZULIP_ADMINISTRATOR" ]; then
+    (
+        cd / # Make sure the current working directory is readable by zulip
+        su zulip -c "git config --global user.email $ZULIP_ADMINISTRATOR"
+        su zulip -c "git config --global user.name 'Zulip Server ($EXTERNAL_HOST)'"
+    )
+fi
+
+if ! has_class "zulip::app_frontend_base"; then
+    set +x
+    cat <<EOF
+
+Success!
+
+Not configuring PostgreSQL, or /home/zulip/deployments, because this
+is not a front-end install.
+
+EOF
+    exit 0
+fi
+
+ln -nsf "$deploy_path" /home/zulip/deployments/current
+ln -nsf /etc/zulip/settings.py "$deploy_path"/zproject/prod_settings.py
+mkdir -p "$deploy_path"/prod-static/serve
+cp -rT "$deploy_path"/prod-static/serve /home/zulip/prod-static
+chown -R zulip:zulip /home/zulip /var/log/zulip /etc/zulip/settings.py
+
+if ! [ -e /home/zulip/deployments/current/zulip-git-version ]; then
+    mkdir /srv/zulip.git
+    chown zulip:zulip /srv/zulip.git
+    su zulip -c 'git clone --bare --mirror /home/zulip/deployments/current /srv/zulip.git'
+    su zulip -c 'cd /home/zulip/deployments/current && ./scripts/lib/update-git-upstream && ./tools/cache-zulip-git-version'
+fi
+
+if ! [ -e "/home/zulip/prod-static/generated" ]; then
+    # If we're installing from a Git checkout, we need to run
+    # `tools/update-prod-static` in order to build the static
+    # assets.
+    su zulip -c '/home/zulip/deployments/current/tools/update-prod-static'
+fi
+
+if [ -n "$NO_INIT_DB" ]; then
+    set +x
+    cat <<EOF
+
+Success!
+
+Stopping because --no-init-db was passed.  To complete the
+installation, configure PostgreSQL by creating the database and
+database user, and then run:
+
+   su zulip -c '/home/zulip/deployments/current/scripts/setup/initialize-database'
+   su zulip -c '/home/zulip/deployments/current/manage.py generate_realm_creation_link'
+EOF
+
+    if [ -n "$PUSH_NOTIFICATIONS" ]; then
+        echo
+        echo "Since you specified you want to enable the push notification service, you'll also need to "
+        echo "manually register your server by running:"
+        echo
+        echo "   su zulip -c '/home/zulip/deployments/current/manage.py register_server'"
+        echo
+    fi
+    exit 0
+else
+    /home/zulip/deployments/current/scripts/setup/create-database
+    su zulip -c '/home/zulip/deployments/current/scripts/setup/initialize-database --quiet'
+
+    if [ -n "$PUSH_NOTIFICATIONS" ]; then
+        if [ -z "$PUSH_NOTIFICATIONS_SERVICE_TOS_AGREED" ]; then
+            # This should be impossible as the installation should have aborted early if the user
+            # passed --push-notifications without accepting the ToS. But we include this as a precaution
+            # to not allow future bugs to cause ToS acceptance to be skipped.
+            SKIP_TOS_AGREEMENT_OPTION=""
+        else
+            SKIP_TOS_AGREEMENT_OPTION="--agree-to-terms-of-service"
+        fi
+        set +x
+        echo "Services enabled, attempting to register server..."
+        set -x
+
+        # Without PYTHONUNBUFFERED=1, stdout/stderr might end up displayed not in chronological order, resulting
+        # in confusing output to the user.
+        su zulip -c "PYTHONUNBUFFERED=1 /home/zulip/deployments/current/manage.py register_server ${SKIP_TOS_AGREEMENT_OPTION}" \
+            || {
+                set +x
+                echo
+                echo "WARNING: Server registration for push notifications failed."
+                echo "This does not affect the rest of your Zulip installation."
+                echo
+                echo "To enable push notifications after resolving the issue, run:"
+                echo "  su zulip -c '/home/zulip/deployments/current/manage.py register_server'"
+                echo
+                set -x
+            }
+    fi
+
+    su zulip -c '/home/zulip/deployments/current/manage.py generate_realm_creation_link'
+fi

scripts/lib/install-node

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+version=24.19.0
+arch="$(uname -m)"
+
+case $arch in
+    x86_64)
+        tarball="node-v$version-linux-x64.tar.xz"
+        sha256=14b342e71204f811bde6153be8e04b62aef63c236fef92b55f9c83154b409647
+        ;;
+
+    aarch64)
+        tarball="node-v$version-linux-arm64.tar.xz"
+        sha256=01443c1e1a29e531ccad5a46fefa6df490d2189c49f7955904aecdbb0fe86fdc
+        ;;
+esac
+
+check_version() {
+    out="$(node --version)" && [ "$out" = "v$version" ] \
+        && [ /usr/local/bin/pnpm -ef /srv/zulip-node/lib/node_modules/corepack/dist/pnpm.js ]
+}
+
+if ! check_version 2>/dev/null; then
+    set -x
+    tmpdir="$(mktemp -d)"
+    trap 'rm -r "$tmpdir"' EXIT
+    cd "$tmpdir"
+    curl -fLO --retry 3 "https://nodejs.org/dist/v$version/$tarball"
+    sha256sum -c <<<"$sha256 $tarball"
+    rm -rf /srv/zulip-node
+    mkdir -p /srv/zulip-node
+    tar -xJf "$tarball" --no-same-owner --strip-components=1 -C /srv/zulip-node
+    ln -sf /srv/zulip-node/bin/{corepack,node,npm,npx} /usr/local/bin
+    COREPACK_DEFAULT_TO_LATEST=0 /usr/local/bin/corepack enable
+
+    # Clean up after previous versions of this script
+    rm -rf /srv/zulip-yarn /usr/bin/yarn /usr/local/nvm
+
+    check_version
+fi

scripts/lib/install-uv

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+set -eu
+
+version=0.12.5
+arch="$(uname -m)"
+tarball="uv-$arch-unknown-linux-gnu.tar.gz"
+declare -A sha256=(
+    [aarch64]=9bf43b4d1a07665bf64d4c4e710930b382321a785e0eb10aac07f46471f86a31
+    [x86_64]=68a509da24b06b4223a1c0175fb5eb5bc79342b76cbeff0cfe51ac3f5b17b6b2
+)
+
+check_version() {
+    out="$(uv --version)" && [[ "$out" = "uv $version ("* ]]
+}
+
+if ! check_version 2>/dev/null; then
+    set -x
+    tmpdir="$(mktemp -d)"
+    trap 'rm -r "$tmpdir"' EXIT
+    cd "$tmpdir"
+    curl -fLO --retry 3 "https://github.com/astral-sh/uv/releases/download/$version/$tarball"
+    sha256sum -c <<<"${sha256[$arch]} $tarball"
+    tar -xzf "$tarball" --no-same-owner --strip-components=1 -C /usr/local/bin "uv-$arch-unknown-linux-gnu"/{uv,uvx}
+    check_version
+fi

scripts/lib/node_cache.py

@@ -0,0 +1,39 @@
+import os
+
+from scripts.lib.zulip_tools import run
+
+DEFAULT_PRODUCTION = False
+
+
+def setup_node_modules(production: bool = DEFAULT_PRODUCTION) -> None:
+    if os.path.islink("node_modules"):
+        os.unlink("node_modules")
+
+    skip = False
+
+    try:
+        with open("node_modules/.pnpm/lock.yaml") as a, open("pnpm-lock.yaml") as b:
+            if a.read() == b.read():
+                skip = True
+    except FileNotFoundError:
+        pass
+
+    # We need this check when switching between branches without `starlight_help`
+    # package. `node_modules` will be removed when working on a non `starlight_help`
+    # branch, but if `node_modules/.pnpm/lock.yaml` has not been updated by that
+    # branch, we will end up in a situation where we might not have `node_modules`
+    # even when we run the provision command.
+    if not os.path.exists("starlight_help/node_modules"):
+        skip = False
+
+    if not skip:
+        run(
+            [
+                "/usr/local/bin/corepack",
+                "pnpm",
+                "install",
+                "--frozen-lockfile",
+                "--prefer-offline",
+                *(["--prod"] if production else []),
+            ],
+        )

scripts/lib/puppet_cache.py

@@ -0,0 +1,107 @@
+import hashlib
+import json
+import os
+import shutil
+import subprocess
+import tempfile
+from urllib.request import urlopen
+
+import yaml
+
+from .zulip_tools import run
+
+ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+ZULIP_SRV_PATH = "/srv"
+
+PUPPET_MODULES_CACHE_PATH = os.path.join(ZULIP_SRV_PATH, "zulip-puppet-cache")
+PUPPET_DEPS_FILE_PATH = os.path.join(ZULIP_PATH, "puppet/deps.yaml")
+PUPPET_THIRDPARTY = os.path.join(PUPPET_MODULES_CACHE_PATH, "current")
+
+
+def generate_sha1sum_puppet_modules() -> str:
+    data = {}
+    with open(PUPPET_DEPS_FILE_PATH) as fb:
+        data["deps.yaml"] = fb.read().strip()
+    data["puppet-version"] = subprocess.check_output(
+        # This is 10x faster than `puppet --version`
+        ["ruby", "-r", "puppet/version", "-e", "puts Puppet.version"],
+        text=True,
+    ).strip()
+
+    sha1sum = hashlib.sha1()
+    sha1sum.update(json.dumps(data, sort_keys=True).encode())
+    return sha1sum.hexdigest()
+
+
+def setup_puppet_modules() -> None:
+    sha1sum = generate_sha1sum_puppet_modules()
+    target_path = os.path.join(PUPPET_MODULES_CACHE_PATH, sha1sum)
+    success_stamp = os.path.join(target_path, ".success-stamp")
+    # Check if a cached version already exists
+    if not os.path.exists(success_stamp):
+        do_puppet_module_install(target_path, success_stamp)
+
+    if os.path.islink(PUPPET_THIRDPARTY):
+        os.remove(PUPPET_THIRDPARTY)
+    elif os.path.isdir(PUPPET_THIRDPARTY):
+        shutil.rmtree(PUPPET_THIRDPARTY)
+    os.symlink(target_path, PUPPET_THIRDPARTY)
+
+
+def do_puppet_module_install(
+    target_path: str,
+    success_stamp: str,
+) -> None:
+    os.makedirs(target_path, exist_ok=True)
+    with open(PUPPET_DEPS_FILE_PATH) as yaml_file:
+        deps = yaml.safe_load(yaml_file)
+    for module, metadata in deps.items():
+        install_puppet_module(target_path, module, metadata["version"], metadata["sha256sum"])
+    with open(success_stamp, "w"):
+        pass
+
+
+def install_puppet_module(
+    target_path: str, module: str, version: str, expected_sha256sum: str
+) -> None:
+    print(f"Fetching {module}-{version} from forgeapi.puppet.com...")
+    with urlopen(
+        f"https://forgeapi.puppet.com/v3/releases/{module}-{version}", timeout=60
+    ) as forge_resp:
+        forge_data = json.load(forge_resp)
+
+    forge_sha256sum = forge_data["file_sha256"]
+    if forge_sha256sum != expected_sha256sum:
+        raise Exception(
+            f"Forge API returned unexpected SHA256 sum for {module}-{version}: "
+            f"expected {expected_sha256sum}, got {forge_sha256sum}"
+        )
+
+    with tempfile.NamedTemporaryFile(
+        prefix=f"zulip-puppet-{module}-{version}-",
+        suffix=".tar.gz",
+    ) as tarball:
+        with urlopen(
+            "https://forgeapi.puppet.com" + forge_data["file_uri"], timeout=60
+        ) as tarball_resp:
+            tarball_content = tarball_resp.read()
+            local_sha256sum = hashlib.sha256(tarball_content).hexdigest()
+            if local_sha256sum != expected_sha256sum:
+                raise Exception(
+                    f"Downloaded file had unexpected SHA256 sum for {module}-{version}: "
+                    f"expected {expected_sha256sum}, got {forge_sha256sum}"
+                )
+            tarball.write(tarball_content)
+            tarball.flush()
+
+        run(
+            [
+                "puppet",
+                "module",
+                "--modulepath",
+                target_path,
+                "install",
+                tarball.name,
+                "--ignore-dependencies",
+            ],
+        )

scripts/lib/queue_workers.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+import os
+import sys
+
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.append(BASE_DIR)
+from scripts.lib.setup_path import setup_path
+
+setup_path()
+
+os.environ["DJANGO_SETTINGS_MODULE"] = "zproject.settings"
+
+import django
+
+django.setup()
+from zerver.worker.queue_processors import get_active_worker_queues
+
+if __name__ == "__main__":
+    for worker in sorted(get_active_worker_queues()):
+        print(worker)

scripts/lib/run_hooks.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+import argparse
+import os
+import subprocess
+import sys
+
+sys.path.append(os.path.join(os.path.dirname(__file__), "..", ".."))
+from scripts.lib.zulip_tools import (
+    DEPLOYMENTS_DIR,
+    assert_running_as_root,
+    get_deploy_root,
+    get_zulip_pwent,
+    parse_version_from,
+    su_to_zulip,
+)
+
+assert_running_as_root()
+
+# Updates to the below choices to add a new hook type should
+# adjust puppet's `app_frontend_base.pp` as well.
+parser = argparse.ArgumentParser()
+parser.add_argument("kind", choices=["pre-deploy", "post-deploy"], help="")
+parser.add_argument("--from-git", action="store_true", help="Upgrading from git")
+args = parser.parse_args()
+
+from version import ZULIP_MERGE_BASE as NEW_ZULIP_MERGE_BASE
+from version import ZULIP_VERSION as NEW_ZULIP_VERSION
+
+deploy_path = get_deploy_root()
+
+if args.kind == "post-deploy":
+    old_dir_name = "last"
+    if not os.path.exists(DEPLOYMENTS_DIR + "/last"):
+        # Fresh installs which are doing an OS upgrade don't have a
+        # "last" yet
+        old_dir_name = "current"
+else:
+    old_dir_name = "current"
+old_version = parse_version_from(DEPLOYMENTS_DIR + "/" + old_dir_name)
+old_merge_base = parse_version_from(DEPLOYMENTS_DIR + "/" + old_dir_name, merge_base=True)
+
+path = f"/etc/zulip/hooks/{args.kind}.d"
+if not os.path.exists(path):
+    sys.exit(0)
+
+# Pass in, via environment variables, the old/new "version
+# string" (which is a `git describe` output)
+env = os.environ.copy()
+env["ZULIP_OLD_VERSION"] = old_version
+env["ZULIP_NEW_VERSION"] = NEW_ZULIP_VERSION
+
+# preexec_fn=su_to_zulip normally handles this, but our explicit
+# env overrides that
+env["HOME"] = get_zulip_pwent().pw_dir
+
+
+def resolve_version_string(version: str) -> str:
+    return subprocess.check_output(
+        ["git", "rev-parse", version], cwd=deploy_path, preexec_fn=su_to_zulip, text=True
+    ).strip()
+
+
+if args.from_git:
+    # If we have a git repo, we also resolve those `git describe`
+    # values to full commit hashes, as well as provide the
+    # merge-base of the old/new commits with mainline.
+    env["ZULIP_OLD_COMMIT"] = resolve_version_string(old_version)
+    env["ZULIP_NEW_COMMIT"] = resolve_version_string(NEW_ZULIP_VERSION)
+    env["ZULIP_OLD_MERGE_BASE_COMMIT"] = resolve_version_string(old_merge_base)
+    env["ZULIP_NEW_MERGE_BASE_COMMIT"] = resolve_version_string(NEW_ZULIP_MERGE_BASE)
+
+failures = []
+for script_name in sorted(f for f in os.listdir(path) if f.endswith(".hook")):
+    result = subprocess.run(
+        [os.path.join(path, script_name)],
+        check=False,
+        cwd=deploy_path,
+        preexec_fn=su_to_zulip,
+        env=env,
+    )
+    if result.returncode != 0:
+        # Pre-deploy hooks abort on the first failure; post-deploy
+        # hooks are best-effort and a failure of one does not abort
+        # the rest of them.
+        if args.kind == "pre-deploy":
+            sys.exit(1)
+        failures.append(script_name)
+
+if failures:
+    print("Failed hooks:")
+    for failed_script in failures:
+        print(f"  {failed_script}")

scripts/lib/setup-apt-repo

@@ -0,0 +1,120 @@
+#!/usr/bin/env bash
+#
+# This script handles adding custom apt repositories into
+# /etc/apt/sources.list.d/ files.  It bundles the GPG keys which are
+# used to verify the repositories (via `apt-key`), to explicitly pin
+# the trusted signing keys, as opposed to blindly trusting HTTPS.
+#
+# Each /etc/apt/sources.list.d/foo.list file is created via `--list
+# foo`, where `foo` defaults to `zulip`.  The default `zulip.list` is
+# installed in `scripts/lib/install` / `tools/lib/provision.py`, and
+# other `.list` files may be installed by Puppet.
+set -x
+set -e
+set -u
+set -o pipefail
+shopt -s extglob
+
+verify=false
+args="$(getopt -o '' --long verify,list: -- "$@")"
+eval "set -- $args"
+LIST=zulip
+while true; do
+    case "$1" in
+        --verify)
+            verify=true
+            shift
+            ;;
+        --list)
+            LIST="$2"
+            shift
+            shift
+            ;;
+        --)
+            shift
+            break
+            ;;
+    esac
+done
+
+# Ensure the directory for LAST_DEPENDENCIES_HASH exists
+mkdir -p /var/lib/zulip
+
+SOURCES_FILE=/etc/apt/sources.list.d/$LIST.list
+PREF_FILE=/etc/apt/preferences.d/$LIST.pref
+STAMP_FILE=/etc/apt/sources.list.d/$LIST.list.apt-update-in-progress
+
+# All paths, for purposes of hashing, should be relative to the deploy
+# root.
+cd "$(dirname "$0")/../.."
+
+LIST_PATH="scripts/setup/apt-repos/$LIST"
+if ! [ -d "$LIST_PATH" ]; then
+    echo "Not a valid value for --list: '$LIST'"
+    echo ""
+    echo "Valid values are:"
+    ls -1 "scripts/setup/apt-repos/"
+    exit 1
+fi
+
+release="$(. /etc/os-release && printf '%s' "$VERSION_CODENAME")"
+
+if [ ! -f "$LIST_PATH/$release.list" ]; then
+    cat <<EOF
+Unsupported release $release for sources.list file $LIST.  To add a
+new release, make a $LIST_PATH/$release.list file based on existing
+.list files in that directory.
+
+EOF
+    exit 1
+fi
+
+DEPENDENCIES_HASH="$(sha256sum "$LIST_PATH"/?(*.asc|"$release.list"|"$LIST.pref"|custom.sh) "scripts/lib/setup-apt-repo")"
+DEPENDENCIES_HASH_FILE="/var/lib/zulip/setup-repositories-state-$LIST"
+# Ensure that DEPENDENCIES_HASH_FILE exists before hashing it.
+touch "$DEPENDENCIES_HASH_FILE"
+LAST_DEPENDENCIES_HASH="$(cat "$DEPENDENCIES_HASH_FILE")"
+
+# First, we only do anything in setup-apt-repo if any of its inputs
+# (apt keys, code, etc.)  changed.
+if [ "$DEPENDENCIES_HASH" = "$LAST_DEPENDENCIES_HASH" ]; then
+    exit 0
+elif [ "$verify" == true ]; then
+    exit 1
+fi
+
+# Hash to check if the configuration is changed by the script later.
+hashes=$(sha256sum "$SOURCES_FILE" "$PREF_FILE" 2>/dev/null || true)
+
+if [ -e "$LIST_PATH/$LIST.pref" ]; then
+    cp "$LIST_PATH/$LIST.pref" "$PREF_FILE"
+else
+    rm -f "$PREF_FILE"
+fi
+
+pre_setup_deps=(apt-transport-https ca-certificates gnupg curl)
+if ! apt-get -dy install "${pre_setup_deps[@]}"; then
+    apt-get update
+fi
+apt-get -y install "${pre_setup_deps[@]}"
+
+cp "$LIST_PATH/"*.asc /etc/apt/keyrings/
+cp "$LIST_PATH/$release.list" "$SOURCES_FILE"
+
+if [ -e "$LIST_PATH/custom.sh" ]; then
+    export LIST_PATH
+    export STAMP_FILE
+    bash "$LIST_PATH/custom.sh"
+fi
+
+if [ "$hashes" = "$(sha256sum "$SOURCES_FILE" "$PREF_FILE" 2>/dev/null || true)" ] && ! [ -e "$STAMP_FILE" ]; then
+    echo "APT configuration did not change; skipping apt-get update"
+else
+    # We create this stamp file to ensure `apt-get update` will be run
+    # the next time this script is invoked, and each time after, until
+    # `apt-get update` finishes successfully.
+    touch "$STAMP_FILE"
+    apt-get update && rm -f "$STAMP_FILE"
+fi
+
+echo "$DEPENDENCIES_HASH" >"$DEPENDENCIES_HASH_FILE"

scripts/lib/setup-yum-repo

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+set -x
+set -e
+
+is_prod=false
+args="$(getopt -o '' --long prod -- "$@")"
+eval "set -- $args"
+while true; do
+    case "$1" in
+        --prod)
+            is_prod=true
+            shift
+            ;;
+        --)
+            shift
+            break
+            ;;
+    esac
+done
+
+is_centos=false
+is_rhel=false
+if [ -e /etc/centos-release ]; then
+    is_centos=true
+    yum install -y epel-release
+
+    if [ "$is_prod" = true ]; then
+        # IUS is needed for installing python36u-mod_wsgi on prod env
+        yum localinstall -y https://centos7.iuscommunity.org/ius-release.rpm
+    fi
+
+elif grep -q "Red Hat" /etc/redhat-release; then
+    is_rhel=true
+    yum localinstall -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+fi
+
+yum update -y
+
+RHVER="$(rpm -qf --queryformat="%{VERSION}" /etc/redhat-release)"
+RHARCH="$(rpm -qf --queryformat="%{ARCH}" /etc/redhat-release)"
+PGVER=10
+if [ "$is_centos" = true ]; then
+    # "Development Tools" is the equivalent of build-essential
+    yum groupinstall -y "Development Tools"
+
+    # PostgreSQL $PGVER
+    yum localinstall -y "https://yum.postgresql.org/$PGVER/redhat/rhel-$RHVER-$RHARCH/pgdg-redhat-repo-latest.noarch.rpm"
+
+    # PGroonga
+    # https://pgroonga.github.io/install/centos.html
+    yum localinstall -y https://packages.groonga.org/centos/groonga-release-latest.noarch.rpm
+elif [ "$is_rhel" = true ]; then
+    yum groupinstall -y "Development Tools"
+    yum localinstall -y https://download.postgresql.org/pub/repos/yum/10/redhat/rhel-latest-x86_64/pgdg-redhat10-10-2.noarch.rpm
+    yum localinstall -y https://packages.groonga.org/centos/groonga-release-latest.noarch.rpm
+else
+    # dnf5 (Fedora 41+) dropped the `groupinstall` alias and no longer
+    # resolves the "Development Tools" display name, so install by group ID.
+    dnf group install -y development-tools
+    # TODO make the postgres version a variable.
+    PGVER=13
+    dnf install -y "https://download.postgresql.org/pub/repos/yum/reporpms/F-$RHVER-x86_64/pgdg-fedora-repo-latest.noarch.rpm"
+fi

scripts/lib/setup_path.py

@@ -0,0 +1,28 @@
+"""
+Use libraries from a virtualenv (by modifying sys.path) in production.
+"""
+
+import os
+import sys
+
+
+def setup_path() -> None:
+    BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+    venv = os.path.realpath(os.path.join(BASE_DIR, ".venv"))
+    if sys.prefix != venv:
+        sys.path = list(
+            filter(
+                # zulip-py3-venv was an historical virtualenv symlink
+                lambda p: "/zulip-py3-venv/" not in p and "/.venv/" not in p,
+                sys.path,
+            )
+        )
+        activate_this = os.path.join(venv, "bin", "activate_this.py")
+        activate_locals = dict(__file__=activate_this)
+        with open(activate_this) as f:
+            exec(f.read(), activate_locals)  # noqa: S102
+        # Check that the python version running this function
+        # is same as python version that created the virtualenv.
+        python_version = "python{}.{}".format(*sys.version_info[:2])
+        if not os.path.exists(os.path.join(venv, "lib", python_version)):
+            raise RuntimeError(venv + " was not set up for this Python version")

scripts/lib/setup_venv.py

@@ -0,0 +1,71 @@
+import os
+
+from scripts.lib.zulip_tools import os_families
+
+ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+VENV_DEPENDENCIES = [
+    "build-essential",
+    "libffi-dev",
+    "libldap2-dev",
+    "python3-dev",  # Needed to install typed-ast dependency of mypy
+    "python3-pip",
+    "virtualenv",
+    "libxml2-dev",  # Used for installing talon-core and python-xmlsec
+    "libxslt1-dev",  # Used for installing talon-core
+    "libpq-dev",  # Needed by psycopg2
+    "libssl-dev",  # Needed to build pycurl and other libraries
+    "libmagic1",  # Used for install python-magic
+    "libyaml-dev",  # For fast YAML parsing in PyYAML
+    # Needed by python-xmlsec:
+    "libxmlsec1-dev",
+    "pkg-config",
+    "jq",  # No longer used in production (clean me up later)
+    "libsasl2-dev",  # For building python-ldap from source
+    "libvips",  # For thumbnailing
+    "libvips-tools",
+    "libicu-dev",  # For building pyicu
+    "libcrypt-dev",  # For building uwsgi
+]
+
+COMMON_YUM_VENV_DEPENDENCIES = [
+    "libffi-devel",
+    "openldap-devel",
+    "libyaml-devel",
+    # Needed by python-xmlsec:
+    "gcc",
+    "python3-devel",
+    "libxml2-devel",
+    "xmlsec1-devel",
+    "xmlsec1-openssl-devel",
+    "libtool-ltdl-devel",
+    "libxslt-devel",
+    "postgresql-libs",  # libpq-dev on apt
+    "openssl-devel",
+    "jq",
+    "vips",  # For thumbnailing
+    "vips-tools",
+]
+
+REDHAT_VENV_DEPENDENCIES = [
+    *COMMON_YUM_VENV_DEPENDENCIES,
+    "python36-devel",
+    "python-virtualenv",
+]
+
+FEDORA_VENV_DEPENDENCIES = [
+    *COMMON_YUM_VENV_DEPENDENCIES,
+    "python3-pip",
+    "virtualenv",  # see https://unix.stackexchange.com/questions/27877/install-virtualenv-on-fedora-16
+]
+
+
+def get_venv_dependencies(vendor: str, os_version: str) -> list[str]:
+    if "debian" in os_families():
+        return VENV_DEPENDENCIES
+    elif "rhel" in os_families():
+        return REDHAT_VENV_DEPENDENCIES
+    elif "fedora" in os_families():
+        return FEDORA_VENV_DEPENDENCIES
+    else:
+        raise AssertionError("Invalid vendor")

scripts/lib/sharding.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+import argparse
+import filecmp
+import grp
+import json
+import os
+import pwd
+import subprocess
+import sys
+
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.append(BASE_DIR)
+from scripts.lib.setup_path import setup_path
+
+setup_path()
+
+from scripts.lib.zulip_tools import get_config_file, get_tornado_ports
+
+
+def nginx_quote(s: str) -> str:
+    return '"' + s.replace("\\", "\\\\").replace('"', '\\"') + '"'
+
+
+# Basic system to do Tornado sharding.  Writes two output .tmp files that need
+# to be renamed to the following files to finalize the changes:
+# * /etc/zulip/nginx_sharding_map.conf; nginx needs to be reloaded after changing.
+# * /etc/zulip/sharding.json; supervisor Django process needs to be reloaded
+# after changing.  TODO: We can probably make this live-reload by statting the file.
+#
+# TODO: Restructure this to automatically generate a sharding layout.
+def write_updated_configs() -> None:
+    config_file = get_config_file()
+    ports = get_tornado_ports(config_file)
+
+    expected_ports = list(range(9800, ports[-1] + 1))
+    assert ports == expected_ports, f"ports ({ports}) must be contiguous, starting with 9800"
+
+    with (
+        open("/etc/zulip/nginx_sharding_map.conf.tmp", "w") as nginx_sharding_conf_f,
+        open("/etc/zulip/sharding.json.tmp", "w") as sharding_json_f,
+    ):
+        if len(ports) == 1:
+            nginx_sharding_conf_f.write('map "" $tornado_server {\n')
+            nginx_sharding_conf_f.write("    default http://tornado;\n")
+            nginx_sharding_conf_f.write("}\n")
+            sharding_json_f.write("{}\n")
+            return
+
+        nginx_sharding_conf_f.write("map $host $tornado_server {\n")
+        nginx_sharding_conf_f.write("    default http://tornado9800;\n")
+        shard_map: dict[str, int | list[int]] = {}
+        shard_regexes: list[tuple[str, int | list[int]]] = []
+        external_host = subprocess.check_output(
+            [os.path.join(BASE_DIR, "scripts/get-django-setting"), "EXTERNAL_HOST"],
+            text=True,
+        ).strip()
+        for key, shards in config_file["tornado_sharding"].items():
+            if key.endswith("_regex"):
+                ports = [int(port) for port in key.removesuffix("_regex").split("_")]
+                shard_regexes.append((shards, ports[0] if len(ports) == 1 else ports))
+                nginx_sharding_conf_f.write(
+                    f"    {nginx_quote('~*' + shards)} http://tornado{'_'.join(map(str, ports))};\n"
+                )
+            else:
+                ports = [int(port) for port in key.split("_")]
+                for shard in shards.split():
+                    if "." in shard:
+                        host = shard
+                    else:
+                        host = f"{shard}.{external_host}"
+                    assert host not in shard_map, f"host {host} duplicated"
+                    shard_map[host] = ports[0] if len(ports) == 1 else ports
+                    nginx_sharding_conf_f.write(
+                        f"    {nginx_quote(host)} http://tornado{'_'.join(map(str, ports))};\n"
+                    )
+            nginx_sharding_conf_f.write("\n")
+        nginx_sharding_conf_f.write("}\n")
+
+        data = {"shard_map": shard_map, "shard_regexes": shard_regexes}
+        sharding_json_f.write(json.dumps(data) + "\n")
+
+        for fh in (nginx_sharding_conf_f, sharding_json_f):
+            os.fchown(
+                fh.fileno(),
+                pwd.getpwnam("zulip").pw_uid,
+                grp.getgrnam("zulip").gr_gid,
+            )
+            os.fchmod(fh.fileno(), 0o644)
+
+
+parser = argparse.ArgumentParser(
+    description="Adjust Tornado sharding configuration",
+)
+parser.add_argument(
+    "--errors-ok",
+    action="store_true",
+    help="Exits 1 if there are no changes; if there are errors or changes, exits 0.",
+)
+options = parser.parse_args()
+
+config_file_path = "/etc/zulip"
+base_files = ["nginx_sharding_map.conf", "sharding.json"]
+full_real_paths = [f"{config_file_path}/{filename}" for filename in base_files]
+full_new_paths = [f"{filename}.tmp" for filename in full_real_paths]
+try:
+    write_updated_configs()
+    for old, new in zip(full_real_paths, full_new_paths, strict=False):
+        if not filecmp.cmp(old, new):
+            # There are changes; leave .tmp files and exit 0
+            if "SUPPRESS_SHARDING_NOTICE" not in os.environ:
+                print("===> Updated sharding; run scripts/refresh-sharding-and-restart")
+            sys.exit(0)
+    # No changes; clean up and exit 1
+    for filename in full_new_paths:
+        os.unlink(filename)
+    sys.exit(1)
+except AssertionError as e:
+    # Clean up whichever files we made
+    for filename in full_new_paths:
+        if os.path.exists(filename):
+            os.unlink(filename)
+    if options.errors_ok:
+        sys.exit(0)
+    else:
+        print(e, file=sys.stderr)
+        sys.exit(2)

scripts/lib/supervisor.py

@@ -0,0 +1,77 @@
+import socket
+import time
+from http.client import HTTPConnection
+from xmlrpc import client
+
+from typing_extensions import override
+
+
+class UnixStreamHTTPConnection(HTTPConnection):
+    @override
+    def connect(self) -> None:
+        self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        connected = False
+        for i in range(2):
+            try:
+                self.sock.connect(self.host)
+                connected = True
+                break
+            except FileNotFoundError:
+                # Backoff and retry
+                time.sleep(2**i)
+        if not connected:
+            raise Exception(
+                "Failed to connect to supervisor -- check that it is running, by running 'service supervisor status'"
+            )
+
+
+class UnixStreamTransport(client.Transport):
+    def __init__(self, socket_path: str) -> None:
+        self.socket_path = socket_path
+        super().__init__()
+
+    @override
+    def make_connection(self, host: tuple[str, dict[str, str]] | str) -> UnixStreamHTTPConnection:
+        return UnixStreamHTTPConnection(self.socket_path)
+
+
+def rpc() -> client.ServerProxy:
+    return client.ServerProxy(
+        "http://localhost", transport=UnixStreamTransport("/var/run/supervisor.sock")
+    )
+
+
+def list_supervisor_processes(
+    filter_names: list[str] | None = None, *, only_running: bool | None = None
+) -> list[str]:
+    results = []
+    processes = rpc().supervisor.getAllProcessInfo()
+    assert isinstance(processes, list)
+    for process in processes:
+        if process["group"] != process["name"]:
+            name = f"{process['group']}:{process['name']}"
+        else:
+            name = process["name"]
+
+        if filter_names:
+            match = False
+            for filter_name in filter_names:
+                # zulip-tornado:* matches zulip-tornado:9800 and zulip-tornado
+                if filter_name.endswith(":*") and (
+                    name.startswith(filter_name.removesuffix("*"))
+                    or name == filter_name.removesuffix(":*")
+                ):
+                    match = True
+                    break
+                if name == filter_name:
+                    match = True
+                    break
+            if not match:
+                continue
+
+        if only_running is None:
+            results.append(name)
+        elif only_running == (process["statename"] in ("RUNNING", "STARTING")):
+            results.append(name)
+
+    return results

scripts/lib/supported-os

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+#
+# This script serves only to verify that the OS is a supported
+# version, before we attempt to rely on that version in
+# upgrade-zulip-stage-3
+
+if [ -f /etc/os-release ]; then
+    os_info="$(
+        . /etc/os-release
+        printf '%s\n' "$ID" "$VERSION_ID"
+    )"
+    {
+        read -r os_id
+        read -r os_version_id
+    } <<<"$os_info"
+fi
+
+case "$os_id $os_version_id" in
+    'debian 12' | 'debian 13' | 'ubuntu 22.04' | 'ubuntu 24.04' | 'ubuntu 26.04')
+        exit 0
+        ;;
+    *)
+        exit 1
+        ;;
+esac

scripts/lib/unpack-zulip

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+import glob
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+
+os.environ["PYTHONUNBUFFERED"] = "y"
+
+sys.path.append(os.path.join(os.path.dirname(__file__), "..", ".."))
+import version
+from scripts.lib.zulip_tools import (
+    DEPLOYMENTS_DIR,
+    ENDC,
+    FAIL,
+    get_deployment_version,
+    is_invalid_upgrade,
+    make_deploy_path,
+    overwrite_symlink,
+)
+
+if len(sys.argv) != 2:
+    print(FAIL + f"Usage: {sys.argv[0]} <tarball>" + ENDC)
+    sys.exit(1)
+
+tarball_path = sys.argv[1]
+
+deploy_path = make_deploy_path()
+
+extract_path = tempfile.mkdtemp()
+subprocess.check_call(["tar", "-xf", tarball_path, "-C", extract_path])
+
+new_version = get_deployment_version(extract_path)
+current_version = version.ZULIP_VERSION
+if is_invalid_upgrade(current_version, new_version):
+    print(
+        FAIL + "Your current version is very old. Please first upgrade to version "
+        f"1.4.3 and then upgrade to {new_version}." + ENDC
+    )
+    shutil.rmtree(extract_path)
+    sys.exit(1)
+
+shutil.move(glob.glob(os.path.join(extract_path, "zulip-server-*"))[0], deploy_path)
+os.rmdir(extract_path)
+overwrite_symlink("/etc/zulip/settings.py", os.path.join(deploy_path, "zproject/prod_settings.py"))
+overwrite_symlink(deploy_path, os.path.join(DEPLOYMENTS_DIR, "next"))
+
+print(deploy_path)
+sys.exit(0)

scripts/lib/update-git-upstream

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -eu
+
+upstream_url="https://github.com/zulip/zulip.git"
+if ! git remote | grep -qx upstream; then
+    git remote add upstream "$upstream_url"
+else
+    git remote set-url upstream "$upstream_url"
+fi
+
+git fetch --prune --quiet --tags --all

scripts/lib/upgrade-zulip

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+import argparse
+import configparser
+import logging
+import os
+import shutil
+import subprocess
+import sys
+import time
+
+TARBALL_ARCHIVE_PATH = "/home/zulip/archives"
+os.environ["PYTHONUNBUFFERED"] = "y"
+
+sys.path.append(os.path.join(os.path.dirname(__file__), "..", ".."))
+from scripts.lib.zulip_tools import (
+    DEPLOYMENTS_DIR,
+    assert_running_as_root,
+    get_config_file,
+    get_deploy_options,
+    get_deployment_lock,
+    release_deployment_lock,
+    su_to_zulip,
+)
+
+config_file: configparser.RawConfigParser = get_config_file()
+deploy_options = get_deploy_options(config_file)
+
+assert_running_as_root(strip_lib_from_paths=True)
+
+# make sure we have appropriate file permissions
+os.umask(0o22)
+
+logging.Formatter.converter = time.gmtime
+logging.basicConfig(format="%(asctime)s upgrade-zulip: %(message)s", level=logging.INFO)
+
+parser = argparse.ArgumentParser()
+parser.add_argument("tarball", help="Path to Zulip Server tarball")
+args, extra_options = parser.parse_known_args()
+
+error_rerun_script = f"{DEPLOYMENTS_DIR}/current/scripts/upgrade-zulip {args.tarball}"
+get_deployment_lock(error_rerun_script)
+
+try:
+    # Copy the release tarball to an archival path that's readable by
+    # the Zulip user, and then unpack it from that directory, so that
+    # we can unpack using the Zulip user even if the original path was
+    # not readable by the Zulip user.
+    logging.info("Archiving the tarball under %s", TARBALL_ARCHIVE_PATH)
+    os.makedirs(TARBALL_ARCHIVE_PATH, exist_ok=True)
+    archived_tarball_path = os.path.join(TARBALL_ARCHIVE_PATH, os.path.basename(args.tarball))
+    shutil.copy(args.tarball, archived_tarball_path)
+    subprocess.check_output(["chown", "-R", "zulip:zulip", TARBALL_ARCHIVE_PATH])
+
+    logging.info("Unpacking the tarball")
+    unpack_zulip = os.path.realpath(os.path.join(os.path.dirname(__file__), "unpack-zulip"))
+    deploy_path = subprocess.check_output(
+        [unpack_zulip, archived_tarball_path], preexec_fn=su_to_zulip, text=True
+    )
+
+    # chdir to deploy_path and then run upgrade-zulip-stage-2 from the
+    # new version of Zulip (having the upgrade logic run from the new
+    # version is much better for fixing bugs in the upgrade process).
+    deploy_path = deploy_path.strip()
+    os.chdir(deploy_path)
+    try:
+        subprocess.check_call(
+            [
+                os.path.abspath("./scripts/lib/upgrade-zulip-stage-2"),
+                deploy_path,
+                *deploy_options,
+                *extra_options,
+            ]
+        )
+    except subprocess.CalledProcessError:
+        # There's no use in showing a stacktrace here; it just hides
+        # the error from stage 2.
+        sys.exit(1)
+finally:
+    release_deployment_lock()

scripts/lib/upgrade-zulip-from-git

@@ -0,0 +1,229 @@
+#!/usr/bin/env python3
+import argparse
+import logging
+import os
+import subprocess
+import sys
+import time
+
+LOCAL_GIT_CACHE_DIR = "/srv/zulip.git"
+os.environ["PYTHONUNBUFFERED"] = "y"
+
+sys.path.append(os.path.join(os.path.dirname(__file__), "..", ".."))
+from scripts.lib.zulip_tools import (
+    DEPLOYMENTS_DIR,
+    assert_running_as_root,
+    get_config,
+    get_config_file,
+    get_deploy_options,
+    get_deploy_root,
+    get_deployment_lock,
+    make_deploy_path,
+    overwrite_symlink,
+    release_deployment_lock,
+    su_to_zulip,
+    upgrade_script_arg_parser,
+)
+
+config_file = get_config_file()
+deploy_options = get_deploy_options(config_file)
+remote_url = get_config(
+    config_file, "deployment", "git_repo_url", "https://github.com/zulip/zulip.git"
+)
+
+assert_running_as_root(strip_lib_from_paths=True)
+
+# make sure we have appropriate file permissions
+os.umask(0o22)
+
+logging.Formatter.converter = time.gmtime
+logging.basicConfig(format="%(asctime)s upgrade-zulip-from-git: %(message)s", level=logging.INFO)
+
+parser = argparse.ArgumentParser(add_help=False)
+parser.add_argument("refname", help="Git reference, e.g. a branch, tag, or commit ID.")
+git_ref = parser.add_mutually_exclusive_group()
+git_ref.add_argument(
+    "--remote-url", help="Override the Git remote URL configured in /etc/zulip/zulip.conf."
+)
+git_ref.add_argument(
+    "--local-ref",
+    action="store_true",
+    help="Provided branch name has been pushed directly to /srv/zulip.git already",
+)
+
+# Any unknown options are passed through verbatim to upgrade-zulip-stage-2
+# and on to upgrade-zulip-stage-3. To make those passthrough options
+# discoverable, when the user runs --help we show a combined help that
+# also documents the stage-3 options.
+if {"-h", "--help"} & set(sys.argv[1:]):
+    argparse.ArgumentParser(parents=[parser, upgrade_script_arg_parser()]).parse_args(["--help"])
+
+args, extra_options = parser.parse_known_args()
+
+refname = args.refname
+# Command line remote URL will be given preference above the one
+# in /etc/zulip/zulip.conf.
+if args.remote_url:
+    remote_url = args.remote_url
+
+os.makedirs(DEPLOYMENTS_DIR, exist_ok=True)
+
+error_rerun_script = f"{DEPLOYMENTS_DIR}/current/scripts/upgrade-zulip-from-git {refname}"
+get_deployment_lock(error_rerun_script)
+
+try:
+    deploy_path = make_deploy_path()
+
+    # Populate LOCAL_GIT_CACHE_DIR with both the requested remote and zulip/zulip.
+    if not os.path.exists(LOCAL_GIT_CACHE_DIR):
+        logging.info("Making local repository cache")
+        subprocess.check_call(
+            ["git", "init", "--bare", "-q", LOCAL_GIT_CACHE_DIR],
+            stdout=subprocess.DEVNULL,
+        )
+        subprocess.check_call(
+            ["git", "remote", "add", "origin", remote_url],
+            cwd=LOCAL_GIT_CACHE_DIR,
+        )
+
+    if os.stat(LOCAL_GIT_CACHE_DIR).st_uid == 0:
+        subprocess.check_call(["chown", "-R", "zulip:zulip", LOCAL_GIT_CACHE_DIR])
+
+    os.chdir(LOCAL_GIT_CACHE_DIR)
+    if not args.local_ref:
+        subprocess.check_call(
+            ["git", "remote", "set-url", "origin", remote_url], preexec_fn=su_to_zulip
+        )
+
+    fetch_spec = subprocess.check_output(
+        ["git", "config", "remote.origin.fetch"],
+        preexec_fn=su_to_zulip,
+        text=True,
+    ).strip()
+    if fetch_spec in ("+refs/*:refs/*", "+refs/heads/*:refs/heads/*"):
+        # The refspec impinges on refs/heads/ -- this is an old mirror
+        # configuration.
+        logging.info("Cleaning up mirrored repository")
+        # remotes.origin.mirror may not be set -- we do not use
+        # check_call to ignore errors if it's already missing
+        subprocess.call(
+            ["git", "config", "--unset", "remote.origin.mirror"],
+            preexec_fn=su_to_zulip,
+        )
+        subprocess.check_call(
+            ["git", "config", "remote.origin.fetch", "+refs/heads/*:refs/remotes/origin/*"],
+            preexec_fn=su_to_zulip,
+        )
+        matching_refs = subprocess.check_output(
+            ["git", "for-each-ref", "--format=%(refname)", "refs/pull/", "refs/heads/"],
+            preexec_fn=su_to_zulip,
+            text=True,
+        ).splitlines()
+
+        # We can't use `git worktree list --porcelain -z` here because
+        # Ubuntu 22.04 only has git 2.34.1, and -z was
+        # introduced in 2.36
+        worktree_data = subprocess.check_output(
+            ["git", "worktree", "list", "--porcelain"],
+            preexec_fn=su_to_zulip,
+            text=True,
+        ).splitlines()
+        keep_refs = set()
+        for worktree_line in worktree_data:
+            if worktree_line.startswith("branch "):
+                keep_refs.add(worktree_line.removeprefix("branch "))
+
+        delete_input = "".join(
+            f"delete {refname}\n" for refname in matching_refs if refname not in keep_refs
+        )
+        subprocess.run(
+            ["git", "update-ref", "--stdin"],
+            check=True,
+            preexec_fn=su_to_zulip,
+            input=delete_input,
+            text=True,
+        )
+
+        logging.info("Repacking repository after pruning unnecessary refs...")
+        subprocess.check_call(
+            ["git", "gc", "--prune=now"],
+            preexec_fn=su_to_zulip,
+        )
+
+    if not args.local_ref:
+        subprocess.check_call(
+            [os.path.join(get_deploy_root(), "scripts/lib/update-git-upstream")],
+            preexec_fn=su_to_zulip,
+        )
+
+    # Generate the deployment directory via git worktree from our local repository.
+    try_refs = [
+        f"refs/tags/{refname}",
+        f"refs/heads/{refname}" if args.local_ref else f"refs/remotes/origin/{refname}",
+    ]
+    for ref in try_refs:
+        result = subprocess.run(
+            ["git", "rev-parse", "--verify", ref],
+            preexec_fn=su_to_zulip,
+            text=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.DEVNULL,
+            check=False,
+        )
+        if result.returncode == 0:
+            fullref = ref
+            commit_hash = result.stdout.strip()
+            break
+        elif result.returncode != 128:
+            result.check_returncode()
+    else:
+        logging.error(
+            "Failed to resolve %s as a tag or %s branch name!",
+            refname,
+            "local" if args.local_ref else "remote",
+        )
+        sys.exit(1)
+    refname = fullref
+
+    subprocess.check_call(
+        ["git", "worktree", "add", "--detach", deploy_path, refname],
+        stdout=subprocess.DEVNULL,
+        preexec_fn=su_to_zulip,
+    )
+    os.chdir(deploy_path)
+    extra_flags = []
+    if not refname.startswith("refs/tags/"):
+        extra_flags = ["-t"]
+    subprocess.check_call(
+        [
+            "git",
+            "checkout",
+            "-q",
+            *extra_flags,
+            "-b",
+            "deployment-" + os.path.basename(deploy_path),
+            refname,
+        ],
+        preexec_fn=su_to_zulip,
+    )
+
+    overwrite_symlink("/etc/zulip/settings.py", "zproject/prod_settings.py")
+
+    overwrite_symlink(deploy_path, os.path.join(DEPLOYMENTS_DIR, "next"))
+
+    try:
+        subprocess.check_call(
+            [
+                os.path.join(deploy_path, "scripts", "lib", "upgrade-zulip-stage-2"),
+                deploy_path,
+                "--from-git",
+                *deploy_options,
+                *extra_options,
+            ]
+        )
+    except subprocess.CalledProcessError:
+        # There's no use in showing a stacktrace here; it just hides
+        # the error from stage 2.
+        sys.exit(1)
+finally:
+    release_deployment_lock()

scripts/lib/upgrade-zulip-stage-2

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+#
+# This script serves only to verify that the OS is a supported
+# version, before we attempt to rely on that version in
+# upgrade-zulip-stage-3
+
+set -eu
+
+ZULIP_PATH="$(readlink -f "$(dirname "$0")"/../..)"
+if ! "$ZULIP_PATH/scripts/lib/supported-os"; then
+    echo "Unsupported platform!"
+    echo
+    echo "Sorry! The support for your OS has been discontinued."
+    echo "Please upgrade your OS to a supported release first,"
+    echo "by following these instructions:"
+    echo
+    echo "https://zulip.readthedocs.io/en/latest/production/upgrade.html#upgrading-the-operating-system"
+    echo
+    exit 1
+fi
+
+exec "$ZULIP_PATH/scripts/lib/upgrade-zulip-stage-3" "$@"

scripts/lib/upgrade-zulip-stage-3

@@ -0,0 +1,372 @@
+#!/usr/bin/env python3
+#
+# This script contains the actual logic for upgrading from an old
+# version of Zulip to the new version.  upgrade-zulip-stage-3 is
+# always run from the new version of Zulip, so any bug fixes take
+# effect on the very next upgrade.
+import argparse
+import logging
+import os
+import re
+import subprocess
+import sys
+import time
+
+os.environ["PYTHONUNBUFFERED"] = "y"
+
+# Force a known locale.  Some packages on PyPI fail to install in some locales.
+os.environ["LC_ALL"] = "C.UTF-8"
+os.environ["LANG"] = "C.UTF-8"
+os.environ["LANGUAGE"] = "C.UTF-8"
+
+sys.path.append(os.path.join(os.path.dirname(__file__), "..", ".."))
+from scripts.lib.zulip_tools import (
+    DEPLOYMENTS_DIR,
+    assert_running_as_root,
+    get_config,
+    get_config_file,
+    parse_version_from,
+    su_to_zulip,
+    upgrade_script_arg_parser,
+)
+
+assert_running_as_root()
+
+# Set a known, reliable PATH
+os.environ["PATH"] = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+
+logging.Formatter.converter = time.gmtime
+logging.basicConfig(format="%(asctime)s upgrade-zulip-stage-3: %(message)s", level=logging.INFO)
+
+# make sure we have appropriate file permissions
+os.umask(0o22)
+
+parser = argparse.ArgumentParser(parents=[upgrade_script_arg_parser()])
+parser.add_argument("deploy_path", metavar="deploy_path", help="Path to deployment directory")
+parser.add_argument(
+    "--from-git", action="store_true", help="Upgrading from git, so run update-prod-static."
+)
+args = parser.parse_args()
+
+if args.skip_restart:
+    if args.less_graceful:
+        logging.warning("Ignored --less-graceful; --skip-restart is always graceful.")
+        args.less_graceful = False
+    if args.skip_migrations:
+        logging.warning(
+            "Ignored --skip-migrations; all upgrades with --skip-restart asserts no migrations."
+        )
+        args.skip_migrations = False
+    if args.skip_puppet:
+        logging.warning(
+            "Ignored --skip-puppet; all upgrades with --skip-restart asserts no puppet changes."
+        )
+        args.skip_puppet = False
+
+if not args.skip_puppet and args.less_graceful:
+    logging.warning("Ignored --less-graceful; all upgrades without --skip-puppet are ungraceful.")
+
+if args.only_django:
+    logging.error(
+        "Only restarting Django into a new deployment leads to inconsistent caches "
+        "with Tornado; use --skip-client-reloads instead."
+    )
+    sys.exit(1)
+
+deploy_path = args.deploy_path
+os.chdir(deploy_path)
+
+config_file = get_config_file()
+
+IS_SERVER_UP = True
+HAS_FILLED_CACHES = False
+
+if args.from_git:
+    logging.info("Caching Zulip Git version...")
+    subprocess.check_call(["./tools/cache-zulip-git-version"], preexec_fn=su_to_zulip)
+
+from version import ZULIP_VERSION as NEW_ZULIP_VERSION
+
+old_version = parse_version_from(DEPLOYMENTS_DIR + "/current")
+logging.info("Upgrading from %s to %s, in %s", old_version, NEW_ZULIP_VERSION, deploy_path)
+
+migrations_needed = False
+
+
+def fill_memcached_caches() -> None:
+    global HAS_FILLED_CACHES
+    if HAS_FILLED_CACHES or migrations_needed:
+        return
+    subprocess.check_call(
+        ["./manage.py", "fill_memcached_caches", "--automated", "--skip-checks"],
+        preexec_fn=su_to_zulip,
+    )
+    HAS_FILLED_CACHES = True
+
+
+def shutdown_server(fill_caches: bool = True) -> None:
+    global IS_SERVER_UP
+
+    if args.skip_restart:
+        logging.info("Upgrade would require shutting down Zulip -- aborting!")
+        sys.exit(1)
+
+    if fill_caches:
+        fill_memcached_caches()
+
+    if IS_SERVER_UP:
+        logging.info("Stopping Zulip...")
+        subprocess.check_call(["./scripts/stop-server"], preexec_fn=su_to_zulip)
+        IS_SERVER_UP = False
+
+
+if not (args.skip_restart or args.skip_puppet):
+    # We need to temporarily hold pgroonga, if installed -- upgrading
+    # it without running the appropriate upgrade SQL can cause
+    # PostgreSQL to crash
+    if get_config(config_file, "machine", "pgroonga", False):
+        subprocess.check_call(["apt-mark", "hold", "postgresql-*-pgdg-pgroonga"])
+    logging.info("Upgrading system packages...")
+    subprocess.check_call(["apt-get", "update"])
+    subprocess.check_call(["apt-get", "-y", "--with-new-pkgs", "upgrade"])
+    if get_config(config_file, "machine", "pgroonga", False):
+        subprocess.check_call(["apt-mark", "unhold", "postgresql-*-pgdg-pgroonga"])
+
+# Now we should have an environment set up where we can run our tools;
+# first, creating the production venv.
+subprocess.check_call(
+    [os.path.join(deploy_path, "scripts", "lib", "create-production-venv"), deploy_path]
+)
+
+# Check to make sure that this upgrade is not actually a database
+# downgrade.
+if not args.skip_downgrade_check:
+    subprocess.check_call(
+        [os.path.join(deploy_path, "scripts", "lib", "check-database-compatibility")],
+        preexec_fn=su_to_zulip,
+    )
+
+# Make sure the right version of node is installed
+subprocess.check_call([os.path.join(deploy_path, "scripts", "lib", "install-node")])
+
+# Generate any new secrets that were added in the new version required.
+# TODO: Do caching to only run this when it has changed.
+subprocess.check_call(
+    [os.path.join(deploy_path, "scripts", "setup", "generate_secrets.py"), "--production"]
+)
+
+# TODO/compatibility: Puppet class names for the manifest renames in
+# the 11.0 release.  Can be removed when we can no longer directly
+# upgrade from 10.x.
+class_renames = {
+    "zulip::postfix_localmail": "zulip::local_mailserver",
+}
+classes = re.split(r"\s*,\s*", get_config(config_file, "machine", "puppet_classes", ""))
+new_classes = [class_renames.get(c, c) for c in classes]
+if classes != new_classes:
+    if args.skip_restart:
+        logging.error("Would need to adjust puppet classes -- aborting!")
+        sys.exit(1)
+    logging.info("Adjusting Puppet classes for renames...")
+    subprocess.check_call(
+        [
+            "crudini",
+            "--set",
+            "/etc/zulip/zulip.conf",
+            "machine",
+            "puppet_classes",
+            ", ".join(new_classes),
+        ]
+    )
+
+# And then, building/installing the static assets.
+if args.ignore_static_assets:
+    # For the OS version upgrade use case, the static assets are
+    # already in place, and we don't need to do anything.  Further,
+    # neither of the options below will work for all installations,
+    # because if we installed from Git, `prod-static/serve` may be
+    # empty so we can't do the non-Git thing, whereas if we installed
+    # from a tarball, we won't have a `tools/` directory and thus
+    # cannot run `tools/update-prod-static`.
+    pass
+elif args.from_git:
+    # Because `upgrade-zulip-from-git` needs to build static assets, it
+    # is at risk of being OOM killed on systems with limited free RAM.
+    mem_bytes = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES")
+    mem_gib = mem_bytes / (1024.0**3)  # e.g. 3.74
+
+    # Ideally, we'd have 2 thresholds here, depending on whether the
+    # system is running queue workers multithreaded or multiprocess.
+    # See puppet/zulip/manifests/app_frontend_base.pp for background.
+    if mem_gib < 4.2:
+        logging.info("Shutting down server to ensure sufficient free RAM for webpack.")
+        shutdown_server(fill_caches=False)
+
+    # Note: The fact that this is before we apply Puppet changes means
+    # that we don't support adding new Puppet dependencies of
+    # update-prod-static with the Git upgrade process.  But it'll fail
+    # safely; this seems like a worthwhile tradeoff to minimize downtime.
+    logging.info("Building static assets...")
+    try:
+        subprocess.check_call(["./tools/update-prod-static"], preexec_fn=su_to_zulip)
+    except subprocess.CalledProcessError:
+        logging.error("Failed to build static assets.")
+        if IS_SERVER_UP:
+            logging.error("Usually the cause is insufficient free RAM to run webpack.")
+            logging.error("Try stopping the Zulip server (scripts/stop-server) and trying again.")
+        sys.exit(1)
+
+else:
+    # Since this doesn't do any actual work, it's likely safe to have
+    # this run before we apply Puppet changes (saving a bit of downtime).
+    logging.info("Installing static assets...")
+    subprocess.check_call(
+        ["cp", "-rT", os.path.join(deploy_path, "prod-static/serve"), "/home/zulip/prod-static"],
+        preexec_fn=su_to_zulip,
+    )
+
+# Ensure the local uploads directory exists before we run system
+# checks, since check_uploads_settings verifies it.  Puppet would
+# create it later in the upgrade, but the system check runs first.
+# `os.path.isdir` follows symlinks, so admins who have replaced the
+# directory with a symlink to alternate storage are not disturbed.
+if not os.path.isdir("/home/zulip/uploads"):
+    subprocess.check_call(
+        ["install", "-d", "-o", "zulip", "-g", "zulip", "-m", "0755", "/home/zulip/uploads"]
+    )
+
+# Perform system checks -- including database checks, so we don't need
+# to do them when we do migrations, below.
+if not args.skip_checks:
+    subprocess.check_call(["./manage.py", "check", "--database", "default"], preexec_fn=su_to_zulip)
+
+# Our next optimization is to check whether any migrations are needed
+# before we start the critical section of the restart.  This saves
+# about 1s of downtime in a no-op upgrade.  We omit this check if we
+# already stopped the server above, due to low memory.
+if not IS_SERVER_UP:
+    migrations_needed = True
+elif not args.skip_migrations:
+    logging.info("Checking for needed migrations")
+    migrations_output = subprocess.check_output(
+        ["./manage.py", "showmigrations", "--skip-checks"], preexec_fn=su_to_zulip, text=True
+    )
+    for ln in migrations_output.split("\n"):
+        line_str = ln.strip()
+        if line_str.startswith("[ ]"):
+            migrations_needed = True
+
+if args.skip_restart and migrations_needed:
+    logging.error("Would need to apply migrations -- aborting!")
+    sys.exit(1)
+
+# Install hooks before we check for puppet changes, so we skip an
+# unnecessary stop/start cycle if hook changes are the only ones.
+logging.info("Installing hooks")
+subprocess.check_call(
+    ["./scripts/zulip-puppet-apply", "--tags", "hooks", "--force"],
+)
+
+# If we are planning on running puppet, we can pre-run it in --noop
+# mode and see if it will actually make any changes; if not, we can
+# skip it during the upgrade.  We omit this check if the server is
+# already stopped, since it's not better than just pressing on.
+has_puppet_changes = True
+if not args.skip_puppet and IS_SERVER_UP:
+    logging.info("Pre-checking for puppet changes...")
+    try_puppet = subprocess.run(
+        ["./scripts/zulip-puppet-apply", "--noop", "--force"],
+        stdout=subprocess.DEVNULL,
+        check=False,
+    )
+    if try_puppet.returncode == 0:
+        if args.skip_restart:
+            logging.info("Verified no Puppet changes are necessary.")
+        else:
+            logging.info("No puppet changes found, skipping!")
+        args.skip_puppet = True
+        has_puppet_changes = False
+    elif try_puppet.returncode == 2:
+        logging.error("Puppet error -- aborting!")
+        sys.exit(1)
+    elif args.skip_restart:
+        logging.error("Would need to apply puppet changes -- aborting!")
+        sys.exit(1)
+
+
+if args.skip_restart:
+    logging.info("Successfully configured in %s!", deploy_path)
+else:
+    # NOTE: Here begins the most likely critical period, where we may be
+    # shutting down the server; we should strive to minimize the number of
+    # steps that happen between here and the "Restarting Zulip" line
+    # below.
+    hooks_args = []
+    if args.from_git:
+        hooks_args = ["--from-git"]
+
+    subprocess.check_call(["./scripts/lib/run_hooks.py", "pre-deploy", *hooks_args])
+
+    if not args.skip_puppet:
+        # Puppet may adjust random services; to minimize risk of issues
+        # due to inconsistent state, we shut down the server first.
+        shutdown_server()
+        logging.info("Applying Puppet changes...")
+        subprocess.check_call(["./scripts/zulip-puppet-apply", "--force"])
+        subprocess.check_call(["apt-get", "-y", "--with-new-pkgs", "upgrade"])
+        # Puppet may have reloaded supervisor, and in so doing started
+        # services; mark as potentially needing to stop the server.
+        IS_SERVER_UP = True
+
+    if migrations_needed:
+        # Database migrations assume that they run on a database in
+        # quiesced state.
+        shutdown_server()
+        logging.info("Applying database migrations...")
+        subprocess.check_call(
+            ["./manage.py", "migrate", "--noinput", "--skip-checks"], preexec_fn=su_to_zulip
+        )
+
+    logging.info("Restarting Zulip...")
+    start_args = ["--skip-checks"]
+    if args.only_django and IS_SERVER_UP:
+        # Calling `start-server --only-django` only starts Django,
+        # which is almost certainly not what we want after an upgrade.
+        # If we had to stop services for other reasons (e.g. puppet),
+        # we ignore --only-django and downgrade it to
+        # --skip-client-reloads
+        start_args.append("--only-django")
+    else:
+        start_args.append("--skip-client-reloads")
+    if not HAS_FILLED_CACHES:
+        start_args.append("--fill-cache")
+    if IS_SERVER_UP:
+        if args.less_graceful:
+            start_args.append("--less-graceful")
+        subprocess.check_call(["./scripts/restart-server", *start_args], preexec_fn=su_to_zulip)
+    else:
+        subprocess.check_call(["./scripts/start-server", *start_args], preexec_fn=su_to_zulip)
+
+    logging.info("Upgrade complete!")
+
+    subprocess.check_call(["./scripts/lib/run_hooks.py", "post-deploy", *hooks_args])
+
+    if not args.skip_client_reloads and not args.only_django:
+        subprocess.check_call(["./scripts/reload-clients", "--background"], preexec_fn=su_to_zulip)
+
+if args.audit_fts_indexes:
+    logging.info("Correcting full-text search indexes for updated dictionary files")
+    logging.info("This may take a while but the server should work while it runs.")
+    subprocess.check_call(
+        ["./manage.py", "audit_fts_indexes", "--automated", "--skip-checks"], preexec_fn=su_to_zulip
+    )
+
+if not args.skip_purge_old_deployments:
+    logging.info("Purging old deployments...")
+    subprocess.check_call(["./scripts/purge-old-deployments"])
+else:
+    logging.info("Skipping purging old deployments.")
+
+if args.skip_puppet and has_puppet_changes:
+    logging.info("Showing un-applied Puppet changes:")
+    subprocess.check_call(["./scripts/zulip-puppet-apply", "--noop", "--show_diff"])

scripts/lib/warn-rabbitmq-nodename-change

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+# For security reasons, we need to configure RabbitMQ to listen only
+# on localhost, which we cannot do if the nodename contains the
+# hostname (e.g. rabbit@zulip-host).  Containing the hostname also
+# makes it brittle to hostname changes, which are (for example) common
+# in containerized environments.
+
+set -eu
+
+# Try to find the current nodename
+CURRENT=$(sh -c 'cd /usr/lib/rabbitmq/bin/ && . ./rabbitmq-env && echo $NODENAME')
+if [ "$CURRENT" != "zulip@localhost" ]; then
+    cat <<EOF
+
+***** WARNING *****
+
+We are renaming the rabbitmq server's nodename from '$CURRENT' to
+'zulip@localhost', as rabbitmq is being reconfigured to listen only on
+localhost.  This will also make the server more resilient to hostname
+changes.  This will only affect you if you were using the rabbitmq
+server for other, non-Zulip uses.
+
+*******************
+
+EOF
+
+    service rabbitmq-server stop || true
+fi

scripts/lib/zulip_tools.py

@@ -0,0 +1,766 @@
+#!/usr/bin/env python3
+import argparse
+import configparser
+import functools
+import hashlib
+import json
+import logging
+import os
+import platform
+import pwd
+import random
+import shlex
+import shutil
+import signal
+import subprocess
+import sys
+import time
+import uuid
+import zoneinfo
+from collections.abc import Sequence
+from datetime import datetime, timedelta
+from typing import IO, Any, Literal, overload
+from urllib.parse import SplitResult
+
+DEPLOYMENTS_DIR = "/home/zulip/deployments"
+LOCK_DIR = os.path.join(DEPLOYMENTS_DIR, "lock")
+TIMESTAMP_FORMAT = "%Y-%m-%d-%H-%M-%S"
+
+# Color codes
+OKBLUE = "\033[94m"
+OKGREEN = "\033[92m"
+WARNING = "\033[93m"
+FAIL = "\033[91m"
+ENDC = "\033[0m"
+BLACKONYELLOW = "\x1b[0;30;43m"
+WHITEONRED = "\x1b[0;37;41m"
+BOLDRED = "\x1b[1;31m"
+BOLD = "\x1b[1m"
+GRAY = "\x1b[90m"
+
+GREEN = "\x1b[32m"
+YELLOW = "\x1b[33m"
+BLUE = "\x1b[34m"
+MAGENTA = "\x1b[35m"
+CYAN = "\x1b[36m"
+
+
+def overwrite_symlink(src: str, dst: str) -> None:
+    dir, base = os.path.split(dst)
+    while True:
+        # Note: creating a temporary filename like this is not generally
+        # secure.  It’s fine in this case because os.symlink refuses to
+        # overwrite an existing target; we handle the error and try again.
+        tmp = os.path.join(dir, f".{base}.{random.randrange(1 << 40):010x}")
+        try:
+            os.symlink(src, tmp)
+        except FileExistsError:
+            continue
+        break
+    try:
+        os.rename(tmp, dst)
+    except BaseException:
+        os.remove(tmp)
+        raise
+
+
+def parse_cache_script_args(description: str) -> argparse.Namespace:
+    # Keep this in sync with clean_unused_caches in provision_inner.py
+    parser = argparse.ArgumentParser(description=description)
+
+    parser.add_argument(
+        "--threshold",
+        dest="threshold_days",
+        type=int,
+        default=14,
+        metavar="<days>",
+        help="Any cache which is not in "
+        "use by a deployment not older than threshold days(current "
+        "installation in dev) and older than threshold days will be "
+        "deleted. (defaults to 14)",
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="If specified then script will only print the caches "
+        "that it will delete/keep back. It will not delete any cache.",
+    )
+    parser.add_argument(
+        "--verbose",
+        action="store_true",
+        help="If specified then script will print a detailed report "
+        "of what is being will deleted/kept back.",
+    )
+    parser.add_argument(
+        "--no-print-headings",
+        dest="no_headings",
+        action="store_true",
+        help="If specified then script will not print headings for what will be deleted/kept back.",
+    )
+
+    args = parser.parse_args()
+    args.verbose |= args.dry_run  # Always print a detailed report in case of dry run.
+    return args
+
+
+def get_deploy_root() -> str:
+    return os.path.realpath(
+        os.path.normpath(os.path.join(os.path.dirname(__file__), "..", "..")),
+    )
+
+
+def parse_version_from(deploy_path: str, merge_base: bool = False) -> str:
+    if not os.path.exists(os.path.join(deploy_path, "zulip-git-version")):
+        try:
+            # Pull this tool from _our_ deploy root, since it may not
+            # exist historically, but run it the cwd of the old
+            # deploy, so we set up its remote.
+            subprocess.check_call(
+                [os.path.join(get_deploy_root(), "scripts", "lib", "update-git-upstream")],
+                cwd=deploy_path,
+                preexec_fn=su_to_zulip,
+            )
+            subprocess.check_call(
+                [os.path.join(deploy_path, "tools", "cache-zulip-git-version")],
+                cwd=deploy_path,
+                preexec_fn=su_to_zulip,
+            )
+        except subprocess.CalledProcessError:
+            pass
+    try:
+        varname = "ZULIP_MERGE_BASE" if merge_base else "ZULIP_VERSION"
+        return subprocess.check_output(
+            [sys.executable, "-c", f"from version import {varname}; print({varname})"],
+            cwd=deploy_path,
+            text=True,
+        ).strip()
+    except subprocess.CalledProcessError:
+        return "0.0.0"
+
+
+def get_deployment_version(extract_path: str) -> str:
+    version = "0.0.0"
+    for item in os.listdir(extract_path):
+        item_path = os.path.join(extract_path, item)
+        if item.startswith("zulip-server") and os.path.isdir(item_path):
+            version = parse_version_from(item_path)
+            break
+    return version
+
+
+def is_invalid_upgrade(current_version: str, new_version: str) -> bool:
+    if new_version > "1.4.3" and current_version <= "1.3.10":
+        return True
+    return False
+
+
+def get_zulip_pwent() -> pwd.struct_passwd:
+    deploy_root_uid = os.stat(get_deploy_root()).st_uid
+    if deploy_root_uid != 0:
+        return pwd.getpwuid(deploy_root_uid)
+
+    # In the case that permissions got messed up and the deployment
+    # directory is unexpectedly owned by root, we fallback to the
+    # `zulip` user as that's the correct value in production.
+    return pwd.getpwnam("zulip")
+
+
+def get_postgres_pwent() -> pwd.struct_passwd:
+    try:
+        return pwd.getpwnam("postgres")
+    except KeyError:
+        return get_zulip_pwent()
+
+
+def su_to_zulip(save_suid: bool = False) -> None:
+    """Warning: su_to_zulip assumes that the zulip checkout is owned by
+    the zulip user (or whatever normal user is running the Zulip
+    installation).  It should never be run from the installer or other
+    production contexts before /home/zulip/deployments/current is
+    created."""
+    pwent = get_zulip_pwent()
+    os.setgid(pwent.pw_gid)
+    if save_suid:
+        os.setresuid(pwent.pw_uid, pwent.pw_uid, os.getuid())
+    else:
+        os.setuid(pwent.pw_uid)
+    os.environ["HOME"] = pwent.pw_dir
+
+
+def make_deploy_path() -> str:
+    timestamp = datetime.now().strftime(TIMESTAMP_FORMAT)  # noqa: DTZ005
+    return os.path.join(DEPLOYMENTS_DIR, timestamp)
+
+
+TEMPLATE_DATABASE_DIR = "test-backend/databases"
+
+
+def get_dev_uuid_var_path(create_if_missing: bool = False) -> str:
+    zulip_path = get_deploy_root()
+    uuid_path = os.path.join(os.path.realpath(os.path.dirname(zulip_path)), ".zulip-dev-uuid")
+    if os.path.exists(uuid_path):
+        with open(uuid_path) as f:
+            zulip_uuid = f.read().strip()
+    else:
+        if create_if_missing:
+            zulip_uuid = str(uuid.uuid4())
+            # We need root access here, since the path will be under /srv/ in the
+            # development environment.
+            run_as_root(["sh", "-c", 'echo "$1" > "$2"', "-", zulip_uuid, uuid_path])
+        else:
+            raise AssertionError("Missing UUID file; please run tools/provision!")
+
+    result_path = os.path.join(zulip_path, "var", zulip_uuid)
+    os.makedirs(result_path, exist_ok=True)
+    return result_path
+
+
+def get_deployment_lock(error_rerun_script: str) -> None:
+    start_time = time.time()
+    got_lock = False
+    while time.time() - start_time < 300:
+        try:
+            os.mkdir(LOCK_DIR)
+            got_lock = True
+            break
+        except OSError:
+            print(
+                WARNING
+                + "Another deployment in progress; waiting for lock... "
+                + f"(If no deployment is running, rmdir {LOCK_DIR})"
+                + ENDC,
+                flush=True,
+            )
+            time.sleep(3)
+
+    if not got_lock:
+        print(
+            FAIL
+            + "Deployment already in progress.  Please run\n"
+            + f"  {error_rerun_script}\n"
+            + "manually when the previous deployment finishes, or run\n"
+            + f"  rmdir {LOCK_DIR}\n"
+            + "if the previous deployment crashed."
+            + ENDC
+        )
+        sys.exit(1)
+
+
+def release_deployment_lock() -> None:
+    shutil.rmtree(LOCK_DIR)
+
+
+def run(args: Sequence[str], **kwargs: Any) -> None:
+    # Output what we're doing in the `set -x` style
+    print(f"+ {shlex.join(args)}", flush=True)
+
+    try:
+        subprocess.check_call(args, **kwargs)
+    except subprocess.CalledProcessError as error:
+        print()
+        if error.returncode < 0:
+            try:
+                signal_name = signal.Signals(-error.returncode).name
+            except ValueError:
+                signal_name = f"unknown signal {-error.returncode}"
+            print(
+                WHITEONRED
+                + f"Subcommand of {sys.argv[0]} died with {signal_name}: {shlex.join(args)}"
+                + ENDC
+            )
+        else:
+            print(
+                WHITEONRED
+                + f"Subcommand of {sys.argv[0]} failed with exit status {error.returncode}: {shlex.join(args)}"
+                + ENDC
+            )
+            print(WHITEONRED + "Actual error output for the subcommand is just above this." + ENDC)
+        print()
+        sys.exit(1)
+
+
+def log_management_command(cmd: Sequence[str], log_path: str) -> None:
+    log_dir = os.path.dirname(log_path)
+    if not os.path.exists(log_dir):
+        os.makedirs(log_dir)
+
+    formatter = logging.Formatter("%(asctime)s: %(message)s")
+    file_handler = logging.FileHandler(log_path)
+    file_handler.setFormatter(formatter)
+    logger = logging.getLogger("zulip.management")
+    logger.addHandler(file_handler)
+    logger.setLevel(logging.INFO)
+
+    logger.info("Ran %s", shlex.join(cmd))
+
+
+def get_environment() -> str:
+    if os.path.exists(DEPLOYMENTS_DIR):
+        return "prod"
+    return "dev"
+
+
+def get_recent_deployments(threshold_days: int | None) -> set[str]:
+    # Returns a list of deployments not older than threshold days
+    # including `/root/zulip` directory if it exists.
+    recent = set()
+    if threshold_days is not None:
+        threshold_date = datetime.now() - timedelta(days=threshold_days)  # noqa: DTZ005
+    else:
+        threshold_date = None
+    if os.path.isdir(DEPLOYMENTS_DIR):
+        for dir_name in os.listdir(DEPLOYMENTS_DIR):
+            target_dir = os.path.join(DEPLOYMENTS_DIR, dir_name)
+            if not os.path.isdir(target_dir):
+                # Skip things like uwsgi sockets, symlinks, etc.
+                continue
+            if not os.path.exists(os.path.join(target_dir, "zerver")):
+                # Skip things like "lock" that aren't actually a deployment directory
+                continue
+            try:
+                date = datetime.strptime(dir_name, TIMESTAMP_FORMAT)  # noqa: DTZ007
+                if threshold_date is None or date >= threshold_date:
+                    recent.add(target_dir)
+            except ValueError:
+                # Always include deployments whose name is not in the format of a timestamp.
+                recent.add(target_dir)
+                # If it is a symlink then include the target as well.
+                if os.path.islink(target_dir):
+                    recent.add(os.path.realpath(target_dir))
+    if os.path.exists("/root/zulip"):
+        recent.add("/root/zulip")
+    return recent
+
+
+def get_threshold_timestamp(threshold_days: int) -> int:
+    # Given number of days, this function returns timestamp corresponding
+    # to the time prior to given number of days.
+    threshold = datetime.now() - timedelta(days=threshold_days)  # noqa: DTZ005
+    threshold_timestamp = int(time.mktime(threshold.utctimetuple()))
+    return threshold_timestamp
+
+
+def get_caches_to_be_purged(
+    caches_dir: str, caches_in_use: set[str], threshold_days: int
+) -> set[str]:
+    # Given a directory containing caches, a list of caches in use
+    # and threshold days, this function return a list of caches
+    # which can be purged. Remove the cache only if it is:
+    # 1: Not in use by the current installation(in dev as well as in prod).
+    # 2: Not in use by a deployment not older than `threshold_days` (in prod).
+    # 3: Not in use by '/root/zulip'.
+    # 4: Not older than `threshold_days`.
+    caches_to_purge = set()
+    threshold_timestamp = get_threshold_timestamp(threshold_days)
+    for cache_dir_base in os.listdir(caches_dir):
+        cache_dir = os.path.join(caches_dir, cache_dir_base)
+        if cache_dir in caches_in_use:
+            # Never purge a cache which is in use.
+            continue
+        if os.path.getctime(cache_dir) < threshold_timestamp:
+            caches_to_purge.add(cache_dir)
+    return caches_to_purge
+
+
+def purge_unused_caches(
+    caches_dir: str,
+    caches_in_use: set[str],
+    cache_type: str,
+    args: argparse.Namespace,
+) -> None:
+    if not os.path.exists(caches_dir):
+        return
+
+    all_caches = {os.path.join(caches_dir, cache) for cache in os.listdir(caches_dir)}
+    caches_to_purge = get_caches_to_be_purged(caches_dir, caches_in_use, args.threshold_days)
+    caches_to_keep = all_caches - caches_to_purge
+
+    maybe_perform_purging(
+        caches_to_purge, caches_to_keep, cache_type, args.dry_run, args.verbose, args.no_headings
+    )
+    if args.verbose:
+        print("Done!")
+
+
+def generate_sha1sum_emoji(zulip_path: str) -> str:
+    sha = hashlib.sha1()
+
+    filenames = [
+        "web/images/zulip-emoji/zulip.png",
+        "tools/setup/emoji/emoji_map.json",
+        "tools/setup/emoji/build_emoji",
+        "tools/setup/emoji/emoji_setup_utils.py",
+        "tools/setup/emoji/emoji_names.py",
+        "zerver/management/data/unified_reactions.json",
+    ]
+
+    for filename in filenames:
+        file_path = os.path.join(zulip_path, filename)
+        with open(file_path, "rb") as reader:
+            sha.update(reader.read())
+
+    # Take into account the version of `emoji-datasource-google` package
+    # while generating success stamp.
+    with open(os.path.join(zulip_path, "node_modules/emoji-datasource-google/package.json")) as fp:
+        emoji_datasource_version = json.load(fp)["version"]
+    sha.update(emoji_datasource_version.encode())
+
+    return sha.hexdigest()
+
+
+def maybe_perform_purging(
+    dirs_to_purge: set[str],
+    dirs_to_keep: set[str],
+    dir_type: str,
+    dry_run: bool,
+    verbose: bool,
+    no_headings: bool,
+) -> None:
+    if dry_run:
+        print("Performing a dry run...")
+    if not no_headings:
+        print(f"Cleaning unused {dir_type}s...")
+
+    for directory in dirs_to_purge:
+        if verbose:
+            print(f"Cleaning unused {dir_type}: {directory}")
+        if not dry_run:
+            run_as_root(["rm", "-rf", directory])
+
+    for directory in dirs_to_keep:
+        if verbose:
+            print(f"Keeping used {dir_type}: {directory}")
+
+
+@functools.lru_cache(None)
+def os_families() -> set[str]:
+    """
+    Known families:
+    debian (includes: debian, ubuntu)
+    ubuntu (includes: ubuntu)
+    fedora (includes: fedora, rhel, centos)
+    rhel (includes: rhel, centos)
+    centos (includes: centos)
+    """
+    distro_info = platform.freedesktop_os_release()
+    return {distro_info["ID"], *distro_info.get("ID_LIKE", "").split()}
+
+
+def get_tzdata_zi() -> IO[str]:
+    for path in zoneinfo.TZPATH:
+        filename = os.path.join(path, "tzdata.zi")
+        if os.path.exists(filename):
+            return open(filename)
+    raise RuntimeError("Missing time zone data (tzdata.zi)")
+
+
+def files_and_string_digest(filenames: Sequence[str], extra_strings: Sequence[str]) -> str:
+    # see is_digest_obsolete for more context
+    sha1sum = hashlib.sha1()
+    for fn in filenames:
+        with open(fn, "rb") as file_to_hash:
+            sha1sum.update(file_to_hash.read())
+
+    for extra_string in extra_strings:
+        sha1sum.update(extra_string.encode())
+
+    return sha1sum.hexdigest()
+
+
+def is_digest_obsolete(
+    hash_name: str, filenames: Sequence[str], extra_strings: Sequence[str] = []
+) -> bool:
+    """
+    In order to determine if we need to run some
+    process, we calculate a digest of the important
+    files and strings whose respective contents
+    or values may indicate such a need.
+
+        filenames = files we should hash the contents of
+        extra_strings = strings we should hash directly
+
+    Grep for callers to see examples of how this is used.
+
+    To elaborate on extra_strings, they will typically
+    be things like:
+
+        - package versions (that we import)
+        - settings values (that we stringify with
+          json, deterministically)
+    """
+    last_hash_path = os.path.join(get_dev_uuid_var_path(), hash_name)
+    try:
+        with open(last_hash_path) as f:
+            old_hash = f.read()
+    except FileNotFoundError:
+        # This is normal for a fresh checkout--a missing
+        # digest is an obsolete digest.
+        return True
+
+    new_hash = files_and_string_digest(filenames, extra_strings)
+
+    return new_hash != old_hash
+
+
+def write_new_digest(
+    hash_name: str, filenames: Sequence[str], extra_strings: Sequence[str] = []
+) -> None:
+    hash_path = os.path.join(get_dev_uuid_var_path(), hash_name)
+    new_hash = files_and_string_digest(filenames, extra_strings)
+    with open(hash_path, "w") as f:
+        f.write(new_hash)
+
+    # Be a little verbose here--our callers ensure we
+    # only write new digests when things have changed, and
+    # making this system more transparent to developers
+    # can help them troubleshoot provisioning glitches.
+    print("New digest written to: " + hash_path)
+
+
+def is_root() -> bool:
+    if "posix" in os.name and os.geteuid() == 0:
+        return True
+    return False
+
+
+def run_as_root(args: list[str], **kwargs: Any) -> None:
+    sudo_args = kwargs.pop("sudo_args", [])
+    if not is_root():
+        args = ["sudo", *sudo_args, "--", *args]
+    run(args, **kwargs)
+
+
+def assert_not_running_as_root() -> None:
+    script_name = os.path.abspath(sys.argv[0])
+    if is_root():
+        pwent = get_zulip_pwent()
+        msg = (
+            f"{os.path.basename(script_name)} should not be run as root. Use `su {pwent.pw_name}` to switch to the 'zulip'\n"
+            f"user before rerunning this, or use \n  su {pwent.pw_name} -c '{script_name} ...'\n"
+            "to switch users and run this as a single command."
+        )
+        print(msg)
+        sys.exit(1)
+
+
+def assert_running_as_root(strip_lib_from_paths: bool = False) -> None:
+    script_name = os.path.abspath(sys.argv[0])
+    # Since these Python scripts are run inside a thin shell wrapper,
+    # we need to replace the paths in order to ensure we instruct
+    # users to (re)run the right command.
+    if strip_lib_from_paths:
+        script_name = script_name.replace("scripts/lib/upgrade", "scripts/upgrade")
+    if not is_root():
+        print(f"{script_name} must be run as root.")
+        sys.exit(1)
+
+
+@overload
+def get_config(
+    config_file: configparser.RawConfigParser,
+    section: str,
+    key: str,
+    default_value: None = None,
+) -> str | None: ...
+@overload
+def get_config(
+    config_file: configparser.RawConfigParser,
+    section: str,
+    key: str,
+    default_value: str,
+) -> str: ...
+@overload
+def get_config(
+    config_file: configparser.RawConfigParser, section: str, key: str, default_value: bool
+) -> bool: ...
+def get_config(
+    config_file: configparser.RawConfigParser,
+    section: str,
+    key: str,
+    default_value: str | bool | None = None,
+) -> str | bool | None:
+    if config_file.has_option(section, key):
+        val = config_file.get(section, key)
+        if isinstance(default_value, bool):
+            # This list is parallel to puppet/zulip/lib/puppet/functions/zulipconf.rb
+            return val.lower() in ["1", "y", "t", "true", "yes", "enable", "enabled"]
+        return val
+    return default_value
+
+
+def get_config_file() -> configparser.RawConfigParser:
+    config_file = configparser.RawConfigParser()
+    config_file.read("/etc/zulip/zulip.conf")
+    return config_file
+
+
+def get_deploy_options(config_file: configparser.RawConfigParser) -> list[str]:
+    return shlex.split(get_config(config_file, "deployment", "deploy_options", ""))
+
+
+def get_tornado_ports(config_file: configparser.RawConfigParser) -> list[int]:
+    ports = []
+    if config_file.has_section("tornado_sharding"):
+        ports = sorted(
+            {
+                int(port)
+                for key in config_file.options("tornado_sharding")
+                for port in key.removesuffix("_regex").split("_")
+            }
+        )
+    if not ports:
+        ports = [9800]
+    return ports
+
+
+def get_or_create_dev_uuid_var_path(path: str) -> str:
+    absolute_path = f"{get_dev_uuid_var_path()}/{path}"
+    os.makedirs(absolute_path, exist_ok=True)
+    return absolute_path
+
+
+def is_vagrant_env_host(path: str) -> bool:
+    return ".vagrant" in os.listdir(path)
+
+
+def has_application_server(once: bool = False) -> bool:
+    if once:
+        return os.path.exists("/etc/supervisor/conf.d/zulip/zulip-once.conf")
+    return (
+        # Current path
+        os.path.exists("/etc/supervisor/conf.d/zulip/zulip.conf")
+        # Old path, relevant for upgrades
+        or os.path.exists("/etc/supervisor/conf.d/zulip.conf")
+    )
+
+
+def has_process_fts_updates() -> bool:
+    return (
+        # Current path
+        os.path.exists("/etc/supervisor/conf.d/zulip/zulip_db.conf")
+        # Old path, relevant for upgrades
+        or os.path.exists("/etc/supervisor/conf.d/zulip_db.conf")
+    )
+
+
+def deport(netloc: str) -> str:
+    """Remove the port from a hostname:port string.  Brackets on a literal
+    IPv6 address are included."""
+    r = SplitResult("", netloc, "", "", "")
+    assert r.hostname is not None
+    return "[" + r.hostname + "]" if ":" in r.hostname else r.hostname
+
+
+def start_script_arg_parser(action: str, add_help: bool = False) -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(add_help=add_help)
+    parser.add_argument("--fill-cache", action="store_true", help="Fill the memcached caches")
+    parser.add_argument(
+        "--skip-checks", action="store_true", help="Skip syntax and database checks"
+    )
+    which_services = parser.add_mutually_exclusive_group()
+    which_services.add_argument(
+        "--skip-client-reloads",
+        action="store_true",
+        help="Do not send reload events to web clients",
+    )
+    which_services.add_argument(
+        "--only-django",
+        action="store_true",
+        help=f"Only {action} Django (not Tornado or workers)",
+    )
+    which_services.add_argument(
+        "--tornado-reshard",
+        action="store_true",
+        help="Restart changed Tornado shards",
+    )
+    if action == "restart":
+        parser.add_argument(
+            "--less-graceful",
+            action="store_true",
+            help="Restart with more concern for expediency than minimizing availability interruption",
+        )
+    return parser
+
+
+def upgrade_script_arg_parser() -> argparse.ArgumentParser:
+    """Options shared by upgrade-zulip-stage-3 and its wrapper scripts.
+
+    Wrappers like upgrade-zulip-from-git use this as a parent parser so
+    that --help advertises the same options stage-3 accepts, and forward
+    the parsed values through to stage-3.
+    """
+    parser = argparse.ArgumentParser(
+        add_help=False, parents=[start_script_arg_parser(action="restart", add_help=False)]
+    )
+    parser.add_argument(
+        "--skip-restart",
+        action="store_true",
+        help="Configure, but do not restart into, the new version; aborts if any system-wide changes would happen.",
+    )
+    parser.add_argument(
+        "--skip-puppet", action="store_true", help="Skip doing puppet/apt upgrades."
+    )
+    parser.add_argument("--skip-migrations", action="store_true", help="Skip doing migrations.")
+    parser.add_argument(
+        "--skip-downgrade-check",
+        action="store_true",
+        help="Skip the safety check to prevent database downgrades.",
+    )
+    parser.add_argument(
+        "--ignore-static-assets",
+        action="store_true",
+        help="Do not attempt to copy/manage static assets.",
+    )
+    parser.add_argument(
+        "--skip-purge-old-deployments",
+        action="store_true",
+        help="Skip purging old deployments.",
+    )
+    parser.add_argument(
+        "--audit-fts-indexes",
+        action="store_true",
+        help="Audit and fix full text search indexes.",
+    )
+    return parser
+
+
+def atomic_nagios_write(
+    name: str,
+    status: Literal["ok", "warning", "critical", "unknown"],
+    message: str | None = None,
+    event_time: int | None = None,
+) -> int:
+    if message is None:
+        message = status
+    if event_time is None:
+        event_time = int(time.time())
+    if status == "ok":
+        status_int = 0
+    elif status == "warning":
+        status_int = 1
+    elif status == "critical":
+        status_int = 2
+    elif status == "unknown":
+        status_int = 3
+
+    path = "/var/lib/nagios_state/" + name
+    with open(path + ".tmp", "w") as fh:
+        fh.write("|".join([str(event_time), str(status_int), status, message]) + "\n")
+    os.rename(path + ".tmp", path)
+
+    # Return code should be if the cron job ran to completion
+    # successfully, not if the result of the check was outside of
+    # bounds ("ok" / "critical"); this prevents the Sentry cron
+    # wrapper from spamming with a "failure" email if the nagios check
+    # requires multiple failures in a row.
+    if status == "unknown":
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    cmd = sys.argv[1]
+    if cmd == "make_deploy_path":
+        print(make_deploy_path())
+    elif cmd == "get_dev_uuid":
+        print(get_dev_uuid_var_path())

scripts/log-search

@@ -0,0 +1,595 @@
+#!/usr/bin/env python3
+
+import argparse
+import calendar
+import gzip
+import logging
+import os
+import re
+import signal
+import statistics
+import sys
+from datetime import date, datetime, timedelta, timezone
+from enum import Enum, auto
+from re import Match
+from typing import TextIO
+
+ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(ZULIP_PATH)
+
+from scripts.lib.setup_path import setup_path
+
+setup_path()
+
+os.environ["DJANGO_SETTINGS_MODULE"] = "zproject.settings"
+
+from typing import Protocol
+
+from django.conf import settings
+
+from scripts.lib.zulip_tools import (
+    BOLD,
+    CYAN,
+    ENDC,
+    FAIL,
+    GRAY,
+    OKBLUE,
+    get_config,
+    get_config_file,
+)
+
+
+def parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="Search logfiles, ignoring commonly-fetched URLs.")
+    log_selection = parser.add_argument_group("File selection")
+    log_selection_options = log_selection.add_mutually_exclusive_group()
+    access_log_retention_days = int(
+        get_config(get_config_file(), "application_server", "access_log_retention_days", "14")
+    )
+    log_selection_options.add_argument(
+        "--log-files",
+        "-n",
+        help="Number of log files to search",
+        choices=range(1, access_log_retention_days + 2),
+        metavar=f"[1-{access_log_retention_days + 1}]",
+        type=int,
+    )
+    log_selection_options.add_argument(
+        "--all-logs",
+        "-A",
+        help="Parse all logfiles, not just most recent",
+        action="store_true",
+    )
+    log_selection_options.add_argument(
+        "--min-hours",
+        "-H",
+        help="Estimated minimum number of hours; includes previous log file, if estimated less than this",
+        type=int,
+        choices=range(24),
+        default=3,
+    )
+    log_selection.add_argument(
+        "--nginx",
+        "-N",
+        help="Parse from NGINX logs, not server.log",
+        action="store_true",
+    )
+
+    filtering = parser.add_argument_group("Filtering")
+    filtering.add_argument(
+        "filter_terms",
+        help="IP address, hostname, user-id, HTTP method, path, datetime prefix, or status code to search for; multiple are AND'ed together",
+        nargs="+",
+    )
+    filtering.add_argument(
+        "--all-lines",
+        "-L",
+        help="Show all matching lines; equivalent to -suaemtpr",
+        action="store_true",
+    )
+    filtering.add_argument("--static", "-s", help="Include static file paths", action="store_true")
+    filtering.add_argument("--uploads", "-u", help="Include file upload paths", action="store_true")
+    filtering.add_argument("--avatars", "-a", help="Include avatar paths", action="store_true")
+    filtering.add_argument("--events", "-e", help="Include event fetch paths", action="store_true")
+    filtering.add_argument("--messages", "-m", help="Include message paths", action="store_true")
+    filtering.add_argument(
+        "--typing",
+        "-t",
+        help="Include typing notification path",
+        action="store_true",
+    )
+    filtering.add_argument("--presence", "-p", help="Include presence paths", action="store_true")
+    filtering.add_argument(
+        "--report", "-r", help="Include Sentry reporting paths", action="store_true"
+    )
+    filtering.add_argument(
+        "--no-other", "-O", help="Exclude paths not explicitly included", action="store_true"
+    )
+    filtering.add_argument(
+        "--client",
+        "--user-agent",
+        "-C",
+        help="Only include requests whose client/user-agent contains this string",
+    )
+    filtering.add_argument(
+        "--extra",
+        "-X",
+        help="Only include requests whose logline includes this in [extra] flags",
+    )
+
+    output = parser.add_argument_group("Output")
+    output.add_argument("--full-line", "-F", help="Show full matching line", action="store_true")
+    output.add_argument("--timeline", "-T", help="Show start, end, and gaps", action="store_true")
+    output.add_argument("--stats", "-S", help="Compute and show statistics", action="store_true")
+    return parser
+
+
+def maybe_gzip(logfile_name: str) -> TextIO:
+    if logfile_name.endswith(".gz"):
+        return gzip.open(logfile_name, "rt")
+    return open(logfile_name)
+
+
+NGINX_LOG_LINE_RE = re.compile(
+    r"""
+      (?P<ip> \S+ ) \s+
+      - \s+
+      (?P<user> \S+ ) \s+
+      \[
+         (?P<date> \d+/\w+/\d+ )
+         :
+         (?P<time> \d+:\d+:\d+ )
+         \s+ [+-]\d+
+      \] \s+
+      "
+         (?P<method> \S+ )
+         \s+
+         (?P<full_path> (?P<path> [^"?]+ ) (\?[^"]*)? )
+         \s+
+         (?P<http_version> HTTP/[^"]+ )
+      " \s+
+      (?P<code> \d+ ) \s+
+      (?P<bytes> \d+ ) \s+
+      "(?P<referer> [^"]* )" \s+
+      "(?P<user_agent> [^"]* )" \s+
+      (?P<hostname> \S+ ) \s+
+      (?P<duration> \S+ )
+    """,
+    re.VERBOSE,
+)
+
+PYTHON_LOG_LINE_RE = re.compile(
+    r"""
+      (?P<date> \d+-\d+-\d+ ) \s+
+      (?P<time> \d+:\d+:\d+\.\d+ ) \s+
+      INFO \s+  # All access log lines are INFO
+      (pid:\d+ \s+) ?
+      \[ (?P<source> zr(:\d+)?) \] \s+
+      (?P<ip>
+        \d{1,3}(\.\d{1,3}){3}
+      | ([a-f0-9:]+:+){1,7}[a-f0-9]*
+      ) \s+
+      (?P<method> [A-Z]+ ) \s+
+      (?P<code> \d+ ) \s+
+      (?P<duration> \S+ ) \s+ # This can be "217ms" or "1.7s"
+      ( \( (?P<perf> [^)]+ ) \) \s+ )*
+      (?P<full_path> (?P<path> /\S* ) ) \s+
+      (?P<extra> .* )   # Multiple extra things can go here
+      \(
+        (?P<user>
+           ( (?P<user_id> \d+ ) | unauth )
+           @
+           (?P<hostname> \S+ )
+         | zulip-server:\S+
+         | scim-client:\S+
+         | internal
+        ) \s+ via \s+ (?P<user_agent> .* )
+      \)
+    """,
+    re.VERBOSE,
+)
+
+
+class FilterType(Enum):
+    HOSTNAME = auto()
+    CLIENT_IP = auto()
+    USER_ID = auto()
+    AUTHED = auto()
+    METHOD = auto()
+    PATH = auto()
+    STATUS = auto()
+    DATETIME = auto()
+
+
+class FilterFunc(Protocol):
+    def __call__(self, m: Match[str], t: str = ...) -> bool: ...
+
+
+def main() -> None:
+    args = parser().parse_args()
+
+    (filter_types, filter_funcs, substr_terms) = parse_filters(args)
+    logfile_names = parse_logfile_names(args)
+    if args.timeline and args.nginx:
+        print("! nginx logs not suggested for timeline, due to imprecision", file=sys.stderr)
+
+    use_color = sys.stdout.isatty()
+    lowered_terms = [term.lower() for term in substr_terms]
+    if args.stats:
+        durations: list[int] | None = []
+    else:
+        durations = None
+    try:
+        for logfile_name in reversed(logfile_names):
+            with maybe_gzip(logfile_name) as logfile:
+                for logline in logfile:
+                    # As a performance optimization, just do a substring
+                    # check before we parse the line fully
+                    lowered = logline.lower()
+                    if not all(f in lowered for f in lowered_terms):
+                        continue
+
+                    if args.nginx:
+                        match = NGINX_LOG_LINE_RE.match(logline)
+                    else:
+                        match = PYTHON_LOG_LINE_RE.match(logline)
+                    if match is None:
+                        # We expect other types of loglines in the Python logfiles
+                        if args.nginx:
+                            print(f"! Failed to parse:\n{logline}", file=sys.stderr)
+                        continue
+                    if passes_filters(filter_funcs, match, args):
+                        print_line(
+                            match,
+                            args,
+                            filter_types=filter_types,
+                            use_color=use_color,
+                            durations=durations,
+                        )
+    except BrokenPipeError:
+        # Python flushes standard streams on exit; redirect remaining output
+        # to devnull to avoid another BrokenPipeError at shutdown
+        devnull = os.open(os.devnull, os.O_WRONLY)
+        os.dup2(devnull, sys.stdout.fileno())
+        sys.exit(1)
+    except KeyboardInterrupt:
+        sys.exit(signal.SIGINT + 128)
+
+    if durations is not None:
+        # Prepend [0] to make the percentiles 1-indexed, instead of 0-indexed
+        percentiles = [0, *statistics.quantiles(durations, n=100)]
+        print()
+        print(f"Total requests: {len(durations)}")
+        print(f"Min duration:   {min(durations):>5}ms")
+        print(f"p50 duration:   {int(percentiles[50]):>5}ms")
+        print(f"p75 duration:   {int(percentiles[75]):>5}ms")
+        print(f"p90 duration:   {int(percentiles[90]):>5}ms")
+        print(f"p95 duration:   {int(percentiles[95]):>5}ms")
+        print(f"p99 duration:   {int(percentiles[99]):>5}ms")
+        print(f"Max duration:   {max(durations):>5}ms")
+
+
+def parse_logfile_names(args: argparse.Namespace) -> list[str]:
+    if args.nginx:
+        base_path = "/var/log/nginx/access.log"
+    else:
+        base_path = "/var/log/zulip/server.log"
+
+    for term in args.filter_terms:
+        date_term = re.match(r"2\d\d\d-\d\d-\d\d", term)
+        if not date_term:
+            continue
+        # They're limiting to a specific day; find the right logfile
+        # which is going to have any matches
+        rotations = int(
+            (datetime.now(tz=timezone.utc).date() - date.fromisoformat(date_term[0]))
+            / timedelta(days=1)
+        )
+        access_log_retention_days = int(
+            get_config(get_config_file(), "application_server", "access_log_retention_days", "14")
+        )
+        if rotations > access_log_retention_days:
+            raise RuntimeError(f"Date is too old (more than {access_log_retention_days} days ago)")
+        if rotations == 0:
+            return [base_path]
+        if rotations == 1:
+            return [f"{base_path}.1"]
+        else:
+            return [f"{base_path}.{rotations}.gz"]
+
+    logfile_names = [base_path]
+    if args.all_logs:
+        logfile_count = 15
+    elif args.log_files is not None:
+        logfile_count = args.log_files
+    else:
+        # Detect if there was a logfile rotation in the last
+        # (min-hours)-ish hours, and if so include the previous
+        # logfile as well.
+        logfile_count = 1
+        try:
+            current_size = os.path.getsize(base_path)
+            past_size = os.path.getsize(base_path + ".1")
+            if current_size < (args.min_hours / 24.0) * past_size:
+                logfile_count = 2
+        except FileNotFoundError:
+            pass
+    for n in range(1, logfile_count):
+        logname = f"{base_path}.{n}"
+        if n > 1:
+            logname += ".gz"
+        logfile_names.append(logname)
+    return logfile_names
+
+
+month_no_to_name_lookup = {f"{k:02d}": v for k, v in enumerate(calendar.month_abbr)}
+month_name_to_no_lookup = {v: k for k, v in month_no_to_name_lookup.items()}
+
+
+def convert_from_nginx_date(date: str) -> str:
+    day_of_month, month_abbr, year = date.split("/")
+    return f"{year}-{month_name_to_no_lookup[month_abbr]}-{day_of_month}"
+
+
+def convert_to_nginx_date(date: str) -> str:
+    year, month_no, day_of_month = date.split("-")
+    return f"{day_of_month}/{month_no_to_name_lookup[month_no]}/{year}"
+
+
+def parse_filters(
+    args: argparse.Namespace,
+) -> tuple[set[FilterType], list[FilterFunc], list[str]]:
+    # The heuristics below are not intended to be precise -- they
+    # certainly count things as "IPv4" or "IPv6" addresses that are
+    # invalid.  However, we expect the input here to already be
+    # reasonably well-formed.
+
+    filter_types = set()
+    filter_funcs = []
+    filter_terms = []
+
+    if args.events and not args.nginx:
+        logging.warning("Adding --nginx -- /events requests do not appear in Django logs.")
+        args.nginx = True
+
+    for filter_term in args.filter_terms:
+        if re.match(r"[1-5][0-9][0-9]$", filter_term):
+            filter_func = lambda m, t=filter_term: m["code"] == t
+            filter_type = FilterType.STATUS
+            if not args.nginx and filter_term == "502":
+                logging.warning("Adding --nginx -- 502's do not appear in Django logs.")
+                args.nginx = True
+        elif re.match(r"[1-5]xx$", filter_term):
+            filter_term = filter_term[0]
+            filter_func = lambda m, t=filter_term: m["code"].startswith(t)
+            filter_type = FilterType.STATUS
+        elif re.match(r"\d+$", filter_term):
+            if args.nginx:
+                raise parser().error("Cannot parse user-ids with nginx logs; try without --nginx")
+            filter_func = lambda m, t=filter_term: m["user_id"] == t
+            filter_type = FilterType.USER_ID
+        elif re.match(r"\d{1,3}(\.\d{1,3}){3}$", filter_term):
+            filter_func = lambda m, t=filter_term: m["ip"] == t
+            filter_type = FilterType.CLIENT_IP
+        elif re.match(r"([a-f0-9:]+:+){1,7}[a-f0-9]+$", filter_term):
+            filter_func = lambda m, t=filter_term: m["ip"] == t
+            filter_type = FilterType.CLIENT_IP
+        elif re.match(r"DELETE|GET|HEAD|OPTIONS|PATCH|POST|PUT", filter_term):
+            filter_func = lambda m, t=filter_term: m["method"].upper() == t
+            filter_type = FilterType.METHOD
+        elif re.match(r"(2\d\d\d-\d\d-\d\d)( \d(\d(:(\d(\d(:(\d\d?)?)?)?)?)?)?)?", filter_term):
+            if args.nginx:
+                datetime_parts = filter_term.split(" ")
+                filter_term = ":".join(
+                    [convert_to_nginx_date(datetime_parts[0]), *datetime_parts[1:]]
+                )
+                filter_func = lambda m, t=filter_term: f"{m['date']}:{m['time']}".startswith(t)
+            else:
+                filter_func = lambda m, t=filter_term: f"{m['date']} {m['time']}".startswith(t)
+            filter_type = FilterType.DATETIME
+        elif filter_term in ["authed", "unauthed"]:
+            if args.nginx:
+                raise parser().error("Cannot parse user-ids with nginx logs; try without --nginx")
+            filter_type = FilterType.USER_ID
+            if filter_term == "authed":
+                filter_func = lambda m, t=None: m["user_id"] is not None and m["user_id"] != ""
+                filter_term = None
+                filter_types.add(FilterType.AUTHED)
+            else:
+                filter_func = lambda m, t=None: m["user_id"] is None
+                filter_term = "(unauth@"
+        elif re.match(r"[a-z0-9]([a-z0-9-]*[a-z0-9])?$", filter_term.lower()):
+            filter_term = filter_term.lower()
+            if args.nginx:
+                filter_func = lambda m, t=filter_term: m["hostname"].startswith(t + ".")
+            else:
+                filter_func = lambda m, t=filter_term: m["hostname"] == t
+            filter_type = FilterType.HOSTNAME
+        elif re.match(r"[a-z0-9-]+(\.[a-z0-9-]+)+$", filter_term.lower()) and re.search(
+            r"[a-z-]", filter_term.lower()
+        ):
+            if not args.nginx:
+                raise parser().error("Cannot parse full domains with Python logs; try --nginx")
+            filter_term = filter_term.lower()
+            filter_func = lambda m, t=filter_term: m["hostname"] == t
+            filter_type = FilterType.HOSTNAME
+        elif re.match(r"/\S*$", filter_term):
+            filter_func = lambda m, t=filter_term: m["path"] == t
+            filter_type = FilterType.PATH
+            args.all_lines = True
+        else:
+            raise RuntimeError(
+                f"Can't parse {filter_term} as an IP, hostname, user-id, HTTP method, path, or status code."
+            )
+        if filter_type in filter_types:
+            parser().error("Supplied the same time of value more than once, which cannot match!")
+        filter_types.add(filter_type)
+        filter_funcs.append(filter_func)
+        if filter_term is not None:
+            filter_terms.append(filter_term)
+
+    if args.client:
+        filter_funcs.append(lambda m, t=args.client: t in m["user_agent"])
+        filter_terms.append(args.client)
+
+    if args.extra:
+        if args.nginx:
+            raise parser().error("nginx logs do not contain [extra] data; try without --nginx")
+        filter_funcs.append(lambda m, t=args.extra: t in m["extra"])
+        filter_terms.append(args.extra)
+
+    return (filter_types, filter_funcs, filter_terms)
+
+
+def passes_filters(
+    string_filters: list[FilterFunc],
+    match: Match[str],
+    args: argparse.Namespace,
+) -> bool:
+    if not all(f(match) for f in string_filters):
+        return False
+
+    if args.all_lines:
+        return True
+
+    path = match["path"]
+    if path.startswith("/static/"):
+        return args.static
+    elif path.startswith("/user_uploads/"):
+        return args.uploads
+    elif path.startswith(("/user_avatars/", "/avatar/")):
+        return args.avatars
+    elif re.match(r"/(json|api/v1)/events($|\?|/)", path):
+        return args.events
+    elif path in ("/api/v1/typing", "/json/typing"):
+        return args.typing
+    elif re.match(r"/(json|api/v1)/messages($|\?|/)", path):
+        return args.messages
+    elif path in ("/api/v1/users/me/presence", "/json/users/me/presence"):
+        return args.presence
+    elif path == "/error_tracing":
+        return args.report
+    else:
+        return not args.no_other
+
+
+last_match_end: datetime | None = None
+
+
+def print_line(
+    match: Match[str],
+    args: argparse.Namespace,
+    filter_types: set[FilterType],
+    use_color: bool,
+    durations: list[int] | None = None,
+) -> None:
+    global last_match_end
+
+    if match["duration"].endswith("ms"):
+        duration_ms = int(match["duration"].removesuffix("ms"))
+    else:
+        duration_ms = int(float(match["duration"].removesuffix("s")) * 1000)
+    if durations is not None:
+        durations.append(duration_ms)
+
+    if args.full_line:
+        print(match.group(0))
+        return
+
+    if args.nginx:
+        date = convert_from_nginx_date(match["date"])
+    else:
+        date = match["date"]
+    if args.all_logs or (args.log_files is not None and args.log_files > 1):
+        ts = date + " " + match["time"]
+    else:
+        ts = match["time"]
+
+    if match["duration"].endswith("ms"):
+        duration_ms = int(match["duration"].removesuffix("ms"))
+    else:
+        duration_ms = int(float(match["duration"].removesuffix("s")) * 1000)
+
+    code = int(match["code"])
+    indicator = " "
+    color = ""
+    if code == 401:
+        indicator = ":"
+        color = CYAN
+    elif code == 499:
+        indicator = "-"
+        color = GRAY
+    elif code >= 400 and code < 499:
+        indicator = ">"
+        color = OKBLUE
+    elif code >= 500 and code <= 599:
+        indicator = "!"
+        color = FAIL
+
+    if use_color:
+        url = f"{BOLD}{match['full_path']}"
+    else:
+        url = match["full_path"]
+        color = ""
+
+    if FilterType.HOSTNAME not in filter_types:
+        hostname = match["hostname"]
+        if hostname is None:
+            hostname = "???." + settings.EXTERNAL_HOST
+        elif not args.nginx:
+            if hostname != "root":
+                hostname += "." + settings.EXTERNAL_HOST
+            elif settings.EXTERNAL_HOST == "zulipchat.com":
+                hostname = "zulip.com"
+            else:
+                hostname = settings.EXTERNAL_HOST
+        url = "https://" + hostname + url
+
+    user_id = ""
+    if not args.nginx and match["user_id"] is not None:
+        user_id = match["user_id"] + "@"
+
+    if args.timeline:
+        logline_end = datetime.fromisoformat(date + " " + match["time"])
+        logline_start = logline_end - timedelta(milliseconds=duration_ms)
+        if last_match_end is not None:
+            gap_ms = int((logline_start - last_match_end) / timedelta(milliseconds=1))
+            if gap_ms > 5000:
+                print()
+                print(f"========== {int(gap_ms / 1000):>4} second gap ==========")
+                print()
+            elif gap_ms > 1000:
+                print(f"============ {gap_ms:>5}ms gap ============")
+            elif gap_ms > 0:
+                print(f"------------ {gap_ms:>5}ms gap ------------")
+            else:
+                print(f"!!!!!!!!!! {abs(gap_ms):>5}ms overlap !!!!!!!!!!")
+        if args.all_logs or (args.log_files is not None and args.log_files > 1):
+            print(logline_start.isoformat(" ", timespec="milliseconds") + " (start)")
+        else:
+            print(logline_start.time().isoformat(timespec="milliseconds") + " (start)")
+        last_match_end = logline_end
+
+    if args.nginx:
+        show_user = False
+    elif FilterType.AUTHED in filter_types:
+        show_user = True
+    elif FilterType.USER_ID in filter_types:
+        show_user = False
+    else:
+        show_user = True
+    parts = [
+        ts,
+        f"{duration_ms:>5}ms",
+        f"{user_id:7}" if show_user else None,
+        f"{match['ip']:39}" if FilterType.CLIENT_IP not in filter_types else None,
+        indicator + match["code"],
+        f"{match['method']:6}",
+        url,
+    ]
+
+    print(color + " ".join(p for p in parts if p is not None) + (ENDC if use_color else ""))
+
+
+if __name__ == "__main__":
+    main()

scripts/nagios/check-rabbitmq-consumers

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+import argparse
+import os
+import re
+import subprocess
+import sys
+import time
+from collections import defaultdict
+
+ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.append(ZULIP_PATH)
+from scripts.lib.check_rabbitmq_queue import normal_queues
+from scripts.lib.zulip_tools import (
+    atomic_nagios_write,
+    get_config,
+    get_config_file,
+    get_tornado_ports,
+)
+
+if "USER" in os.environ and os.environ["USER"] not in ["root", "rabbitmq"]:
+    print("This script must be run as the root or rabbitmq user")
+
+
+parser = argparse.ArgumentParser()
+parser.parse_args()
+
+config_file = get_config_file()
+TORNADO_PROCESSES = len(get_tornado_ports(config_file))
+
+output = subprocess.check_output(["/usr/sbin/rabbitmqctl", "list_consumers"], text=True)
+
+consumers: dict[str, int] = defaultdict(int)
+
+queues = {
+    *normal_queues,
+    # All tornado queues get grouped into this, even if sharded
+    "notify_tornado",
+}
+
+for queue_name in queues:
+    queue_name = queue_name.strip()
+    consumers[queue_name] = 0
+
+for line in output.split("\n"):
+    parts = line.split("\t")
+    if len(parts) >= 2:
+        queue_name = parts[0]
+        if queue_name.startswith("notify_tornado_"):
+            queue_name = "notify_tornado"
+
+        # Collapse sharded queues into a single queue for this check.
+        queue_name = re.sub(r"_shard(\d+)", "", queue_name)
+        consumers[queue_name] += 1
+
+now = int(time.time())
+
+for queue_name, count in consumers.items():
+    target_count = 1
+    if queue_name == "notify_tornado":
+        target_count = TORNADO_PROCESSES
+    elif queue_name == "missedmessage_mobile_notifications":
+        target_count = int(
+            get_config(config_file, "application_server", "mobile_notification_shards", "1")
+        )
+    elif queue_name == "user_activity":
+        target_count = int(
+            get_config(config_file, "application_server", "user_activity_shards", "1")
+        )
+    else:
+        target_count = int(
+            get_config(config_file, "application_server", f"{queue_name}_workers", "1")
+        )
+
+    atomic_nagios_write(
+        "check-rabbitmq-consumers-" + queue_name,
+        "critical" if count < target_count else "ok",
+        f"queue {queue_name} has {count} consumers, needs {target_count}",
+    )

scripts/nagios/check-rabbitmq-queue

@@ -0,0 +1,12 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+
+ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.append(ZULIP_PATH)
+
+from scripts.lib.check_rabbitmq_queue import check_rabbitmq_queues
+
+if __name__ == "__main__":
+    check_rabbitmq_queues()

scripts/purge-old-deployments

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+import argparse
+import os
+import subprocess
+import sys
+
+ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(ZULIP_PATH)
+from scripts.lib import clean_unused_caches
+from scripts.lib.zulip_tools import (
+    DEPLOYMENTS_DIR,
+    get_recent_deployments,
+    maybe_perform_purging,
+    su_to_zulip,
+)
+
+LOCAL_GIT_CACHE_DIR = "/srv/zulip.git"
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="This script can be used for cleaning old unused deployments.",
+        epilog="Orphaned/unused caches older than threshold days will be automatically "
+        "examined and removed.",
+    )
+    parser.add_argument(
+        "--threshold",
+        dest="threshold_days",
+        type=int,
+        default=14,
+        metavar="<days>",
+        help="Deployments older than threshold days will be deleted. (defaults to 14)",
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="If specified then script will only print the deployments and "
+        "caches that it will delete/keep back. It will not delete anything.",
+    )
+    parser.add_argument(
+        "--verbose",
+        action="store_true",
+        help="If specified then script will print a detailed report of what is going on.",
+    )
+    parser.add_argument(
+        "--no-print-headings",
+        dest="no_headings",
+        action="store_true",
+        help="If specified then script will not print headings for what will be deleted/kept back.",
+    )
+
+    args = parser.parse_args()
+    args.verbose |= args.dry_run  # Always print a detailed report in case of dry run.
+    return args
+
+
+def get_deployments_to_be_purged(recent_deployments: set[str]) -> set[str]:
+    all_deployments = {
+        os.path.join(DEPLOYMENTS_DIR, deployment) for deployment in os.listdir(DEPLOYMENTS_DIR)
+    }
+    deployments_to_purge = set()
+    for deployment in all_deployments:
+        # Deployments whose name is not in the format of a timestamp are
+        # always included in the recent_deployments and are not deleted.
+        if not os.path.isdir(deployment):
+            # Skip things like uwsgi sockets.
+            continue
+        if not os.path.exists(os.path.join(deployment, "zerver")):
+            # Skip things like "lock" that aren't actually a deployment directory
+            continue
+        if deployment not in recent_deployments:
+            deployments_to_purge.add(deployment)
+    return deployments_to_purge
+
+
+def main() -> None:
+    args = parse_args()
+    deployments_to_keep = get_recent_deployments(args.threshold_days)
+    deployments_to_purge = get_deployments_to_be_purged(deployments_to_keep)
+
+    maybe_perform_purging(
+        deployments_to_purge, deployments_to_keep, "deployment", args.dry_run, args.verbose, True
+    )
+
+    if not args.dry_run:
+        if os.path.exists(LOCAL_GIT_CACHE_DIR):
+            subprocess.check_call(
+                ["git", "worktree", "prune"], cwd=LOCAL_GIT_CACHE_DIR, preexec_fn=su_to_zulip
+            )
+
+        if not args.no_headings:
+            print("Deployments cleaned successfully...")
+            print("Cleaning orphaned/unused caches...")
+
+    # Call 'clean_unused_caches.py' script to clean any orphaned/unused caches.
+    clean_unused_caches.main(args)
+    print("Done!")
+
+
+if __name__ == "__main__":
+    main()

scripts/refresh-sharding-and-restart

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Stand up the new zulip-tornado supervisor instances, and write out
+# the newly generated config files, with .tmp suffix
+SUPPRESS_SHARDING_NOTICE=1 "$(dirname "$0")/zulip-puppet-apply" -f
+
+# Verify, before we move them into place
+if ! [ -e /etc/zulip/nginx_sharding_map.conf.tmp ] || ! [ -e /etc/zulip/sharding.json.tmp ]; then
+    echo "No sharding updates found to apply."
+    exit 1
+fi
+
+# In the ordering of operations below, the crucial detail is that
+# Django, Tornado, and workers need to be restarted before reloading
+# nginx. Django and Tornado have in-memory maps of which realm belongs
+# to which shard. Reloading nginx will cause users' tornado requests
+# to be routed according to the new sharding scheme. If that happens
+# before Django is restarted, updating its realm->shard map, users on
+# realms whose shard has changed will have their tornado requests
+# handled by the new tornado process, while Django will still use the
+# old process for its internal communication with tornado when
+# servicing the user's requests.  That's a bad state that leads to
+# clients getting into reload loops ending in crashing on 500 response
+# while Django is restarting.  For this reason it's important to
+# reload nginx only after Django and Tornado.
+"$(dirname "$0")/restart-server" --tornado-reshard
+service nginx reload

scripts/reload-clients

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+import argparse
+import configparser
+import logging
+import os
+import sys
+import time
+
+sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
+from scripts.lib.setup_path import setup_path
+
+setup_path()
+
+import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util import Retry
+
+from scripts.lib.zulip_tools import get_config, get_config_file, get_tornado_ports
+
+config_file = get_config_file()
+reload_rate = int(
+    get_config(
+        config_file,
+        "application_server",
+        "client_reload_rate",
+        "50",
+    )
+)
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--rate", type=int, help="Number of clients to reload per second", default=reload_rate
+)
+parser.add_argument("--background", action="store_true", help="Daemonize the process")
+
+args = parser.parse_args()
+reload_rate = args.rate
+
+secret_config_file = configparser.RawConfigParser()
+secret_config_file.read("/etc/zulip/zulip-secrets.conf")
+shared_secret = get_config(secret_config_file, "secrets", "shared_secret")
+assert shared_secret
+
+# Perform relatively slow retries (2s, 4s, 8s) with backoff, including
+# on POST requests.  Failure to send this request successfully means
+# that clients may fail to reload, so we want to be somewhat resilient
+# to failures.  Since we are on localhost, we do not expect network
+# failures, only Tornado restarts, to cause failures here.
+retry = Retry(total=3, backoff_factor=1, allowed_methods=Retry.DEFAULT_ALLOWED_METHODS | {"POST"})
+c = requests.Session()
+c.mount("http://", HTTPAdapter(max_retries=retry))
+
+log_filename = None
+if args.background:
+    # Double-fork to daemonize
+    pid = os.fork()
+    if pid > 0:
+        sys.exit(0)
+    os.setsid()
+    pid = os.fork()
+    if pid > 0:
+        sys.exit(0)
+    log_filename = "/var/log/zulip/reload-clients.log"
+
+logging.Formatter.converter = time.gmtime
+logging.basicConfig(
+    format="%(asctime)s reload-clients: %(message)s",
+    level=logging.INFO,
+    filename=log_filename,
+)
+
+first = True
+server_total = 0
+for port in get_tornado_ports(config_file):
+    logging.info("Starting to send client reload events to Tornado port %d", port)
+    try:
+        shard_total = 0
+        complete = False
+        # Rather than make a sustained one request per second, we batch
+        # into 5-second chunks of 5 times the client_reload_rate
+        SECONDS_PER_BATCH = 5
+        while not complete:
+            if not first:
+                time.sleep(SECONDS_PER_BATCH)
+            first = False
+
+            logging.info("Sending reload events to %d clients", reload_rate * SECONDS_PER_BATCH)
+            resp = c.post(
+                f"http://127.0.0.1:{port}/api/internal/web_reload_clients",
+                data={"client_count": reload_rate * SECONDS_PER_BATCH, "secret": shared_secret},
+                timeout=5,
+                proxies={"http": ""},  # Make sure we don't go through Smokescreen
+            )
+            resp.raise_for_status()
+            shard_total += resp.json()["sent_events"]
+            complete = resp.json()["complete"]
+        logging.info("Sent %d reload events to Tornado port %d", shard_total, port)
+        server_total += shard_total
+    except requests.exceptions.HTTPError:
+        # Failures in one shard likely won't affect other shards --
+        # give up on this shard, and try the next one,
+        logging.exception("Failed to send web_reload_clients request to Tornado port %d", port)
+
+if len(get_tornado_ports(config_file)) > 1:
+    logging.info("Sent total of %d reload events, across all Tornado instances", server_total)

scripts/restart-server

@@ -0,0 +1,337 @@
+#!/usr/bin/env python3
+import contextlib
+import json
+import logging
+import os
+import pwd
+import shlex
+import subprocess
+import sys
+import time
+
+sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
+from scripts.lib.setup_path import setup_path
+
+setup_path()
+
+from scripts.lib.supervisor import list_supervisor_processes
+from scripts.lib.zulip_tools import (
+    DEPLOYMENTS_DIR,
+    ENDC,
+    LOCK_DIR,
+    OKGREEN,
+    WARNING,
+    get_config,
+    get_config_file,
+    get_tornado_ports,
+    has_application_server,
+    has_process_fts_updates,
+    overwrite_symlink,
+    start_script_arg_parser,
+    su_to_zulip,
+)
+
+action = "restart"
+if not sys.argv[0].endswith("restart-server"):
+    action = "start"
+verbing = action.title() + "ing"
+
+logging.Formatter.converter = time.gmtime
+logging.basicConfig(format=f"%(asctime)s {action}-server: %(message)s", level=logging.INFO)
+
+parser = start_script_arg_parser(action=action, add_help=True)
+args = parser.parse_args()
+
+deploy_path = os.path.realpath(os.path.join(os.path.dirname(__file__), ".."))
+os.chdir(deploy_path)
+
+username = pwd.getpwuid(os.getuid()).pw_name
+if username == "root":
+    su_to_zulip()
+elif username != "zulip":
+    logging.error("Must be run as user 'zulip'.")
+    sys.exit(1)
+
+if os.environ.get("RUNNING_UNDER_CRON") and os.path.exists(LOCK_DIR):
+    logging.info("Skipping cron-triggered restart during deploy.")
+    sys.exit(1)
+
+if args.tornado_reshard:
+    tornado_reshard_files = ["/etc/zulip/sharding.json", "/etc/zulip/nginx_sharding_map.conf"]
+    if not all(os.path.exists(f"{path}.tmp") for path in tornado_reshard_files):
+        logging.info(
+            "No resharding changes to apply!  Edit zulip.conf and run refresh-sharding-and-restart"
+        )
+        sys.exit(1)
+
+if not args.skip_checks:
+    logging.info("Running syntax and database checks")
+    subprocess.check_call(["./manage.py", "check", "--database", "default"])
+
+if args.fill_cache:
+    logging.info("Filling memcached caches")
+    subprocess.check_call(["./manage.py", "fill_memcached_caches", "--automated", "--skip-checks"])
+
+current_symlink = os.path.join(DEPLOYMENTS_DIR, "current")
+last_symlink = os.path.join(DEPLOYMENTS_DIR, "last")
+change_symlink = os.readlink(current_symlink) != deploy_path
+if change_symlink:
+    overwrite_symlink(os.readlink(current_symlink), last_symlink)
+    overwrite_symlink(deploy_path, current_symlink)
+
+config_file = get_config_file()
+tornado_ports = get_tornado_ports(config_file)
+workers = []
+
+if has_application_server():
+    # Start by restarting the workers and similar processes, one at a
+    # time.  Workers can always support processing events with old event
+    # contents, but cannot necessarily understand events enqueued by a
+    # newer Django process.  Restarting them one at a time, rather than
+    # all-at-once, minimizes the downtime of each, and reduces startup
+    # contention.
+    #
+    # For "start" or less-graceful circumstances, we don't need to
+    # iterate; we'll stop all of them at once, and start them all later.
+    # In those cases, using the glob form is faster -- but if we do need
+    # to iterate, we need to expand the glob.
+    if action == "start" or args.less_graceful:
+        workers.append("zulip-workers:*")
+    else:
+        workers.extend(list_supervisor_processes(["zulip-workers:*"]))
+
+    if has_application_server(once=True):
+        # These used to be included in "zulip-workers:*"; since we may
+        # be restarting an older version of Zulip, which has not
+        # applied puppet to reload the new list of processes, only
+        # stop them if they currently exist according to
+        # `supervisorctl`.
+        workers.extend(
+            list_supervisor_processes(
+                [
+                    "zulip_deliver_scheduled_emails",
+                    "zulip_deliver_scheduled_messages",
+                ]
+            )
+        )
+
+    # This is an optional service, so may or may not exist
+    workers.extend(list_supervisor_processes(["zulip-katex"]))
+
+    # This does have some Python code, which reads from settings.py,
+    # so we need to restart it on every deploy.  A short outage during
+    # the restart is fine, as clients will transparently retry.
+    workers.append("zulip-tus")
+
+    workers.extend(list_supervisor_processes(["zulip-email-server"]))
+
+if has_process_fts_updates():
+    workers.append("process-fts-updates")
+
+# Before we start (re)starting main services, make sure to start any
+# optional auxiliary services that we don't stop, but do expect to be
+# running, and aren't currently.  These get new versions by getting
+# updated supervisor files, and puppet restarts them -- so we never
+# restart them in here, only start them.
+aux_services = list_supervisor_processes(["go-camo", "smokescreen"], only_running=False)
+if aux_services:
+    subprocess.check_call(["supervisorctl", "start", *aux_services])
+
+if args.only_django:
+    workers = []
+    check_services = ["zulip-django"]
+else:
+    check_services = [*workers, "zulip-django", "zulip-tornado:*"]
+
+# If none of the workers nor the application servers are running, this
+# is actually a "start," not a restart, which means we will defer
+# workers to later.
+running_services = list_supervisor_processes(check_services, only_running=True)
+if action == "restart" and len(running_services) == 0:
+    action = "start"
+    verbing = "Starting"
+elif action == "start":
+    existing_services = list_supervisor_processes(check_services)
+    if existing_services == running_services:
+        # Check if the version that is currently running is the
+        # version that we would have started.  We do this via
+        # comparing our CWD to the CWD of the oldest uwsgi worker.
+        oldest_pid = subprocess.check_output(
+            ["pgrep", "--oldest", "--full", "zulip-django uWSGI worker"], text=True
+        ).strip()
+        running_cwd = os.readlink(f"/proc/{oldest_pid}/cwd")
+        if deploy_path != running_cwd:
+            logging.warning("Mismatch between 'current' symlink and currently-running code!")
+            logging.warning("Already-running deploy: %s", running_cwd)
+            logging.warning("We would have started:  %s", deploy_path)
+            logging.warning("To restart, call %s/scripts/restart-server", deploy_path)
+        else:
+            logging.info("Zulip is already started; nothing to do!")
+        sys.exit(0)
+
+
+def restart_or_start(service: str) -> None:
+    our_verb = action
+    existing_services = list_supervisor_processes([service])
+    running_services = list_supervisor_processes([service], only_running=True)
+    if our_verb == "restart" and len(running_services) == 0:
+        our_verb = "start"
+    elif our_verb == "start" and existing_services == running_services:
+        logging.info("%s already started!", service)
+        return
+    subprocess.check_call(["supervisorctl", our_verb, service])
+
+
+def ports_as_set(val: int | list[int]) -> frozenset[int]:
+    return frozenset(val) if isinstance(val, list) else frozenset([val])
+
+
+def update_tornado_sharding() -> list[int]:
+    with open("/etc/zulip/sharding.json") as old_shard_fh:
+        old_sharding = json.load(old_shard_fh)
+    with open("/etc/zulip/sharding.json.tmp") as new_shard_fh:
+        new_sharding = json.load(new_shard_fh)
+    affected_tornados: set[int] = set()
+    for realm in set().union(old_sharding["shard_map"], new_sharding["shard_map"]):
+        old_ports = ports_as_set(old_sharding["shard_map"].get(realm, []))
+        new_ports = ports_as_set(new_sharding["shard_map"].get(realm, []))
+        if old_ports != new_ports:
+            # A realm sharded across multiple ports gets requests at
+            # random from nginx, and each does an X-Accel-Redirect to
+            # the user's right Tornado instance. So all ports in the
+            # set need a restart when we add or remove ports.
+            affected_tornados |= old_ports | new_ports
+    old_regex_set = {
+        (regex, ports_as_set(ports)) for (regex, ports) in old_sharding["shard_regexes"]
+    }
+    new_regex_set = {
+        (regex, ports_as_set(ports)) for (regex, ports) in new_sharding["shard_regexes"]
+    }
+    for regex, ports in old_regex_set ^ new_regex_set:
+        affected_tornados |= ports
+
+    tornado_reshard_files = ["/etc/zulip/sharding.json", "/etc/zulip/nginx_sharding_map.conf"]
+    for path in tornado_reshard_files:
+        os.rename(f"{path}.tmp", path)
+
+    return list(affected_tornados)
+
+
+if action == "restart" and len(workers) > 0:
+    if args.less_graceful:
+        # The less graceful form stops every worker now; we start them
+        # back up at the end.
+        logging.info("Stopping workers")
+        subprocess.check_call(["supervisorctl", "stop", *workers])
+    else:
+        # We cannot pass all of these to one `supervisorctl restart`
+        # because that takes them all down at once, waits until they are
+        # all down, and then brings them back up; doing them sequentially
+        # requires multiple `supervisorctl restart` calls.
+        for worker in workers:
+            logging.info("Restarting %s", worker)
+            restart_or_start(worker)
+
+if has_application_server():
+    if args.tornado_reshard:
+        affected_tornado_ports = update_tornado_sharding()
+        logging.info("Tornado ports affected by this resharding: %s", affected_tornado_ports)
+    else:
+        affected_tornado_ports = tornado_ports
+    if not args.only_django:
+        # Next, we restart the Tornado processes sequentially, in order to
+        # minimize downtime of the tornado service caused by too many
+        # Python processes restarting at the same time, resulting in each
+        # receiving insufficient priority.  This is important, because
+        # Tornado being unavailable for too long is the main source of
+        # user-visible downtime when we restart a Zulip server.  We do
+        # this before restarting Django, in case there are new event types
+        # which it will need to know how to deal with.
+        if len(tornado_ports) > 1:
+            for p in affected_tornado_ports:
+                # Restart Tornado processes individually for a better rate of
+                # restarts.  This also avoids behavior with restarting a whole
+                # supervisord group where if any individual process is slow to
+                # stop, the whole bundle stays stopped for an extended time.
+                logging.info("%s Tornado process on port %s", verbing, p)
+                restart_or_start(f"zulip-tornado:zulip-tornado-port-{p}")
+        else:
+            logging.info("%s Tornado process", verbing)
+            restart_or_start("zulip-tornado:*")
+
+    # Finally, restart the Django uWSGI processes.
+    if (
+        action == "restart"
+        and not args.less_graceful
+        and get_config(config_file, "application_server", "rolling_restart", False)
+        and os.path.exists("/home/zulip/deployments/uwsgi-control")
+    ):
+        # See if it's currently running
+        uwsgi_status = subprocess.run(
+            ["supervisorctl", "status", "zulip-django"],
+            stdout=subprocess.DEVNULL,
+            check=False,
+        )
+        if uwsgi_status.returncode == 0:
+            logging.info("Starting rolling restart of django server")
+            with contextlib.suppress(FileNotFoundError):
+                os.unlink("/var/lib/zulip/django-workers.ready")
+            with open("/home/zulip/deployments/uwsgi-control", "w") as control_socket:
+                # "c" is chain-reloading:
+                # https://uwsgi-docs.readthedocs.io/en/latest/MasterFIFO.html#available-commands
+                control_socket.write("c")
+            n = 0
+            while not os.path.exists("/var/lib/zulip/django-workers.ready"):
+                time.sleep(1)
+                n += 1
+                if n % 5 == 0:
+                    logging.info("...")
+            logging.info("Chain reloading complete")
+        else:
+            logging.info("Starting django server")
+            subprocess.check_call(["supervisorctl", "start", "zulip-django"])
+    else:
+        logging.info("%s django server", verbing)
+        restart_or_start("zulip-django")
+
+    using_sso = subprocess.check_output(["./scripts/get-django-setting", "USING_APACHE_SSO"])
+    if using_sso.strip() == b"True":
+        logging.info("Restarting Apache WSGI process...")
+        subprocess.check_call(["pkill", "-x", "apache2", "-u", "zulip"])
+
+# If we were doing this non-gracefully, or starting as opposed to
+# restarting, we need to turn the workers (back) on.  There's no
+# advantage to doing this not-all-at-once.
+if (action == "start" or args.less_graceful) and not args.only_django:
+    workers = list_supervisor_processes(workers, only_running=False)
+    if workers:
+        logging.info("Starting workers")
+        subprocess.check_call(["supervisorctl", "start", *workers])
+
+if (
+    has_application_server()
+    and not args.skip_client_reloads
+    and not args.only_django
+    and not args.tornado_reshard
+):
+    # All of the servers have been (re)started; now enqueue events in
+    # the Tornado servers to tell clients to reload.
+    logging.info("Sending reload events to clients in the background")
+    subprocess.check_call(["./scripts/reload-clients", "--background"])
+
+logging.info("Done!")
+print(OKGREEN + f"Zulip {action}ed successfully!" + ENDC)
+
+if change_symlink and "PWD" in os.environ:
+    for symlink in [last_symlink, current_symlink]:
+        if os.path.commonprefix([os.environ["PWD"], symlink]) == symlink:
+            print(
+                """
+{}Your shell entered its current directory through a symlink:
+  {}
+which has now changed. Your shell will not see this change until you run:
+  cd {}
+to traverse the symlink again.{}
+""".format(WARNING, symlink, shlex.quote(os.environ["PWD"]), ENDC),
+                file=sys.stderr,
+            )

scripts/setup/apt-repos/ksplice/jammy.list

@@ -0,0 +1,2 @@
+deb [signed-by=/etc/apt/keyrings/ksplice.asc] http://www.ksplice.com/apt jammy ksplice
+deb-src [signed-by=/etc/apt/keyrings/ksplice.asc] http://www.ksplice.com/apt jammy ksplice

scripts/setup/apt-repos/ksplice/ksplice.asc

@@ -0,0 +1,65 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQENBEoTaW8BCADXQtpKT5gzOC+/Me50Z07GHfZqkjAThrY+XGhKenklDrZA8nXe
+FDcmlmMvfeSViP5UH+X7tzjUFT2FcUh65+Onggi/J9nFIDweQXxpzDYyWCK+B0RX
+InKsq3TfEs5G0yIfYuKi/pgLYkFBls0stWC+1BS+3Lx4uDRTb/44D4LgzHKoAfy1
+Soho8nDDL1pWEpQAq/5yVSgRc1Vvs1s+CmR8zE5gVi3cfGS0kigdfZJVEdAY/w99
+t3abgYo1Eq3+Vc1bb+5DiEQZlZsWxWglQlvSyx60U2oxr05Ki+3ZyBomfFCTfL2m
+fzzJ8cyglzNhFKhyFQIHqzoPR+Sxl8ppcnEJABEBAAG0NktzcGxpY2UgQVBUIFJl
+cG9zaXRvcnkgU2lnbmluZyBLZXkgPGRldmVsQGtzcGxpY2UuY29tPokBNgQTAQgA
+IAUCShNpbwIbAwULCQgHAwQVCgkIBRYCAwEAAh4BAheAAAoJEPfKYmW21AOO/pUH
+/jKDtB3iRU2B4jii71CSFyFaz3BvJvgRMmIf53L85h3sUvqeVJiy8MoreWeoxst9
+uJBnp8W61QwolCbU6awqdZ2ywRi7JyYNopaEKptxJ3EgBYm+Dq0S7srQK0qCMdRX
+k7OrhCoJEmev7SazhpdIkMWPtRyksgktBMlwQ5/PyLyW+mP3a8ujYDjMIqzScyDV
+YBTKK8HtXaLb6Y2Fu4jinAm4YLP3XfnAyNE1Xi9fkzTBWgC4AZ4wctQWxViu6Q91
+HBB1xBjQYD6aCrPLB8/EtYO6n9UoIov6We8qwDDq7oufEKt8/uLXsomEbaWgOqAv
+wZzpU6ZHueA8JEmNQYzf6pWZAg0EXKv2DQEQANWkHff3Mp7btrQsBCfiNYNh9fi2
+0KBhtfWyDI4pyU7ZkzF0sgXZPPUquYuKbRqbqW1NghWk/SFUewfWLLsxpWDUr+9p
+ghLx2MvdKuaNfvQ/dAoiu7kevyIY4q9fiMwdtRmaCFnJVF2+XZA1z2iH6X6LcLPI
+KEWU1Xd0aWaxoFFPqjkRy+dlDxxV2xsWdEBikIM7rnA4K6NY1V7YXl4DrHLiZB9U
+4K4XuNjWxvjNFqdNUTSFnLKKDo55NmO62OvtX6QOtPkrc91efaQ+xVZwR0kk61r6
+Gon3CcDVqJMk02m9E/p8m2+LDymgmokgPtVQ9N8anfyTqw997gGaoR9FJRs1Pkko
+IW+Wnhjf2kfOYp9f7yON5nZeAHH9ngaxbqr+0A6SxnyccH9cg9mSvpX61ddk/gPm
+l40hYvGHNrnzkUOIaLx3Vngogyl6omFS7bi+t72uZifbA4U/oZhl+LUo4wiYCNAL
+XcGS2kCVKoM3MJB6mg1++gaI7y/Sw7yYfLXp+mn6GTtPiG95JyhhggFpMxx1MSW8
++MDmaBdNoEX+q20XUuUV/nU+82QpBWgJHtX36m5kaxZ6r0q/4ZpRgLe4qj3owoI7
+gbfi5K725ijh5nfKvsayVIzqsHQWLjJ8NP1H2ZLxgem4IqGBDkhQGXrHvaHWzTb6
+ZqF54fQJkxtX/uETABEBAAG0RE9yYWNsZSBPU1MgZ3JvdXAgKE9wZW4gU291cmNl
+IFNvZnR3YXJlIGdyb3VwKSA8YnVpbGRAb3NzLm9yYWNsZS5jb20+iQI+BBMBAgAo
+BQJcq/YNAhsDBQklmAYABgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRCCVi6p
+rZhto3nfD/4vHhyoqJ+yURoGlbPjKodwC27PbmwJbjBsfhXnoR8pkCH5ZD8nA8XJ
+tLLCpvOAhSsiXODwIy5ScozkESVSQo8Ngj4KO9S0/QH14VOGqOntY2XLQhBfyoLq
+n+BMsNe4RsouP4R8u8qKGpwva0khpBaJABp+0bkMUchqdmlWvzx5cnAAwKV7+bb3
+HsriDRz3n29l0UXuKCAhweVMncZYZsvNFeLR9dVNAkCW1HkbH4WdGdKYCmUvqPVQ
+bVyB6xmt8lls2yXT7kSrigdQ3exOBqpNhoxZHuMfocUX7l3Tmd2mW0tdsbohrVzV
+nh76FrZ6EGdx4HFIK3lVPnO2a8kKbhBPj5LwAqx1AunHddZNazkHjUNjLfJAdpDP
+5KjjZgS28YJ6Y+wJ+SJ3xk0SW2X0ozSIdglsV6G/ZyRl7hFU0QNWC6uWcQQogK+F
+/BLhvPYhBk9JhAsYuZRjCmmR/ZWOQOFNBQynWKoteyiUKMN9NxmuVRoARc/sDXC4
+sGUAQcT/Jk5lupyATgBkqRWclia7aWtKQ2GKww5WxWEPILIUTDX58P5Ge9H240c6
+qB5NX/qQ7Ia76cLx2fArKrTAsnO77wQ116Zy+V32nDHcU9ZMZDgYY0ncxV3B/Cdi
+SDm8oYNI6Y8O4SefGRo8mtMkgdIld+NKD8zQ+IsZdw4ykZU15ulJArkCDQRcq/YN
+ARAAodmaW82j7/5qZiH06CeXNJRy2osQ2R7ybtDsddRqQRmBN9FTRqf71OZ+hQLI
+dLXWrcDSX4WgH8UFPjkHLFR1/znShB3Q8Cmqjk3E2lAKpiA4I6lMdPRKdGH2BAIM
+aDN9hJmXwwT6LMRTlY6NDnWD/ZqM4NcYhYc/BgTyVnIXu0TtsU0TC97uwitB58BH
+R4BLPw8wV1DlRL+9hlD6N4tTZ1mp+XYHsCc/sy5elrfUySEHeVph0f69ZpAs9uT9
+uHty8q2QNsMdjXc1LadOlbJ+N5QIWkMe6nMw2RyVzQh/jhYoDVrSw7t3qYFbJUzQ
+iCsLGJ5cn8RlUWSFcS6Vwa74vSIeGRH00Dp1Fe8L/AmewIBKPPEWrLOWFN81HVDB
+Z1kmkLwiX2gfdVytPhO0S8kPG5dMyp4xI581Kx2pqIT27q2BsLXeoFO9uygGD1Gz
+aFjadGpSE2G8yhFu3VTWpfCGf/2DV/7WLca8QPqPYC5YydT3N6FHfaK4ZCXjySjj
+bxtEQ/PTwBj76/f+fhT9xuygnMC8KDX5ZhB7bq/SYgki7M6Z4VGZdxpMdRm/Jjpq
+pK9B46ejSHVyNFkA31PpnyqVhvCHzKEY2V/JtA+aV3+h6IM1WvjexKXpbTZM4sVn
+fqHZ4am3YspRXP7MVDCsB0W7pSj/WWAZEZvMF7M2BQKRAIsAEQEAAYkCJQQYAQIA
+DwUCXKv2DQIbDAUJJZgGAAAKCRCCVi6prZhto7gSD/9ZESN0eiy9Ms9uMPCa0fRH
+dPCKz96oc9Krnsj2MNI69ENaS/j8KJ0G7X4WxMOkiefjCIAgT14xv8vz0JzZjkvL
+MeXM5EkwSDMSpyMh7CpFwTK8xvJOfHgZziEqIyFFwwtZC5anr8lPT34Heg/NAtce
++4C4q7RmMUmXXqht2gvu0BMA4+2qbGTC3bYbWUGQZRUI6IS7CDX70CCIyEMe3oaD
+zAeMqhCIe/il4YMrFyV19MVMAfTe/H7abBPrVr9GMTViofOaWqZNrz1IM0NK2sbZ
+WKRIHRh0O6pLMHoUxxRGS0nDDKE4oSMnhzbTBkbnFB+Il85yKPZBg9bm9i1A0Kcp
++ymwXsEI/8Zd1gBODJqMLGnimQ2wBmVHIdTHXM8xHUTX6x76XmzXzLRX5v7VgESY
+CZwQwv1F6/5FvJ35heYn4/2sNOGS89fFX7gdmCXSZe9N3UJRSc2d3jRlLMWjyFOa
+v/6PZPuJHfBzGejK/93ww5Sq5iwoMt0Gv2eD4K9t//yU0knp1sJABwRe9GfwUqOr
+6I/6Ec9dc6H8Wsy8EmtsPdXoXrl7K/Isw3vgJrF3YHau7TXIs0YBFmvyI4fdx23h
+vILSVIDnXI14+ih7od+AIQCwUS+i+KWvuQVuykMas/j3CHR6+1EM+ap+MwuKJpHE
+5d586NuHxeqt80YNMJDN0Q==
+=Y2MU
+-----END PGP PUBLIC KEY BLOCK-----

scripts/setup/apt-repos/ksplice/noble.list

@@ -0,0 +1,2 @@
+deb [signed-by=/etc/apt/keyrings/ksplice.asc] http://www.ksplice.com/apt noble ksplice
+deb-src [signed-by=/etc/apt/keyrings/ksplice.asc] http://www.ksplice.com/apt noble ksplice

scripts/setup/apt-repos/teleport/jammy.list

@@ -0,0 +1 @@
+deb [signed-by=/etc/apt/keyrings/teleport-pubkey.asc] https://apt.releases.teleport.dev/ubuntu jammy stable/v17

scripts/setup/apt-repos/teleport/noble.list

@@ -0,0 +1 @@
+deb [signed-by=/etc/apt/keyrings/teleport-pubkey.asc] https://apt.releases.teleport.dev/ubuntu noble stable/v17

scripts/setup/apt-repos/teleport/teleport-pubkey.asc

@@ -0,0 +1,52 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQINBF+R3LYBEADOEO9i3Dm5rEAiXONchX3M54QzZX0yHArSpYQ5aJDdJRQbqzqT
++e2os8NpSjVDZFNz5ul8xkZsnCLX7pgrAYqq+vsXL4bMWDP96S6PjfVIAyV4ylv0
+DBReMdkaAZb/IoPhkSTT+ayw4eGEtUz/k7mxMpQ9ob7qFtGs8aNVT/An5LfFR1Lx
+9WOlFPPIAJKcHVIyRD+4EoCSn1R1c61UHFIRatbAnwOLs3iz4/GU+w9wdbuWbDuk
+nGdG0Lmlzp42HHxeJJFQlOTed97+trktvAiuzA/0lbQHEcWvxfWAy5//cjORp+H3
+RGLp8fJ+fFRAyA4WP6O3wIC4gAAgsEn8WpVT8wZYlLMRf694SeawBtyUSlcsn9i1
+LuOh5akOY3iQtH01+rMBjOaMkCmpT2nQaUH+HS2iZBddBHdAMMQtj2UolMRbUSxH
++GJczes1t9/WH3vbvh5ESMOy0fH14Tjo+9yQYa4EhFNNloAG10DYFLlCj47fWDdS
+o/++vhZsKaS7yLHDGOLPT+x15ComG2gupmRkbATvUddztlsfF+tD97laT9eaLB1W
+zxszqr8+LxP961wmbS2j+ZBbXyrPr1Fln/TdyFAhkIMJ+J5hZB+NcjRUwUoB7nOd
++FbTxtnyJb2iaJNCJHJQVA85IYzUpXA3CDdgUHF810kVBcBPBtLhZC5ybQARAQAB
+tCtHcmF2aXRhdGlvbmFsLCBJbmMgPGluZm9AZ3Jhdml0YXRpb25hbC5jb20+iQJU
+BBMBCAA+FiEEDF6LpWWOMg0bAxF5yH7VOmKCxBEFAl+R3LYCGwMFCRLMAwAFCwkI
+BwIGFQoJCAsCBBYCAwECHgECF4AACgkQyH7VOmKCxBFfxxAAiXWJm86oZtVdAlp1
+pzpKeV0pwgrnt7Uk8fu5tYpdE/oVMnwcdsDDQucItGtHGfjmzs3Cr8/praekenf1
+9iHSz422OpIGzCI4VfXaFPVfzbV1w7cSOnceY6lPnKUMrRBKKJX5Nw/6LZS40gsQ
+BoeZxe0MXB4tBc4dY30f1MQ44amRYmtTA7wep+ymVRfkPnHNnIrsdYGldbfPsbPO
+PUX8ZnWZiuI0+NgX3oBOl6YY4JehBJj61Ukx1DPHHLhhundHumChYFn+LBIZxD3O
+B9uoRzUzwUIM0N9IUjpGvtkqtm7Vbs6/bDxI4Owgsa7vXpEXZ2qD0AIle7sD0Fjl
+F19o2mXmEeQp9Fl4OrkZCURCQvPq9UCh6Nu0a1+SnbG+qXyyvqszy2tkV4xmcF4w
+Gib0SVT8RR08NeJXkHtBscnecgUA1BTH8J8RnUeQXZhUn51bVJk4JaDnEXp8VEP2
+gNce+oUY2XQtLDVzHysGhexDrWk8ycl/zvwyxKv+kj5QhjXugHkOMnW53mdMe3N/
+gwsV+kJUm6NdtLtTAOkky/GfkIGTWNQPD2/42T+0cA9lTVxihh+wz9tgA1ZbtVOK
+P2DNA10rsCuzGPFn8d6Khymt0o66dgfEloy9Y14leoqUCMPU3ibLP6bYuow2AJUz
+KcvTgmfjP1/ghNXI7E2vgNi8wta5Ag0EX5HctgEQALx4btbP47LwrIqB4loog2sT
+pac7fdbA+YVeqP/9KoLw1ZB+5DeqNKmtUHSau9mRVh8a8g7slpGhH6hxlEHr7ek/
+mA/o91jB4RGo5mfyuWcJQKRyHS4pWciEM/gK+o6lEceTdUwvKI6OrJ4koPd3HZth
+mw+xPyAdGKY3oBmrXeZ6XkuDfME8doRmuwlw/tbmje63/2j97ebiFfQcyWLH32d8
+T+yEpAj+55Qxp6aJZaDOeAuzBtyAopxGRjGsxBUF/VSUwxYb0bmwWgPIhPC77oEk
+AEMPsIsI9LJ8fQY/sOzwhyNNt+b7rgto6AFskz7urezzCuuIwMeupmC78QWGw9jM
+zHFf3R6O1KQ0v8PBYYb6BHkjzho6hTcOZO9Zh+XO4k6uEwlu+Zc0AmyHmQeQ3I8Z
+tAb//LJk9X62yNPE/8wjtEUzXqyzlLpGjRFr6kQv+6nqs8JxyCnS34Q+au2IqOnn
+iFkHj/w79mtmzR4G43wo3x1nGjyz+vTpsurmJ+qFMO0bLcE/HV8aGxs0YeQsByOc
+SU8TK6v+Wkn58LT4cvjIO5G/2UM7kucXl56hqvguvnFTLNqewWtqgS7IRuykcYgK
+HrBYb/iVH+Fb+9Th9VX7bl0ZeoH7O8RbvxKGkd90+DPsurBeIQ7S4zM9w7WnAsAC
+Sgs8owYZpHpyrK8QFD4zABEBAAGJAjwEGAEIACYWIQQMXoulZY4yDRsDEXnIftU6
+YoLEEQUCX5HctgIbDAUJEswDAAAKCRDIftU6YoLEEURID/4oQhZZPindZJHiwQqm
+0a8H1ssgZAz6E8PejoN0gbsblbOrtkGDLU8gvzksvd/9luSLRgPw++m6ut87PeMv
+MKc4UIyRb5oSgh5WE0bW9191Gkfge9DRrIdtUDG8N+oTlIWYHTXC5zlwmfMobtQE
+kFUdPbedhytYx1wgbh8KP8sLXGPXut5VqDy/EgNzqERnI5kLeiDvMsLz0xjdHpGW
+ASfJMNX120GU8Mwqa6gWvP52BB20pU9bC1VQX1qiqD6V1GpxQJ2jACKke6boiqbL
+Bdb0UgmW4XYIp4ZjLC842e0qSyfd8rt3PzYrbK/NPuXAV7f+wAhPSC18v+1Ap5Kh
+KKHRLvyUVGxwaBVedOuuC/OqJwSSLa0cQKytFK+3OJAdTYoHtsh++ScgEL/wOCXs
+gM5xmlI6Pk/6Ev0Hz/kDY5F0w4/VvSEaS/7TSkmf5JvxdueVObf5ry5O+L4J7t7y
+JwdtPhXgHR0PHidnh/02SVn8XIzHdB9OZ2i6Wr12loFZGltWdmJVkQC/cj/HBr5I
+ZizQril+7cXDI/8Hyk04d19rmjSIU49FderpNYYOv38dqaAsosYge6JzYdIzJrJH
+/DIKnSAU/a14sFUrNm+TYJmZto35hSltUxLEzLIWeR9TjpOh6VS1UzdGQh32NP+h
+oq8y1SJMCrfC9Ub5q2/ijiJWUw==
+=+Ne5
+-----END PGP PUBLIC KEY BLOCK-----

scripts/setup/apt-repos/zulip/bookworm.list

@@ -0,0 +1,5 @@
+deb [signed-by=/etc/apt/keyrings/pgdg.asc] http://apt.postgresql.org/pub/repos/apt/ bookworm-pgdg main
+deb-src [signed-by=/etc/apt/keyrings/pgdg.asc] http://apt.postgresql.org/pub/repos/apt/ bookworm-pgdg main
+
+deb http://deb.debian.org/debian bookworm-backports main
+deb-src http://deb.debian.org/debian bookworm-backports main

scripts/setup/apt-repos/zulip/custom.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+os_info="$(. /etc/os-release && printf '%s\n' "$ID" "$VERSION_ID" "$VERSION_CODENAME")"
+{
+    read -r os_id
+    read -r os_version_id
+    read -r os_version_codename
+} <<<"$os_info"
+
+if [ ! -e /usr/share/doc/groonga-apt-source/copyright ] && [ "$os_id $os_version_id" != "ubuntu 26.04" ]; then
+    pgroonga_apt_sign_key=$(readlink -f "$LIST_PATH/pgroonga-packages.groonga.org.asc")
+
+    remove_pgroonga_apt_tmp_dir() {
+        rm -rf "$pgroonga_apt_tmp_dir"
+    }
+    pgroonga_apt_tmp_dir=$(mktemp --directory)
+    trap remove_pgroonga_apt_tmp_dir EXIT
+
+    {
+        cd "$pgroonga_apt_tmp_dir" || exit 1
+        tmp_gpg_home=.gnupg
+        gpg --homedir="$tmp_gpg_home" --import "$pgroonga_apt_sign_key"
+        # Find fingerprint of the first key.
+        pgroonga_apt_sign_key_fingerprint=$(
+            gpg --homedir="$tmp_gpg_home" --with-colons --list-keys \
+                | grep '^fpr:' \
+                | cut --delimiter=: --fields=10 \
+                | head --lines=1
+        )
+
+        groonga_apt_source_deb="groonga-apt-source-latest-$os_version_codename.deb"
+        groonga_apt_source_deb_sign="$groonga_apt_source_deb.asc.$pgroonga_apt_sign_key_fingerprint"
+        curl -fLO --retry 3 "https://packages.groonga.org/$os_id/$groonga_apt_source_deb"
+        curl -fLO --retry 3 "https://packages.groonga.org/$os_id/$groonga_apt_source_deb_sign"
+        gpg \
+            --homedir="$tmp_gpg_home" \
+            --verify \
+            "$groonga_apt_source_deb_sign" \
+            "$groonga_apt_source_deb"
+        # To suppress the following warning by "apt-get install":
+        #   N: Download is performed unsandboxed as root as file
+        #   '.../groonga-apt-source-latest-$os_version_codename.deb' couldn't be
+        #   accessed by user '_apt'. - pkgAcquire::Run (13: Permission denied)
+        chown _apt .
+
+        apt-get -y install "./$groonga_apt_source_deb"
+    }
+    touch "$STAMP_FILE"
+fi

scripts/setup/apt-repos/zulip/jammy.list

@@ -0,0 +1,11 @@
+deb [signed-by=/etc/apt/keyrings/pgdg.asc] http://apt.postgresql.org/pub/repos/apt/ jammy-pgdg main
+deb-src [signed-by=/etc/apt/keyrings/pgdg.asc] http://apt.postgresql.org/pub/repos/apt/ jammy-pgdg main
+
+deb [signed-by=/etc/apt/keyrings/pgroonga-ppa.asc] http://ppa.launchpad.net/groonga/ppa/ubuntu jammy main
+deb-src [signed-by=/etc/apt/keyrings/pgroonga-ppa.asc] http://ppa.launchpad.net/groonga/ppa/ubuntu jammy main
+
+deb [signed-by=/etc/apt/keyrings/strukturag.asc] https://ppa.launchpadcontent.net/strukturag/libheif/ubuntu/ jammy main
+deb-src [signed-by=/etc/apt/keyrings/strukturag.asc] https://ppa.launchpadcontent.net/strukturag/libheif/ubuntu/ jammy main
+
+deb [signed-by=/etc/apt/keyrings/strukturag.asc] https://ppa.launchpadcontent.net/strukturag/libde265/ubuntu/ jammy main
+deb-src [signed-by=/etc/apt/keyrings/strukturag.asc] https://ppa.launchpadcontent.net/strukturag/libde265/ubuntu/ jammy main

scripts/setup/apt-repos/zulip/noble.list

@@ -0,0 +1,12 @@
+deb [signed-by=/etc/apt/keyrings/pgdg.asc] http://apt.postgresql.org/pub/repos/apt/ noble-pgdg main
+deb-src [signed-by=/etc/apt/keyrings/pgdg.asc] http://apt.postgresql.org/pub/repos/apt/ noble-pgdg main
+
+deb [signed-by=/etc/apt/keyrings/pgroonga-ppa.asc] http://ppa.launchpad.net/groonga/ppa/ubuntu noble main
+deb-src [signed-by=/etc/apt/keyrings/pgroonga-ppa.asc] http://ppa.launchpad.net/groonga/ppa/ubuntu noble main
+
+deb [signed-by=/etc/apt/keyrings/strukturag.asc] https://ppa.launchpadcontent.net/strukturag/libheif/ubuntu/ noble main
+deb-src [signed-by=/etc/apt/keyrings/strukturag.asc] https://ppa.launchpadcontent.net/strukturag/libheif/ubuntu/ noble main
+
+# libde265 in 24.04 is up-to-date, so the PPA doesn't provide 24.04 packages
+#deb [signed-by=/etc/apt/keyrings/strukturag.asc] https://ppa.launchpadcontent.net/strukturag/libde265/ubuntu/ noble main
+#deb-src [signed-by=/etc/apt/keyrings/strukturag.asc] https://ppa.launchpadcontent.net/strukturag/libde265/ubuntu/ noble main

scripts/setup/apt-repos/zulip/pgdg.asc

@@ -0,0 +1,77 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQINBE6XR8IBEACVdDKT2HEH1IyHzXkb4nIWAY7echjRxo7MTcj4vbXAyBKOfjja
+UrBEJWHN6fjKJXOYWXHLIYg0hOGeW9qcSiaa1/rYIbOzjfGfhE4x0Y+NJHS1db0V
+G6GUj3qXaeyqIJGS2z7m0Thy4Lgr/LpZlZ78Nf1fliSzBlMo1sV7PpP/7zUO+aA4
+bKa8Rio3weMXQOZgclzgeSdqtwKnyKTQdXY5MkH1QXyFIk1nTfWwyqpJjHlgtwMi
+c2cxjqG5nnV9rIYlTTjYG6RBglq0SmzF/raBnF4Lwjxq4qRqvRllBXdFu5+2pMfC
+IZ10HPRdqDCTN60DUix+BTzBUT30NzaLhZbOMT5RvQtvTVgWpeIn20i2NrPWNCUh
+hj490dKDLpK/v+A5/i8zPvN4c6MkDHi1FZfaoz3863dylUBR3Ip26oM0hHXf4/2U
+A/oA4pCl2W0hc4aNtozjKHkVjRx5Q8/hVYu+39csFWxo6YSB/KgIEw+0W8DiTII3
+RQj/OlD68ZDmGLyQPiJvaEtY9fDrcSpI0Esm0i4sjkNbuuh0Cvwwwqo5EF1zfkVj
+Tqz2REYQGMJGc5LUbIpk5sMHo1HWV038TWxlDRwtOdzw08zQA6BeWe9FOokRPeR2
+AqhyaJJwOZJodKZ76S+LDwFkTLzEKnYPCzkoRwLrEdNt1M7wQBThnC5z6wARAQAB
+tBxQb3N0Z3JlU1FMIERlYmlhbiBSZXBvc2l0b3J5iQJOBBMBCAA4AhsDBQsJCAcD
+BRUKCQgLBRYCAwEAAh4BAheAFiEEuXsK/KoaR/BE8kSgf8x9RqzMTPgFAlhtCD8A
+CgkQf8x9RqzMTPgECxAAk8uL+dwveTv6eH21tIHcltt8U3Ofajdo+D/ayO53LiYO
+xi27kdHD0zvFMUWXLGxQtWyeqqDRvDagfWglHucIcaLxoxNwL8+e+9hVFIEskQAY
+kVToBCKMXTQDLarz8/J030Pmcv3ihbwB+jhnykMuyyNmht4kq0CNgnlcMCdVz0d3
+z/09puryIHJrD+A8y3TD4RM74snQuwc9u5bsckvRtRJKbP3GX5JaFZAqUyZNRJRJ
+Tn2OQRBhCpxhlZ2afkAPFIq2aVnEt/Ie6tmeRCzsW3lOxEH2K7MQSfSu/kRz7ELf
+Cz3NJHj7rMzC+76Rhsas60t9CjmvMuGONEpctijDWONLCuch3Pdj6XpC+MVxpgBy
+2VUdkunb48YhXNW0jgFGM/BFRj+dMQOUbY8PjJjsmVV0joDruWATQG/M4C7O8iU0
+B7o6yVv4m8LDEN9CiR6r7H17m4xZseT3f+0QpMe7iQjz6XxTUFRQxXqzmNnloA1T
+7VjwPqIIzkj/u0V8nICG/ktLzp1OsCFatWXh7LbU+hwYl6gsFH/mFDqVxJ3+DKQi
+vyf1NatzEwl62foVjGUSpvh3ymtmtUQ4JUkNDsXiRBWczaiGSuzD9Qi0ONdkAX3b
+ewqmN4TfE+XIpCPxxHXwGq9Rv1IFjOdCX0iG436GHyTLC1tTUIKF5xV4Y0+cXIOI
+RgQQEQgABgUCTpdI7gAKCRDFr3dKWFELWqaPAKD1TtT5c3sZz92Fj97KYmqbNQZP
++ACfSC6+hfvlj4GxmUjp1aepoVTo3weJAhwEEAEIAAYFAk6XSQsACgkQTFprqxLS
+p64F8Q//cCcutwrH50UoRFejg0EIZav6LUKejC6kpLeubbEtuaIH3r2zMblPGc4i
++eMQKo/PqyQrceRXeNNlqO6/exHozYi2meudxa6IudhwJIOn1MQykJbNMSC2sGUp
+1W5M1N5EYgt4hy+qhlfnD66LR4G+9t5FscTJSy84SdiOuqgCOpQmPkVRm1HX5X1+
+dmnzMOCk5LHHQuiacV0qeGO7JcBCVEIDr+uhU1H2u5GPFNHm5u15n25tOxVivb94
+xg6NDjouECBH7cCVuW79YcExH/0X3/9G45rjdHlKPH1OIUJiiX47OTxdG3dAbB4Q
+fnViRJhjehFscFvYWSqXo3pgWqUsEvv9qJac2ZEMSz9x2mj0ekWxuM6/hGWxJdB+
++985rIelPmc7VRAXOjIxWknrXnPCZAMlPlDLu6+vZ5BhFX0Be3y38f7GNCxFkJzl
+hWZ4Cj3WojMj+0DaC1eKTj3rJ7OJlt9S9xnO7OOPEUTGyzgNIDAyCiu8F4huLPaT
+ape6RupxOMHZeoCVlqx3ouWctelB2oNXcxxiQ/8y+21aHfD4n/CiIFwDvIQjl7dg
+mT3u5Lr6yxuosR3QJx1P6rP5ZrDTP9khT30t+HZCbvs5Pq+v/9m6XDmi+NlU7Zuh
+Ehy97tL3uBDgoL4b/5BpFL5U9nruPlQzGq1P9jj40dxAaDAX/WKJAj0EEwEIACcC
+GwMFCwkIBwMFFQoJCAsFFgIDAQACHgECF4AFAlB5KywFCQPDFt8ACgkQf8x9RqzM
+TPhuCQ//QAjRSAOCQ02qmUAikT+mTB6baOAakkYq6uHbEO7qPZkv4E/M+HPIJ4wd
+nBNeSQjfvdNcZBA/x0hr5EMcBneKKPDj4hJ0panOIRQmNSTThQw9OU351gm3YQct
+AMPRUu1fTJAL/AuZUQf9ESmhyVtWNlH/56HBfYjE4iVeaRkkNLJyX3vkWdJSMwC/
+LO3Lw/0M3R8itDsm74F8w4xOdSQ52nSRFRh7PunFtREl+QzQ3EA/WB4AIj3VohIG
+kWDfPFCzV3cyZQiEnjAe9gG5pHsXHUWQsDFZ12t784JgkGyO5wT26pzTiuApWM3k
+/9V+o3HJSgH5hn7wuTi3TelEFwP1fNzI5iUUtZdtxbFOfWMnZAypEhaLmXNkg4zD
+kH44r0ss9fR0DAgUav1a25UnbOn4PgIEQy2fgHKHwRpCy20d6oCSlmgyWsR40EPP
+YvtGq49A2aK6ibXmdvvFT+Ts8Z+q2SkFpoYFX20mR2nsF0fbt1lfH65P64dukxeR
+GteWIeNakDD40bAAOH8+OaoTGVBJ2ACJfLVNM53PEoftavAwUYMrR910qvwYfd/4
+6rh46g1Frr9SFMKYE9uvIJIgDsQB3QBp71houU4H55M5GD8XURYs+bfiQpJG1p7e
+B8e5jZx1SagNWc4XwL2FzQ9svrkbg1Y+359buUiP7T6QXX2zY++JAj0EEwEIACcC
+GwMFCwkIBwMFFQoJCAsFFgIDAQACHgECF4AFAlEqbZUFCQg2wEEACgkQf8x9RqzM
+TPhFMQ//WxAfKMdpSIA9oIC/yPD/dJpY/+DyouOljpE6MucMy/ArBECjFTBwi/j9
+NYM4ynAk34IkhuNexc1i9/05f5RM6+riLCLgAOsADDbHD4miZzoSxiVr6GQ3YXMb
+OGld9kV9Sy6mGNjcUov7iFcf5Hy5w3AjPfKuR9zXswyfzIU1YXObiiZT38l55pp/
+BSgvGVQsvbNjsff5CbEKXS7q3xW+WzN0QWF6YsfNVhFjRGj8hKtHvwKcA02wwjLe
+LXVTm6915ZUKhZXUFc0vM4Pj4EgNswH8Ojw9AJaKWJIZmLyW+aP+wpu6YwVCicxB
+Y59CzBO2pPJDfKFQzUtrErk9irXeuCCLesDyirxJhv8o0JAvmnMAKOLhNFUrSQ2m
++3EnF7zhfz70gHW+EG8X8mL/EN3/dUM09j6TVrjtw43RLxBzwMDeariFF9yC+5bL
+tnGgxjsB9Ik6GV5v34/NEEGf1qBiAzFmDVFRZlrNDkq6gmpvGnA5hUWNr+y0i01L
+jGyaLSWHYjgw2UEQOqcUtTFK9MNzbZze4mVaHMEz9/aMfX25R6qbiNqCChveIm8m
+Yr5Ds2zdZx+G5bAKdzX7nx2IUAxFQJEE94VLSp3npAaTWv3sHr7dR8tSyUJ9poDw
+gw4W9BIcnAM7zvFYbLF5FNggg/26njHCCN70sHt8zGxKQINMc6SJAj0EEwEIACcC
+GwMFCwkIBwMFFQoJCAsFFgIDAQACHgECF4AFAlLpFRkFCQ6EJy0ACgkQf8x9RqzM
+TPjOZA//Zp0e25pcvle7cLc0YuFr9pBv2JIkLzPm83nkcwKmxaWayUIG4Sv6pH6h
+m8+S/CHQij/yFCX+o3ngMw2J9HBUvafZ4bnbI0RGJ70GsAwraQ0VlkIfg7GUw3Tz
+voGYO42rZTru9S0K/6nFP6D1HUu+U+AsJONLeb6oypQgInfXQExPZyliUnHdipei
+4WR1YFW6sjSkZT/5C3J1wkAvPl5lvOVthI9Zs6bZlJLZwusKxU0UM4Btgu1Sf3nn
+JcHmzisixwS9PMHE+AgPWIGSec/N27a0KmTTvImV6K6nEjXJey0K2+EYJuIBsYUN
+orOGBwDFIhfRk9qGlpgt0KRyguV+AP5qvgry95IrYtrOuE7307SidEbSnvO5ezNe
+mE7gT9Z1tM7IMPfmoKph4BfpNoH7aXiQh1Wo+ChdP92hZUtQrY2Nm13cmkxYjQ4Z
+gMWfYMC+DA/GooSgZM5i6hYqyyfAuUD9kwRN6BqTbuAUAp+hCWYeN4D88sLYpFh3
+paDYNKJ+Gf7Yyi6gThcV956RUFDH3ys5Dk0vDL9NiWwdebWfRFbzoRM3dyGP889a
+OyLzS3mh6nHzZrNGhW73kslSQek8tjKrB+56hXOnb4HaElTZGDvD5wmrrhN94kby
+Gtz3cydIohvNO9d90+29h0eGEDYti7j7maHkBKUAwlcPvMg5m3Y=
+=DA1T
+-----END PGP PUBLIC KEY BLOCK-----

scripts/setup/apt-repos/zulip/pgroonga-packages.groonga.org.asc

@@ -0,0 +1,52 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQINBFpuo48BEAC/4UomZcZt2wjDb4WoFGxVxDISaf4k/xliQCz3V11CHN53If3d
+GsqjOQrJTRpu5euXEoecb0YiiUWI28eD1IajTruunGDjI672/u3+iR6Ga1wk7nia
+/5tlo4LMYl5va/BQkxVIx95OcoHBh46loVAnbaq+OyfV+r9dILJUf1txAJ65RIGZ
+SikgkoSCoVnRwNhhLo4ssu65M6o4lRsI6dUP8wpny8dK8EpRlSwrbicrll+lBDxF
+ZIh1OKll2u5qZ7mtQKOMbCwVhildyHrdUG1PI4ynIIDv9/8r+Wf4weoCqlnsKdKr
+eHf990yWde0qSMTZnyjJfyY8pKNPkz2cWHKpxiKz6DaDSXJA/soAAmneRh370L0+
+q9YbUL/TKa3iBmNZOaiJxS+w2pt4n7Z0GPocz56MtGHY0zLplmrm/+z8bBH+Vv1M
+DQEt1hIE+fdB2tGIDCgjvegEVRckCdWb6ZHeRvcmE7/VKoY0GZZC2088AEyc6Hd9
+6B0NWEDpooDFTfkaZ5IVc1VW3K7FF6l0Xm+9fLzmL7L8vzAnMZD5xB1AnDw/Woix
+7Hl7uTdVXBlFWckiuU2N83mtT/IflsK3oBfZ7UVJJWzXiY1iGiEDmAqnENmSiN+D
+NKGZusJM75rCcUM47XIxC1tV+bYbVf/ToPdmXfn5ElR9nMjl77jY+TEmqQARAQAB
+tEFHcm9vbmdhIEtleSAoR3Jvb25nYSBPZmZpY2lhbCBTaWduaW5nIEtleSkgPHBh
+Y2thZ2VzQGdyb29uZ2Eub3JnPokCTgQTAQgAOBYhBCcB8xfPzMuXXK3pwmJM93Q0
+g5IlBQJabqOPAhsDBQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEGJM93Q0g5Il
+v5IP/0K+YEP051EH82R/9F2Krhc4+cI/ktrrqLk1dvshkpU5m5jc+p8Ynjdc9Orq
+oEotKUIDCQVtyVdk/bOWd75tgCScclslsXLdpx6hGYA6eRKAV4yPtXQnSd8FutQW
+iu9poaMPMgODH8bf8PuDGrkLjNhdA0ZuLf1eiabIO94HaF1o1tZ0jVg6UbZ8MwLt
+LhU9ox81VjEPOssGiR8MCBgHZCxhfe2gSR1mmxDE9d80Kb2DYb4kCke2MjoSku0S
+h2BtIl/9Gn2VGp5E1klf+VRjoxK6MjvefY7yG5mSWBsE8YmKholzVD4QUSXWY0e3
+ed8LPt38WXM43eUJt1uD5fd+XoVfTR+ZSaU2P9cEbYb8iec23t2AfXHowhpOcUqy
+LSnTGPdTeVmZQmp6LSjtIAYwz5Bjdq3kDkPZikYKGYykYuoO4gIuOiBCoWEKUil+
+bTIG3eU1rfCLjWsWvTJfmsxrB9hX+DjLQUNP7GH/DGAxwt4oSu0hhlGrpAs1ir09
+KeIQreCyw+yP8t2yFwQ4Lo264Q1euAl29DauB8jfAzS4VVR6U08TpxtzrzTBpKgI
+0wY+Xg2TgTG1JrdZFAYfgnppkeRfwvppZ51EKgtMtDYsHj07Ds6XcoTrpd9Ttj4k
+F/Oe8Y8xg7NkRmPF3CXK1HRsdTkfpvR2lUEMKCiMBWmFGWnIuQINBFpuo48BEAC1
+IHivA5uPRnHpyWUuljjwRQ38TGTNuHeDW0XGychifUSaJMuAkE2ONkGwCeFd2wv6
+fZA/YVv8OmngwehvBLleDUCS9ANXcuQA9DeH1ZmA032zc3tA8yUq7NXgsIRMWAxo
+L3Ed+rZa5ivWTL86vOXo9ZkP4PIlD4Dz5KRSOQbYDxypR4Icg5ivyWHBcC5o89Ed
+XG7Cj1hFzAHREquVKIwR+dcZ1zCsAd2lB8lNoDu4qkMX+adZVzzDl+LLJzLos9wx
+llJGkIctbICWbc2Il906NTU4o23UipUSfgl5Rv7xWoHsYA1nzGK423Ji0JZSjbfk
+nNBT1HFKhrVyhcUiNJ8t7VL/nkxfHeZKkmiZOu53xwemm8S45ly6FDMDLZZ8ij8R
+6SlyRHu/lJUw6PFBk/GSC7HH7QFmT1tli5/BTgl0EHCAxvxv+wHYt++Lv1kKToYu
+WPOjD52y1edwcdqT/l022AyhXP9sHyjlacSxww1giLDNwy+2GOtwdaUreQd5SVJ7
+48bh4V7VjRvVCIJGhQs72PI4AFyFQZ1gCXHZK7lAsDg9LV5iuOWD8iu3zS7XuJFn
+Di8aIe6zQ6qP8FMivaQeRvwnRPCdRvk/noBfuqlDPSBCEfDiVkLfbkdNKIDF3QgF
+c7ZuGSJl+m6IluUm1hMx587+JnR9FpW0Lc46ypMApQARAQABiQI2BBgBCAAgFiEE
+JwHzF8/My5dcrenCYkz3dDSDkiUFAlpuo48CGwwACgkQYkz3dDSDkiV7ZA//SLYc
+hegp0AWszEDvySB6dyn3v/ZZfuIUkknPbaYo2M516TUFb960EMFaKC96FeV6AEyv
+B+pL9iDsobRqtrMzLfWENibhnY3Dm+THNgPkFg905+ygnMKSHTloi++BFgSl+sjO
+SRtdN5yNF8tVuRUsgiepQdcQ6AioFHaq4HxMAIJLfJA3qIue4/cGIpbdneflCvOa
+V6jYDjCsUdLSwh3J4DdVyquIq34wVVZYdtOao7EwKVcy62GHtJxmnv8no5hsAizL
+t+8nf14jX4zIueoBGLyFP2XoaiUH9NDuApp3oILuQPighu3BFC7bjy3MFIaO7MHb
+Jvm0wjQADAr4UhXk1ALNWrSDiLMNa0raJzzzr43dCIN6lsmlbNj4CvMYdtWi4ObS
+F5p9m03oHJhSRhn7+mAJJDwCqrYyf0T1WWR/dz86LkefdHQF/5mVmWEX6OoqcYL+
+gC4S+dc+5RP2anHYO00pBtrWNjSZ6Fy0TqV4PQbO4BZG+omu2HkiU890NLgW0DKl
+pEMq49UQfItfJtvCffzpC4kq8obMi4eFFC6puz1DcjSjFd9U2iAUFb7rSrS2fxi1
+pj3P7QUmt1MuQgRuaE64dcaP92TtCUVFtgmFPoDL00aMJiz9roVOyTgLc90AHq6R
+6TYbUXXz7NBwM+kX8Uq9KpPXkHnP1D4TgEw7o5A=
+=7PCC
+-----END PGP PUBLIC KEY BLOCK-----

scripts/setup/apt-repos/zulip/pgroonga-ppa.asc

@@ -0,0 +1,27 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: SKS 1.1.5
+Comment: Hostname: keyserver.ubuntu.com
+
+mQINBFMyaEABEADEeUoP5AmhJZLHujySJaLKd1EXpIxkope3l4f/3tkbPTvV+jkyYvOSX/ii
+NAHyVGPrg4UjJI+kDLru1glFoabBs+J4xk2Lql9DcHW0x7YPgBXKCuj2sg84Teb0KTnAkAFI
+gd8w2kzs6l9itQbtCWei0wH+pGpp9YJqEqJhqc3U51LqUoMmYjTd2WTaN+BNVZOE3ws4Pa4+
+JyikyVOkuj/8q+jRF8LHWmeb76SuycvZMyGjJBnIs65wmmSN6VpeHtQA3NzIAV2FKuirEEuW
+5j7fD9L8WNTXy++K6NTvcd2InXgEWMG3rSGvfE2CvZSxKc/b9/xVdaG0LDtTTWPTosNdluUA
+K4M91Rz7UEvC06SWIdxqL8LdLOy/6OYaJxgh1exrEadRPWAKr+Ak7LDMiYPHq80npQzMh8Hn
+oUbQTaQ7KDFD17BZ7HauXn8rRmG/vpdQvKzswRoc3+mroh5hA6otjKdmOWAaD5AmNNxcvR3O
+iNjDTnX804pjrjHzmTAAxrPKwPxbPzFUeNB4Dt4BzUdesdbbZYJw5ojcvt9qCfLU0DOZXWk4
+t1nHvIQvGoG8cE63x1buVHcwFXwxocMUdn4iFam9srBlDx1G8TMlaKITNHvoc//evBU0arz0
+12UMDb8TXAT5zOW1W6szFSt1ZFn0By9Ab+I69UCIgB2iknMUtwARAQABtBlMYXVuY2hwYWQg
+UFBBIGZvciBHcm9vbmdhiQI4BBMBAgAiBQJTMmhAAhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIe
+AQIXgAAKCRAzWefOy2ShV5p5EACWQ5QBQV86SAhjJHfRe1MSYVZtrA3v+wwS52V/C9sV9xlc
+Sn2IGIpUQM2816fVMTM23pvj9lzuP3PpcHl0vc5/gqSliRPAzXawgs8Gl+wB9FcwnKPPQfyR
+CWflFg5mc37+5wM/fTagL7V5S+mBcDrAyMOr7AFgt03nTCwoSaNHatF+pBZRqpyWByM6sqD+
+IF2ChI4O3QTI5H6HkbQStEHLGuz8G6UX1MBRZ9LrRl0hUqdc2F/dcogrtV5NWkimXvMcTBYB
+9dTeuvj7eSma6JemqJAXfCARq54KjYwcZHapzczJe3Kgj1tSqD8O1Z8fsT16P2b0tUnDgNjX
+PB9UQGsP5NLK+vGnVBg5xJA5GqaN69lsUsvPaFFngmKmykQQNlDk7Q3yrAvMJWwRPQ1c8eMq
+kkrkKBal8Rq/EJaoaoBG6SFx2cSAD4GMxPEL7G3qJeNJPSLVSOP9j9nGIZ4SETeGIKCNVgVF
+16iXIxwflvgJP+/1qX3/Qin2YDYCeGvFmJiy5s0VFM1IdK+znrUu0apoCSYVia1D9NJGEdKf
+l7m/Odtxycq9VP1DKD/k4spWqsdpiBkJMch7XWfa7KF12bwV/tHP4KqJ02/xqhSqk+Ru4A9R
+LjQFWXs4o/Qwf5qF7FlGhpFridr0xdxtLVmK7bV4Hogldx1bKbkOpCL29BY8GA==
+=NjhU
+-----END PGP PUBLIC KEY BLOCK-----

scripts/setup/apt-repos/zulip/resolute.list

@@ -0,0 +1,6 @@
+deb [signed-by=/etc/apt/keyrings/pgdg.asc] http://apt.postgresql.org/pub/repos/apt/ resolute-pgdg main
+deb-src [signed-by=/etc/apt/keyrings/pgdg.asc] http://apt.postgresql.org/pub/repos/apt/ resolute-pgdg main
+
+# Not available for Ubuntu 26.04 yet
+#deb [signed-by=/etc/apt/keyrings/pgroonga-ppa.asc] http://ppa.launchpad.net/groonga/ppa/ubuntu resolute main
+#deb-src [signed-by=/etc/apt/keyrings/pgroonga-ppa.asc] http://ppa.launchpad.net/groonga/ppa/ubuntu resolute main

scripts/setup/apt-repos/zulip/strukturag.asc

@@ -0,0 +1,28 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQINBGY0lEwBEAD9qq6oOeRJvCi0fHNqJELvS8ykqfuXxWqn1B9DLt0VuTxgmaxG
+QNaG0B2pfrhoVDs+9wwfOuW0rtUtjWpccJjZaooTIEFSR4AKZAh3O4gUtvI7y95r
+q7+wWFct4RjbFyZOpmeinhIgnItTbkVjhjJvvQ66H4gUfNeJOoAE7CYgnMhZ1Sn8
+E6zxWJVLo0c5hXkkc4x4T1Aq02diO0Zg6T6HesxsCLeb5Sbjrca7ukUBGV3WFKAF
+QC22r8oCC4LiN5TX1WNQJjDSw0M2AY31AF+ebQvZGa1kcClWjqBCD7717+zbevPe
+/bx88Yr6eh+xuJ3ZJQ209phdeNq9RRkIDnap7TIVn21VoCQolH+adP9boxgG1iIz
+0gBx1KXBjRgV9owoYan4Mz9+c+5MMgJ975D+8GoRcNr1oycjOO/VVXw7ST9UiXqo
+TuOU2c1P8KmM3IUF41Igy5rd7MLy9uizKC/WY/lfLOH8pzT3LoWVmhMDJHooF9fq
+UC80GRIIaBM3f7sQH873zdqzaRPqtgJDxOAzLoZqUaFwLXuMfvYxhwMLWhe3b6Wp
+shCe3xifqZIHf9iZkqZ9b2TsstXervs0TlW9Xy8FT8RMuEzsX8DLNsUert+PzWpq
+C8Qse1AElMgSSxEsucWMbW5cnKj6ViEf10gy5VwlTgEgvt9KpZ2Icm43mwARAQAB
+tB1MYXVuY2hwYWQgUFBBIGZvciBzdHJ1a3R1ciBBR4kCTgQTAQoAOBYhBEfpK5wT
+yXuwgfX/wXF9tBYRYATJBQJmNJRMAhsDBQsJCAcCBhUKCQgLAgQWAgMBAh4BAheA
+AAoJEHF9tBYRYATJ/oMQAPVJWmzF+UKwkCA/kFeihXdA6OWl1BttFQpwTAAmxr9u
+SCg/w1ltIYqTfZcAjMUhY52oDix/NxzZZb/B4dxOkUM4gYmkbzq4wlT0QyrRgjNZ
+wflwdNjXGI9Ew9ta/3xvNE98LQWyV64t+Q8PihDD7VwOE/c2XVPvXQk8GWmsHJ48
+nGLaAPzn68kolv3XpkDB+ul07OLC7FXlADxWFOsobwula/zytFK2oyuQ1NiXoJ5I
+3+IA5sAp7O1mDOZ3m8RROfw56IbwHlNXAP8/REbqWEsXVNkyMf56snpsIr8FPlqY
+C74QbR/PHcb7vEdeWZgHmuIGNF37QPMx4XUGWjh/9NNi7AHGMa2cRsSF1xm1R99Q
+Zr1vmRPJM2z54gPq/JwY7WlcL6yEFMBL+WtNuiua5xxZ3rkMs956IuFS55zjrNeO
+MPmK77tdmFEui0GiTIyAJpoZIwp0AgYBdmP4/2Y6j6LN1sV+VkOzQz2CqDT87yO1
+BoN65jFuw61H3Y/+OBQMtoxamxs1m/Jjv0sEYf21nKgfukOTLecRWH1tXu0moTAf
+eEf0MOXzhH1teyfv/21haprwxJn0cG515oHB/JutFNKrXx3qtNH5QjRXIbz9EWAs
+u1mi4ERxSWAELxiJQWztCOhB5sI8Auq+Kk4uzbCutK6BQrUQORz2FGEJwEXy1f7g
+=76Dg
+-----END PGP PUBLIC KEY BLOCK-----

scripts/setup/apt-repos/zulip/trixie.list

@@ -0,0 +1,2 @@
+deb [signed-by=/etc/apt/keyrings/pgdg.asc] http://apt.postgresql.org/pub/repos/apt/ trixie-pgdg main
+deb-src [signed-by=/etc/apt/keyrings/pgdg.asc] http://apt.postgresql.org/pub/repos/apt/ trixie-pgdg main

scripts/setup/apt-repos/zulip/zulip.pref

@@ -0,0 +1,10 @@
+# libarrow1600 is missing on Debian 12
+Package: libgroonga0
+Pin: version 14.0.3-1
+Pin-Priority: 499
+
+
+# On bookworm, pull libheif1 from backports
+Package: libheif1
+Pin: release n=bookworm-backports
+Pin-Priority: 500

scripts/setup/compare-settings-to-template

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+
+import difflib
+import os
+import re
+import sys
+
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.append(BASE_DIR)
+from scripts.lib.setup_path import setup_path
+
+setup_path()
+
+
+current_path = "/etc/zulip/settings.py"
+if len(sys.argv) >= 2:
+    current_path = sys.argv[1]
+
+print(f"Reading current configuration from {current_path}...")
+with open(current_path) as f:
+    current_contents = f.read()
+
+import requests
+
+print("Fetching old versions of setting templates from Github...")
+templ = {}
+url: str | None = "https://api.github.com/repos/zulip/zulip/tags?per_page=100"
+
+while url is not None:
+    resp = requests.get(url)
+    if resp.status_code != 200:
+        print(resp.content)
+        sys.exit(1)
+
+    url = resp.links.get("next", {}).get("url")
+    for tag in [t["name"] for t in resp.json()]:
+        if not re.match(r"^\d+\.\d+(\.\d+)?$", tag):
+            continue
+
+        filename = "zproject/prod_settings_template.py"
+        if [int(x) for x in tag.split(".")] < [1, 4]:
+            filename = "zproject/local_settings_template.py"
+
+        print(f"  - {tag}")
+        resp = requests.get(
+            f"https://raw.githubusercontent.com/zulip/zulip/{tag}/{filename}",
+        )
+        if resp.status_code == 200:
+            templ[tag] = resp.content.decode()
+        else:
+            print("Failure: ")
+            print(resp)
+
+print("Computing minimal difference...")
+sequence_matchers = {}
+for version, contents in templ.items():
+    sm = difflib.SequenceMatcher(None, current_contents, contents)
+    sequence_matchers[version] = sm
+
+likely = sorted(sequence_matchers.keys(), key=lambda v: sequence_matchers[v].ratio(), reverse=True)
+print(f"Most likely version: {likely[0]}")
+
+
+diff = difflib.unified_diff(
+    templ[likely[0]].splitlines(keepends=True),
+    current_contents.splitlines(keepends=True),
+    fromfile=f"zulip-server-{version}/zproject/prod_settings_template.py",
+    tofile=current_path,
+)
+
+print()
+print("".join(diff), end="")

scripts/setup/configure-rabbitmq

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+#
+# Delete the "guest" default user and replace it with a Zulip user
+# with a real password
+set -eu
+
+# If the RabbitMQ distribution cookie is insecure, reset it and
+# restart RabbitMQ.
+"$(dirname "$0")/generate-rabbitmq-cookie"
+
+RABBITMQ_USERNAME=$("$(dirname "$0")/../get-django-setting" RABBITMQ_USERNAME)
+RABBITMQ_PASSWORD=$("$(dirname "$0")/../get-django-setting" RABBITMQ_PASSWORD)
+RABBITMQ_VHOST=$("$(dirname "$0")/../get-django-setting" RABBITMQ_VHOST)
+
+tries=0
+while ((tries < 30)) && ! rabbitmqctl ping -q 2>/dev/null; do
+    ((tries++)) || echo "Waiting for RabbitMQ to start up..."
+    sleep 1
+done
+rabbitmqctl await_startup
+
+rabbitmqctl delete_user "$RABBITMQ_USERNAME" || true
+rabbitmqctl delete_user zulip || true
+rabbitmqctl delete_user guest || true
+rabbitmqctl add_user "$RABBITMQ_USERNAME" "$RABBITMQ_PASSWORD"
+rabbitmqctl set_user_tags "$RABBITMQ_USERNAME" administrator
+if ! rabbitmqctl list_vhosts --no-table-headers --quiet | grep -qx "$RABBITMQ_VHOST"; then
+    rabbitmqctl add_vhost "$RABBITMQ_VHOST"
+fi
+rabbitmqctl set_permissions -p "$RABBITMQ_VHOST" "$RABBITMQ_USERNAME" '.*' '.*' '.*'

scripts/setup/create-database

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+set -e
+
+if [ "$EUID" -ne 0 ]; then
+    echo "Error: This script must be run as root" >&2
+    exit 1
+fi
+set -x
+
+# What user should we use for connecting to the database
+POSTGRES_USER="${POSTGRES_USER:-postgres}"
+
+# What database name and username to use when connecting to the database
+DATABASE_NAME=$(crudini --get /etc/zulip/zulip.conf postgresql database_name 2>/dev/null || echo zulip)
+DATABASE_USER=$(crudini --get /etc/zulip/zulip.conf postgresql database_user 2>/dev/null || echo zulip)
+
+if [ "$(su "$POSTGRES_USER" -c "cd / && psql -v ON_ERROR_STOP=1 -Atc \"SELECT 1 FROM pg_database WHERE datname='$DATABASE_NAME';\"")" = "1" ]; then
+    records="$(su "$POSTGRES_USER" -c "cd / && psql -v ON_ERROR_STOP=1 -Atc 'SELECT COUNT(*) FROM $DATABASE_NAME.zerver_message;' $DATABASE_USER" || echo 0)"
+    if [ "$records" -gt 200 ]; then
+        set +x
+        echo "WARNING: This will delete your Zulip database which currently contains $records messages."
+        read -p "Do you want to proceed? [y/N] " -r
+        echo
+        if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+            exit 1
+        fi
+        set -x
+    fi
+fi
+
+# Shut down all services to ensure a quiescent state.
+if [ -e "/home/zulip/deployments" ]; then
+    su zulip -c "/home/zulip/deployments/current/scripts/stop-server"
+fi
+
+# Drop any open connections to any old database.
+# Send the script via stdin in case the postgres user lacks permission to read it.
+su -s /usr/bin/env - -- "$POSTGRES_USER" \
+    bash -s - zulip zulip_base <"$(dirname "$0")/terminate-psql-sessions"
+
+(
+    cd / # Make sure the current working directory is readable by postgres
+    su "$POSTGRES_USER" -c "psql -v ON_ERROR_STOP=1 -v dbname=$DATABASE_NAME -v dbuser=$DATABASE_USER -e"
+) <"$(dirname "$0")/create-db.sql"
+
+# Set a postgres password if the postgres username is not "zulip".
+# When the username is zulip, we rely on running as the zulip system
+# user for authentication via postgres' peer authentication.
+if [ "$DATABASE_USER" != "zulip" ]; then
+    PASSWORD=$(crudini --get /etc/zulip/zulip-secrets.conf secrets postgres_password)
+    su "$POSTGRES_USER" -c "psql -v ON_ERROR_STOP=1  -e postgres" <<EOF
+ALTER ROLE $DATABASE_USER PASSWORD '$PASSWORD';
+EOF
+fi
+
+# Clear memcached to avoid contamination from previous database state
+if [ -e "/home/zulip/deployments" ]; then
+    /home/zulip/deployments/current/scripts/setup/flush-memcached
+fi
+
+echo "Database created"

scripts/setup/create-db.sql

@@ -0,0 +1,20 @@
+\set dbuser :dbuser
+SELECT CASE WHEN :'dbuser' = ':dbuser' THEN 'zulip' ELSE :'dbuser' END AS dbuser \gset
+\set dbname :dbname
+SELECT CASE WHEN :'dbname' = ':dbname' THEN 'zulip' ELSE :'dbname' END AS dbname \gset
+
+\connect postgres
+DROP DATABASE IF EXISTS :"dbname";
+SELECT format($$BEGIN
+    CREATE USER %I;
+EXCEPTION WHEN duplicate_object THEN
+    RAISE NOTICE 'user already exists';
+END$$, :'dbuser') AS code \gset
+DO :'code';
+CREATE DATABASE :"dbname"
+    OWNER=:dbuser
+    ENCODING=UTF8
+    LC_COLLATE='C.UTF-8'
+    LC_CTYPE='C.UTF-8'
+    TEMPLATE=template0;
+\connect :"dbname"

scripts/setup/create-pgroonga.sql

@@ -0,0 +1,2 @@
+\connect zulip
+CREATE EXTENSION pgroonga;

scripts/setup/flush-memcached

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+import os
+import sys
+
+BASE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../..")
+sys.path.append(BASE_DIR)
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "zproject.settings")
+
+from scripts.lib.setup_path import setup_path
+
+setup_path()
+
+import bmemcached
+from django.conf import settings
+
+cache = settings.CACHES["default"]
+match cache["BACKEND"]:
+    case "django.core.cache.backends.locmem.LocMemCache":
+        pass
+    case "zerver.lib.singleton_bmemcached.SingletonBMemcached":
+        assert isinstance(cache["OPTIONS"], dict)
+        client = bmemcached.Client((cache["LOCATION"],), **cache["OPTIONS"])
+        try:
+            client.flush_all()
+        finally:
+            client.disconnect_all()
+    case _:
+        raise ValueError(f"Unrecognized cache backend {cache['BACKEND']}")

scripts/setup/generate-rabbitmq-cookie

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+#
+# rabbitmq sets an insecure "magic cookie" which is used for auth;
+# reset it to be longer.
+set -eu
+
+cookiefile=/var/lib/rabbitmq/.erlang.cookie
+# If the RabbitMQ distribution cookie is insecure, reset it
+if [ ! -f "$cookiefile" ] || ! size="$(wc -c "$cookiefile")" || [ "${size%% *}" -le 20 ]; then
+    running=0
+    if service rabbitmq-server status >/dev/null; then
+        running=1
+        service rabbitmq-server stop
+    fi
+
+    echo "Setting a more secure RabbitMQ distribution magic cookie"
+    cookie="$(LC_ALL=C tr -dc '[:alnum:]' </dev/urandom | head -c255)"
+    [ "${#cookie}" -eq 255 ] # make sure tr wasn’t OOM-killed
+    tmpfile="$(mktemp "$cookiefile.XXXXXXXXXX")"
+    chown rabbitmq: "$tmpfile"
+    printf '%s' "$cookie" >"$tmpfile"
+    mv "$tmpfile" "$cookiefile"
+
+    if [ "$running" == "1" ]; then
+        service rabbitmq-server start
+    fi
+fi

scripts/setup/generate-self-signed-cert

@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+set -e
+
+usage() {
+    echo "usage: $0 [--force] [--exists-ok] EXTERNAL_HOST" >&2
+    exit 1
+}
+
+args="$(getopt -o '' --long help,force,exists-ok -- "$@")"
+eval "set -- $args"
+while true; do
+    case "$1" in
+        --help) usage ;;
+        --force)
+            FORCE=1
+            shift
+            ;;
+        --exists-ok)
+            EXISTS_OK=1
+            shift
+            ;;
+        --)
+            shift
+            break
+            ;;
+        *) usage ;;
+    esac
+done
+EXTERNAL_HOST="$1"
+
+if [ -z "$EXTERNAL_HOST" ]; then
+    usage
+fi
+
+if [ "$EUID" -ne 0 ]; then
+    echo "error: this script must be run as root" >&2
+    exit 1
+fi
+
+set -x
+
+is_redhat=false
+if [ -e /etc/redhat-release ]; then
+    is_redhat=true
+    SSLDIR=/etc/pki/tls
+else
+    SSLDIR=/etc/ssl
+fi
+KEYFILE=$SSLDIR/private/zulip.key
+CERTFILE=$SSLDIR/certs/zulip.combined-chain.crt
+
+if [ -n "$EXISTS_OK" ] && [ -e "$KEYFILE" ] && [ -e "$CERTFILE" ]; then
+    exit 0
+fi
+
+if [ -z "$FORCE" ] && { [ -e "$KEYFILE" ] || [ -e "$CERTFILE" ]; }; then
+    echo "$0: certificate and/or key already exists; use --force to overwrite." >&2
+    exit 1
+fi
+rm -f "$KEYFILE" "$CERTFILE"
+
+if [[ "$EXTERNAL_HOST" =~ ^(([0-9]+\.){3}[0-9]+)(:[0-9]+)?$ ]]; then
+    subjectAltName="IP:${BASH_REMATCH[1]}" # IPv4 address
+elif [[ "$EXTERNAL_HOST" =~ ^\[([^][]*)\](:[0-9]+)?$ ]]; then
+    subjectAltName="IP:${BASH_REMATCH[1]}" # IPv6 address
+elif [[ "$EXTERNAL_HOST" =~ ^([^:]+)(:[0-9]+)?$ ]]; then
+    subjectAltName="DNS:${BASH_REMATCH[1]}"
+else
+    echo "$0: invalid host $EXTERNAL_HOST" >&2
+    exit 1
+fi
+
+config="$(mktemp)" || exit 1
+trap 'rm -f "$config"' EXIT
+
+cat >"$config" <<EOF
+# Based on /usr/share/ssl-cert/ssleay.cnf from Debian's \`ssl-cert\`
+# package, which is used for the system's snakeoil cert in /etc/ssl/.
+
+[ req ]
+default_bits        = 2048
+distinguished_name  = req_distinguished_name
+prompt              = no
+policy              = policy_anything
+req_extensions      = v3_req
+x509_extensions     = v3_req
+
+[ req_distinguished_name ]
+commonName          = $EXTERNAL_HOST
+
+[ v3_req ]
+basicConstraints    = CA:FALSE
+subjectAltName      = $subjectAltName
+EOF
+
+if [ "$is_redhat" = true ]; then
+    yum install -y openssl
+else
+    apt-get install -y openssl
+fi
+
+# Based on /usr/sbin/make-ssl-cert from Debian's `ssl-cert` package.
+openssl req -new -x509 \
+    -config "$config" -days 3650 -nodes -sha256 \
+    -out "$CERTFILE" -keyout "$KEYFILE"
+
+chmod 644 "$CERTFILE"
+chmod 640 "$KEYFILE"

scripts/setup/generate_secrets.py

@@ -0,0 +1,228 @@
+#!/usr/bin/env python3
+# This tools generates /etc/zulip/zulip-secrets.conf
+import json
+import os
+import sys
+from contextlib import suppress
+
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.append(BASE_DIR)
+from scripts.lib.setup_path import setup_path
+from scripts.lib.zulip_tools import get_config, get_config_file
+
+setup_path()
+
+os.environ["DISABLE_MANDATORY_SECRET_CHECK"] = "True"
+os.environ["DJANGO_SETTINGS_MODULE"] = "zproject.settings"
+
+import argparse
+import configparser
+import uuid
+
+from nacl.encoding import Base64Encoder
+from nacl.public import PrivateKey
+
+os.chdir(os.path.join(os.path.dirname(__file__), "..", ".."))
+
+# Standard, 64-bit tokens
+AUTOGENERATED_SETTINGS = [
+    "avatar_salt",
+    "rabbitmq_password",
+    "shared_secret",
+]
+
+
+def random_string(cnt: int) -> str:
+    # We do in-function imports so that we only do the expensive work
+    # of importing cryptography modules when necessary.
+    #
+    # This helps optimize noop provision performance.
+    from django.utils.crypto import get_random_string
+
+    return get_random_string(cnt)
+
+
+def random_token() -> str:
+    # We do in-function imports so that we only do the expensive work
+    # of importing cryptography modules when necessary.
+    #
+    # This helps optimize noop provision performance.
+    import secrets
+
+    return secrets.token_hex(32)
+
+
+def generate_django_secretkey() -> str:
+    """Secret key generation taken from Django's startproject.py"""
+
+    # We do in-function imports so that we only do the expensive work
+    # of importing cryptography modules when necessary.
+    #
+    # This helps optimize noop provision performance.
+    from django.utils.crypto import get_random_string
+
+    chars = "abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)"
+    return get_random_string(50, chars)
+
+
+def get_old_conf(output_filename: str) -> dict[str, str]:
+    if not os.path.exists(output_filename) or os.path.getsize(output_filename) == 0:
+        return {}
+
+    secrets_file = configparser.RawConfigParser()
+    secrets_file.read(output_filename)
+
+    return dict(secrets_file.items("secrets"))
+
+
+def generate_secrets(development: bool = False) -> None:
+    if development:
+        OUTPUT_SETTINGS_FILENAME = "zproject/dev-secrets.conf"
+    else:
+        OUTPUT_SETTINGS_FILENAME = "/etc/zulip/zulip-secrets.conf"
+    current_conf = get_old_conf(OUTPUT_SETTINGS_FILENAME)
+
+    lines: list[str] = []
+    if len(current_conf) == 0:
+        lines = ["[secrets]\n"]
+
+    def need_secret(name: str) -> bool:
+        return name not in current_conf
+
+    def add_secret(name: str, value: str) -> None:
+        lines.append(f"{name} = {value}\n")
+        current_conf[name] = value
+
+    for name in AUTOGENERATED_SETTINGS:
+        if need_secret(name):
+            add_secret(name, random_token())
+
+    # These secrets are exclusive to a Zulip development environment.
+    # We use PostgreSQL peer authentication by default in production,
+    # and initial_password_salt is used to generate passwords for the
+    # test/development database users.  See `manage.py
+    # print_initial_password`.
+    if development and need_secret("initial_password_salt"):
+        add_secret("initial_password_salt", random_token())
+    if development and need_secret("local_database_password"):
+        add_secret("local_database_password", random_token())
+
+    # We only need a secret if the database username does not match
+    # the OS username, as identd auth works in that case.
+    if get_config(
+        get_config_file(), "postgresql", "database_user", "zulip"
+    ) != "zulip" and need_secret("postgres_password"):
+        add_secret("postgres_password", random_token())
+
+    # The core Django SECRET_KEY setting, used by Django internally to
+    # secure sessions.  If this gets changed, all users will be logged out.
+    if need_secret("secret_key"):
+        secret_key = generate_django_secretkey()
+        add_secret("secret_key", secret_key)
+        # To prevent Django ImproperlyConfigured error
+        from zproject import settings
+
+        settings.SECRET_KEY = secret_key
+
+    # Secret key for the Camo HTTPS proxy.
+    if need_secret("camo_key"):
+        add_secret("camo_key", random_string(64))
+
+    # We enable Altcha in development
+    if development and need_secret("altcha_hmac"):
+        add_secret("altcha_hmac", random_token())
+
+    if not development:
+        # The memcached_password and redis_password secrets are only
+        # required/relevant in production.
+
+        # Password for authentication to memcached.
+        if need_secret("memcached_password"):
+            # We defer importing settings unless we need it, because
+            # importing settings is expensive (mostly because of
+            # django-auth-ldap) and we want the noop case to be fast.
+            from zproject import settings
+
+            if settings.MEMCACHED_LOCATION == "127.0.0.1:11211":
+                add_secret("memcached_password", random_token())
+
+        # Password for authentication to Redis.
+        if need_secret("redis_password"):
+            # We defer importing settings unless we need it, because
+            # importing settings is expensive (mostly because of
+            # django-auth-ldap) and we want the noop case to be fast.
+            from zproject import settings
+
+            if settings.REDIS_HOST == "127.0.0.1":
+                # To prevent Puppet from restarting Redis, which would lose
+                # data because we configured Redis to disable persistence, set
+                # the Redis password on the running server and edit the config
+                # file directly.
+
+                import redis
+
+                from zerver.lib.redis_utils import get_redis_client
+
+                redis_password = random_token()
+
+                for filename in ["/etc/redis/zuli-redis.conf", "/etc/redis/zulip-redis.conf"]:
+                    if os.path.exists(filename):
+                        with open(filename, "a") as f:
+                            f.write(
+                                "# Set a Redis password based on zulip-secrets.conf\n"
+                                f"requirepass '{redis_password}'\n",
+                            )
+                        break
+
+                with suppress(redis.exceptions.ConnectionError):
+                    get_redis_client().config_set("requirepass", redis_password)
+
+                add_secret("redis_password", redis_password)
+
+    # Random id and secret used to identify this installation when
+    # accessing the Zulip mobile push notifications service.
+    # * zulip_org_key is generated using os.urandom().
+    # * zulip_org_id only needs to be unique, so we use a UUID.
+    if need_secret("zulip_org_key"):
+        add_secret("zulip_org_key", random_string(64))
+    if need_secret("zulip_org_id"):
+        add_secret("zulip_org_id", str(uuid.uuid4()))
+
+    if development and need_secret("push_registration_encryption_keys"):
+        # 'settings.ZILENCER_ENABLED' would be a better check than
+        # 'development' for whether we need push bouncer secrets,
+        # but we're trying to avoid importing settings.
+        private_key = PrivateKey.generate()
+        private_key_str = Base64Encoder.encode(bytes(private_key)).decode("utf-8")
+        public_key_str = Base64Encoder.encode(bytes(private_key.public_key)).decode("utf-8")
+        add_secret(
+            "push_registration_encryption_keys", json.dumps({public_key_str: private_key_str})
+        )
+
+    if len(lines) == 0:
+        print("generate_secrets: No new secrets to generate.")
+        return
+
+    with open(OUTPUT_SETTINGS_FILENAME, "a") as f:
+        # Write a newline at the start, in case there was no newline at
+        # the end of the file due to human editing.
+        f.write("\n" + "".join(lines))
+
+    print(f"Generated new secrets in {OUTPUT_SETTINGS_FILENAME}.")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    group = parser.add_mutually_exclusive_group(required=True)
+    group.add_argument(
+        "--development", action="store_true", help="For setting up the developer env for zulip"
+    )
+    group.add_argument(
+        "--production",
+        action="store_false",
+        dest="development",
+        help="For setting up the production env for zulip",
+    )
+    results = parser.parse_args()
+
+    generate_secrets(results.development)

scripts/setup/initialize-database

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+set -e
+
+usage() {
+    echo "usage: initialize-database [--quiet]" >&2
+    exit 1
+}
+
+args="$(getopt -o '' --long help,quiet -- "$@")"
+eval "set -- $args"
+while true; do
+    case "$1" in
+        --help) usage ;;
+        --quiet)
+            QUIET=1
+            shift
+            ;;
+        --)
+            shift
+            break
+            ;;
+        *) usage ;;
+    esac
+done
+
+if [ "$#" -gt 0 ]; then
+    usage
+fi
+
+set -x
+
+# Change to root directory of the checkout that we're running from
+THIS_DIR="$(dirname "$(readlink -f "$0")")"
+cd "$THIS_DIR/../.."
+
+./manage.py migrate --noinput
+./manage.py createcachetable third_party_api_results
+
+# Check if the supervisor socket exists.  If not, it could be:
+#
+# A) A normal installation went bad (supervisor hasn't started)
+# B) We are in a Docker container and don't have supervisor running
+#
+# In either case, it doesn't make sense to restart supervisor jobs
+if [ -e "/var/run/supervisor.sock" ]; then
+    ./scripts/restart-server
+fi
+
+set +x
+if [ -z "$QUIET" ]; then
+    echo "Congratulations!  You have successfully configured your Zulip database."
+    echo "If you haven't already, you should configure email in /etc/zulip/settings.py."
+    echo
+    echo "Next, run as the zulip user (use 'su zulip' if needed):"
+    echo
+    echo "    /home/zulip/deployments/current/manage.py generate_realm_creation_link"
+    echo
+    echo "This generates a secure, single-use link that you can use to set up "
+    echo "a Zulip organization from the convenience of your web browser."
+fi

scripts/setup/install

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+#
+# Thin wrapper around the actual install script (scripts/lib/install).
+# The purpose of this wrapper is to log the full install script output
+# to /var/log/zulip/install.log for easy debugging.
+
+set -e
+if [ "$EUID" -ne 0 ]; then
+    echo "Error: The installation script must be run as root." >&2
+    exit 1
+fi
+umask 022
+mkdir -p /var/log/zulip
+
+"$(dirname "$(dirname "$0")")/lib/install" "$@" 2>&1 | tee -a /var/log/zulip/install.log
+failed=${PIPESTATUS[0]}
+
+if [ "$failed" -ne 0 ]; then
+    echo -e '\033[0;31m'
+    echo "Zulip installation failed (exit code $failed)!"
+    echo
+    echo -n "The install process is designed to be idempotent, so you can retry "
+    echo -n "after resolving whatever issue caused the failure (there should be a traceback above). "
+    echo -n "A log of this installation is available in /var/log/zulip/install.log"
+    echo -e '\033[0m'
+    exit "$failed"
+fi

scripts/setup/pgroonga-config

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+set -eux
+
+dbversion=$(crudini --get /etc/zulip/zulip.conf postgresql version)
+dbname=$(crudini --get /etc/zulip/zulip.conf postgresql database_name 2>/dev/null || echo zulip)
+
+if ! su postgres -c "psql -At -c 'SELECT datname FROM pg_database WHERE NOT datistemplate'" | grep -Fx "$dbname"; then
+    echo "No database to install into!"
+    exit 0
+fi
+
+sharedir="${1:-/usr/share/postgresql/$dbversion}"
+applied_file="$sharedir/pgroonga_setup.sql.applied"
+
+installed_version=$(dpkg-query --show --showformat='${Version}' "postgresql-$dbversion-pgdg-pgroonga")
+
+if [ ! -f "$applied_file" ]; then
+    sql="CREATE EXTENSION PGROONGA"
+else
+    sql="ALTER EXTENSION pgroonga UPDATE"
+fi
+
+echo "$sql" | su postgres -c "psql -v ON_ERROR_STOP=1 $dbname"
+
+echo "$installed_version" >"$applied_file"

scripts/setup/reindex-textual-data

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+
+import argparse
+import logging
+import os
+import sys
+import time
+
+sys.path.append(os.path.join(os.path.dirname(__file__), "../.."))
+from scripts.lib.setup_path import setup_path
+
+setup_path()
+
+import psycopg2
+import psycopg2.extensions
+from psycopg2.sql import SQL, Identifier
+
+from scripts.lib.zulip_tools import su_to_zulip
+
+su_to_zulip()
+os.environ["DJANGO_SETTINGS_MODULE"] = "zproject.settings"
+from django.conf import settings
+
+parser = argparse.ArgumentParser(
+    description="Reindex all text indexes, for glibc upgrades.  This will take a write lock on every table, unless --concurrently is passed."
+)
+parser.add_argument(
+    "--concurrently", action="store_true", help="reindex concurrently, on Pg ≥ 11; takes longer"
+)
+parser.add_argument("--force", action="store_true", help="run the reindexing")
+options = parser.parse_args()
+
+logging.Formatter.converter = time.gmtime
+logging.basicConfig(format="%(asctime)s %(levelname)s: %(message)s")
+logger = logging.getLogger("reindex-textual-data")
+logger.setLevel(logging.DEBUG)
+
+pg_args = {}
+pg_args["host"] = settings.DATABASES["default"]["HOST"]
+pg_args["port"] = settings.DATABASES["default"].get("PORT")
+pg_args["password"] = settings.DATABASES["default"].get("PASSWORD")
+pg_args["user"] = settings.DATABASES["default"]["USER"]
+pg_args["dbname"] = settings.DATABASES["default"]["NAME"]
+pg_args["sslmode"] = settings.DATABASES["default"]["OPTIONS"].get("sslmode")
+pg_args["connect_timeout"] = "600"
+
+# connection_factory=None lets mypy understand the return type
+conn = psycopg2.connect(connection_factory=None, **pg_args)
+conn.autocommit = True
+
+cursor = conn.cursor()
+cursor.execute(
+    """
+SELECT
+        irel.relname AS index_name,
+        trel.relname AS table_name,
+        pg_size_pretty(pg_table_size(i.indrelid)) AS size
+FROM
+        pg_index AS i
+        JOIN pg_class AS trel ON trel.oid = i.indrelid
+        JOIN pg_namespace AS tnsp ON trel.relnamespace = tnsp.oid
+        JOIN pg_class AS irel ON irel.oid = i.indexrelid
+        JOIN pg_attribute AS a ON a.attrelid = i.indexrelid
+        WHERE tnsp.nspname = 'zulip'
+        AND a.attcollation != 0
+GROUP BY 1, 2, i.indrelid
+ORDER BY pg_table_size(i.indrelid) ASC
+"""
+)
+reindex_targets = cursor.fetchall()
+
+if not options.force:
+    print("Would reindex the following indexes:")
+    for index, _, _ in reindex_targets:
+        print(f"  {index}")
+    print(
+        """
+
+Re-run with --force to reindex these.  Without --concurrently, these
+index rebuilds will TAKE A WRITE LOCK on their respective tables,
+which means that production traffic will be affected.
+
+On PostgreSQL 11 and above, you can pass --concurrently, which will do
+the index rebuilds without taking such a lock. We recommend only using
+--concurrently if the Zulip server will be serving traffic while
+running this tool, because concurrent reindexing takes longer to
+complete even if the server is not running.
+"""
+    )
+    sys.exit(0)
+
+had_failures = False
+for index_name, table_name, index_size in reindex_targets:
+    if options.concurrently:
+        command = SQL("REINDEX INDEX CONCURRENTLY {}").format(Identifier(index_name))
+    else:
+        command = SQL("REINDEX INDEX {}").format(Identifier(index_name))
+
+    logger.info("%s -- on %s, %s", command, table_name, index_size)
+    try:
+        cursor.execute(command)
+    except psycopg2.OperationalError as e:
+        logger.warning("Failed to reindex %s: %s", index_name, e)
+        had_failures = True
+
+if not had_failures:
+    sys.exit(0)
+
+print(
+    """
+=========================> REINDEXING FAILED <=========================
+
+Reindexing failed on one or more indexes; this is generally caused
+by duplicate rows that had been incorrectly inserted because of
+corrupted database indexes.  This happens, for example, when glibc was
+upgraded to 2.28 or higher, which has a different ordering of text
+("collation") than earlier versions.  Because of the new ordering,
+entries in the indexes are not found (because they exist in a
+different place), which results in duplicates being created in the
+database for columns which should be unique.
+
+There is no generalized tool to fix these duplicate rows, but in most cases
+the database can be repaired with some manual work.  We are using
+this chat.zulip.org for support around such repairs:
+
+  https://chat.zulip.org/#narrow/channel/31-production-help/topic/database.20corruption
+
+This tool has regenerated all database indexes that do not have duplicate rows.
+Once you have manually repaired duplicate rows, you can rerun this command
+to regenerate the rest.
+
+It is reasonable to run your Zulip server to avoid downtime while you
+plan the manual repair work. Additional duplicate rows may be created
+for any corrupted index that has not yet been regenerated. But any new
+duplicate rows will be similar to existing duplicate rows that you
+already need to manually repair.
+    """
+)
+sys.exit(1)

scripts/setup/restore-backup

@@ -0,0 +1,187 @@
+#!/usr/bin/env python3
+import argparse
+import os
+import re
+import subprocess
+import sys
+import tempfile
+from typing import IO
+
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.append(BASE_DIR)
+from scripts.lib.zulip_tools import assert_running_as_root, get_postgres_pwent, run, su_to_zulip
+
+POSTGRES_PWENT = get_postgres_pwent()
+
+parser = argparse.ArgumentParser()
+parser.add_argument("tarball", help="Filename of input tarball")
+parser.add_argument(
+    "--keep-settings", help="Do not overwrite local /etc/zulip/settings.py", action="store_true"
+)
+parser.add_argument(
+    "--keep-zulipconf", help="Do not overwrite local /etc/zulip/zulip.conf", action="store_true"
+)
+
+
+def restore_backup(tarball_file: IO[bytes], keep_settings: bool, keep_zulipconf: bool) -> None:
+    assert_running_as_root()
+    su_to_zulip(save_suid=True)
+
+    from scripts.lib.setup_path import setup_path
+
+    setup_path()
+
+    # First, we unpack the /etc/zulip configuration, so we know how
+    # this server is supposed to be configured (and can import
+    # /etc/zulip/settings.py via `from django.conf import settings`,
+    # next).  Ignore errors if zulip-backup/settings is not present
+    # (E.g. because this is a development backup).
+    tarball_file.seek(0, 0)
+    excludes = []
+    if keep_settings:
+        excludes += ["--exclude=settings.py"]
+    if keep_zulipconf:
+        excludes += ["--exclude=zulip.conf"]
+    run(
+        [
+            "tar",
+            "--directory=/etc/zulip",
+            "--keep-directory-symlink",
+            *excludes,
+            "--strip-components=2",
+            "-xz",
+            "zulip-backup/settings",
+        ],
+        stdin=tarball_file,
+    )
+
+    os.environ["DJANGO_SETTINGS_MODULE"] = "zproject.settings"
+    from django.conf import settings
+
+    paths = [
+        # zproject will only be present for development environment backups.
+        ("zproject", os.path.join(settings.DEPLOY_ROOT, "zproject")),
+    ]
+    if settings.LOCAL_UPLOADS_DIR is not None:
+        # We only need to restore LOCAL_UPLOADS_DIR if the system is
+        # configured to locally host uploads.
+        paths.append(("uploads", os.path.join(settings.DEPLOY_ROOT, settings.LOCAL_UPLOADS_DIR)))
+
+    with tempfile.TemporaryDirectory(prefix="zulip-restore-backup-") as tmp:
+        uid = os.getuid()
+        gid = os.getgid()
+        os.setresuid(0, 0, 0)
+        for name, path in paths:
+            os.makedirs(path, exist_ok=True)
+            os.chown(path, uid, gid)
+        os.setresuid(uid, uid, 0)
+
+        assert not any("|" in name or "|" in path for name, path in paths)
+        transform_args = [
+            r"--transform=s|^zulip-backup/{}(/.*)?$|{}\1|x".format(
+                re.escape(name),
+                path.replace("\\", r"\\"),
+            )
+            for name, path in paths
+        ]
+
+        os.mkdir(os.path.join(tmp, "zulip-backup"))
+        tarball_file.seek(0, 0)
+        run(
+            ["tar", "-C", tmp, "--keep-directory-symlink", *transform_args, "-xPz"],
+            stdin=tarball_file,
+        )
+
+        # Now, extract the database backup, destroy the old
+        # database, and create a new, empty database.
+        db = settings.DATABASES["default"]
+        assert isinstance(db["NAME"], str)
+        db_dir = os.path.join(tmp, "zulip-backup", "database")
+        os.setresuid(0, 0, 0)
+        run(["chown", "-R", POSTGRES_PWENT.pw_name, "--", tmp])
+        # Open the terminate-psql-sessions script while still running
+        # as root, since the postgres user may lack permission to read
+        # it -- e.g. because /home/zulip is not world-readable, which
+        # is the default for home directories on Debian 13 and Ubuntu
+        # 21.04 and newer.
+        terminate_psql_sessions_fd = os.open(
+            os.path.join(settings.DEPLOY_ROOT, "scripts", "setup", "terminate-psql-sessions"),
+            os.O_RDONLY,
+        )
+        os.setresuid(POSTGRES_PWENT.pw_uid, POSTGRES_PWENT.pw_uid, 0)
+
+        postgresql_env = dict(os.environ)
+        if db["HOST"] not in ["", "localhost"]:
+            postgresql_env["PGHOST"] = db["HOST"]
+            if "PORT" in db:
+                postgresql_env["PGPORT"] = str(db["PORT"])
+            postgresql_env["PGUSER"] = db["USER"]
+            if "PASSWORD" in db:
+                postgresql_env["PGPASSWORD"] = db["PASSWORD"]
+
+        run(
+            ["bash", "-s", "-", db["NAME"]],
+            stdin=terminate_psql_sessions_fd,
+            env=postgresql_env,
+        )
+        os.close(terminate_psql_sessions_fd)
+        run(["dropdb", "--if-exists", "--", db["NAME"]], cwd="/", env=postgresql_env)
+        run(
+            ["createdb", "--owner=zulip", "--template=template0", "--", db["NAME"]],
+            cwd="/",
+            env=postgresql_env,
+        )
+        os.setresuid(0, 0, 0)
+
+        if settings.PRODUCTION:
+            # In case we are restoring a backup from an older Zulip
+            # version, there may be new secrets to generate.
+            run(
+                [
+                    os.path.join(settings.DEPLOY_ROOT, "scripts", "setup", "generate_secrets.py"),
+                    "--production",
+                ]
+            )
+
+            # If there is a local RabbitMQ, we need to reconfigure it
+            # to ensure the RabbitMQ password matches the value in the
+            # restored zulip-secrets.conf.  We need to be careful to
+            # only do this if RabbitMQ is configured to run locally on
+            # the system.
+            rabbitmq_host = subprocess.check_output(
+                [
+                    os.path.join(settings.DEPLOY_ROOT, "scripts", "get-django-setting"),
+                    "RABBITMQ_HOST",
+                ],
+                text=True,
+            ).strip()
+            if rabbitmq_host in ["127.0.0.1", "::1", "localhost", "localhost6"]:
+                run([os.path.join(settings.DEPLOY_ROOT, "scripts", "setup", "configure-rabbitmq")])
+
+            # In production, we also need to do a `zulip-puppet-apply`
+            # in order to apply any configuration from
+            # /etc/zulip/zulip.conf to this system, since it was
+            # originally installed without the restored copy of that
+            # file.
+            run([os.path.join(settings.DEPLOY_ROOT, "scripts", "zulip-puppet-apply"), "-f"])
+
+        # Now, restore the database backup using pg_restore.  This
+        # needs to run after zulip-puppet-apply to ensure full-text
+        # search extensions are available and installed.
+        os.setresuid(POSTGRES_PWENT.pw_uid, POSTGRES_PWENT.pw_uid, 0)
+        run(["pg_restore", "--dbname=" + db["NAME"], "--", db_dir], cwd="/", env=postgresql_env)
+        os.setresuid(0, 0, 0)
+        run(["chown", "-R", str(uid), "--", tmp])
+        os.setresuid(uid, uid, 0)
+
+        if settings.PRODUCTION:
+            run([os.path.join(settings.DEPLOY_ROOT, "scripts", "restart-server")])
+
+        run([os.path.join(settings.DEPLOY_ROOT, "scripts", "setup", "flush-memcached")])
+
+
+if __name__ == "__main__":
+    args = parser.parse_args()
+
+    with open(args.tarball, "rb") as tarball_file:
+        restore_backup(tarball_file, args.keep_settings, args.keep_zulipconf)

scripts/setup/setup-certbot

@@ -0,0 +1,112 @@
+#!/usr/bin/env bash
+
+set -e
+
+usage() {
+    cat <<EOF >&2
+Usage: $0 --email=admin@example.com [--method={webroot|standalone}] \
+hostname.example.com [another.example.com]
+EOF
+    exit 1
+}
+
+if [ "$EUID" -ne 0 ]; then
+    echo "Error: This script must be run as root" >&2
+    exit 1
+fi
+
+method=webroot
+args="$(getopt -o '' --long help,email:,method:,agree-tos -n "$0" -- "$@")"
+eval "set -- $args"
+agree_tos=()
+while true; do
+    case "$1" in
+        --email)
+            EMAIL="$2"
+            shift
+            shift
+            ;;
+        --method)
+            method="$2"
+            shift
+            shift
+            ;;
+        --agree-tos)
+            agree_tos=(--agree-tos)
+            shift
+            ;;
+        --help)
+            show_help=1
+            shift
+            ;;
+        --)
+            shift
+            break
+            ;;
+    esac
+done
+
+# Parse the remaining arguments as Subject Alternative Names to pass to certbot
+HOSTNAMES=()
+for arg; do
+    HOSTNAMES+=(-d "$arg")
+done
+DOMAIN=$1
+
+if [ -n "$show_help" ]; then
+    usage
+fi
+
+if [ -z "$DOMAIN" ] || [ -z "$EMAIL" ]; then
+    usage
+fi
+
+case "$method" in
+    standalone)
+        method_args=(--standalone --no-directory-hooks)
+        ;;
+    webroot)
+        method_args=(--webroot '--webroot-path=/var/lib/zulip/certbot-webroot/')
+        ;;
+    *)
+        usage
+        ;;
+esac
+
+# If we aren't being run interactively, default to keeping the
+# existing certificate (rather than burning through a renewal)
+# If run interactively, certbot will prompt.
+default_keep=()
+if [ ! -t 0 ]; then
+    default_keep=(--keep-until-expiring)
+fi
+
+# We need to know _which_ domain is Zulip's, in the symlink deploy
+# hook, so we pass this down
+export ZULIP_DOMAIN="$DOMAIN"
+
+# Certbot does not run deploy hooks on new certificates
+# (certbot/certbot#9978) so we will need to fake it if so
+if [ -d "/etc/letsencrypt/live/$DOMAIN/" ]; then
+    needs_hooks=0
+else
+    needs_hooks=1
+fi
+
+certbot certonly "${method_args[@]}" \
+    "${HOSTNAMES[@]}" -m "$EMAIL" \
+    "${agree_tos[@]}" \
+    "${default_keep[@]}" \
+    --no-eff-email
+
+# "certbot certonly" before version 3.2.0 does not run deploy hooks,
+# so we fake running them.
+if [ "$needs_hooks" = "1" ]; then
+    export RENEWED_DOMAINS="$*"
+    if [ "$method" == "webroot" ]; then
+        for deploy_hook in /etc/letsencrypt/renewal-hooks/deploy/*; do
+            "$deploy_hook"
+        done
+    fi
+fi
+echo "Certbot SSL certificate configuration succeeded."

scripts/setup/sha256-file-to

@@ -0,0 +1,32 @@
+#!/bin/sh
+
+if [ "$#" -ne 3 ]; then
+    echo "Usage:"
+    echo "  sha256-file-to SHA256 http://FETCH/FROM DST"
+    echo
+    echo "SHA256 is the sha256sum of the file fetched; the file is"
+    echo "placed in DST."
+    exit 1
+fi
+
+set -e
+set -x
+
+SHA256="$1"
+URL="$2"
+DST="$3"
+
+# Work in a tmpdir which we clean up at the end
+tmpdir="$(mktemp -d)"
+trap 'rm -r "$tmpdir"' EXIT
+cd "$tmpdir"
+
+# Fetch to a predictable name, not whatever curl guesses from the URL
+LOCALFILE="output"
+curl -fL --retry 3 -o "$LOCALFILE" "$URL"
+
+# Check the hash against what was passed in
+echo "$SHA256  $LOCALFILE" >"$LOCALFILE.sha256"
+sha256sum -c "$LOCALFILE.sha256"
+
+mv "$LOCALFILE" "$DST"

scripts/setup/sha256-tarball-to

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+
+if [ "$#" -lt 4 ]; then
+    echo "Usage:"
+    echo "  sha256-tarball-to SHA256 http://FETCH/FROM.tar.gz SRC1 DST1 [SRC2 DST2 [...]]"
+    echo
+    echo "SHA256 is the sha256sum of the tarball fetched; each SRC (which may be a"
+    echo "directory) is expected to be a relative path into the unpacked tarball,"
+    echo "and each DST is the absolute path it should be moved to."
+    exit 1
+fi
+
+set -e
+set -x
+
+SHA256="$1"
+URL="$2"
+shift
+shift
+
+# Work in a tmpdir which we clean up at the end
+tmpdir="$(mktemp -d)"
+trap 'rm -r "$tmpdir"' EXIT
+cd "$tmpdir"
+
+# We support both .tar.gz and .zip files
+if [[ "$URL" == *.tar.gz ]] || [[ "$URL" == *.tgz ]]; then
+    extension="tar.gz"
+elif [[ "$URL" == *.zip ]]; then
+    extension="zip"
+else
+    echo "Can't determine archive type from URL: $URL"
+    exit 1
+fi
+
+# Fetch to a predictable name, not whatever curl guesses from the URL
+LOCALFILE="archive.$extension"
+curl -fL --retry 3 -o "$LOCALFILE" "$URL"
+
+# Check the hash against what was passed in
+echo "$SHA256  $LOCALFILE" >"$LOCALFILE.sha256"
+sha256sum -c "$LOCALFILE.sha256"
+
+if [[ "$extension" == "tar.gz" ]]; then
+    tar xzf "$LOCALFILE"
+else
+    unzip "$LOCALFILE"
+fi
+
+# Take the rest of the arguments two-at-a-time, as source and
+# destination to move out of the unpacked tarball.
+while [ "$#" -gt 0 ]; do
+    mv "$1" "$2"
+    shift
+    shift
+done

scripts/setup/terminate-psql-sessions

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -e
+
+cd /
+
+tables="$(printf "'%s'," "${@//\'/\'\'}")"
+tables="${tables%,}"
+
+psql postgres -v ON_ERROR_STOP=1 <<EOF
+SELECT pg_terminate_backend(s.pid)
+    FROM pg_stat_activity s, pg_roles r
+    WHERE
+        s.datname IN ($tables)
+        AND r.rolname = CURRENT_USER
+        AND (s.usename = r.rolname OR r.rolsuper = 't')
+        AND s.pid <> pg_backend_pid();
+EOF

scripts/setup/upgrade-postgresql

@@ -0,0 +1,149 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [ "$EUID" -ne 0 ]; then
+    echo "Error: This script must be run as root" >&2
+    exit 1
+fi
+
+# Force a known locale; some OS X terminals set an invalid LC_CTYPE
+# which crashes pg_upgradecluster
+export LC_ALL=C.UTF-8
+export LANG=C.UTF-8
+export LANGUAGE=C.UTF-8
+
+LATEST_SUPPORTED_VERSION=18
+UPGRADE_TO=${1:-$LATEST_SUPPORTED_VERSION}
+UPGRADE_FROM=$(crudini --get /etc/zulip/zulip.conf postgresql version)
+ZULIP_PATH="$(dirname "$0")/../.."
+
+if [ "$UPGRADE_TO" = "$UPGRADE_FROM" ]; then
+    echo "Already running PostgreSQL $UPGRADE_TO!"
+    exit 1
+fi
+
+if [[ "$UPGRADE_TO" -lt "$UPGRADE_FROM" ]]; then
+    echo "Refusing to downgrade PostgreSQL $UPGRADE_FROM to $UPGRADE_TO!"
+    exit 1
+fi
+
+if [[ "$UPGRADE_TO" -gt "$LATEST_SUPPORTED_VERSION" ]]; then
+    echo "This version of Zulip does not support PostgreSQL $UPGRADE_TO."
+    echo "You may need to upgrade Zulip before you can upgrade to PostgreSQL $UPGRADE_TO."
+    exit 1
+fi
+
+# Verify that the version in /etc/zulip/zulip.conf is the version that
+# Django actually stores its data in.  We can only do that if the
+# database server is on the same host as the application server.
+if [ -d /home/zulip/deployments/current ]; then
+    DATA_IS_IN=$(
+        su -s /usr/bin/env -- zulip \
+            DJANGO_SETTINGS_MODULE=zproject.settings \
+            uv run --directory=/home/zulip/deployments/current --no-sync \
+            python -c 'from django.db import connection; print(connection.cursor().connection.server_version // 10000)'
+    )
+
+    if [ "$UPGRADE_FROM" != "$DATA_IS_IN" ]; then
+        cat <<EOF
+
+/etc/zulip/zulip.conf claims that Zulip is running PostgreSQL
+$UPGRADE_FROM, but the server is connected to a PostgreSQL running
+version $DATA_IS_IN.  Check the output from pg_lsclusters to verify
+which clusters are running, and update /etc/zulip/zulip.conf to match.
+
+In general, this results from manually upgrading PostgreSQL; you
+should use this tool for all PostgreSQL upgrades.
+EOF
+        exit 1
+    fi
+fi
+
+set -x
+
+"$ZULIP_PATH"/scripts/lib/setup-apt-repo
+apt-get install -y "postgresql-$UPGRADE_TO" "postgresql-client-$UPGRADE_TO"
+drop_main_cluster="$(
+    pg_lsclusters --json \
+        | jq --arg pg_version "$UPGRADE_TO" -r \
+            '.[] | select((.version|tostring == $ARGS.named.pg_version) and (.cluster == "main")) | .cluster'
+)"
+if [ -n "$drop_main_cluster" ]; then
+    pg_dropcluster "$UPGRADE_TO" main --stop
+fi
+
+(
+    # Two-stage application of Puppet; we apply the bare-bones
+    # PostgreSQL configuration first, so that FTS will be configured
+    # prior to the pg_upgradecluster.
+    TEMP_CONF_DIR=$(mktemp -d)
+    cp /etc/zulip/zulip.conf "$TEMP_CONF_DIR"
+    ZULIP_CONF="${TEMP_CONF_DIR}/zulip.conf"
+    crudini --set "$ZULIP_CONF" postgresql version "$UPGRADE_TO"
+
+    if [ -f "/usr/share/postgresql/$UPGRADE_FROM/pgroonga_setup.sql.applied" ]; then
+        # This file is intentionally blank, to trigger always running
+        # `ALTER EXTENSION pgroonga UPDATE` and not `CREATE EXTENSION pgroonga`.
+        touch "/usr/share/postgresql/$UPGRADE_TO/pgroonga_setup.sql.applied"
+    fi
+
+    "$ZULIP_PATH"/scripts/zulip-puppet-apply -f --config "$ZULIP_CONF" --tags postgresql_upgrade
+    rm -rf "$TEMP_CONF_DIR"
+)
+
+# Capture the output so we know where the path to the post-upgrade scripts is
+UPGRADE_LOG=$(mktemp "/var/log/zulip/upgrade-postgresql-$UPGRADE_FROM-$UPGRADE_TO.XXXXXXXXX.log")
+pg_upgradecluster -v "$UPGRADE_TO" "$UPGRADE_FROM" main --method=upgrade --link --no-start | tee "$UPGRADE_LOG"
+SCRIPTS_PATH=$(grep -o "/var/log/postgresql/pg_upgradecluster-$UPGRADE_FROM-$UPGRADE_TO-main.*" "$UPGRADE_LOG" || true)
+
+# If the upgrade completed successfully, lock in the new version in
+# our configuration immediately
+crudini --set /etc/zulip/zulip.conf postgresql version "$UPGRADE_TO"
+
+# Make sure the new PostgreSQL is running
+pg_ctlcluster "$UPGRADE_TO" main start
+
+# Update the statistics
+su postgres -c "/usr/lib/postgresql/$UPGRADE_TO/bin/vacuumdb --all --analyze-only --jobs 10"
+
+# Update extensions
+if [ -n "$SCRIPTS_PATH" ] && [ -f "$SCRIPTS_PATH/update_extensions.sql" ]; then
+    su postgres -c "psql --file $SCRIPTS_PATH/update_extensions.sql"
+    rm "$SCRIPTS_PATH/update_extensions.sql"
+fi
+
+# Start the database up cleanly
+"$ZULIP_PATH"/scripts/zulip-puppet-apply -f
+
+# Drop the old data, binaries, and scripts
+pg_dropcluster "$UPGRADE_FROM" main
+apt remove -y "postgresql-$UPGRADE_FROM"
+if [ -n "$SCRIPTS_PATH" ] && [ -f "$SCRIPTS_PATH/delete_old_cluster.sh" ]; then
+    su postgres -c "$SCRIPTS_PATH/delete_old_cluster.sh"
+    rm "$SCRIPTS_PATH/delete_old_cluster.sh"
+fi
+
+if [ -n "$SCRIPTS_PATH" ]; then
+    if [ -n "$(ls -A "$SCRIPTS_PATH")" ]; then
+        set +x
+        echo
+        echo
+        echo ">>>>> pg_upgradecluster succeeded, but unexpected post-upgrade scripts"
+        echo "      were found:"
+        ls -A1 "$SCRIPTS_PATH"
+    else
+        rmdir "$SCRIPTS_PATH"
+    fi
+else
+    set +x
+    echo
+    echo
+    echo ">>>>> pg_upgradecluster succeeded, but post-upgrade scripts path could not"
+    echo "      be parsed out!  Please read the pg_upgradecluster output to understand"
+    echo "      the current status of your cluster:"
+    echo "          $UPGRADE_LOG"
+    echo "      and report this bug with the PostgreSQL $UPGRADE_FROM -> $UPGRADE_TO upgrade to:"
+    echo "          https://github.com/zulip/zulip/issues"
+    echo
+    echo
+fi

scripts/start-server

@@ -0,0 +1 @@
+restart-server

scripts/stop-server

@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+import argparse
+import logging
+import os
+import pwd
+import subprocess
+import sys
+import time
+
+sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
+from scripts.lib.setup_path import setup_path
+
+setup_path()
+
+from scripts.lib.supervisor import list_supervisor_processes
+from scripts.lib.zulip_tools import (
+    ENDC,
+    OKGREEN,
+    WARNING,
+    has_application_server,
+    has_process_fts_updates,
+    su_to_zulip,
+)
+
+# Parse arguments before any side effects, so `--help` prints usage
+# instead of stopping the server.
+argparse.ArgumentParser(
+    description="Stop the Zulip server (Django, Tornado and workers).",
+).parse_args()
+
+deploy_path = os.path.realpath(os.path.join(os.path.dirname(__file__), ".."))
+os.chdir(deploy_path)
+
+username = pwd.getpwuid(os.getuid()).pw_name
+if username == "root":
+    su_to_zulip()
+elif username != "zulip":
+    logging.error("Must be run as user 'zulip'.")
+    sys.exit(1)
+
+logging.Formatter.converter = time.gmtime
+logging.basicConfig(format="%(asctime)s stop-server: %(message)s", level=logging.INFO)
+
+services = []
+
+# Start with the least-critical services:
+if has_process_fts_updates():
+    services.append("process-fts-updates")
+
+if has_application_server():
+    # Contrary to the order in (re)start-server, we stop django before the
+    # workers, to increase the chance that we finish processing any work
+    # that may have been enqueued by the Django, leaving the final state
+    # closer to "empty."  We stop Django before Tornado so it doesn't try
+    # to make requests to make queues with a down'd Tornado.
+    services.append("zulip-django")
+    services.append("zulip-tornado:*")
+    services.append("zulip-workers:*")
+    services.append("zulip-tus")
+    services.append("zulip-katex")
+    services.append("zulip-email-server")
+    if has_application_server(once=True):
+        # These used to be included in "zulip-workers:*"; we may be
+        # stopping an older version of Zulip, which has not applied
+        # puppet to reload the new list of processes, but the
+        # list_supervisor_processes below will filter them out if they
+        # do not exist.
+        services.append("zulip_deliver_scheduled_emails")
+        services.append("zulip_deliver_scheduled_messages")
+
+services = list_supervisor_processes(services, only_running=True)
+if services:
+    subprocess.check_call(["supervisorctl", "stop", *services])
+
+print()
+print(OKGREEN + "Zulip stopped successfully!" + ENDC)
+
+using_sso = subprocess.check_output(["./scripts/get-django-setting", "USING_APACHE_SSO"])
+if using_sso.strip() == b"True":
+    print()
+    print(WARNING + "Apache2 needs to be shut down; as root, run:" + ENDC)
+    print("    service apache2 stop")
+    print()

scripts/upgrade-zulip

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+#
+# This is a thin wrapper around the upgrade script (scripts/lib/upgrade-zulip).
+# This wrapper exists to log output to /var/log/zulip/upgrade.log for debugging.
+
+set -e
+
+if [ "$EUID" -ne 0 ]; then
+    basename=$(basename "$0")
+    echo "Error: $basename must be run as root." >&2
+    exit 1
+fi
+
+"$(dirname "$0")/lib/upgrade-zulip" "$@" 2>&1 | tee -a /var/log/zulip/upgrade.log
+failed=${PIPESTATUS[0]}
+
+if [ "$failed" -ne 0 ]; then
+    echo -e '\033[0;31m'
+    echo "Zulip upgrade failed (exit code $failed)!"
+    echo
+    echo -n "The upgrade process is designed to be idempotent, so you can retry "
+    echo -n "after resolving whatever issue caused the failure (there should be a traceback above). "
+    echo -n "A log of this installation is available in /var/log/zulip/upgrade.log"
+    echo -e '\033[0m'
+    exit "$failed"
+fi

scripts/upgrade-zulip-from-git

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+#
+# This is a thin wrapper around the upgrade-from-git script (scripts/lib/upgrade-zulip-from-git).
+# This wrapper exists to log output to /var/log/zulip/upgrade.log for debugging.
+
+set -e
+
+if [ "$EUID" -ne 0 ]; then
+    basename=$(basename "$0")
+    echo "Error: $basename must be run as root." >&2
+    exit 1
+fi
+
+"$(dirname "$0")/lib/upgrade-zulip-from-git" "$@" 2>&1 | tee -a /var/log/zulip/upgrade.log
+failed=${PIPESTATUS[0]}
+
+if [ "$failed" -ne 0 ]; then
+    echo -e '\033[0;31m'
+    echo "Zulip upgrade failed (exit code $failed)!"
+    echo
+    echo -n "The upgrade process is designed to be idempotent, so you can retry "
+    echo -n "after resolving whatever issue caused the failure (there should be a traceback above). "
+    echo -n "A log of this installation is available in /var/log/zulip/upgrade.log"
+    echo -e '\033[0m'
+    exit "$failed"
+fi

scripts/zulip-puppet-apply

@@ -0,0 +1,114 @@
+#!/usr/bin/env python3
+import argparse
+import configparser
+import os
+import platform
+import re
+import subprocess
+import sys
+import tempfile
+
+import yaml
+
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, BASE_DIR)
+
+from scripts.lib.puppet_cache import setup_puppet_modules
+from scripts.lib.zulip_tools import assert_running_as_root
+
+assert_running_as_root()
+
+parser = argparse.ArgumentParser(description="Run Puppet")
+parser.add_argument(
+    "--force", "-f", action="store_true", help="Do not prompt with proposed changes"
+)
+parser.add_argument("--noop", action="store_true", help="Do not apply the changes")
+parser.add_argument("--config", default="/etc/zulip/zulip.conf", help="Alternate zulip.conf path")
+args, extra_args = parser.parse_known_args()
+
+config = configparser.RawConfigParser()
+config.read(args.config)
+
+setup_puppet_modules()
+
+distro_info = platform.freedesktop_os_release()
+puppet_config = """
+Exec { path => "/usr/sbin:/usr/bin:/sbin:/bin" }
+"""
+
+for pclass in re.split(r"\s*,\s*", config.get("machine", "puppet_classes")):
+    if " " in pclass:
+        print(
+            f"The `machine.puppet_classes` setting in {args.config} must be comma-separated, not space-separated!"
+        )
+        sys.exit(1)
+    puppet_config += f"include {pclass}\n"
+
+# We use the Puppet configuration from the same Zulip checkout as this script
+scripts_path = os.path.join(BASE_DIR, "scripts")
+puppet_module_path = os.path.join(BASE_DIR, "puppet")
+puppet_cmd = [
+    "puppet",
+    "apply",
+    f"--modulepath={puppet_module_path}:/srv/zulip-puppet-cache/current",
+    "-e",
+    puppet_config,
+]
+if args.noop:
+    puppet_cmd += ["--noop"]
+puppet_cmd += extra_args
+
+# Set the scripts path to be a factor so it can be used by Puppet code
+puppet_env = os.environ.copy()
+puppet_env["FACTER_zulip_conf_path"] = args.config
+puppet_env["FACTER_zulip_scripts_path"] = scripts_path
+
+
+def noop_would_change(puppet_cmd: list[str]) -> bool:
+    # --noop does not work with --detailed-exitcodes; see
+    # https://tickets.puppetlabs.com/browse/PUP-686
+    with tempfile.NamedTemporaryFile() as lastrun_file:
+        try:
+            subprocess.check_call(
+                # puppet_cmd may already contain --noop, but it is safe to
+                # supply twice
+                [*puppet_cmd, "--noop", "--lastrunfile", lastrun_file.name],
+                env=puppet_env,
+            )
+        except subprocess.CalledProcessError:
+            sys.exit(2)
+
+        # Reopen the file since Puppet rewrote it with a new inode
+        with open(lastrun_file.name) as lastrun:
+            lastrun_data = yaml.safe_load(lastrun)
+            resources = lastrun_data.get("resources", {})
+            if resources.get("failed", 0) != 0:
+                sys.exit(2)
+            return resources.get("out_of_sync", 0) != 0
+
+
+if not args.noop and not args.force:
+    if not noop_would_change([*puppet_cmd, "--show_diff"]):
+        sys.exit(0)
+
+    do_apply = None
+    while do_apply != "y":
+        sys.stdout.write("Apply changes? [y/N] ")
+        sys.stdout.flush()
+        do_apply = sys.stdin.readline().strip().lower()
+        if do_apply in ("", "n"):
+            sys.exit(0)
+
+if args.noop and args.force:
+    if noop_would_change(puppet_cmd):
+        sys.exit(1)
+    else:
+        sys.exit(0)
+
+ret = subprocess.call([*puppet_cmd, "--detailed-exitcodes"], env=puppet_env)
+# ret = 0 => no changes, no errors
+# ret = 2 => changes, no errors
+# ret = 4 => no changes, yes errors
+# ret = 6 => changes, yes errors
+if ret not in (0, 2):
+    sys.exit(2)