snapshot of https://github.com/zulip/zulip.git @ 48c69911c7dd4e03b20552eb3bd77ad8151c1357 (2026-09-15)

scripts/lib/build-pgroonga

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+version="4.0.6"
+sha256=d0048944763c18f91bc67e043aafa64c2c53f6246547c9474311efbc05ccfe66
+
+tmpdir="$(mktemp -d)"
+trap 'rm -r "$tmpdir"' EXIT
+cd "$tmpdir"
+tarball="pgroonga-$version.tar.gz"
+curl -fLO --retry 3 "https://github.com/pgroonga/pgroonga/releases/download/$version/$tarball"
+sha256sum -c <<<"$sha256 $tarball"
+tar -xzf "$tarball"
+cd "pgroonga-$version"
+
+meson setup build .
+meson install -C build

scripts/lib/check-database-compatibility

@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+import logging
+import os
+import subprocess
+import sys
+
+ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.insert(0, ZULIP_PATH)
+
+from scripts.lib.setup_path import setup_path
+from scripts.lib.zulip_tools import (
+    DEPLOYMENTS_DIR,
+    assert_not_running_as_root,
+    get_config,
+    get_config_file,
+    parse_version_from,
+)
+from version import ZULIP_VERSION as NEW_VERSION
+
+assert_not_running_as_root()
+setup_path()
+os.environ["DJANGO_SETTINGS_MODULE"] = "zproject.settings"
+
+import django
+from django.db import connection
+from django.db.migrations.loader import MigrationLoader
+
+from zerver.lib.migration_status import STALE_MIGRATIONS
+
+django.setup()
+
+django_pg_version = connection.cursor().connection.server_version // 10000
+if os.path.exists("/etc/init.d/postgresql") and os.path.exists("/etc/zulip/zulip.conf"):
+    postgresql_version = int(get_config(get_config_file(), "postgresql", "version", "0"))
+    if postgresql_version == 0:
+        postgresql_version = django_pg_version
+        subprocess.check_call(
+            [
+                "crudini",
+                "--set",
+                "/etc/zulip/zulip.conf",
+                "postgresql",
+                "version",
+                str(postgresql_version),
+            ]
+        )
+    elif postgresql_version != django_pg_version:
+        logging.critical(
+            "PostgreSQL version mismatch: %d (running) vs %d (configured)",
+            django_pg_version,
+            postgresql_version,
+        )
+        logging.info(
+            "/etc/zulip/zulip.conf claims that Zulip is running PostgreSQL\n"
+            "%d, but the server is connected to a PostgreSQL running\n"
+            "version %d.  Check the output from pg_lsclusters to verify\n"
+            "which clusters are running, and update /etc/zulip/zulip.conf to match.\n"
+            "\n"
+            "In general, this results from manually upgrading PostgreSQL; you\n"
+            "should follow our instructions for using our tool to do so:\n"
+            "https://zulip.readthedocs.io/en/stable/production/upgrade.html#upgrading-postgresql",
+            postgresql_version,
+            django_pg_version,
+        )
+        sys.exit(1)
+
+if django_pg_version < 14:
+    logging.critical("Unsupported PostgreSQL version: %d", postgresql_version)
+    logging.info(
+        "Please upgrade to PostgreSQL 14 or newer first.\n"
+        "See https://zulip.readthedocs.io/en/stable/production/"
+        "upgrade.html#upgrading-postgresql"
+    )
+    sys.exit(1)
+
+loader = MigrationLoader(connection)
+missing = set(loader.applied_migrations)
+
+missing.difference_update(STALE_MIGRATIONS)
+
+for key, migration in loader.disk_migrations.items():
+    missing.discard(key)
+    missing.difference_update(migration.replaces)
+if not missing:
+    sys.exit(0)
+
+print("Migrations which are currently applied, but missing in the new version:")
+for app, migration_name in sorted(missing):
+    print(f"  {app} - {migration_name}")
+
+current_version = parse_version_from(os.path.join(DEPLOYMENTS_DIR, "current"))
+logging.error(
+    "This is not an upgrade -- the current deployment (version %s) "
+    "contains %s database migrations which %s (version %s) does not.",
+    current_version,
+    len(missing),
+    ZULIP_PATH,
+    NEW_VERSION,
+)
+sys.exit(1)

scripts/lib/check_rabbitmq_queue.py

@@ -0,0 +1,233 @@
+import json
+import os
+import re
+import subprocess
+import sys
+import time
+from collections import defaultdict
+from typing import Any
+
+ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+sys.path.append(ZULIP_PATH)
+from scripts.lib.zulip_tools import atomic_nagios_write, get_config, get_config_file
+
+normal_queues = [
+    "deferred_work",
+    "deferred_email_senders",
+    "digest_emails",
+    "email_mirror",
+    "email_senders",
+    "embed_links",
+    "embedded_bots",
+    "missedmessage_emails",
+    "missedmessage_mobile_notifications",
+    "outgoing_webhooks",
+    "thumbnail",
+    "user_activity",
+    "user_activity_interval",
+]
+
+# Soft reactivations share the deferred_work queue unless a server opts
+# into a dedicated queue; only then is there a consumer to monitor.
+if get_config(get_config_file(), "application_server", "dedicated_soft_reactivation_queue", False):
+    normal_queues.append("soft_reactivation")
+
+mobile_notification_shards = int(
+    get_config(get_config_file(), "application_server", "mobile_notification_shards", "1")
+)
+user_activity_shards = int(
+    get_config(get_config_file(), "application_server", "user_activity_shards", "1")
+)
+
+OK = 0
+WARNING = 1
+CRITICAL = 2
+UNKNOWN = 3
+
+states = {
+    0: "OK",
+    1: "WARNING",
+    2: "CRITICAL",
+    3: "UNKNOWN",
+}
+
+MAX_SECONDS_TO_CLEAR: defaultdict[str, int] = defaultdict(
+    lambda: 30,
+    deferred_work=600,
+    # Soft reactivations are the deferred_work backfills split into their
+    # own queue; keep deferred_work's generous clear-time budget so the slow
+    # backfills that motivated the split don't trip false alerts.
+    soft_reactivation=600,
+    digest_emails=1200,
+    missedmessage_mobile_notifications=120,
+    embed_links=60,
+    email_senders=90,
+    deferred_email_senders=3600,
+)
+CRITICAL_SECONDS_TO_CLEAR: defaultdict[str, int] = defaultdict(
+    lambda: 60,
+    deferred_work=900,
+    soft_reactivation=900,
+    missedmessage_mobile_notifications=180,
+    digest_emails=1800,
+    embed_links=90,
+    email_senders=300,
+    deferred_email_senders=4500,
+)
+
+
+def analyze_queue_stats(
+    queue_name: str, stats: dict[str, Any], queue_count_rabbitmqctl: int
+) -> dict[str, Any]:
+    now = int(time.time())
+    if stats == {}:
+        return dict(status=UNKNOWN, name=queue_name, message="invalid or no stats data")
+
+    if now - stats["update_time"] > 180 and queue_count_rabbitmqctl > 10:
+        # Queue isn't updating the stats file and has some events in
+        # the backlog, it's likely stuck.
+        #
+        # TODO: There's an unlikely race condition here - if the queue
+        # was fully emptied and was idle due to no new events coming
+        # for over 180 seconds, suddenly gets a burst of events and
+        # this code runs exactly in the very small time window between
+        # those events popping up and the queue beginning to process
+        # the first one (which will refresh the stats file at the very
+        # start), we'll incorrectly return the CRITICAL status. The
+        # chance of that happening should be negligible because the queue
+        # worker should wake up immediately and log statistics before
+        # starting to process the first event.
+        return dict(
+            status=CRITICAL,
+            name=queue_name,
+            message="queue appears to be stuck, last update {}, queue size {}".format(
+                stats["update_time"], queue_count_rabbitmqctl
+            ),
+        )
+
+    current_size = queue_count_rabbitmqctl
+    average_consume_time = stats["recent_average_consume_time"]
+    if average_consume_time is None:
+        # Queue just started; we can't effectively estimate anything.
+        #
+        # If the queue is stuck in this state and not processing
+        # anything, eventually the `update_time` rule above will fire.
+        return dict(status=OK, name=queue_name, message="")
+
+    expected_time_to_clear_backlog = current_size * average_consume_time
+    if expected_time_to_clear_backlog > MAX_SECONDS_TO_CLEAR[queue_name]:
+        if expected_time_to_clear_backlog > CRITICAL_SECONDS_TO_CLEAR[queue_name]:
+            status = CRITICAL
+        else:
+            status = WARNING
+
+        return dict(
+            status=status,
+            name=queue_name,
+            message=f"clearing the backlog will take too long: {expected_time_to_clear_backlog}s, size: {current_size}",
+        )
+
+    return dict(status=OK, name=queue_name, message="")
+
+
+WARN_COUNT_THRESHOLD_DEFAULT = 10
+CRITICAL_COUNT_THRESHOLD_DEFAULT = 50
+
+
+def check_other_queues(queue_counts_dict: dict[str, int]) -> list[dict[str, Any]]:
+    """Do a simple queue size check for queues whose workers don't publish stats files."""
+
+    results = []
+    for queue, count in queue_counts_dict.items():
+        if queue in normal_queues:
+            continue
+
+        if count > CRITICAL_COUNT_THRESHOLD_DEFAULT:
+            results.append(dict(status=CRITICAL, name=queue, message=f"count critical: {count}"))
+        elif count > WARN_COUNT_THRESHOLD_DEFAULT:
+            results.append(dict(status=WARNING, name=queue, message=f"count warning: {count}"))
+        else:
+            results.append(dict(status=OK, name=queue, message=""))
+
+    return results
+
+
+def check_rabbitmq_queues() -> None:
+    pattern = re.compile(r"(\w+)\t(\d+)\t(\d+)")
+    if "USER" in os.environ and os.environ["USER"] not in ["root", "rabbitmq"]:
+        print("This script must be run as the root or rabbitmq user")
+
+    list_queues_output = subprocess.check_output(
+        ["/usr/sbin/rabbitmqctl", "list_queues", "name", "messages", "consumers"],
+        text=True,
+    )
+    queue_counts_rabbitmqctl = {}
+    queues_with_consumers = []
+    for line in list_queues_output.split("\n"):
+        line = line.strip()
+        m = pattern.match(line)
+        if m:
+            queue = m.group(1)
+            count = int(m.group(2))
+            consumers = int(m.group(3))
+            queue_counts_rabbitmqctl[queue] = count
+            if consumers > 0 and not queue.startswith("notify_tornado"):
+                queues_with_consumers.append(queue)
+
+    queue_stats_dir = subprocess.check_output(
+        [os.path.join(ZULIP_PATH, "scripts/get-django-setting"), "QUEUE_STATS_DIR"],
+        text=True,
+    ).strip()
+    queue_stats: dict[str, dict[str, Any]] = {}
+
+    check_queues = normal_queues
+    if mobile_notification_shards > 1:
+        # For sharded queue workers, where there's a separate queue
+        # for each shard, we need to make sure none of those are
+        # backlogged.
+        check_queues += [
+            f"missedmessage_mobile_notifications_shard{d}"
+            for d in range(1, mobile_notification_shards + 1)
+        ]
+    if user_activity_shards > 1:
+        check_queues += [f"user_activity_shard{d}" for d in range(1, user_activity_shards + 1)]
+
+    queues_to_check = set(check_queues).intersection(set(queues_with_consumers))
+    for queue in queues_to_check:
+        fn = queue + ".stats"
+        file_path = os.path.join(queue_stats_dir, fn)
+        if not os.path.exists(file_path):
+            queue_stats[queue] = {}
+            continue
+
+        with open(file_path) as f:
+            try:
+                queue_stats[queue] = json.load(f)
+            except json.decoder.JSONDecodeError:
+                queue_stats[queue] = {}
+
+    results = []
+    for queue_name, stats in queue_stats.items():
+        results.append(analyze_queue_stats(queue_name, stats, queue_counts_rabbitmqctl[queue_name]))
+
+    results.extend(check_other_queues(queue_counts_rabbitmqctl))
+
+    status = max(result["status"] for result in results)
+
+    if status > 0:
+        queue_error_template = "queue {} problem: {}:{}"
+        error_message = "; ".join(
+            queue_error_template.format(result["name"], states[result["status"]], result["message"])
+            for result in results
+            if result["status"] > 0
+        )
+        sys.exit(
+            atomic_nagios_write(
+                "check-rabbitmq-results",
+                "critical" if status == CRITICAL else "warning",
+                error_message,
+            )
+        )
+    else:
+        atomic_nagios_write("check-rabbitmq-results", "ok", "queues normal")

scripts/lib/clean_emoji_cache.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+import argparse
+import os
+import sys
+
+ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.append(ZULIP_PATH)
+from scripts.lib.zulip_tools import (
+    get_environment,
+    get_recent_deployments,
+    parse_cache_script_args,
+    purge_unused_caches,
+)
+
+ENV = get_environment()
+EMOJI_CACHE_PATH = "/srv/zulip-emoji-cache"
+
+
+def get_caches_in_use(threshold_days: int) -> set[str]:
+    setups_to_check = {ZULIP_PATH}
+    caches_in_use = set()
+
+    if ENV == "prod":
+        setups_to_check |= get_recent_deployments(threshold_days)
+    if ENV == "dev":
+        CACHE_SYMLINK = os.path.join(ZULIP_PATH, "static", "generated", "emoji")
+        CURRENT_CACHE = os.path.dirname(os.path.realpath(CACHE_SYMLINK))
+        caches_in_use.add(CURRENT_CACHE)
+
+    for setup_dir in setups_to_check:
+        emoji_link_path = os.path.join(setup_dir, "static/generated/emoji")
+        if not os.path.islink(emoji_link_path):
+            # This happens for a deployment directory extracted from a
+            # tarball, which just has a copy of the emoji data, not a symlink.
+            continue
+        # The actual cache path doesn't include the /emoji
+        caches_in_use.add(os.path.dirname(os.readlink(emoji_link_path)))
+    return caches_in_use
+
+
+def main(args: argparse.Namespace) -> None:
+    caches_in_use = get_caches_in_use(args.threshold_days)
+    purge_unused_caches(EMOJI_CACHE_PATH, caches_in_use, "emoji cache", args)
+
+
+if __name__ == "__main__":
+    args = parse_cache_script_args("This script cleans unused Zulip emoji caches.")
+    main(args)

scripts/lib/clean_unused_caches.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+import argparse
+import os
+import shutil
+import sys
+
+ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.append(ZULIP_PATH)
+from scripts.lib import clean_emoji_cache
+from scripts.lib.zulip_tools import parse_cache_script_args
+
+
+def main(args: argparse.Namespace) -> None:
+    os.chdir(ZULIP_PATH)
+    shutil.rmtree("/srv/zulip-venv-cache", ignore_errors=True)  # Replaced as of 10.0
+    shutil.rmtree("/srv/zulip-npm-cache", ignore_errors=True)  # Replaced as of 7.0
+    clean_emoji_cache.main(args)
+
+
+if __name__ == "__main__":
+    args = parse_cache_script_args("This script cleans unused Zulip caches.")
+    main(args)

scripts/lib/create-production-venv

@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+import argparse
+import os
+import platform
+import sys
+
+ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+if ZULIP_PATH not in sys.path:
+    sys.path.append(ZULIP_PATH)
+
+from scripts.lib.setup_venv import get_venv_dependencies
+from scripts.lib.zulip_tools import os_families, run
+
+parser = argparse.ArgumentParser(description="Create a production virtualenv with caching")
+parser.add_argument("deploy_path")
+args = parser.parse_args()
+
+# install dependencies for setting up the virtualenv
+distro_info = platform.freedesktop_os_release()
+vendor = distro_info["ID"]
+os_version = distro_info["VERSION_ID"]
+VENV_DEPENDENCIES = get_venv_dependencies(vendor, os_version)
+if "debian" in os_families():
+    run(["apt-get", "-y", "install", "--no-install-recommends", *VENV_DEPENDENCIES])
+elif "fedora" in os_families():
+    run(["yum", "-y", "install", *VENV_DEPENDENCIES])
+else:
+    print("Unsupported platform: {}".format(distro_info["ID"]))
+    sys.exit(1)
+
+os.chdir(ZULIP_PATH)
+run(["scripts/lib/install-uv"])
+run(
+    ["uv", "sync", "--frozen", "--no-managed-python", "--only-group=prod"],
+    env={k: v for k, v in os.environ.items() if k not in {"PYTHONDEVMODE", "PYTHONWARNINGS"}},
+)

scripts/lib/install

@@ -0,0 +1,790 @@
+#!/usr/bin/env bash
+set -e
+
+usage() {
+    # A subset of this documentation also appears in docs/production/install.md
+    cat <<'EOF'
+Usage:
+  install --hostname=zulip.example.com --email=zulip-admin@example.com [options...]
+  install --help
+
+Options:
+  --hostname=zulip.example.com
+      The user-accessible domain name for this Zulip server, i.e.,
+      what users will type in their web browser.  Required, unless
+      --no-init-db or --puppet-classes is set, and --certbot is not.
+      Use Punycode for non-ASCII domains.
+
+  --email=zulip-admin@example.com
+      The email address of the person or team who should get support
+      and error emails from this Zulip server.  Required, unless
+      --no-init-db or --puppet-classes is set and --certbot is not.
+
+  --certbot
+      Obtains a free SSL certificate for the server using Certbot,
+      https://certbot.eff.org/  Recommended.  Conflicts with --self-signed-cert.
+  --self-signed-cert
+      Generate a self-signed SSL certificate for the server. This isn’t suitable for
+      production use, but may be convenient for testing.  Conflicts with --certbot.
+
+  --postgresql-database-name=zulip
+      Sets the PostgreSQL database name.
+  --postgresql-database-user=zulip
+      Sets the PostgreSQL database user.
+  --postgresql-version=18
+      Sets the version of PostgreSQL that will be installed.
+
+  --no-init-db
+      Does not do any database initialization; use when you already have a Zulip
+      database.
+  --puppet-classes
+      Comma-separated list of Puppet classes to install; defaults to
+      'zulip:::profile::standalone'.  Implies --no-init-db.
+
+  --no-overwrite-settings
+      Preserve existing `/etc/zulip` configuration files.
+  --no-dist-upgrade
+      Skip the initial `apt-get dist-upgrade`.
+
+  --push-notifications
+      With this option, the Zulip installer registers your server for the Mobile
+      Push Notification Service, and sets up the initial default configuration.
+      You will be immediately prompted to agree to the Terms of Service, and your
+      server will be registered at the end of the installation process.
+  --no-push-notifications
+      Disable push notifications registration (the default if neither
+      flag is provided).
+
+  --no-submit-usage-statistics
+      If you enable push notifications, by default your server
+      will submit basic metadata (required for billing and for
+      determining free plan eligibility), as well as aggregate usage
+      statistics. You can disable submitting usage statistics by passing
+      this flag.
+      If push notifications are not enabled, data won't be submitted, so
+      this flag is redundant.
+
+  --agree-to-terms-of-service
+      If you enable push notifications, you can pass this flag to indicate
+      that you have read and agree to the Zulip Terms of Service:
+      <https://zulip.com/policies/terms>.
+      This skips the Terms of Service prompt, allowing for running the installer
+      with --push-notifications in scripts without requiring user input.
+
+
+EOF
+}
+
+system_requirements_failure() {
+    set +x
+    echo >&2
+    cat >&2
+    cat <<EOF >&2
+
+For more information, see:
+  https://zulip.readthedocs.io/en/latest/production/requirements.html
+EOF
+    exit 1
+}
+
+# Shell option parsing.  Over time, we'll want to move some of the
+# environment variables below into this self-documenting system.
+args="$(getopt -o '' --long help,hostname:,email:,certbot,self-signed-cert,cacert:,postgresql-database-name:,postgresql-database-user:,postgresql-version:,no-init-db,puppet-classes:,no-overwrite-settings,no-dist-upgrade,push-notifications,no-push-notifications,no-submit-usage-statistics,agree-to-terms-of-service -n "$0" -- "$@")"
+
+eval "set -- $args"
+while true; do
+    case "$1" in
+        --help)
+            usage
+            exit 0
+            ;;
+
+        --hostname)
+            EXTERNAL_HOST="$2"
+            shift
+            shift
+            ;;
+        --email)
+            ZULIP_ADMINISTRATOR="$2"
+            shift
+            shift
+            ;;
+
+        --certbot)
+            USE_CERTBOT=1
+            shift
+            ;;
+        --self-signed-cert)
+            SELF_SIGNED_CERT=1
+            shift
+            ;;
+        --postgresql-database-name)
+            POSTGRESQL_DATABASE_NAME="$2"
+            shift
+            shift
+            ;;
+        --postgresql-database-user)
+            POSTGRESQL_DATABASE_USER="$2"
+            shift
+            shift
+            ;;
+        --postgresql-version)
+            POSTGRESQL_VERSION="$2"
+            shift
+            shift
+            ;;
+        --no-init-db)
+            NO_INIT_DB=1
+            shift
+            ;;
+        --puppet-classes)
+            PUPPET_CLASSES="$2"
+            NO_INIT_DB=1
+            shift
+            shift
+            ;;
+
+        --no-overwrite-settings)
+            NO_OVERWRITE_SETTINGS=1
+            shift
+            ;;
+        --no-dist-upgrade)
+            NO_DIST_UPGRADE=1
+            shift
+            ;;
+        --push-notifications)
+            if [ -n "$NO_PUSH_NOTIFICATIONS" ]; then
+                echo "error: --push-notifications and --no-push-notifications are incompatible." >&2
+                exit 1
+            fi
+            PUSH_NOTIFICATIONS=1
+            shift
+            ;;
+        --no-push-notifications)
+            if [ -n "$PUSH_NOTIFICATIONS" ]; then
+                echo "error: --push-notifications and --no-push-notifications are incompatible." >&2
+                exit 1
+            fi
+            NO_PUSH_NOTIFICATIONS=1
+            shift
+            ;;
+        --no-submit-usage-statistics)
+            NO_SUBMIT_USAGE_STATISTICS=1
+            shift
+            ;;
+        --agree-to-terms-of-service)
+            AGREE_TO_TERMS_OF_SERVICE_FLAG=1
+            shift
+            ;;
+        --)
+            shift
+            break
+            ;;
+    esac
+done
+
+if [ "$#" -gt 0 ]; then
+    usage >&2
+    exit 1
+fi
+
+## Options from environment variables.
+#
+# Specify options for apt.
+read -r -a APT_OPTIONS <<<"${APT_OPTIONS:-}"
+# Install additional packages.
+read -r -a ADDITIONAL_PACKAGES <<<"${ADDITIONAL_PACKAGES:-}"
+# Comma-separated list of Puppet manifests to install.  The default is
+# zulip::profile::standalone for an all-in-one system or
+# zulip::profile::docker for Docker.  Use
+# e.g. zulip::profile::app_frontend for a Zulip frontend server.
+PUPPET_CLASSES="${PUPPET_CLASSES:-zulip::profile::standalone}"
+POSTGRESQL_VERSION="${POSTGRESQL_VERSION:-18}"
+
+if [ -n "$SELF_SIGNED_CERT" ] && [ -n "$USE_CERTBOT" ]; then
+    set +x
+    echo "error: --self-signed-cert and --certbot are incompatible" >&2
+    echo >&2
+    usage >&2
+    exit 1
+fi
+
+if [ -z "$EXTERNAL_HOST" ] || [ -z "$ZULIP_ADMINISTRATOR" ]; then
+    if [ -n "$USE_CERTBOT" ] || [ -z "$NO_INIT_DB" ]; then
+        usage >&2
+        exit 1
+    fi
+fi
+
+if [ "$EXTERNAL_HOST" = zulip.example.com ] \
+    || [ "$ZULIP_ADMINISTRATOR" = zulip-admin@example.com ]; then
+    # These example values are specifically checked for and would fail
+    # later; see zerver/checks.py
+    echo 'error: The example hostname and email must be replaced with real values.' >&2
+    echo >&2
+    usage >&2
+    exit 1
+fi
+
+case "$EXTERNAL_HOST" in
+    *[![:ascii:]]*)
+        # Non-ASCII hostnames are specifically checked for and would
+        # fail later; see zerver/checks.py
+        echo "error: The hostname ($EXTERNAL_HOST) contains non-ASCII characters." >&2
+        echo 'Internationalized domain names must be specified in punycode ("xn--") form:' >&2
+        # python3 is not installed until later, so it may not be
+        # available to compute the suggested encoding.
+        if command -v python3 >/dev/null 2>&1; then
+            suggestion="$(python3 -c '
+import sys
+
+hostname, sep, port = sys.argv[1].partition(":")
+print(
+    ".".join(
+        "xn--" + label.encode("punycode").decode() if not label.isascii() else label
+        for label in hostname.split(".")
+    )
+    + sep
+    + port
+)' "$EXTERNAL_HOST")"
+            echo "    --hostname=$suggestion" >&2
+        fi
+        exit 1
+        ;;
+esac
+
+case "$POSTGRESQL_VERSION" in
+    [0-9] | [0-9].* | 1[0-3] | 1[0-3].*)
+        echo "error: PostgreSQL 14 or newer is required." >&2
+        exit 1
+        ;;
+esac
+
+if [ -z "$PUSH_NOTIFICATIONS" ] && [ -z "$NO_PUSH_NOTIFICATIONS" ]; then
+    # Unless specified, we default to --no-push-notifications
+    NO_PUSH_NOTIFICATIONS=1
+fi
+
+# We set these to Python-style True/False string values, for easy
+# insertion into the settings.py file.
+if [ -n "$PUSH_NOTIFICATIONS" ]; then
+    SERVICE_MOBILE_PUSH="True"
+
+    # --push-notifications also enables SUBMIT_USAGE_STATISTICS, unless the user
+    # explicitly opted out by passing --no-submit-usage-statistics.
+    if [ -n "$NO_SUBMIT_USAGE_STATISTICS" ]; then
+        SERVICE_SUBMIT_USAGE_STATISTICS="False"
+    else
+        SERVICE_SUBMIT_USAGE_STATISTICS="True"
+    fi
+else
+    SERVICE_MOBILE_PUSH="False"
+    # SUBMIT_USAGE_STATISTICS without PUSH_NOTIFICATIONS is an unusual
+    # configuration that we don't need to offer in the installer.
+    SERVICE_SUBMIT_USAGE_STATISTICS="False"
+fi
+
+# ToS acceptance needs to be ensured if --push-notifications is passed.
+if [ -n "$PUSH_NOTIFICATIONS" ]; then
+    # If the user provided the --agree-to-terms-of-service flag, we can just proceed.
+    if [ -n "$AGREE_TO_TERMS_OF_SERVICE_FLAG" ]; then
+        PUSH_NOTIFICATIONS_SERVICE_TOS_AGREED=1
+        echo "Push notifications will be enabled, as you agreed to the Zulip Terms of Service"
+        echo "by passing the --agree-to-terms-of-service flag."
+        sleep 2
+    else
+        # If user asked for push notifications, prompt for ToS acceptance.
+        echo
+        echo "You chose to register your server for the Mobile Push Notifications Service."
+        echo "Doing so will share basic metadata with the service's maintainers, including:"
+        echo
+        echo "* The server's configured hostname: $EXTERNAL_HOST"
+        echo "* The server's configured contact email address: $ZULIP_ADMINISTRATOR"
+        echo "* Basic metadata about each organization hosted by the server; see:"
+        echo "    <https://zulip.com/doc-permalinks/basic-metadata>"
+        if [ -z "$NO_SUBMIT_USAGE_STATISTICS" ]; then
+            echo "* The server's usage statistics; see:"
+            echo "    <https://zulip.com/doc-permalinks/usage-statistics>"
+        fi
+        echo
+        echo "For details on why a centralized push notification service is necessary, see:"
+        echo "    <https://zulip.com/doc-permalinks/why-service>"
+        echo
+        echo "Use of this service is governed by the Zulip Terms of Service:"
+        echo "    <https://zulip.com/policies/terms>"
+        echo
+        read -r -p "Do you want to agree to the Zulip Terms of Service and proceed? [Y/n] " tos_prompt
+        echo
+
+        # Normalize the user’s response to lowercase.
+        case "${tos_prompt,,}" in
+            "" | y | yes)
+                echo "Great! Push notifications will be enabled; continuing with installation..."
+                PUSH_NOTIFICATIONS_SERVICE_TOS_AGREED=1
+                sleep 2
+                ;;
+            *)
+                echo "In order to enable push notifications, you must agree to the Terms of Service."
+                echo "If you do not want to enable push notifications, run the command without the --push-notifications flag."
+                exit 1
+                ;;
+        esac
+    fi
+fi
+
+# Do set -x after option parsing is complete
+set -x
+
+ZULIP_PATH="$(readlink -f "$(dirname "$0")"/../..)"
+
+# Force a known locale.  Some packages on PyPI fail to install in some locales.
+export LC_ALL="C.UTF-8"
+export LANG="C.UTF-8"
+export LANGUAGE="C.UTF-8"
+
+# Force a known path; this fixes problems on Debian where `su` from
+# non-root may not adjust `$PATH` to root's.
+export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+
+# Check for a supported OS release.
+if [ -f /etc/os-release ]; then
+    os_info="$(
+        . /etc/os-release
+        printf '%s\n' "$ID" "$ID_LIKE" "$VERSION_ID" "$VERSION_CODENAME"
+    )"
+    {
+        read -r os_id
+        read -r os_id_like
+        read -r os_version_id
+        read -r os_version_codename || true
+    } <<<"$os_info"
+    case " $os_id $os_id_like " in
+        *' debian '*)
+            package_system="apt"
+            ;;
+        *' rhel '*)
+            package_system="yum"
+            ;;
+    esac
+fi
+
+if ! "$ZULIP_PATH/scripts/lib/supported-os"; then
+    system_requirements_failure <<EOF
+Unsupported OS release: $os_id $os_version_id
+
+Zulip in production is supported only on:
+ - Debian 12
+ - Debian 13
+ - Ubuntu 22.04 LTS
+ - Ubuntu 24.04 LTS
+ - Ubuntu 26.04 LTS
+EOF
+fi
+
+machine="$(uname -m)"
+if [ "$machine" != x86_64 ] && [ "$machine" != aarch64 ]; then
+    system_requirements_failure <<EOF
+Unsupported CPU architecture: $machine (expected x86_64 or aarch64).
+EOF
+fi
+
+dpkg_architecture="$(dpkg --print-architecture)"
+if [ "$dpkg_architecture" != amd64 ] && [ "$dpkg_architecture" != arm64 ]; then
+    system_requirements_failure <<EOF
+Unsupported OS architecture: $dpkg_architecture (expected amd64 or arm64).
+EOF
+fi
+
+has_universe() {
+    apt-cache policy \
+        | grep -q "^     release v=$os_version_id,o=Ubuntu,a=$os_version_codename,n=$os_version_codename,l=Ubuntu,c=universe"
+}
+
+if [ "$os_id" = ubuntu ] && ! has_universe && ! { apt-get update && has_universe; }; then
+    system_requirements_failure <<EOF
+You must enable the Ubuntu Universe repository before installing
+Zulip.  You can do this with:
+
+    sudo add-apt-repository universe
+    sudo apt update
+EOF
+fi
+
+case ",$PUPPET_CLASSES," in
+    *,zulip::profile::standalone,* | *,zulip::profile::postgresql,*)
+        if [ "$package_system" = apt ]; then
+            # We're going to install PostgreSQL from the PostgreSQL apt
+            # repository; this may conflict with the existing PostgreSQL.
+            OTHER_PG="$(dpkg --get-selections \
+                | grep -E '^postgresql-[0-9]+\s+install$' \
+                | grep -v "^postgresql-$POSTGRESQL_VERSION\b" \
+                | cut -f 1)" || true
+            if [ -n "$OTHER_PG" ]; then
+                INDENTED="${OTHER_PG//$'\n'/$'\n'    }"
+                SPACED="${OTHER_PG//$'\n'/ }"
+                cat <<EOF
+
+The following PostgreSQL servers were found to already be installed:
+
+    $INDENTED
+
+Zulip needs to install PostgreSQL $POSTGRESQL_VERSION, but does not wish
+to uninstall existing databases in order to do so.  Remove all other
+PostgreSQL servers manually before running the installer:
+
+    sudo apt-get remove $SPACED
+
+EOF
+                exit 1
+            fi
+        fi
+        ;;
+esac
+
+# Check for at least ~1.86GB of RAM before starting installation;
+# otherwise users will find out about insufficient RAM via weird
+# errors like a segfault running `pip install`.
+# Additionally, some AWS images that are advertised to be 2 GB
+# are actually 1880000B in size.
+mem_kb=$(head -n1 /proc/meminfo | awk '{print $2}')
+if [ "$mem_kb" -lt 1860000 ]; then
+    set +x
+    echo -e '\033[0;31m' >&2
+    echo "Insufficient RAM.  Zulip requires at least 2GB of RAM." >&2
+    echo >&2
+    echo -e '\033[0m' >&2
+    exit 1
+fi
+
+# Anything under 5GB, we recommended allocating 2GB of swap.  Error
+# out if there's no swap.
+swap_kb=$(grep SwapTotal /proc/meminfo | awk '{print $2}')
+if [ "$mem_kb" -lt 5000000 ] && [ "$swap_kb" -eq 0 ]; then
+    set +x
+    echo -e '\033[0;31m' >&2
+    echo "No swap allocated; when running with < 5GB of RAM, we recommend at least 2GB of swap." >&2
+    echo "https://www.digitalocean.com/community/tutorials/how-to-add-swap-space-on-ubuntu-22-04" >&2
+    echo >&2
+    echo -e '\033[0m' >&2
+    exit 1
+fi
+
+# Do package update, e.g. do `apt-get update` on Debian
+if [ "$package_system" = apt ]; then
+    # setup-apt-repo does an `apt-get update`
+    "$ZULIP_PATH"/scripts/lib/setup-apt-repo
+elif [ "$package_system" = yum ]; then
+    "$ZULIP_PATH"/scripts/lib/setup-yum-repo
+fi
+
+# Check early for missing SSL certificates
+if [ "$PUPPET_CLASSES" = "zulip::profile::standalone" ] && [ -z "$USE_CERTBOT""$SELF_SIGNED_CERT" ] && { ! [ -e "/etc/ssl/private/zulip.key" ] || ! [ -e "/etc/ssl/certs/zulip.combined-chain.crt" ]; }; then
+    set +x
+    cat <<EOF
+
+No SSL certificate found.  One or both required files is missing:
+    /etc/ssl/private/zulip.key
+    /etc/ssl/certs/zulip.combined-chain.crt
+
+Suggested solutions:
+ * For most sites, the --certbot option is recommended.
+ * If you have your own key and cert, see docs linked below
+   for how to install them.
+ * For non-production testing, try the --self-signed-cert option.
+
+For help and more details, see our SSL documentation:
+  https://zulip.readthedocs.io/en/latest/production/ssl-certificates.html
+
+Once fixed, just rerun scripts/setup/install; it'll pick up from here!
+
+EOF
+    exit 1
+fi
+
+# don't run dist-upgrade in one click apps to make the
+# installation process more seamless.
+if [ -z "$NO_DIST_UPGRADE" ]; then
+    if [ "$package_system" = apt ]; then
+        apt-get -y dist-upgrade "${APT_OPTIONS[@]}"
+    elif [ "$package_system" = yum ]; then
+        # On CentOS, there is no need to do `yum -y upgrade` because `yum -y
+        # update` already does the same thing.
+        :
+    fi
+fi
+
+if [ "$package_system" = apt ]; then
+    # Note that any additions to these lists must also be added to
+    # `zulip::profile::base` such that the new dependency is seen by
+    # upgrades, as well as new installs.
+    if ! apt-get install -y --no-install-recommends \
+        python3 python3-yaml puppet git curl jq crudini \
+        "${ADDITIONAL_PACKAGES[@]}"; then
+        set +x
+        echo -e '\033[0;31m' >&2
+        echo "Installing packages failed; is network working and (on Ubuntu) the universe repository enabled?" >&2
+        echo >&2
+        echo -e '\033[0m' >&2
+        exit 1
+    fi
+elif [ "$package_system" = yum ]; then
+    if ! yum install -y \
+        python3 python3-pyyaml puppet git curl jq crudini \
+        "${ADDITIONAL_PACKAGES[@]}"; then
+        set +x
+        echo -e '\033[0;31m' >&2
+        echo "Installing packages failed; is network working?" >&2
+        echo >&2
+        echo -e '\033[0m' >&2
+        exit 1
+    fi
+fi
+
+# We generate a self-signed cert even with certbot, so we can use the
+# webroot authenticator, which requires nginx be set up with a
+# certificate.
+if [ -n "$SELF_SIGNED_CERT" ] || [ -n "$USE_CERTBOT" ]; then
+    "$ZULIP_PATH"/scripts/setup/generate-self-signed-cert \
+        --exists-ok "${EXTERNAL_HOST:-$(hostname)}"
+fi
+
+# Generate /etc/zulip/zulip.conf .
+mkdir -p /etc/zulip
+has_class() {
+    grep -qx "$1" /var/lib/puppet/classes.txt
+}
+
+# puppet apply --noop fails unless the user that it _would_ chown
+# files to exists; https://tickets.puppetlabs.com/browse/PUP-3907
+#
+# The home directory here should match what's declared in base.pp.
+id -u zulip &>/dev/null || useradd -m zulip --home-dir /home/zulip
+if [ -n "$NO_OVERWRITE_SETTINGS" ] && [ -e "/etc/zulip/zulip.conf" ]; then
+    "$ZULIP_PATH"/scripts/zulip-puppet-apply --noop \
+        --write-catalog-summary \
+        --classfile=/var/lib/puppet/classes.txt
+else
+    # Write out more than we need, and remove sections that are not
+    # applicable to the classes that are actually necessary.
+    cat <<EOF >/etc/zulip/zulip.conf
+[machine]
+puppet_classes = $PUPPET_CLASSES
+deploy_type = production
+
+[postgresql]
+version = $POSTGRESQL_VERSION
+EOF
+
+    "$ZULIP_PATH"/scripts/zulip-puppet-apply --noop \
+        --write-catalog-summary \
+        --classfile=/var/lib/puppet/classes.txt
+
+    # We only need the PostgreSQL version setting on database hosts,
+    # or hosts which talk directly to the database (e.g. application
+    # hosts); but we don't know if this is a database host until we
+    # have the catalog summary.
+    if (! has_class "zulip::postgresql_common" && ! has_class "zulip::postgresql_client") || [ "$package_system" != apt ]; then
+        crudini --del /etc/zulip/zulip.conf postgresql
+    fi
+
+    if [ -n "$POSTGRESQL_DATABASE_NAME" ]; then
+        crudini --set /etc/zulip/zulip.conf postgresql database_name "$POSTGRESQL_DATABASE_NAME"
+    fi
+
+    if [ -n "$POSTGRESQL_DATABASE_USER" ]; then
+        crudini --set /etc/zulip/zulip.conf postgresql database_user "$POSTGRESQL_DATABASE_USER"
+    fi
+fi
+
+if has_class "zulip::app_frontend_base"; then
+    # Frontend deploys use /home/zulip/deployments; without this, the
+    # install directory is also only readable by root.
+    mkdir -p /home/zulip/deployments
+    deploy_path=$("$ZULIP_PATH"/scripts/lib/zulip_tools.py make_deploy_path)
+    mv "$ZULIP_PATH" "$deploy_path"
+    ln -nsf /home/zulip/deployments/next "$ZULIP_PATH"
+    ln -nsf "$deploy_path" /home/zulip/deployments/next
+
+    # Create and activate a virtualenv
+    "$deploy_path"/scripts/lib/create-production-venv "$deploy_path"
+
+    "$deploy_path"/scripts/lib/install-node
+
+    if [ -z "$NO_OVERWRITE_SETTINGS" ] || ! [ -e "/etc/zulip/settings.py" ]; then
+        cp -a "$deploy_path"/zproject/prod_settings_template.py /etc/zulip/settings.py
+        if [ -n "$EXTERNAL_HOST" ]; then
+            sed -i "s/^EXTERNAL_HOST =.*/EXTERNAL_HOST = '$EXTERNAL_HOST'/" /etc/zulip/settings.py
+        fi
+        if [ -n "$ZULIP_ADMINISTRATOR" ]; then
+            sed -i "s/^ZULIP_ADMINISTRATOR =.*/ZULIP_ADMINISTRATOR = '$ZULIP_ADMINISTRATOR'/" /etc/zulip/settings.py
+        fi
+
+        # Set SERVICE settings based on what the user provided.
+        if grep -q -E "^#?\s*ZULIP_SERVICE_PUSH_NOTIFICATIONS = " /etc/zulip/settings.py; then
+            sed -i -E "s/^#?\s*ZULIP_SERVICE_PUSH_NOTIFICATIONS = .*/ZULIP_SERVICE_PUSH_NOTIFICATIONS = $SERVICE_MOBILE_PUSH/" /etc/zulip/settings.py
+        else
+            echo "ZULIP_SERVICE_PUSH_NOTIFICATIONS = $SERVICE_MOBILE_PUSH" >>/etc/zulip/settings.py
+        fi
+
+        if grep -q -E "^#?\s*ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS = " /etc/zulip/settings.py; then
+            sed -i -E "s/^#?\s*ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS = .*/ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS = $SERVICE_SUBMIT_USAGE_STATISTICS/" /etc/zulip/settings.py
+        else
+            echo "ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS = $SERVICE_SUBMIT_USAGE_STATISTICS" >>/etc/zulip/settings.py
+        fi
+    fi
+    ln -nsf /etc/zulip/settings.py "$deploy_path"/zproject/prod_settings.py
+    "$deploy_path"/scripts/setup/generate_secrets.py --production
+else
+    deploy_path="$ZULIP_PATH"
+fi
+
+"$deploy_path"/scripts/zulip-puppet-apply -f
+
+if [ "$package_system" = apt ]; then
+    apt-get -y --with-new-pkgs upgrade
+elif [ "$package_system" = yum ]; then
+    # No action is required because `yum update` already does upgrade.
+    :
+fi
+
+if [ -n "$USE_CERTBOT" ]; then
+    "$deploy_path"/scripts/setup/setup-certbot \
+        "$EXTERNAL_HOST" --email "$ZULIP_ADMINISTRATOR"
+fi
+
+if has_class "zulip::nginx" && ! has_class "zulip::profile::docker"; then
+    # Check nginx was configured properly now that we've installed it.
+    # Most common failure mode is certs not having been installed.
+    if ! nginx -t; then
+        (
+            set +x
+            cat <<EOF
+
+Verifying the Zulip nginx configuration failed!
+
+This is almost always a problem with your SSL certificates.  See:
+  https://zulip.readthedocs.io/en/latest/production/ssl-certificates.html
+
+Once fixed, just rerun scripts/setup/install; it'll pick up from here!
+
+EOF
+            exit 1
+        )
+    fi
+fi
+
+if has_class "zulip::profile::rabbitmq"; then
+    if ! rabbitmqctl status >/dev/null; then
+        set +x
+        cat <<EOF
+
+RabbitMQ seems to not have started properly after the installation process.
+Often this is caused by misconfigured /etc/hosts in virtualized environments.
+For more information, see:
+  https://github.com/zulip/zulip/issues/53#issuecomment-143805121
+
+EOF
+        exit 1
+    fi
+fi
+
+# Set up a basic .gitconfig for the 'zulip' user
+if [ -n "$ZULIP_ADMINISTRATOR" ]; then
+    (
+        cd / # Make sure the current working directory is readable by zulip
+        su zulip -c "git config --global user.email $ZULIP_ADMINISTRATOR"
+        su zulip -c "git config --global user.name 'Zulip Server ($EXTERNAL_HOST)'"
+    )
+fi
+
+if ! has_class "zulip::app_frontend_base"; then
+    set +x
+    cat <<EOF
+
+Success!
+
+Not configuring PostgreSQL, or /home/zulip/deployments, because this
+is not a front-end install.
+
+EOF
+    exit 0
+fi
+
+ln -nsf "$deploy_path" /home/zulip/deployments/current
+ln -nsf /etc/zulip/settings.py "$deploy_path"/zproject/prod_settings.py
+mkdir -p "$deploy_path"/prod-static/serve
+cp -rT "$deploy_path"/prod-static/serve /home/zulip/prod-static
+chown -R zulip:zulip /home/zulip /var/log/zulip /etc/zulip/settings.py
+
+if ! [ -e /home/zulip/deployments/current/zulip-git-version ]; then
+    mkdir /srv/zulip.git
+    chown zulip:zulip /srv/zulip.git
+    su zulip -c 'git clone --bare --mirror /home/zulip/deployments/current /srv/zulip.git'
+    su zulip -c 'cd /home/zulip/deployments/current && ./scripts/lib/update-git-upstream && ./tools/cache-zulip-git-version'
+fi
+
+if ! [ -e "/home/zulip/prod-static/generated" ]; then
+    # If we're installing from a Git checkout, we need to run
+    # `tools/update-prod-static` in order to build the static
+    # assets.
+    su zulip -c '/home/zulip/deployments/current/tools/update-prod-static'
+fi
+
+if [ -n "$NO_INIT_DB" ]; then
+    set +x
+    cat <<EOF
+
+Success!
+
+Stopping because --no-init-db was passed.  To complete the
+installation, configure PostgreSQL by creating the database and
+database user, and then run:
+
+   su zulip -c '/home/zulip/deployments/current/scripts/setup/initialize-database'
+   su zulip -c '/home/zulip/deployments/current/manage.py generate_realm_creation_link'
+EOF
+
+    if [ -n "$PUSH_NOTIFICATIONS" ]; then
+        echo
+        echo "Since you specified you want to enable the push notification service, you'll also need to "
+        echo "manually register your server by running:"
+        echo
+        echo "   su zulip -c '/home/zulip/deployments/current/manage.py register_server'"
+        echo
+    fi
+    exit 0
+else
+    /home/zulip/deployments/current/scripts/setup/create-database
+    su zulip -c '/home/zulip/deployments/current/scripts/setup/initialize-database --quiet'
+
+    if [ -n "$PUSH_NOTIFICATIONS" ]; then
+        if [ -z "$PUSH_NOTIFICATIONS_SERVICE_TOS_AGREED" ]; then
+            # This should be impossible as the installation should have aborted early if the user
+            # passed --push-notifications without accepting the ToS. But we include this as a precaution
+            # to not allow future bugs to cause ToS acceptance to be skipped.
+            SKIP_TOS_AGREEMENT_OPTION=""
+        else
+            SKIP_TOS_AGREEMENT_OPTION="--agree-to-terms-of-service"
+        fi
+        set +x
+        echo "Services enabled, attempting to register server..."
+        set -x
+
+        # Without PYTHONUNBUFFERED=1, stdout/stderr might end up displayed not in chronological order, resulting
+        # in confusing output to the user.
+        su zulip -c "PYTHONUNBUFFERED=1 /home/zulip/deployments/current/manage.py register_server ${SKIP_TOS_AGREEMENT_OPTION}" \
+            || {
+                set +x
+                echo
+                echo "WARNING: Server registration for push notifications failed."
+                echo "This does not affect the rest of your Zulip installation."
+                echo
+                echo "To enable push notifications after resolving the issue, run:"
+                echo "  su zulip -c '/home/zulip/deployments/current/manage.py register_server'"
+                echo
+                set -x
+            }
+    fi
+
+    su zulip -c '/home/zulip/deployments/current/manage.py generate_realm_creation_link'
+fi

scripts/lib/install-node

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+version=24.19.0
+arch="$(uname -m)"
+
+case $arch in
+    x86_64)
+        tarball="node-v$version-linux-x64.tar.xz"
+        sha256=14b342e71204f811bde6153be8e04b62aef63c236fef92b55f9c83154b409647
+        ;;
+
+    aarch64)
+        tarball="node-v$version-linux-arm64.tar.xz"
+        sha256=01443c1e1a29e531ccad5a46fefa6df490d2189c49f7955904aecdbb0fe86fdc
+        ;;
+esac
+
+check_version() {
+    out="$(node --version)" && [ "$out" = "v$version" ] \
+        && [ /usr/local/bin/pnpm -ef /srv/zulip-node/lib/node_modules/corepack/dist/pnpm.js ]
+}
+
+if ! check_version 2>/dev/null; then
+    set -x
+    tmpdir="$(mktemp -d)"
+    trap 'rm -r "$tmpdir"' EXIT
+    cd "$tmpdir"
+    curl -fLO --retry 3 "https://nodejs.org/dist/v$version/$tarball"
+    sha256sum -c <<<"$sha256 $tarball"
+    rm -rf /srv/zulip-node
+    mkdir -p /srv/zulip-node
+    tar -xJf "$tarball" --no-same-owner --strip-components=1 -C /srv/zulip-node
+    ln -sf /srv/zulip-node/bin/{corepack,node,npm,npx} /usr/local/bin
+    COREPACK_DEFAULT_TO_LATEST=0 /usr/local/bin/corepack enable
+
+    # Clean up after previous versions of this script
+    rm -rf /srv/zulip-yarn /usr/bin/yarn /usr/local/nvm
+
+    check_version
+fi

scripts/lib/install-uv

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+set -eu
+
+version=0.12.5
+arch="$(uname -m)"
+tarball="uv-$arch-unknown-linux-gnu.tar.gz"
+declare -A sha256=(
+    [aarch64]=9bf43b4d1a07665bf64d4c4e710930b382321a785e0eb10aac07f46471f86a31
+    [x86_64]=68a509da24b06b4223a1c0175fb5eb5bc79342b76cbeff0cfe51ac3f5b17b6b2
+)
+
+check_version() {
+    out="$(uv --version)" && [[ "$out" = "uv $version ("* ]]
+}
+
+if ! check_version 2>/dev/null; then
+    set -x
+    tmpdir="$(mktemp -d)"
+    trap 'rm -r "$tmpdir"' EXIT
+    cd "$tmpdir"
+    curl -fLO --retry 3 "https://github.com/astral-sh/uv/releases/download/$version/$tarball"
+    sha256sum -c <<<"${sha256[$arch]} $tarball"
+    tar -xzf "$tarball" --no-same-owner --strip-components=1 -C /usr/local/bin "uv-$arch-unknown-linux-gnu"/{uv,uvx}
+    check_version
+fi

scripts/lib/node_cache.py

@@ -0,0 +1,39 @@
+import os
+
+from scripts.lib.zulip_tools import run
+
+DEFAULT_PRODUCTION = False
+
+
+def setup_node_modules(production: bool = DEFAULT_PRODUCTION) -> None:
+    if os.path.islink("node_modules"):
+        os.unlink("node_modules")
+
+    skip = False
+
+    try:
+        with open("node_modules/.pnpm/lock.yaml") as a, open("pnpm-lock.yaml") as b:
+            if a.read() == b.read():
+                skip = True
+    except FileNotFoundError:
+        pass
+
+    # We need this check when switching between branches without `starlight_help`
+    # package. `node_modules` will be removed when working on a non `starlight_help`
+    # branch, but if `node_modules/.pnpm/lock.yaml` has not been updated by that
+    # branch, we will end up in a situation where we might not have `node_modules`
+    # even when we run the provision command.
+    if not os.path.exists("starlight_help/node_modules"):
+        skip = False
+
+    if not skip:
+        run(
+            [
+                "/usr/local/bin/corepack",
+                "pnpm",
+                "install",
+                "--frozen-lockfile",
+                "--prefer-offline",
+                *(["--prod"] if production else []),
+            ],
+        )

scripts/lib/puppet_cache.py

@@ -0,0 +1,107 @@
+import hashlib
+import json
+import os
+import shutil
+import subprocess
+import tempfile
+from urllib.request import urlopen
+
+import yaml
+
+from .zulip_tools import run
+
+ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+ZULIP_SRV_PATH = "/srv"
+
+PUPPET_MODULES_CACHE_PATH = os.path.join(ZULIP_SRV_PATH, "zulip-puppet-cache")
+PUPPET_DEPS_FILE_PATH = os.path.join(ZULIP_PATH, "puppet/deps.yaml")
+PUPPET_THIRDPARTY = os.path.join(PUPPET_MODULES_CACHE_PATH, "current")
+
+
+def generate_sha1sum_puppet_modules() -> str:
+    data = {}
+    with open(PUPPET_DEPS_FILE_PATH) as fb:
+        data["deps.yaml"] = fb.read().strip()
+    data["puppet-version"] = subprocess.check_output(
+        # This is 10x faster than `puppet --version`
+        ["ruby", "-r", "puppet/version", "-e", "puts Puppet.version"],
+        text=True,
+    ).strip()
+
+    sha1sum = hashlib.sha1()
+    sha1sum.update(json.dumps(data, sort_keys=True).encode())
+    return sha1sum.hexdigest()
+
+
+def setup_puppet_modules() -> None:
+    sha1sum = generate_sha1sum_puppet_modules()
+    target_path = os.path.join(PUPPET_MODULES_CACHE_PATH, sha1sum)
+    success_stamp = os.path.join(target_path, ".success-stamp")
+    # Check if a cached version already exists
+    if not os.path.exists(success_stamp):
+        do_puppet_module_install(target_path, success_stamp)
+
+    if os.path.islink(PUPPET_THIRDPARTY):
+        os.remove(PUPPET_THIRDPARTY)
+    elif os.path.isdir(PUPPET_THIRDPARTY):
+        shutil.rmtree(PUPPET_THIRDPARTY)
+    os.symlink(target_path, PUPPET_THIRDPARTY)
+
+
+def do_puppet_module_install(
+    target_path: str,
+    success_stamp: str,
+) -> None:
+    os.makedirs(target_path, exist_ok=True)
+    with open(PUPPET_DEPS_FILE_PATH) as yaml_file:
+        deps = yaml.safe_load(yaml_file)
+    for module, metadata in deps.items():
+        install_puppet_module(target_path, module, metadata["version"], metadata["sha256sum"])
+    with open(success_stamp, "w"):
+        pass
+
+
+def install_puppet_module(
+    target_path: str, module: str, version: str, expected_sha256sum: str
+) -> None:
+    print(f"Fetching {module}-{version} from forgeapi.puppet.com...")
+    with urlopen(
+        f"https://forgeapi.puppet.com/v3/releases/{module}-{version}", timeout=60
+    ) as forge_resp:
+        forge_data = json.load(forge_resp)
+
+    forge_sha256sum = forge_data["file_sha256"]
+    if forge_sha256sum != expected_sha256sum:
+        raise Exception(
+            f"Forge API returned unexpected SHA256 sum for {module}-{version}: "
+            f"expected {expected_sha256sum}, got {forge_sha256sum}"
+        )
+
+    with tempfile.NamedTemporaryFile(
+        prefix=f"zulip-puppet-{module}-{version}-",
+        suffix=".tar.gz",
+    ) as tarball:
+        with urlopen(
+            "https://forgeapi.puppet.com" + forge_data["file_uri"], timeout=60
+        ) as tarball_resp:
+            tarball_content = tarball_resp.read()
+            local_sha256sum = hashlib.sha256(tarball_content).hexdigest()
+            if local_sha256sum != expected_sha256sum:
+                raise Exception(
+                    f"Downloaded file had unexpected SHA256 sum for {module}-{version}: "
+                    f"expected {expected_sha256sum}, got {forge_sha256sum}"
+                )
+            tarball.write(tarball_content)
+            tarball.flush()
+
+        run(
+            [
+                "puppet",
+                "module",
+                "--modulepath",
+                target_path,
+                "install",
+                tarball.name,
+                "--ignore-dependencies",
+            ],
+        )

scripts/lib/queue_workers.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+import os
+import sys
+
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.append(BASE_DIR)
+from scripts.lib.setup_path import setup_path
+
+setup_path()
+
+os.environ["DJANGO_SETTINGS_MODULE"] = "zproject.settings"
+
+import django
+
+django.setup()
+from zerver.worker.queue_processors import get_active_worker_queues
+
+if __name__ == "__main__":
+    for worker in sorted(get_active_worker_queues()):
+        print(worker)

scripts/lib/run_hooks.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+import argparse
+import os
+import subprocess
+import sys
+
+sys.path.append(os.path.join(os.path.dirname(__file__), "..", ".."))
+from scripts.lib.zulip_tools import (
+    DEPLOYMENTS_DIR,
+    assert_running_as_root,
+    get_deploy_root,
+    get_zulip_pwent,
+    parse_version_from,
+    su_to_zulip,
+)
+
+assert_running_as_root()
+
+# Updates to the below choices to add a new hook type should
+# adjust puppet's `app_frontend_base.pp` as well.
+parser = argparse.ArgumentParser()
+parser.add_argument("kind", choices=["pre-deploy", "post-deploy"], help="")
+parser.add_argument("--from-git", action="store_true", help="Upgrading from git")
+args = parser.parse_args()
+
+from version import ZULIP_MERGE_BASE as NEW_ZULIP_MERGE_BASE
+from version import ZULIP_VERSION as NEW_ZULIP_VERSION
+
+deploy_path = get_deploy_root()
+
+if args.kind == "post-deploy":
+    old_dir_name = "last"
+    if not os.path.exists(DEPLOYMENTS_DIR + "/last"):
+        # Fresh installs which are doing an OS upgrade don't have a
+        # "last" yet
+        old_dir_name = "current"
+else:
+    old_dir_name = "current"
+old_version = parse_version_from(DEPLOYMENTS_DIR + "/" + old_dir_name)
+old_merge_base = parse_version_from(DEPLOYMENTS_DIR + "/" + old_dir_name, merge_base=True)
+
+path = f"/etc/zulip/hooks/{args.kind}.d"
+if not os.path.exists(path):
+    sys.exit(0)
+
+# Pass in, via environment variables, the old/new "version
+# string" (which is a `git describe` output)
+env = os.environ.copy()
+env["ZULIP_OLD_VERSION"] = old_version
+env["ZULIP_NEW_VERSION"] = NEW_ZULIP_VERSION
+
+# preexec_fn=su_to_zulip normally handles this, but our explicit
+# env overrides that
+env["HOME"] = get_zulip_pwent().pw_dir
+
+
+def resolve_version_string(version: str) -> str:
+    return subprocess.check_output(
+        ["git", "rev-parse", version], cwd=deploy_path, preexec_fn=su_to_zulip, text=True
+    ).strip()
+
+
+if args.from_git:
+    # If we have a git repo, we also resolve those `git describe`
+    # values to full commit hashes, as well as provide the
+    # merge-base of the old/new commits with mainline.
+    env["ZULIP_OLD_COMMIT"] = resolve_version_string(old_version)
+    env["ZULIP_NEW_COMMIT"] = resolve_version_string(NEW_ZULIP_VERSION)
+    env["ZULIP_OLD_MERGE_BASE_COMMIT"] = resolve_version_string(old_merge_base)
+    env["ZULIP_NEW_MERGE_BASE_COMMIT"] = resolve_version_string(NEW_ZULIP_MERGE_BASE)
+
+failures = []
+for script_name in sorted(f for f in os.listdir(path) if f.endswith(".hook")):
+    result = subprocess.run(
+        [os.path.join(path, script_name)],
+        check=False,
+        cwd=deploy_path,
+        preexec_fn=su_to_zulip,
+        env=env,
+    )
+    if result.returncode != 0:
+        # Pre-deploy hooks abort on the first failure; post-deploy
+        # hooks are best-effort and a failure of one does not abort
+        # the rest of them.
+        if args.kind == "pre-deploy":
+            sys.exit(1)
+        failures.append(script_name)
+
+if failures:
+    print("Failed hooks:")
+    for failed_script in failures:
+        print(f"  {failed_script}")

scripts/lib/setup-apt-repo

@@ -0,0 +1,120 @@
+#!/usr/bin/env bash
+#
+# This script handles adding custom apt repositories into
+# /etc/apt/sources.list.d/ files.  It bundles the GPG keys which are
+# used to verify the repositories (via `apt-key`), to explicitly pin
+# the trusted signing keys, as opposed to blindly trusting HTTPS.
+#
+# Each /etc/apt/sources.list.d/foo.list file is created via `--list
+# foo`, where `foo` defaults to `zulip`.  The default `zulip.list` is
+# installed in `scripts/lib/install` / `tools/lib/provision.py`, and
+# other `.list` files may be installed by Puppet.
+set -x
+set -e
+set -u
+set -o pipefail
+shopt -s extglob
+
+verify=false
+args="$(getopt -o '' --long verify,list: -- "$@")"
+eval "set -- $args"
+LIST=zulip
+while true; do
+    case "$1" in
+        --verify)
+            verify=true
+            shift
+            ;;
+        --list)
+            LIST="$2"
+            shift
+            shift
+            ;;
+        --)
+            shift
+            break
+            ;;
+    esac
+done
+
+# Ensure the directory for LAST_DEPENDENCIES_HASH exists
+mkdir -p /var/lib/zulip
+
+SOURCES_FILE=/etc/apt/sources.list.d/$LIST.list
+PREF_FILE=/etc/apt/preferences.d/$LIST.pref
+STAMP_FILE=/etc/apt/sources.list.d/$LIST.list.apt-update-in-progress
+
+# All paths, for purposes of hashing, should be relative to the deploy
+# root.
+cd "$(dirname "$0")/../.."
+
+LIST_PATH="scripts/setup/apt-repos/$LIST"
+if ! [ -d "$LIST_PATH" ]; then
+    echo "Not a valid value for --list: '$LIST'"
+    echo ""
+    echo "Valid values are:"
+    ls -1 "scripts/setup/apt-repos/"
+    exit 1
+fi
+
+release="$(. /etc/os-release && printf '%s' "$VERSION_CODENAME")"
+
+if [ ! -f "$LIST_PATH/$release.list" ]; then
+    cat <<EOF
+Unsupported release $release for sources.list file $LIST.  To add a
+new release, make a $LIST_PATH/$release.list file based on existing
+.list files in that directory.
+
+EOF
+    exit 1
+fi
+
+DEPENDENCIES_HASH="$(sha256sum "$LIST_PATH"/?(*.asc|"$release.list"|"$LIST.pref"|custom.sh) "scripts/lib/setup-apt-repo")"
+DEPENDENCIES_HASH_FILE="/var/lib/zulip/setup-repositories-state-$LIST"
+# Ensure that DEPENDENCIES_HASH_FILE exists before hashing it.
+touch "$DEPENDENCIES_HASH_FILE"
+LAST_DEPENDENCIES_HASH="$(cat "$DEPENDENCIES_HASH_FILE")"
+
+# First, we only do anything in setup-apt-repo if any of its inputs
+# (apt keys, code, etc.)  changed.
+if [ "$DEPENDENCIES_HASH" = "$LAST_DEPENDENCIES_HASH" ]; then
+    exit 0
+elif [ "$verify" == true ]; then
+    exit 1
+fi
+
+# Hash to check if the configuration is changed by the script later.
+hashes=$(sha256sum "$SOURCES_FILE" "$PREF_FILE" 2>/dev/null || true)
+
+if [ -e "$LIST_PATH/$LIST.pref" ]; then
+    cp "$LIST_PATH/$LIST.pref" "$PREF_FILE"
+else
+    rm -f "$PREF_FILE"
+fi
+
+pre_setup_deps=(apt-transport-https ca-certificates gnupg curl)
+if ! apt-get -dy install "${pre_setup_deps[@]}"; then
+    apt-get update
+fi
+apt-get -y install "${pre_setup_deps[@]}"
+
+cp "$LIST_PATH/"*.asc /etc/apt/keyrings/
+cp "$LIST_PATH/$release.list" "$SOURCES_FILE"
+
+if [ -e "$LIST_PATH/custom.sh" ]; then
+    export LIST_PATH
+    export STAMP_FILE
+    bash "$LIST_PATH/custom.sh"
+fi
+
+if [ "$hashes" = "$(sha256sum "$SOURCES_FILE" "$PREF_FILE" 2>/dev/null || true)" ] && ! [ -e "$STAMP_FILE" ]; then
+    echo "APT configuration did not change; skipping apt-get update"
+else
+    # We create this stamp file to ensure `apt-get update` will be run
+    # the next time this script is invoked, and each time after, until
+    # `apt-get update` finishes successfully.
+    touch "$STAMP_FILE"
+    apt-get update && rm -f "$STAMP_FILE"
+fi
+
+echo "$DEPENDENCIES_HASH" >"$DEPENDENCIES_HASH_FILE"

scripts/lib/setup-yum-repo

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+set -x
+set -e
+
+is_prod=false
+args="$(getopt -o '' --long prod -- "$@")"
+eval "set -- $args"
+while true; do
+    case "$1" in
+        --prod)
+            is_prod=true
+            shift
+            ;;
+        --)
+            shift
+            break
+            ;;
+    esac
+done
+
+is_centos=false
+is_rhel=false
+if [ -e /etc/centos-release ]; then
+    is_centos=true
+    yum install -y epel-release
+
+    if [ "$is_prod" = true ]; then
+        # IUS is needed for installing python36u-mod_wsgi on prod env
+        yum localinstall -y https://centos7.iuscommunity.org/ius-release.rpm
+    fi
+
+elif grep -q "Red Hat" /etc/redhat-release; then
+    is_rhel=true
+    yum localinstall -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+fi
+
+yum update -y
+
+RHVER="$(rpm -qf --queryformat="%{VERSION}" /etc/redhat-release)"
+RHARCH="$(rpm -qf --queryformat="%{ARCH}" /etc/redhat-release)"
+PGVER=10
+if [ "$is_centos" = true ]; then
+    # "Development Tools" is the equivalent of build-essential
+    yum groupinstall -y "Development Tools"
+
+    # PostgreSQL $PGVER
+    yum localinstall -y "https://yum.postgresql.org/$PGVER/redhat/rhel-$RHVER-$RHARCH/pgdg-redhat-repo-latest.noarch.rpm"
+
+    # PGroonga
+    # https://pgroonga.github.io/install/centos.html
+    yum localinstall -y https://packages.groonga.org/centos/groonga-release-latest.noarch.rpm
+elif [ "$is_rhel" = true ]; then
+    yum groupinstall -y "Development Tools"
+    yum localinstall -y https://download.postgresql.org/pub/repos/yum/10/redhat/rhel-latest-x86_64/pgdg-redhat10-10-2.noarch.rpm
+    yum localinstall -y https://packages.groonga.org/centos/groonga-release-latest.noarch.rpm
+else
+    # dnf5 (Fedora 41+) dropped the `groupinstall` alias and no longer
+    # resolves the "Development Tools" display name, so install by group ID.
+    dnf group install -y development-tools
+    # TODO make the postgres version a variable.
+    PGVER=13
+    dnf install -y "https://download.postgresql.org/pub/repos/yum/reporpms/F-$RHVER-x86_64/pgdg-fedora-repo-latest.noarch.rpm"
+fi

scripts/lib/setup_path.py

@@ -0,0 +1,28 @@
+"""
+Use libraries from a virtualenv (by modifying sys.path) in production.
+"""
+
+import os
+import sys
+
+
+def setup_path() -> None:
+    BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+    venv = os.path.realpath(os.path.join(BASE_DIR, ".venv"))
+    if sys.prefix != venv:
+        sys.path = list(
+            filter(
+                # zulip-py3-venv was an historical virtualenv symlink
+                lambda p: "/zulip-py3-venv/" not in p and "/.venv/" not in p,
+                sys.path,
+            )
+        )
+        activate_this = os.path.join(venv, "bin", "activate_this.py")
+        activate_locals = dict(__file__=activate_this)
+        with open(activate_this) as f:
+            exec(f.read(), activate_locals)  # noqa: S102
+        # Check that the python version running this function
+        # is same as python version that created the virtualenv.
+        python_version = "python{}.{}".format(*sys.version_info[:2])
+        if not os.path.exists(os.path.join(venv, "lib", python_version)):
+            raise RuntimeError(venv + " was not set up for this Python version")

scripts/lib/setup_venv.py

@@ -0,0 +1,71 @@
+import os
+
+from scripts.lib.zulip_tools import os_families
+
+ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+VENV_DEPENDENCIES = [
+    "build-essential",
+    "libffi-dev",
+    "libldap2-dev",
+    "python3-dev",  # Needed to install typed-ast dependency of mypy
+    "python3-pip",
+    "virtualenv",
+    "libxml2-dev",  # Used for installing talon-core and python-xmlsec
+    "libxslt1-dev",  # Used for installing talon-core
+    "libpq-dev",  # Needed by psycopg2
+    "libssl-dev",  # Needed to build pycurl and other libraries
+    "libmagic1",  # Used for install python-magic
+    "libyaml-dev",  # For fast YAML parsing in PyYAML
+    # Needed by python-xmlsec:
+    "libxmlsec1-dev",
+    "pkg-config",
+    "jq",  # No longer used in production (clean me up later)
+    "libsasl2-dev",  # For building python-ldap from source
+    "libvips",  # For thumbnailing
+    "libvips-tools",
+    "libicu-dev",  # For building pyicu
+    "libcrypt-dev",  # For building uwsgi
+]
+
+COMMON_YUM_VENV_DEPENDENCIES = [
+    "libffi-devel",
+    "openldap-devel",
+    "libyaml-devel",
+    # Needed by python-xmlsec:
+    "gcc",
+    "python3-devel",
+    "libxml2-devel",
+    "xmlsec1-devel",
+    "xmlsec1-openssl-devel",
+    "libtool-ltdl-devel",
+    "libxslt-devel",
+    "postgresql-libs",  # libpq-dev on apt
+    "openssl-devel",
+    "jq",
+    "vips",  # For thumbnailing
+    "vips-tools",
+]
+
+REDHAT_VENV_DEPENDENCIES = [
+    *COMMON_YUM_VENV_DEPENDENCIES,
+    "python36-devel",
+    "python-virtualenv",
+]
+
+FEDORA_VENV_DEPENDENCIES = [
+    *COMMON_YUM_VENV_DEPENDENCIES,
+    "python3-pip",
+    "virtualenv",  # see https://unix.stackexchange.com/questions/27877/install-virtualenv-on-fedora-16
+]
+
+
+def get_venv_dependencies(vendor: str, os_version: str) -> list[str]:
+    if "debian" in os_families():
+        return VENV_DEPENDENCIES
+    elif "rhel" in os_families():
+        return REDHAT_VENV_DEPENDENCIES
+    elif "fedora" in os_families():
+        return FEDORA_VENV_DEPENDENCIES
+    else:
+        raise AssertionError("Invalid vendor")

scripts/lib/sharding.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+import argparse
+import filecmp
+import grp
+import json
+import os
+import pwd
+import subprocess
+import sys
+
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.append(BASE_DIR)
+from scripts.lib.setup_path import setup_path
+
+setup_path()
+
+from scripts.lib.zulip_tools import get_config_file, get_tornado_ports
+
+
+def nginx_quote(s: str) -> str:
+    return '"' + s.replace("\\", "\\\\").replace('"', '\\"') + '"'
+
+
+# Basic system to do Tornado sharding.  Writes two output .tmp files that need
+# to be renamed to the following files to finalize the changes:
+# * /etc/zulip/nginx_sharding_map.conf; nginx needs to be reloaded after changing.
+# * /etc/zulip/sharding.json; supervisor Django process needs to be reloaded
+# after changing.  TODO: We can probably make this live-reload by statting the file.
+#
+# TODO: Restructure this to automatically generate a sharding layout.
+def write_updated_configs() -> None:
+    config_file = get_config_file()
+    ports = get_tornado_ports(config_file)
+
+    expected_ports = list(range(9800, ports[-1] + 1))
+    assert ports == expected_ports, f"ports ({ports}) must be contiguous, starting with 9800"
+
+    with (
+        open("/etc/zulip/nginx_sharding_map.conf.tmp", "w") as nginx_sharding_conf_f,
+        open("/etc/zulip/sharding.json.tmp", "w") as sharding_json_f,
+    ):
+        if len(ports) == 1:
+            nginx_sharding_conf_f.write('map "" $tornado_server {\n')
+            nginx_sharding_conf_f.write("    default http://tornado;\n")
+            nginx_sharding_conf_f.write("}\n")
+            sharding_json_f.write("{}\n")
+            return
+
+        nginx_sharding_conf_f.write("map $host $tornado_server {\n")
+        nginx_sharding_conf_f.write("    default http://tornado9800;\n")
+        shard_map: dict[str, int | list[int]] = {}
+        shard_regexes: list[tuple[str, int | list[int]]] = []
+        external_host = subprocess.check_output(
+            [os.path.join(BASE_DIR, "scripts/get-django-setting"), "EXTERNAL_HOST"],
+            text=True,
+        ).strip()
+        for key, shards in config_file["tornado_sharding"].items():
+            if key.endswith("_regex"):
+                ports = [int(port) for port in key.removesuffix("_regex").split("_")]
+                shard_regexes.append((shards, ports[0] if len(ports) == 1 else ports))
+                nginx_sharding_conf_f.write(
+                    f"    {nginx_quote('~*' + shards)} http://tornado{'_'.join(map(str, ports))};\n"
+                )
+            else:
+                ports = [int(port) for port in key.split("_")]
+                for shard in shards.split():
+                    if "." in shard:
+                        host = shard
+                    else:
+                        host = f"{shard}.{external_host}"
+                    assert host not in shard_map, f"host {host} duplicated"
+                    shard_map[host] = ports[0] if len(ports) == 1 else ports
+                    nginx_sharding_conf_f.write(
+                        f"    {nginx_quote(host)} http://tornado{'_'.join(map(str, ports))};\n"
+                    )
+            nginx_sharding_conf_f.write("\n")
+        nginx_sharding_conf_f.write("}\n")
+
+        data = {"shard_map": shard_map, "shard_regexes": shard_regexes}
+        sharding_json_f.write(json.dumps(data) + "\n")
+
+        for fh in (nginx_sharding_conf_f, sharding_json_f):
+            os.fchown(
+                fh.fileno(),
+                pwd.getpwnam("zulip").pw_uid,
+                grp.getgrnam("zulip").gr_gid,
+            )
+            os.fchmod(fh.fileno(), 0o644)
+
+
+parser = argparse.ArgumentParser(
+    description="Adjust Tornado sharding configuration",
+)
+parser.add_argument(
+    "--errors-ok",
+    action="store_true",
+    help="Exits 1 if there are no changes; if there are errors or changes, exits 0.",
+)
+options = parser.parse_args()
+
+config_file_path = "/etc/zulip"
+base_files = ["nginx_sharding_map.conf", "sharding.json"]
+full_real_paths = [f"{config_file_path}/{filename}" for filename in base_files]
+full_new_paths = [f"{filename}.tmp" for filename in full_real_paths]
+try:
+    write_updated_configs()
+    for old, new in zip(full_real_paths, full_new_paths, strict=False):
+        if not filecmp.cmp(old, new):
+            # There are changes; leave .tmp files and exit 0
+            if "SUPPRESS_SHARDING_NOTICE" not in os.environ:
+                print("===> Updated sharding; run scripts/refresh-sharding-and-restart")
+            sys.exit(0)
+    # No changes; clean up and exit 1
+    for filename in full_new_paths:
+        os.unlink(filename)
+    sys.exit(1)
+except AssertionError as e:
+    # Clean up whichever files we made
+    for filename in full_new_paths:
+        if os.path.exists(filename):
+            os.unlink(filename)
+    if options.errors_ok:
+        sys.exit(0)
+    else:
+        print(e, file=sys.stderr)
+        sys.exit(2)

scripts/lib/supervisor.py

@@ -0,0 +1,77 @@
+import socket
+import time
+from http.client import HTTPConnection
+from xmlrpc import client
+
+from typing_extensions import override
+
+
+class UnixStreamHTTPConnection(HTTPConnection):
+    @override
+    def connect(self) -> None:
+        self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        connected = False
+        for i in range(2):
+            try:
+                self.sock.connect(self.host)
+                connected = True
+                break
+            except FileNotFoundError:
+                # Backoff and retry
+                time.sleep(2**i)
+        if not connected:
+            raise Exception(
+                "Failed to connect to supervisor -- check that it is running, by running 'service supervisor status'"
+            )
+
+
+class UnixStreamTransport(client.Transport):
+    def __init__(self, socket_path: str) -> None:
+        self.socket_path = socket_path
+        super().__init__()
+
+    @override
+    def make_connection(self, host: tuple[str, dict[str, str]] | str) -> UnixStreamHTTPConnection:
+        return UnixStreamHTTPConnection(self.socket_path)
+
+
+def rpc() -> client.ServerProxy:
+    return client.ServerProxy(
+        "http://localhost", transport=UnixStreamTransport("/var/run/supervisor.sock")
+    )
+
+
+def list_supervisor_processes(
+    filter_names: list[str] | None = None, *, only_running: bool | None = None
+) -> list[str]:
+    results = []
+    processes = rpc().supervisor.getAllProcessInfo()
+    assert isinstance(processes, list)
+    for process in processes:
+        if process["group"] != process["name"]:
+            name = f"{process['group']}:{process['name']}"
+        else:
+            name = process["name"]
+
+        if filter_names:
+            match = False
+            for filter_name in filter_names:
+                # zulip-tornado:* matches zulip-tornado:9800 and zulip-tornado
+                if filter_name.endswith(":*") and (
+                    name.startswith(filter_name.removesuffix("*"))
+                    or name == filter_name.removesuffix(":*")
+                ):
+                    match = True
+                    break
+                if name == filter_name:
+                    match = True
+                    break
+            if not match:
+                continue
+
+        if only_running is None:
+            results.append(name)
+        elif only_running == (process["statename"] in ("RUNNING", "STARTING")):
+            results.append(name)
+
+    return results

scripts/lib/supported-os

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+#
+# This script serves only to verify that the OS is a supported
+# version, before we attempt to rely on that version in
+# upgrade-zulip-stage-3
+
+if [ -f /etc/os-release ]; then
+    os_info="$(
+        . /etc/os-release
+        printf '%s\n' "$ID" "$VERSION_ID"
+    )"
+    {
+        read -r os_id
+        read -r os_version_id
+    } <<<"$os_info"
+fi
+
+case "$os_id $os_version_id" in
+    'debian 12' | 'debian 13' | 'ubuntu 22.04' | 'ubuntu 24.04' | 'ubuntu 26.04')
+        exit 0
+        ;;
+    *)
+        exit 1
+        ;;
+esac

scripts/lib/unpack-zulip

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+import glob
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+
+os.environ["PYTHONUNBUFFERED"] = "y"
+
+sys.path.append(os.path.join(os.path.dirname(__file__), "..", ".."))
+import version
+from scripts.lib.zulip_tools import (
+    DEPLOYMENTS_DIR,
+    ENDC,
+    FAIL,
+    get_deployment_version,
+    is_invalid_upgrade,
+    make_deploy_path,
+    overwrite_symlink,
+)
+
+if len(sys.argv) != 2:
+    print(FAIL + f"Usage: {sys.argv[0]} <tarball>" + ENDC)
+    sys.exit(1)
+
+tarball_path = sys.argv[1]
+
+deploy_path = make_deploy_path()
+
+extract_path = tempfile.mkdtemp()
+subprocess.check_call(["tar", "-xf", tarball_path, "-C", extract_path])
+
+new_version = get_deployment_version(extract_path)
+current_version = version.ZULIP_VERSION
+if is_invalid_upgrade(current_version, new_version):
+    print(
+        FAIL + "Your current version is very old. Please first upgrade to version "
+        f"1.4.3 and then upgrade to {new_version}." + ENDC
+    )
+    shutil.rmtree(extract_path)
+    sys.exit(1)
+
+shutil.move(glob.glob(os.path.join(extract_path, "zulip-server-*"))[0], deploy_path)
+os.rmdir(extract_path)
+overwrite_symlink("/etc/zulip/settings.py", os.path.join(deploy_path, "zproject/prod_settings.py"))
+overwrite_symlink(deploy_path, os.path.join(DEPLOYMENTS_DIR, "next"))
+
+print(deploy_path)
+sys.exit(0)

scripts/lib/update-git-upstream

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -eu
+
+upstream_url="https://github.com/zulip/zulip.git"
+if ! git remote | grep -qx upstream; then
+    git remote add upstream "$upstream_url"
+else
+    git remote set-url upstream "$upstream_url"
+fi
+
+git fetch --prune --quiet --tags --all

scripts/lib/upgrade-zulip

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+import argparse
+import configparser
+import logging
+import os
+import shutil
+import subprocess
+import sys
+import time
+
+TARBALL_ARCHIVE_PATH = "/home/zulip/archives"
+os.environ["PYTHONUNBUFFERED"] = "y"
+
+sys.path.append(os.path.join(os.path.dirname(__file__), "..", ".."))
+from scripts.lib.zulip_tools import (
+    DEPLOYMENTS_DIR,
+    assert_running_as_root,
+    get_config_file,
+    get_deploy_options,
+    get_deployment_lock,
+    release_deployment_lock,
+    su_to_zulip,
+)
+
+config_file: configparser.RawConfigParser = get_config_file()
+deploy_options = get_deploy_options(config_file)
+
+assert_running_as_root(strip_lib_from_paths=True)
+
+# make sure we have appropriate file permissions
+os.umask(0o22)
+
+logging.Formatter.converter = time.gmtime
+logging.basicConfig(format="%(asctime)s upgrade-zulip: %(message)s", level=logging.INFO)
+
+parser = argparse.ArgumentParser()
+parser.add_argument("tarball", help="Path to Zulip Server tarball")
+args, extra_options = parser.parse_known_args()
+
+error_rerun_script = f"{DEPLOYMENTS_DIR}/current/scripts/upgrade-zulip {args.tarball}"
+get_deployment_lock(error_rerun_script)
+
+try:
+    # Copy the release tarball to an archival path that's readable by
+    # the Zulip user, and then unpack it from that directory, so that
+    # we can unpack using the Zulip user even if the original path was
+    # not readable by the Zulip user.
+    logging.info("Archiving the tarball under %s", TARBALL_ARCHIVE_PATH)
+    os.makedirs(TARBALL_ARCHIVE_PATH, exist_ok=True)
+    archived_tarball_path = os.path.join(TARBALL_ARCHIVE_PATH, os.path.basename(args.tarball))
+    shutil.copy(args.tarball, archived_tarball_path)
+    subprocess.check_output(["chown", "-R", "zulip:zulip", TARBALL_ARCHIVE_PATH])
+
+    logging.info("Unpacking the tarball")
+    unpack_zulip = os.path.realpath(os.path.join(os.path.dirname(__file__), "unpack-zulip"))
+    deploy_path = subprocess.check_output(
+        [unpack_zulip, archived_tarball_path], preexec_fn=su_to_zulip, text=True
+    )
+
+    # chdir to deploy_path and then run upgrade-zulip-stage-2 from the
+    # new version of Zulip (having the upgrade logic run from the new
+    # version is much better for fixing bugs in the upgrade process).
+    deploy_path = deploy_path.strip()
+    os.chdir(deploy_path)
+    try:
+        subprocess.check_call(
+            [
+                os.path.abspath("./scripts/lib/upgrade-zulip-stage-2"),
+                deploy_path,
+                *deploy_options,
+                *extra_options,
+            ]
+        )
+    except subprocess.CalledProcessError:
+        # There's no use in showing a stacktrace here; it just hides
+        # the error from stage 2.
+        sys.exit(1)
+finally:
+    release_deployment_lock()

scripts/lib/upgrade-zulip-from-git

@@ -0,0 +1,229 @@
+#!/usr/bin/env python3
+import argparse
+import logging
+import os
+import subprocess
+import sys
+import time
+
+LOCAL_GIT_CACHE_DIR = "/srv/zulip.git"
+os.environ["PYTHONUNBUFFERED"] = "y"
+
+sys.path.append(os.path.join(os.path.dirname(__file__), "..", ".."))
+from scripts.lib.zulip_tools import (
+    DEPLOYMENTS_DIR,
+    assert_running_as_root,
+    get_config,
+    get_config_file,
+    get_deploy_options,
+    get_deploy_root,
+    get_deployment_lock,
+    make_deploy_path,
+    overwrite_symlink,
+    release_deployment_lock,
+    su_to_zulip,
+    upgrade_script_arg_parser,
+)
+
+config_file = get_config_file()
+deploy_options = get_deploy_options(config_file)
+remote_url = get_config(
+    config_file, "deployment", "git_repo_url", "https://github.com/zulip/zulip.git"
+)
+
+assert_running_as_root(strip_lib_from_paths=True)
+
+# make sure we have appropriate file permissions
+os.umask(0o22)
+
+logging.Formatter.converter = time.gmtime
+logging.basicConfig(format="%(asctime)s upgrade-zulip-from-git: %(message)s", level=logging.INFO)
+
+parser = argparse.ArgumentParser(add_help=False)
+parser.add_argument("refname", help="Git reference, e.g. a branch, tag, or commit ID.")
+git_ref = parser.add_mutually_exclusive_group()
+git_ref.add_argument(
+    "--remote-url", help="Override the Git remote URL configured in /etc/zulip/zulip.conf."
+)
+git_ref.add_argument(
+    "--local-ref",
+    action="store_true",
+    help="Provided branch name has been pushed directly to /srv/zulip.git already",
+)
+
+# Any unknown options are passed through verbatim to upgrade-zulip-stage-2
+# and on to upgrade-zulip-stage-3. To make those passthrough options
+# discoverable, when the user runs --help we show a combined help that
+# also documents the stage-3 options.
+if {"-h", "--help"} & set(sys.argv[1:]):
+    argparse.ArgumentParser(parents=[parser, upgrade_script_arg_parser()]).parse_args(["--help"])
+
+args, extra_options = parser.parse_known_args()
+
+refname = args.refname
+# Command line remote URL will be given preference above the one
+# in /etc/zulip/zulip.conf.
+if args.remote_url:
+    remote_url = args.remote_url
+
+os.makedirs(DEPLOYMENTS_DIR, exist_ok=True)
+
+error_rerun_script = f"{DEPLOYMENTS_DIR}/current/scripts/upgrade-zulip-from-git {refname}"
+get_deployment_lock(error_rerun_script)
+
+try:
+    deploy_path = make_deploy_path()
+
+    # Populate LOCAL_GIT_CACHE_DIR with both the requested remote and zulip/zulip.
+    if not os.path.exists(LOCAL_GIT_CACHE_DIR):
+        logging.info("Making local repository cache")
+        subprocess.check_call(
+            ["git", "init", "--bare", "-q", LOCAL_GIT_CACHE_DIR],
+            stdout=subprocess.DEVNULL,
+        )
+        subprocess.check_call(
+            ["git", "remote", "add", "origin", remote_url],
+            cwd=LOCAL_GIT_CACHE_DIR,
+        )
+
+    if os.stat(LOCAL_GIT_CACHE_DIR).st_uid == 0:
+        subprocess.check_call(["chown", "-R", "zulip:zulip", LOCAL_GIT_CACHE_DIR])
+
+    os.chdir(LOCAL_GIT_CACHE_DIR)
+    if not args.local_ref:
+        subprocess.check_call(
+            ["git", "remote", "set-url", "origin", remote_url], preexec_fn=su_to_zulip
+        )
+
+    fetch_spec = subprocess.check_output(
+        ["git", "config", "remote.origin.fetch"],
+        preexec_fn=su_to_zulip,
+        text=True,
+    ).strip()
+    if fetch_spec in ("+refs/*:refs/*", "+refs/heads/*:refs/heads/*"):
+        # The refspec impinges on refs/heads/ -- this is an old mirror
+        # configuration.
+        logging.info("Cleaning up mirrored repository")
+        # remotes.origin.mirror may not be set -- we do not use
+        # check_call to ignore errors if it's already missing
+        subprocess.call(
+            ["git", "config", "--unset", "remote.origin.mirror"],
+            preexec_fn=su_to_zulip,
+        )
+        subprocess.check_call(
+            ["git", "config", "remote.origin.fetch", "+refs/heads/*:refs/remotes/origin/*"],
+            preexec_fn=su_to_zulip,
+        )
+        matching_refs = subprocess.check_output(
+            ["git", "for-each-ref", "--format=%(refname)", "refs/pull/", "refs/heads/"],
+            preexec_fn=su_to_zulip,
+            text=True,
+        ).splitlines()
+
+        # We can't use `git worktree list --porcelain -z` here because
+        # Ubuntu 22.04 only has git 2.34.1, and -z was
+        # introduced in 2.36
+        worktree_data = subprocess.check_output(
+            ["git", "worktree", "list", "--porcelain"],
+            preexec_fn=su_to_zulip,
+            text=True,
+        ).splitlines()
+        keep_refs = set()
+        for worktree_line in worktree_data:
+            if worktree_line.startswith("branch "):
+                keep_refs.add(worktree_line.removeprefix("branch "))
+
+        delete_input = "".join(
+            f"delete {refname}\n" for refname in matching_refs if refname not in keep_refs
+        )
+        subprocess.run(
+            ["git", "update-ref", "--stdin"],
+            check=True,
+            preexec_fn=su_to_zulip,
+            input=delete_input,
+            text=True,
+        )
+
+        logging.info("Repacking repository after pruning unnecessary refs...")
+        subprocess.check_call(
+            ["git", "gc", "--prune=now"],
+            preexec_fn=su_to_zulip,
+        )
+
+    if not args.local_ref:
+        subprocess.check_call(
+            [os.path.join(get_deploy_root(), "scripts/lib/update-git-upstream")],
+            preexec_fn=su_to_zulip,
+        )
+
+    # Generate the deployment directory via git worktree from our local repository.
+    try_refs = [
+        f"refs/tags/{refname}",
+        f"refs/heads/{refname}" if args.local_ref else f"refs/remotes/origin/{refname}",
+    ]
+    for ref in try_refs:
+        result = subprocess.run(
+            ["git", "rev-parse", "--verify", ref],
+            preexec_fn=su_to_zulip,
+            text=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.DEVNULL,
+            check=False,
+        )
+        if result.returncode == 0:
+            fullref = ref
+            commit_hash = result.stdout.strip()
+            break
+        elif result.returncode != 128:
+            result.check_returncode()
+    else:
+        logging.error(
+            "Failed to resolve %s as a tag or %s branch name!",
+            refname,
+            "local" if args.local_ref else "remote",
+        )
+        sys.exit(1)
+    refname = fullref
+
+    subprocess.check_call(
+        ["git", "worktree", "add", "--detach", deploy_path, refname],
+        stdout=subprocess.DEVNULL,
+        preexec_fn=su_to_zulip,
+    )
+    os.chdir(deploy_path)
+    extra_flags = []
+    if not refname.startswith("refs/tags/"):
+        extra_flags = ["-t"]
+    subprocess.check_call(
+        [
+            "git",
+            "checkout",
+            "-q",
+            *extra_flags,
+            "-b",
+            "deployment-" + os.path.basename(deploy_path),
+            refname,
+        ],
+        preexec_fn=su_to_zulip,
+    )
+
+    overwrite_symlink("/etc/zulip/settings.py", "zproject/prod_settings.py")
+
+    overwrite_symlink(deploy_path, os.path.join(DEPLOYMENTS_DIR, "next"))
+
+    try:
+        subprocess.check_call(
+            [
+                os.path.join(deploy_path, "scripts", "lib", "upgrade-zulip-stage-2"),
+                deploy_path,
+                "--from-git",
+                *deploy_options,
+                *extra_options,
+            ]
+        )
+    except subprocess.CalledProcessError:
+        # There's no use in showing a stacktrace here; it just hides
+        # the error from stage 2.
+        sys.exit(1)
+finally:
+    release_deployment_lock()

scripts/lib/upgrade-zulip-stage-2

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+#
+# This script serves only to verify that the OS is a supported
+# version, before we attempt to rely on that version in
+# upgrade-zulip-stage-3
+
+set -eu
+
+ZULIP_PATH="$(readlink -f "$(dirname "$0")"/../..)"
+if ! "$ZULIP_PATH/scripts/lib/supported-os"; then
+    echo "Unsupported platform!"
+    echo
+    echo "Sorry! The support for your OS has been discontinued."
+    echo "Please upgrade your OS to a supported release first,"
+    echo "by following these instructions:"
+    echo
+    echo "https://zulip.readthedocs.io/en/latest/production/upgrade.html#upgrading-the-operating-system"
+    echo
+    exit 1
+fi
+
+exec "$ZULIP_PATH/scripts/lib/upgrade-zulip-stage-3" "$@"

scripts/lib/upgrade-zulip-stage-3

@@ -0,0 +1,372 @@
+#!/usr/bin/env python3
+#
+# This script contains the actual logic for upgrading from an old
+# version of Zulip to the new version.  upgrade-zulip-stage-3 is
+# always run from the new version of Zulip, so any bug fixes take
+# effect on the very next upgrade.
+import argparse
+import logging
+import os
+import re
+import subprocess
+import sys
+import time
+
+os.environ["PYTHONUNBUFFERED"] = "y"
+
+# Force a known locale.  Some packages on PyPI fail to install in some locales.
+os.environ["LC_ALL"] = "C.UTF-8"
+os.environ["LANG"] = "C.UTF-8"
+os.environ["LANGUAGE"] = "C.UTF-8"
+
+sys.path.append(os.path.join(os.path.dirname(__file__), "..", ".."))
+from scripts.lib.zulip_tools import (
+    DEPLOYMENTS_DIR,
+    assert_running_as_root,
+    get_config,
+    get_config_file,
+    parse_version_from,
+    su_to_zulip,
+    upgrade_script_arg_parser,
+)
+
+assert_running_as_root()
+
+# Set a known, reliable PATH
+os.environ["PATH"] = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+
+logging.Formatter.converter = time.gmtime
+logging.basicConfig(format="%(asctime)s upgrade-zulip-stage-3: %(message)s", level=logging.INFO)
+
+# make sure we have appropriate file permissions
+os.umask(0o22)
+
+parser = argparse.ArgumentParser(parents=[upgrade_script_arg_parser()])
+parser.add_argument("deploy_path", metavar="deploy_path", help="Path to deployment directory")
+parser.add_argument(
+    "--from-git", action="store_true", help="Upgrading from git, so run update-prod-static."
+)
+args = parser.parse_args()
+
+if args.skip_restart:
+    if args.less_graceful:
+        logging.warning("Ignored --less-graceful; --skip-restart is always graceful.")
+        args.less_graceful = False
+    if args.skip_migrations:
+        logging.warning(
+            "Ignored --skip-migrations; all upgrades with --skip-restart asserts no migrations."
+        )
+        args.skip_migrations = False
+    if args.skip_puppet:
+        logging.warning(
+            "Ignored --skip-puppet; all upgrades with --skip-restart asserts no puppet changes."
+        )
+        args.skip_puppet = False
+
+if not args.skip_puppet and args.less_graceful:
+    logging.warning("Ignored --less-graceful; all upgrades without --skip-puppet are ungraceful.")
+
+if args.only_django:
+    logging.error(
+        "Only restarting Django into a new deployment leads to inconsistent caches "
+        "with Tornado; use --skip-client-reloads instead."
+    )
+    sys.exit(1)
+
+deploy_path = args.deploy_path
+os.chdir(deploy_path)
+
+config_file = get_config_file()
+
+IS_SERVER_UP = True
+HAS_FILLED_CACHES = False
+
+if args.from_git:
+    logging.info("Caching Zulip Git version...")
+    subprocess.check_call(["./tools/cache-zulip-git-version"], preexec_fn=su_to_zulip)
+
+from version import ZULIP_VERSION as NEW_ZULIP_VERSION
+
+old_version = parse_version_from(DEPLOYMENTS_DIR + "/current")
+logging.info("Upgrading from %s to %s, in %s", old_version, NEW_ZULIP_VERSION, deploy_path)
+
+migrations_needed = False
+
+
+def fill_memcached_caches() -> None:
+    global HAS_FILLED_CACHES
+    if HAS_FILLED_CACHES or migrations_needed:
+        return
+    subprocess.check_call(
+        ["./manage.py", "fill_memcached_caches", "--automated", "--skip-checks"],
+        preexec_fn=su_to_zulip,
+    )
+    HAS_FILLED_CACHES = True
+
+
+def shutdown_server(fill_caches: bool = True) -> None:
+    global IS_SERVER_UP
+
+    if args.skip_restart:
+        logging.info("Upgrade would require shutting down Zulip -- aborting!")
+        sys.exit(1)
+
+    if fill_caches:
+        fill_memcached_caches()
+
+    if IS_SERVER_UP:
+        logging.info("Stopping Zulip...")
+        subprocess.check_call(["./scripts/stop-server"], preexec_fn=su_to_zulip)
+        IS_SERVER_UP = False
+
+
+if not (args.skip_restart or args.skip_puppet):
+    # We need to temporarily hold pgroonga, if installed -- upgrading
+    # it without running the appropriate upgrade SQL can cause
+    # PostgreSQL to crash
+    if get_config(config_file, "machine", "pgroonga", False):
+        subprocess.check_call(["apt-mark", "hold", "postgresql-*-pgdg-pgroonga"])
+    logging.info("Upgrading system packages...")
+    subprocess.check_call(["apt-get", "update"])
+    subprocess.check_call(["apt-get", "-y", "--with-new-pkgs", "upgrade"])
+    if get_config(config_file, "machine", "pgroonga", False):
+        subprocess.check_call(["apt-mark", "unhold", "postgresql-*-pgdg-pgroonga"])
+
+# Now we should have an environment set up where we can run our tools;
+# first, creating the production venv.
+subprocess.check_call(
+    [os.path.join(deploy_path, "scripts", "lib", "create-production-venv"), deploy_path]
+)
+
+# Check to make sure that this upgrade is not actually a database
+# downgrade.
+if not args.skip_downgrade_check:
+    subprocess.check_call(
+        [os.path.join(deploy_path, "scripts", "lib", "check-database-compatibility")],
+        preexec_fn=su_to_zulip,
+    )
+
+# Make sure the right version of node is installed
+subprocess.check_call([os.path.join(deploy_path, "scripts", "lib", "install-node")])
+
+# Generate any new secrets that were added in the new version required.
+# TODO: Do caching to only run this when it has changed.
+subprocess.check_call(
+    [os.path.join(deploy_path, "scripts", "setup", "generate_secrets.py"), "--production"]
+)
+
+# TODO/compatibility: Puppet class names for the manifest renames in
+# the 11.0 release.  Can be removed when we can no longer directly
+# upgrade from 10.x.
+class_renames = {
+    "zulip::postfix_localmail": "zulip::local_mailserver",
+}
+classes = re.split(r"\s*,\s*", get_config(config_file, "machine", "puppet_classes", ""))
+new_classes = [class_renames.get(c, c) for c in classes]
+if classes != new_classes:
+    if args.skip_restart:
+        logging.error("Would need to adjust puppet classes -- aborting!")
+        sys.exit(1)
+    logging.info("Adjusting Puppet classes for renames...")
+    subprocess.check_call(
+        [
+            "crudini",
+            "--set",
+            "/etc/zulip/zulip.conf",
+            "machine",
+            "puppet_classes",
+            ", ".join(new_classes),
+        ]
+    )
+
+# And then, building/installing the static assets.
+if args.ignore_static_assets:
+    # For the OS version upgrade use case, the static assets are
+    # already in place, and we don't need to do anything.  Further,
+    # neither of the options below will work for all installations,
+    # because if we installed from Git, `prod-static/serve` may be
+    # empty so we can't do the non-Git thing, whereas if we installed
+    # from a tarball, we won't have a `tools/` directory and thus
+    # cannot run `tools/update-prod-static`.
+    pass
+elif args.from_git:
+    # Because `upgrade-zulip-from-git` needs to build static assets, it
+    # is at risk of being OOM killed on systems with limited free RAM.
+    mem_bytes = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES")
+    mem_gib = mem_bytes / (1024.0**3)  # e.g. 3.74
+
+    # Ideally, we'd have 2 thresholds here, depending on whether the
+    # system is running queue workers multithreaded or multiprocess.
+    # See puppet/zulip/manifests/app_frontend_base.pp for background.
+    if mem_gib < 4.2:
+        logging.info("Shutting down server to ensure sufficient free RAM for webpack.")
+        shutdown_server(fill_caches=False)
+
+    # Note: The fact that this is before we apply Puppet changes means
+    # that we don't support adding new Puppet dependencies of
+    # update-prod-static with the Git upgrade process.  But it'll fail
+    # safely; this seems like a worthwhile tradeoff to minimize downtime.
+    logging.info("Building static assets...")
+    try:
+        subprocess.check_call(["./tools/update-prod-static"], preexec_fn=su_to_zulip)
+    except subprocess.CalledProcessError:
+        logging.error("Failed to build static assets.")
+        if IS_SERVER_UP:
+            logging.error("Usually the cause is insufficient free RAM to run webpack.")
+            logging.error("Try stopping the Zulip server (scripts/stop-server) and trying again.")
+        sys.exit(1)
+
+else:
+    # Since this doesn't do any actual work, it's likely safe to have
+    # this run before we apply Puppet changes (saving a bit of downtime).
+    logging.info("Installing static assets...")
+    subprocess.check_call(
+        ["cp", "-rT", os.path.join(deploy_path, "prod-static/serve"), "/home/zulip/prod-static"],
+        preexec_fn=su_to_zulip,
+    )
+
+# Ensure the local uploads directory exists before we run system
+# checks, since check_uploads_settings verifies it.  Puppet would
+# create it later in the upgrade, but the system check runs first.
+# `os.path.isdir` follows symlinks, so admins who have replaced the
+# directory with a symlink to alternate storage are not disturbed.
+if not os.path.isdir("/home/zulip/uploads"):
+    subprocess.check_call(
+        ["install", "-d", "-o", "zulip", "-g", "zulip", "-m", "0755", "/home/zulip/uploads"]
+    )
+
+# Perform system checks -- including database checks, so we don't need
+# to do them when we do migrations, below.
+if not args.skip_checks:
+    subprocess.check_call(["./manage.py", "check", "--database", "default"], preexec_fn=su_to_zulip)
+
+# Our next optimization is to check whether any migrations are needed
+# before we start the critical section of the restart.  This saves
+# about 1s of downtime in a no-op upgrade.  We omit this check if we
+# already stopped the server above, due to low memory.
+if not IS_SERVER_UP:
+    migrations_needed = True
+elif not args.skip_migrations:
+    logging.info("Checking for needed migrations")
+    migrations_output = subprocess.check_output(
+        ["./manage.py", "showmigrations", "--skip-checks"], preexec_fn=su_to_zulip, text=True
+    )
+    for ln in migrations_output.split("\n"):
+        line_str = ln.strip()
+        if line_str.startswith("[ ]"):
+            migrations_needed = True
+
+if args.skip_restart and migrations_needed:
+    logging.error("Would need to apply migrations -- aborting!")
+    sys.exit(1)
+
+# Install hooks before we check for puppet changes, so we skip an
+# unnecessary stop/start cycle if hook changes are the only ones.
+logging.info("Installing hooks")
+subprocess.check_call(
+    ["./scripts/zulip-puppet-apply", "--tags", "hooks", "--force"],
+)
+
+# If we are planning on running puppet, we can pre-run it in --noop
+# mode and see if it will actually make any changes; if not, we can
+# skip it during the upgrade.  We omit this check if the server is
+# already stopped, since it's not better than just pressing on.
+has_puppet_changes = True
+if not args.skip_puppet and IS_SERVER_UP:
+    logging.info("Pre-checking for puppet changes...")
+    try_puppet = subprocess.run(
+        ["./scripts/zulip-puppet-apply", "--noop", "--force"],
+        stdout=subprocess.DEVNULL,
+        check=False,
+    )
+    if try_puppet.returncode == 0:
+        if args.skip_restart:
+            logging.info("Verified no Puppet changes are necessary.")
+        else:
+            logging.info("No puppet changes found, skipping!")
+        args.skip_puppet = True
+        has_puppet_changes = False
+    elif try_puppet.returncode == 2:
+        logging.error("Puppet error -- aborting!")
+        sys.exit(1)
+    elif args.skip_restart:
+        logging.error("Would need to apply puppet changes -- aborting!")
+        sys.exit(1)
+
+
+if args.skip_restart:
+    logging.info("Successfully configured in %s!", deploy_path)
+else:
+    # NOTE: Here begins the most likely critical period, where we may be
+    # shutting down the server; we should strive to minimize the number of
+    # steps that happen between here and the "Restarting Zulip" line
+    # below.
+    hooks_args = []
+    if args.from_git:
+        hooks_args = ["--from-git"]
+
+    subprocess.check_call(["./scripts/lib/run_hooks.py", "pre-deploy", *hooks_args])
+
+    if not args.skip_puppet:
+        # Puppet may adjust random services; to minimize risk of issues
+        # due to inconsistent state, we shut down the server first.
+        shutdown_server()
+        logging.info("Applying Puppet changes...")
+        subprocess.check_call(["./scripts/zulip-puppet-apply", "--force"])
+        subprocess.check_call(["apt-get", "-y", "--with-new-pkgs", "upgrade"])
+        # Puppet may have reloaded supervisor, and in so doing started
+        # services; mark as potentially needing to stop the server.
+        IS_SERVER_UP = True
+
+    if migrations_needed:
+        # Database migrations assume that they run on a database in
+        # quiesced state.
+        shutdown_server()
+        logging.info("Applying database migrations...")
+        subprocess.check_call(
+            ["./manage.py", "migrate", "--noinput", "--skip-checks"], preexec_fn=su_to_zulip
+        )
+
+    logging.info("Restarting Zulip...")
+    start_args = ["--skip-checks"]
+    if args.only_django and IS_SERVER_UP:
+        # Calling `start-server --only-django` only starts Django,
+        # which is almost certainly not what we want after an upgrade.
+        # If we had to stop services for other reasons (e.g. puppet),
+        # we ignore --only-django and downgrade it to
+        # --skip-client-reloads
+        start_args.append("--only-django")
+    else:
+        start_args.append("--skip-client-reloads")
+    if not HAS_FILLED_CACHES:
+        start_args.append("--fill-cache")
+    if IS_SERVER_UP:
+        if args.less_graceful:
+            start_args.append("--less-graceful")
+        subprocess.check_call(["./scripts/restart-server", *start_args], preexec_fn=su_to_zulip)
+    else:
+        subprocess.check_call(["./scripts/start-server", *start_args], preexec_fn=su_to_zulip)
+
+    logging.info("Upgrade complete!")
+
+    subprocess.check_call(["./scripts/lib/run_hooks.py", "post-deploy", *hooks_args])
+
+    if not args.skip_client_reloads and not args.only_django:
+        subprocess.check_call(["./scripts/reload-clients", "--background"], preexec_fn=su_to_zulip)
+
+if args.audit_fts_indexes:
+    logging.info("Correcting full-text search indexes for updated dictionary files")
+    logging.info("This may take a while but the server should work while it runs.")
+    subprocess.check_call(
+        ["./manage.py", "audit_fts_indexes", "--automated", "--skip-checks"], preexec_fn=su_to_zulip
+    )
+
+if not args.skip_purge_old_deployments:
+    logging.info("Purging old deployments...")
+    subprocess.check_call(["./scripts/purge-old-deployments"])
+else:
+    logging.info("Skipping purging old deployments.")
+
+if args.skip_puppet and has_puppet_changes:
+    logging.info("Showing un-applied Puppet changes:")
+    subprocess.check_call(["./scripts/zulip-puppet-apply", "--noop", "--show_diff"])

scripts/lib/warn-rabbitmq-nodename-change

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+# For security reasons, we need to configure RabbitMQ to listen only
+# on localhost, which we cannot do if the nodename contains the
+# hostname (e.g. rabbit@zulip-host).  Containing the hostname also
+# makes it brittle to hostname changes, which are (for example) common
+# in containerized environments.
+
+set -eu
+
+# Try to find the current nodename
+CURRENT=$(sh -c 'cd /usr/lib/rabbitmq/bin/ && . ./rabbitmq-env && echo $NODENAME')
+if [ "$CURRENT" != "zulip@localhost" ]; then
+    cat <<EOF
+
+***** WARNING *****
+
+We are renaming the rabbitmq server's nodename from '$CURRENT' to
+'zulip@localhost', as rabbitmq is being reconfigured to listen only on
+localhost.  This will also make the server more resilient to hostname
+changes.  This will only affect you if you were using the rabbitmq
+server for other, non-Zulip uses.
+
+*******************
+
+EOF
+
+    service rabbitmq-server stop || true
+fi

scripts/lib/zulip_tools.py

@@ -0,0 +1,766 @@
+#!/usr/bin/env python3
+import argparse
+import configparser
+import functools
+import hashlib
+import json
+import logging
+import os
+import platform
+import pwd
+import random
+import shlex
+import shutil
+import signal
+import subprocess
+import sys
+import time
+import uuid
+import zoneinfo
+from collections.abc import Sequence
+from datetime import datetime, timedelta
+from typing import IO, Any, Literal, overload
+from urllib.parse import SplitResult
+
+DEPLOYMENTS_DIR = "/home/zulip/deployments"
+LOCK_DIR = os.path.join(DEPLOYMENTS_DIR, "lock")
+TIMESTAMP_FORMAT = "%Y-%m-%d-%H-%M-%S"
+
+# Color codes
+OKBLUE = "\033[94m"
+OKGREEN = "\033[92m"
+WARNING = "\033[93m"
+FAIL = "\033[91m"
+ENDC = "\033[0m"
+BLACKONYELLOW = "\x1b[0;30;43m"
+WHITEONRED = "\x1b[0;37;41m"
+BOLDRED = "\x1b[1;31m"
+BOLD = "\x1b[1m"
+GRAY = "\x1b[90m"
+
+GREEN = "\x1b[32m"
+YELLOW = "\x1b[33m"
+BLUE = "\x1b[34m"
+MAGENTA = "\x1b[35m"
+CYAN = "\x1b[36m"
+
+
+def overwrite_symlink(src: str, dst: str) -> None:
+    dir, base = os.path.split(dst)
+    while True:
+        # Note: creating a temporary filename like this is not generally
+        # secure.  It’s fine in this case because os.symlink refuses to
+        # overwrite an existing target; we handle the error and try again.
+        tmp = os.path.join(dir, f".{base}.{random.randrange(1 << 40):010x}")
+        try:
+            os.symlink(src, tmp)
+        except FileExistsError:
+            continue
+        break
+    try:
+        os.rename(tmp, dst)
+    except BaseException:
+        os.remove(tmp)
+        raise
+
+
+def parse_cache_script_args(description: str) -> argparse.Namespace:
+    # Keep this in sync with clean_unused_caches in provision_inner.py
+    parser = argparse.ArgumentParser(description=description)
+
+    parser.add_argument(
+        "--threshold",
+        dest="threshold_days",
+        type=int,
+        default=14,
+        metavar="<days>",
+        help="Any cache which is not in "
+        "use by a deployment not older than threshold days(current "
+        "installation in dev) and older than threshold days will be "
+        "deleted. (defaults to 14)",
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="If specified then script will only print the caches "
+        "that it will delete/keep back. It will not delete any cache.",
+    )
+    parser.add_argument(
+        "--verbose",
+        action="store_true",
+        help="If specified then script will print a detailed report "
+        "of what is being will deleted/kept back.",
+    )
+    parser.add_argument(
+        "--no-print-headings",
+        dest="no_headings",
+        action="store_true",
+        help="If specified then script will not print headings for what will be deleted/kept back.",
+    )
+
+    args = parser.parse_args()
+    args.verbose |= args.dry_run  # Always print a detailed report in case of dry run.
+    return args
+
+
+def get_deploy_root() -> str:
+    return os.path.realpath(
+        os.path.normpath(os.path.join(os.path.dirname(__file__), "..", "..")),
+    )
+
+
+def parse_version_from(deploy_path: str, merge_base: bool = False) -> str:
+    if not os.path.exists(os.path.join(deploy_path, "zulip-git-version")):
+        try:
+            # Pull this tool from _our_ deploy root, since it may not
+            # exist historically, but run it the cwd of the old
+            # deploy, so we set up its remote.
+            subprocess.check_call(
+                [os.path.join(get_deploy_root(), "scripts", "lib", "update-git-upstream")],
+                cwd=deploy_path,
+                preexec_fn=su_to_zulip,
+            )
+            subprocess.check_call(
+                [os.path.join(deploy_path, "tools", "cache-zulip-git-version")],
+                cwd=deploy_path,
+                preexec_fn=su_to_zulip,
+            )
+        except subprocess.CalledProcessError:
+            pass
+    try:
+        varname = "ZULIP_MERGE_BASE" if merge_base else "ZULIP_VERSION"
+        return subprocess.check_output(
+            [sys.executable, "-c", f"from version import {varname}; print({varname})"],
+            cwd=deploy_path,
+            text=True,
+        ).strip()
+    except subprocess.CalledProcessError:
+        return "0.0.0"
+
+
+def get_deployment_version(extract_path: str) -> str:
+    version = "0.0.0"
+    for item in os.listdir(extract_path):
+        item_path = os.path.join(extract_path, item)
+        if item.startswith("zulip-server") and os.path.isdir(item_path):
+            version = parse_version_from(item_path)
+            break
+    return version
+
+
+def is_invalid_upgrade(current_version: str, new_version: str) -> bool:
+    if new_version > "1.4.3" and current_version <= "1.3.10":
+        return True
+    return False
+
+
+def get_zulip_pwent() -> pwd.struct_passwd:
+    deploy_root_uid = os.stat(get_deploy_root()).st_uid
+    if deploy_root_uid != 0:
+        return pwd.getpwuid(deploy_root_uid)
+
+    # In the case that permissions got messed up and the deployment
+    # directory is unexpectedly owned by root, we fallback to the
+    # `zulip` user as that's the correct value in production.
+    return pwd.getpwnam("zulip")
+
+
+def get_postgres_pwent() -> pwd.struct_passwd:
+    try:
+        return pwd.getpwnam("postgres")
+    except KeyError:
+        return get_zulip_pwent()
+
+
+def su_to_zulip(save_suid: bool = False) -> None:
+    """Warning: su_to_zulip assumes that the zulip checkout is owned by
+    the zulip user (or whatever normal user is running the Zulip
+    installation).  It should never be run from the installer or other
+    production contexts before /home/zulip/deployments/current is
+    created."""
+    pwent = get_zulip_pwent()
+    os.setgid(pwent.pw_gid)
+    if save_suid:
+        os.setresuid(pwent.pw_uid, pwent.pw_uid, os.getuid())
+    else:
+        os.setuid(pwent.pw_uid)
+    os.environ["HOME"] = pwent.pw_dir
+
+
+def make_deploy_path() -> str:
+    timestamp = datetime.now().strftime(TIMESTAMP_FORMAT)  # noqa: DTZ005
+    return os.path.join(DEPLOYMENTS_DIR, timestamp)
+
+
+TEMPLATE_DATABASE_DIR = "test-backend/databases"
+
+
+def get_dev_uuid_var_path(create_if_missing: bool = False) -> str:
+    zulip_path = get_deploy_root()
+    uuid_path = os.path.join(os.path.realpath(os.path.dirname(zulip_path)), ".zulip-dev-uuid")
+    if os.path.exists(uuid_path):
+        with open(uuid_path) as f:
+            zulip_uuid = f.read().strip()
+    else:
+        if create_if_missing:
+            zulip_uuid = str(uuid.uuid4())
+            # We need root access here, since the path will be under /srv/ in the
+            # development environment.
+            run_as_root(["sh", "-c", 'echo "$1" > "$2"', "-", zulip_uuid, uuid_path])
+        else:
+            raise AssertionError("Missing UUID file; please run tools/provision!")
+
+    result_path = os.path.join(zulip_path, "var", zulip_uuid)
+    os.makedirs(result_path, exist_ok=True)
+    return result_path
+
+
+def get_deployment_lock(error_rerun_script: str) -> None:
+    start_time = time.time()
+    got_lock = False
+    while time.time() - start_time < 300:
+        try:
+            os.mkdir(LOCK_DIR)
+            got_lock = True
+            break
+        except OSError:
+            print(
+                WARNING
+                + "Another deployment in progress; waiting for lock... "
+                + f"(If no deployment is running, rmdir {LOCK_DIR})"
+                + ENDC,
+                flush=True,
+            )
+            time.sleep(3)
+
+    if not got_lock:
+        print(
+            FAIL
+            + "Deployment already in progress.  Please run\n"
+            + f"  {error_rerun_script}\n"
+            + "manually when the previous deployment finishes, or run\n"
+            + f"  rmdir {LOCK_DIR}\n"
+            + "if the previous deployment crashed."
+            + ENDC
+        )
+        sys.exit(1)
+
+
+def release_deployment_lock() -> None:
+    shutil.rmtree(LOCK_DIR)
+
+
+def run(args: Sequence[str], **kwargs: Any) -> None:
+    # Output what we're doing in the `set -x` style
+    print(f"+ {shlex.join(args)}", flush=True)
+
+    try:
+        subprocess.check_call(args, **kwargs)
+    except subprocess.CalledProcessError as error:
+        print()
+        if error.returncode < 0:
+            try:
+                signal_name = signal.Signals(-error.returncode).name
+            except ValueError:
+                signal_name = f"unknown signal {-error.returncode}"
+            print(
+                WHITEONRED
+                + f"Subcommand of {sys.argv[0]} died with {signal_name}: {shlex.join(args)}"
+                + ENDC
+            )
+        else:
+            print(
+                WHITEONRED
+                + f"Subcommand of {sys.argv[0]} failed with exit status {error.returncode}: {shlex.join(args)}"
+                + ENDC
+            )
+            print(WHITEONRED + "Actual error output for the subcommand is just above this." + ENDC)
+        print()
+        sys.exit(1)
+
+
+def log_management_command(cmd: Sequence[str], log_path: str) -> None:
+    log_dir = os.path.dirname(log_path)
+    if not os.path.exists(log_dir):
+        os.makedirs(log_dir)
+
+    formatter = logging.Formatter("%(asctime)s: %(message)s")
+    file_handler = logging.FileHandler(log_path)
+    file_handler.setFormatter(formatter)
+    logger = logging.getLogger("zulip.management")
+    logger.addHandler(file_handler)
+    logger.setLevel(logging.INFO)
+
+    logger.info("Ran %s", shlex.join(cmd))
+
+
+def get_environment() -> str:
+    if os.path.exists(DEPLOYMENTS_DIR):
+        return "prod"
+    return "dev"
+
+
+def get_recent_deployments(threshold_days: int | None) -> set[str]:
+    # Returns a list of deployments not older than threshold days
+    # including `/root/zulip` directory if it exists.
+    recent = set()
+    if threshold_days is not None:
+        threshold_date = datetime.now() - timedelta(days=threshold_days)  # noqa: DTZ005
+    else:
+        threshold_date = None
+    if os.path.isdir(DEPLOYMENTS_DIR):
+        for dir_name in os.listdir(DEPLOYMENTS_DIR):
+            target_dir = os.path.join(DEPLOYMENTS_DIR, dir_name)
+            if not os.path.isdir(target_dir):
+                # Skip things like uwsgi sockets, symlinks, etc.
+                continue
+            if not os.path.exists(os.path.join(target_dir, "zerver")):
+                # Skip things like "lock" that aren't actually a deployment directory
+                continue
+            try:
+                date = datetime.strptime(dir_name, TIMESTAMP_FORMAT)  # noqa: DTZ007
+                if threshold_date is None or date >= threshold_date:
+                    recent.add(target_dir)
+            except ValueError:
+                # Always include deployments whose name is not in the format of a timestamp.
+                recent.add(target_dir)
+                # If it is a symlink then include the target as well.
+                if os.path.islink(target_dir):
+                    recent.add(os.path.realpath(target_dir))
+    if os.path.exists("/root/zulip"):
+        recent.add("/root/zulip")
+    return recent
+
+
+def get_threshold_timestamp(threshold_days: int) -> int:
+    # Given number of days, this function returns timestamp corresponding
+    # to the time prior to given number of days.
+    threshold = datetime.now() - timedelta(days=threshold_days)  # noqa: DTZ005
+    threshold_timestamp = int(time.mktime(threshold.utctimetuple()))
+    return threshold_timestamp
+
+
+def get_caches_to_be_purged(
+    caches_dir: str, caches_in_use: set[str], threshold_days: int
+) -> set[str]:
+    # Given a directory containing caches, a list of caches in use
+    # and threshold days, this function return a list of caches
+    # which can be purged. Remove the cache only if it is:
+    # 1: Not in use by the current installation(in dev as well as in prod).
+    # 2: Not in use by a deployment not older than `threshold_days` (in prod).
+    # 3: Not in use by '/root/zulip'.
+    # 4: Not older than `threshold_days`.
+    caches_to_purge = set()
+    threshold_timestamp = get_threshold_timestamp(threshold_days)
+    for cache_dir_base in os.listdir(caches_dir):
+        cache_dir = os.path.join(caches_dir, cache_dir_base)
+        if cache_dir in caches_in_use:
+            # Never purge a cache which is in use.
+            continue
+        if os.path.getctime(cache_dir) < threshold_timestamp:
+            caches_to_purge.add(cache_dir)
+    return caches_to_purge
+
+
+def purge_unused_caches(
+    caches_dir: str,
+    caches_in_use: set[str],
+    cache_type: str,
+    args: argparse.Namespace,
+) -> None:
+    if not os.path.exists(caches_dir):
+        return
+
+    all_caches = {os.path.join(caches_dir, cache) for cache in os.listdir(caches_dir)}
+    caches_to_purge = get_caches_to_be_purged(caches_dir, caches_in_use, args.threshold_days)
+    caches_to_keep = all_caches - caches_to_purge
+
+    maybe_perform_purging(
+        caches_to_purge, caches_to_keep, cache_type, args.dry_run, args.verbose, args.no_headings
+    )
+    if args.verbose:
+        print("Done!")
+
+
+def generate_sha1sum_emoji(zulip_path: str) -> str:
+    sha = hashlib.sha1()
+
+    filenames = [
+        "web/images/zulip-emoji/zulip.png",
+        "tools/setup/emoji/emoji_map.json",
+        "tools/setup/emoji/build_emoji",
+        "tools/setup/emoji/emoji_setup_utils.py",
+        "tools/setup/emoji/emoji_names.py",
+        "zerver/management/data/unified_reactions.json",
+    ]
+
+    for filename in filenames:
+        file_path = os.path.join(zulip_path, filename)
+        with open(file_path, "rb") as reader:
+            sha.update(reader.read())
+
+    # Take into account the version of `emoji-datasource-google` package
+    # while generating success stamp.
+    with open(os.path.join(zulip_path, "node_modules/emoji-datasource-google/package.json")) as fp:
+        emoji_datasource_version = json.load(fp)["version"]
+    sha.update(emoji_datasource_version.encode())
+
+    return sha.hexdigest()
+
+
+def maybe_perform_purging(
+    dirs_to_purge: set[str],
+    dirs_to_keep: set[str],
+    dir_type: str,
+    dry_run: bool,
+    verbose: bool,
+    no_headings: bool,
+) -> None:
+    if dry_run:
+        print("Performing a dry run...")
+    if not no_headings:
+        print(f"Cleaning unused {dir_type}s...")
+
+    for directory in dirs_to_purge:
+        if verbose:
+            print(f"Cleaning unused {dir_type}: {directory}")
+        if not dry_run:
+            run_as_root(["rm", "-rf", directory])
+
+    for directory in dirs_to_keep:
+        if verbose:
+            print(f"Keeping used {dir_type}: {directory}")
+
+
+@functools.lru_cache(None)
+def os_families() -> set[str]:
+    """
+    Known families:
+    debian (includes: debian, ubuntu)
+    ubuntu (includes: ubuntu)
+    fedora (includes: fedora, rhel, centos)
+    rhel (includes: rhel, centos)
+    centos (includes: centos)
+    """
+    distro_info = platform.freedesktop_os_release()
+    return {distro_info["ID"], *distro_info.get("ID_LIKE", "").split()}
+
+
+def get_tzdata_zi() -> IO[str]:
+    for path in zoneinfo.TZPATH:
+        filename = os.path.join(path, "tzdata.zi")
+        if os.path.exists(filename):
+            return open(filename)
+    raise RuntimeError("Missing time zone data (tzdata.zi)")
+
+
+def files_and_string_digest(filenames: Sequence[str], extra_strings: Sequence[str]) -> str:
+    # see is_digest_obsolete for more context
+    sha1sum = hashlib.sha1()
+    for fn in filenames:
+        with open(fn, "rb") as file_to_hash:
+            sha1sum.update(file_to_hash.read())
+
+    for extra_string in extra_strings:
+        sha1sum.update(extra_string.encode())
+
+    return sha1sum.hexdigest()
+
+
+def is_digest_obsolete(
+    hash_name: str, filenames: Sequence[str], extra_strings: Sequence[str] = []
+) -> bool:
+    """
+    In order to determine if we need to run some
+    process, we calculate a digest of the important
+    files and strings whose respective contents
+    or values may indicate such a need.
+
+        filenames = files we should hash the contents of
+        extra_strings = strings we should hash directly
+
+    Grep for callers to see examples of how this is used.
+
+    To elaborate on extra_strings, they will typically
+    be things like:
+
+        - package versions (that we import)
+        - settings values (that we stringify with
+          json, deterministically)
+    """
+    last_hash_path = os.path.join(get_dev_uuid_var_path(), hash_name)
+    try:
+        with open(last_hash_path) as f:
+            old_hash = f.read()
+    except FileNotFoundError:
+        # This is normal for a fresh checkout--a missing
+        # digest is an obsolete digest.
+        return True
+
+    new_hash = files_and_string_digest(filenames, extra_strings)
+
+    return new_hash != old_hash
+
+
+def write_new_digest(
+    hash_name: str, filenames: Sequence[str], extra_strings: Sequence[str] = []
+) -> None:
+    hash_path = os.path.join(get_dev_uuid_var_path(), hash_name)
+    new_hash = files_and_string_digest(filenames, extra_strings)
+    with open(hash_path, "w") as f:
+        f.write(new_hash)
+
+    # Be a little verbose here--our callers ensure we
+    # only write new digests when things have changed, and
+    # making this system more transparent to developers
+    # can help them troubleshoot provisioning glitches.
+    print("New digest written to: " + hash_path)
+
+
+def is_root() -> bool:
+    if "posix" in os.name and os.geteuid() == 0:
+        return True
+    return False
+
+
+def run_as_root(args: list[str], **kwargs: Any) -> None:
+    sudo_args = kwargs.pop("sudo_args", [])
+    if not is_root():
+        args = ["sudo", *sudo_args, "--", *args]
+    run(args, **kwargs)
+
+
+def assert_not_running_as_root() -> None:
+    script_name = os.path.abspath(sys.argv[0])
+    if is_root():
+        pwent = get_zulip_pwent()
+        msg = (
+            f"{os.path.basename(script_name)} should not be run as root. Use `su {pwent.pw_name}` to switch to the 'zulip'\n"
+            f"user before rerunning this, or use \n  su {pwent.pw_name} -c '{script_name} ...'\n"
+            "to switch users and run this as a single command."
+        )
+        print(msg)
+        sys.exit(1)
+
+
+def assert_running_as_root(strip_lib_from_paths: bool = False) -> None:
+    script_name = os.path.abspath(sys.argv[0])
+    # Since these Python scripts are run inside a thin shell wrapper,
+    # we need to replace the paths in order to ensure we instruct
+    # users to (re)run the right command.
+    if strip_lib_from_paths:
+        script_name = script_name.replace("scripts/lib/upgrade", "scripts/upgrade")
+    if not is_root():
+        print(f"{script_name} must be run as root.")
+        sys.exit(1)
+
+
+@overload
+def get_config(
+    config_file: configparser.RawConfigParser,
+    section: str,
+    key: str,
+    default_value: None = None,
+) -> str | None: ...
+@overload
+def get_config(
+    config_file: configparser.RawConfigParser,
+    section: str,
+    key: str,
+    default_value: str,
+) -> str: ...
+@overload
+def get_config(
+    config_file: configparser.RawConfigParser, section: str, key: str, default_value: bool
+) -> bool: ...
+def get_config(
+    config_file: configparser.RawConfigParser,
+    section: str,
+    key: str,
+    default_value: str | bool | None = None,
+) -> str | bool | None:
+    if config_file.has_option(section, key):
+        val = config_file.get(section, key)
+        if isinstance(default_value, bool):
+            # This list is parallel to puppet/zulip/lib/puppet/functions/zulipconf.rb
+            return val.lower() in ["1", "y", "t", "true", "yes", "enable", "enabled"]
+        return val
+    return default_value
+
+
+def get_config_file() -> configparser.RawConfigParser:
+    config_file = configparser.RawConfigParser()
+    config_file.read("/etc/zulip/zulip.conf")
+    return config_file
+
+
+def get_deploy_options(config_file: configparser.RawConfigParser) -> list[str]:
+    return shlex.split(get_config(config_file, "deployment", "deploy_options", ""))
+
+
+def get_tornado_ports(config_file: configparser.RawConfigParser) -> list[int]:
+    ports = []
+    if config_file.has_section("tornado_sharding"):
+        ports = sorted(
+            {
+                int(port)
+                for key in config_file.options("tornado_sharding")
+                for port in key.removesuffix("_regex").split("_")
+            }
+        )
+    if not ports:
+        ports = [9800]
+    return ports
+
+
+def get_or_create_dev_uuid_var_path(path: str) -> str:
+    absolute_path = f"{get_dev_uuid_var_path()}/{path}"
+    os.makedirs(absolute_path, exist_ok=True)
+    return absolute_path
+
+
+def is_vagrant_env_host(path: str) -> bool:
+    return ".vagrant" in os.listdir(path)
+
+
+def has_application_server(once: bool = False) -> bool:
+    if once:
+        return os.path.exists("/etc/supervisor/conf.d/zulip/zulip-once.conf")
+    return (
+        # Current path
+        os.path.exists("/etc/supervisor/conf.d/zulip/zulip.conf")
+        # Old path, relevant for upgrades
+        or os.path.exists("/etc/supervisor/conf.d/zulip.conf")
+    )
+
+
+def has_process_fts_updates() -> bool:
+    return (
+        # Current path
+        os.path.exists("/etc/supervisor/conf.d/zulip/zulip_db.conf")
+        # Old path, relevant for upgrades
+        or os.path.exists("/etc/supervisor/conf.d/zulip_db.conf")
+    )
+
+
+def deport(netloc: str) -> str:
+    """Remove the port from a hostname:port string.  Brackets on a literal
+    IPv6 address are included."""
+    r = SplitResult("", netloc, "", "", "")
+    assert r.hostname is not None
+    return "[" + r.hostname + "]" if ":" in r.hostname else r.hostname
+
+
+def start_script_arg_parser(action: str, add_help: bool = False) -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(add_help=add_help)
+    parser.add_argument("--fill-cache", action="store_true", help="Fill the memcached caches")
+    parser.add_argument(
+        "--skip-checks", action="store_true", help="Skip syntax and database checks"
+    )
+    which_services = parser.add_mutually_exclusive_group()
+    which_services.add_argument(
+        "--skip-client-reloads",
+        action="store_true",
+        help="Do not send reload events to web clients",
+    )
+    which_services.add_argument(
+        "--only-django",
+        action="store_true",
+        help=f"Only {action} Django (not Tornado or workers)",
+    )
+    which_services.add_argument(
+        "--tornado-reshard",
+        action="store_true",
+        help="Restart changed Tornado shards",
+    )
+    if action == "restart":
+        parser.add_argument(
+            "--less-graceful",
+            action="store_true",
+            help="Restart with more concern for expediency than minimizing availability interruption",
+        )
+    return parser
+
+
+def upgrade_script_arg_parser() -> argparse.ArgumentParser:
+    """Options shared by upgrade-zulip-stage-3 and its wrapper scripts.
+
+    Wrappers like upgrade-zulip-from-git use this as a parent parser so
+    that --help advertises the same options stage-3 accepts, and forward
+    the parsed values through to stage-3.
+    """
+    parser = argparse.ArgumentParser(
+        add_help=False, parents=[start_script_arg_parser(action="restart", add_help=False)]
+    )
+    parser.add_argument(
+        "--skip-restart",
+        action="store_true",
+        help="Configure, but do not restart into, the new version; aborts if any system-wide changes would happen.",
+    )
+    parser.add_argument(
+        "--skip-puppet", action="store_true", help="Skip doing puppet/apt upgrades."
+    )
+    parser.add_argument("--skip-migrations", action="store_true", help="Skip doing migrations.")
+    parser.add_argument(
+        "--skip-downgrade-check",
+        action="store_true",
+        help="Skip the safety check to prevent database downgrades.",
+    )
+    parser.add_argument(
+        "--ignore-static-assets",
+        action="store_true",
+        help="Do not attempt to copy/manage static assets.",
+    )
+    parser.add_argument(
+        "--skip-purge-old-deployments",
+        action="store_true",
+        help="Skip purging old deployments.",
+    )
+    parser.add_argument(
+        "--audit-fts-indexes",
+        action="store_true",
+        help="Audit and fix full text search indexes.",
+    )
+    return parser
+
+
+def atomic_nagios_write(
+    name: str,
+    status: Literal["ok", "warning", "critical", "unknown"],
+    message: str | None = None,
+    event_time: int | None = None,
+) -> int:
+    if message is None:
+        message = status
+    if event_time is None:
+        event_time = int(time.time())
+    if status == "ok":
+        status_int = 0
+    elif status == "warning":
+        status_int = 1
+    elif status == "critical":
+        status_int = 2
+    elif status == "unknown":
+        status_int = 3
+
+    path = "/var/lib/nagios_state/" + name
+    with open(path + ".tmp", "w") as fh:
+        fh.write("|".join([str(event_time), str(status_int), status, message]) + "\n")
+    os.rename(path + ".tmp", path)
+
+    # Return code should be if the cron job ran to completion
+    # successfully, not if the result of the check was outside of
+    # bounds ("ok" / "critical"); this prevents the Sentry cron
+    # wrapper from spamming with a "failure" email if the nagios check
+    # requires multiple failures in a row.
+    if status == "unknown":
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    cmd = sys.argv[1]
+    if cmd == "make_deploy_path":
+        print(make_deploy_path())
+    elif cmd == "get_dev_uuid":
+        print(get_dev_uuid_var_path())