snapshot of https://github.com/zulip/zulip.git @ 48c69911c7dd4e03b20552eb3bd77ad8151c1357 (2026-09-15)

tools/README.md

@@ -0,0 +1,7 @@
+This directory contains scripts that are used in building, managing,
+testing, and other forms of work in a Zulip development environment.
+Note that tools that are also useful in production belong in
+`scripts/` or should be Django management commands.
+
+For more details, see
+https://zulip.readthedocs.io/en/latest/subsystems/directory-structure.html.

tools/backport-all-prs

@@ -0,0 +1,156 @@
+#!/usr/bin/env -S uv run --script --frozen --only-group release-tools  # -*-python-*-
+
+import re
+import subprocess
+import time
+from typing import Annotated
+
+import typer
+from github import Auth, Github
+from github.PullRequest import PullRequest
+from github.Repository import Repository
+from rich.console import Console
+from rich.progress import track
+
+
+def prs_to_backport(repo: Repository, console: Console, skip: set[int]) -> list[int]:
+    backport_prs = []
+    with console.status("Getting list of closed backport candidate PRs..."):
+        issues = list(repo.get_issues(labels=["backport candidate"], state="closed"))
+    for issue in track(issues, console=console, description="Fetching PR metadata..."):
+        if issue.number in skip:
+            continue
+        if issue.pull_request is None:
+            continue
+        pr = repo.get_pull(issue.number)
+
+        if pr.merged_at is None:
+            print(f"PR {pr.number} does not have a merged_at time!")
+            continue
+
+        backport_prs.append((pr.number, pr.merged_at))
+
+    backport_prs.sort(key=lambda x: x[1])
+    return [pr[0] for pr in backport_prs]
+
+
+def wait_for_complete(repo: Repository, pr: PullRequest, console: Console) -> None:
+    commit = repo.get_commit(pr.head.sha)
+    with console.status("Waiting for tests to pass..."):
+        while True:
+            time.sleep(10)
+            check_runs = commit.get_check_runs()
+            if check_runs.totalCount == 0:
+                continue
+            all_completed = True
+            for check in check_runs:
+                if check.status != "completed":
+                    all_completed = False
+                    break
+                elif check.conclusion not in ["success", "neutral", "skipped"]:
+                    raise Exception(f"{check.name} failed!")
+            if all_completed:
+                break
+    pr.merge(merge_method="rebase")
+
+
+def mark_as_backported(repo: Repository, backport_pr: int, pr_number: int, commit: str) -> None:
+    pr = repo.get_pull(pr_number)
+    pr.create_issue_comment(f"Backported in #{backport_pr} ({commit})")
+    pr.remove_from_labels("backport candidate")
+
+
+def validate_github_token(value: str) -> str:
+    # https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/about-authentication-to-github#githubs-token-formats
+    if value.startswith("github_"):
+        return value
+    if re.match(r"gh[pousr]_", value):
+        return value
+    raise typer.BadParameter("Github access tokens start with `github_`, or `gh`")
+
+
+def main(
+    token: Annotated[
+        str,
+        typer.Option(
+            metavar="TOKEN",
+            envvar="GITHUB_TOKEN",
+            show_envvar=True,
+            callback=validate_github_token,
+            help="Github access token",
+        ),
+    ],
+    skip: Annotated[list[int], typer.Option(default_factory=list)],
+) -> int:
+    """Make a backport PR"""
+
+    console = Console(stderr=True, log_path=False)
+
+    latest_tag = subprocess.check_output(
+        ["git", "tag", "-l", "--sort=-committerdate", "--no-merged", "upstream/main"],
+        text=True,
+    ).splitlines()[0]
+    target_branch = latest_tag.split(".")[0] + ".x"
+
+    gh = Github(auth=Auth.Token(token))
+    repo = gh.get_repo("zulip/zulip")
+    pr_ids = prs_to_backport(repo, console, set(skip))
+
+    branchname = f"backports-{target_branch}"
+    subprocess.check_call(["git", "fetch", "upstream"])
+    subprocess.check_call(
+        [
+            "git",
+            "checkout",
+            "-b",
+            branchname,
+            "--track",
+            f"upstream/{target_branch}",
+        ]
+    )
+    successful_pr_id_commits = []
+    for pr_number in track(pr_ids, console=console, description="Backporting..."):
+        try:
+            subprocess.check_call(
+                ["./tools/backport-pull-request", str(pr_number)],
+                stderr=subprocess.DEVNULL,
+                stdout=subprocess.DEVNULL,
+            )
+            current_commit = subprocess.check_output(
+                ["git", "rev-parse", "HEAD"], text=True
+            ).strip()
+            successful_pr_id_commits.append((pr_number, current_commit))
+        except subprocess.CalledProcessError:
+            subprocess.check_call(["git", "cherry-pick", "--abort"])
+
+    if not successful_pr_id_commits:
+        print("No PRs successfully backported!")
+        return 1
+
+    body = f"Backport to {target_branch}:\n"
+    for pr_number, _ in successful_pr_id_commits:
+        body += f"- #{pr_number}\n"
+
+    subprocess.check_call(["git", "push", "origin", f"HEAD:{branchname}"])
+    backport_pr = repo.create_pull(
+        title=f"{target_branch} backports",
+        body=body,
+        head=f"{gh.get_user().login}:{branchname}",
+        base=target_branch,
+    )
+    wait_for_complete(repo, backport_pr, console)
+
+    for pr_number, commit in track(
+        successful_pr_id_commits, console=console, description="Commenting on backported PRs..."
+    ):
+        mark_as_backported(repo, backport_pr.number, pr_number, commit)
+
+    subprocess.check_call(["git", "push", "origin", "--delete", branchname])
+    subprocess.check_call(["git", "checkout", target_branch])
+    subprocess.check_call(["git", "branch", "--delete", "--force", branchname])
+    subprocess.check_call(["git", "pull"])
+    return 0
+
+
+if __name__ == "__main__":
+    typer.run(main)

tools/backport-pull-request

@@ -0,0 +1,103 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+    cat >&2 <<EOF
+usage: $0 PULL_REQUEST_ID
+
+Fetch the given GitHub pull request branch and backport it to
+the current branch using 'git cherry-pick -x'.
+
+Typical usage is:
+  git fetch upstream
+  git checkout -b 8.x upstream/8.x
+  $0 FIRST_PR_ID
+  $0 SECOND_PR_ID
+  git push origin +HEAD:backport-changes
+EOF
+    exit 1
+}
+
+pr_id="${1-}"
+
+if [ -z "$pr_id" ]; then
+    usage
+fi
+
+fail() {
+    echo "$1"
+    exit 1
+}
+
+type gh >/dev/null 2>&1 \
+    || fail "The 'gh' CLI tool is not installed; see https://cli.github.com/"
+gh auth status 2>/dev/null || fail "Not authenticated to github"
+
+# Find the last commit that was merged.  We will look back in `main`
+# for other commits from the same PR.
+# shellcheck disable=SC2016
+merge_commit="$(
+    gh api graphql \
+        -q '.data.repository.pullRequest.timelineItems.nodes[0].commit.oid' \
+        -F pr_id="$1" \
+        -f query='
+query($pr_id:Int!) {
+  repository(name: "zulip", owner: "zulip") {
+    pullRequest(number:$pr_id) {
+      timelineItems(last:1, itemTypes: [MERGED_EVENT]) {
+        nodes {
+          ... on MergedEvent {
+            commit {
+              oid
+            }
+          }
+        }
+      }
+    }
+  }
+}
+'
+)"
+
+[ -n "$merge_commit" ] || fail "PR #$pr_id is not merged into 'main'. Merge it first before backporting."
+
+# We cannot trust the "commits" count on the PR, since only part of it
+# may get merged, or it may have commits squashed during the merge.
+# Walk backwards on `main` from the merge commit we found, checking
+# that each of those commits is still associated with the same PR.
+commit_id="$merge_commit"
+while true; do
+    # shellcheck disable=SC2016
+    this_pr="$(gh api graphql -F "commit_id=$commit_id 0" \
+        --jq '.data.repository.ref.target.history.edges[].node.associatedPullRequests.nodes[].number' \
+        -f query='
+query($commit_id: String!) {
+  repository(owner: "zulip", name:"zulip") {
+    ref(qualifiedName:"main") {
+      target {
+        ... on Commit {
+          history(first:1, after: $commit_id) {
+            edges {
+              node {
+                oid
+                associatedPullRequests(first: 1) {
+                  nodes {
+                    number
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}')"
+    if [ "$this_pr" != "$pr_id" ]; then
+        break
+    fi
+    commit_id="$(git rev-parse "$commit_id"~)"
+done
+
+set -x
+git cherry-pick -x "$commit_id~..$merge_commit"

tools/build-demo-organization-wordlist

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+import re
+
+import orjson
+import requests
+
+EXCLUDED_NOUNS = [
+    "Fightings",
+    "Followings",
+    "Harassments",
+    "Injustices",
+    "Insults",
+    "Outrages",
+    "Pleas",
+    "Poisons",
+    "Prejudices",
+    "Punishments",
+    "Robberies",
+    "Sins",
+    "Softwares",
+    "Steams",
+    "Theses",
+]
+
+EXCLUDED_ADVERBS = [
+    "Obnoxiously",
+    "Rudely",
+    "Ruthlessly",
+    "Scornfully",
+    "Selfishly",
+]
+
+EXCLUDED_ADJECTIVES = [
+    "Cruel",
+    "Destructive",
+    "Dishonest",
+    "Illegal",
+    "Infamous",
+    "Shocking",
+]
+
+
+def clean_word(word: str) -> str:
+    return word.replace("\\n", "").replace("'", "").replace("]", "").replace("[", "").strip()
+
+
+def run() -> None:
+    url = "https://raw.githubusercontent.com/jitsi/js-utils/refs/heads/master/random/roomNameGenerator.ts"
+    response = requests.get(url)
+    content = str(response.content)
+
+    raw_plural_nouns = re.findall(r"const _PLURALNOUN_.*?=\s*(.*?);", content)
+    plural_nouns = raw_plural_nouns[0].split(",")
+    cleaned_nouns: list[str] = [
+        clean_word(noun).lower() for noun in plural_nouns if clean_word(noun) not in EXCLUDED_NOUNS
+    ]
+
+    raw_adverbs = re.findall(r"const _ADVERB_.*?=\s*(.*?);", content)
+    adverbs = raw_adverbs[0].split(",")
+    cleaned_adverbs: list[str] = [
+        clean_word(adverb).lower()
+        for adverb in adverbs
+        if clean_word(adverb) not in EXCLUDED_ADVERBS
+    ]
+
+    raw_adjectives = re.findall(r"const _ADJECTIVE_.*?=\s*(.*?);", content)
+    adjectives = raw_adjectives[0].split(",")
+    cleaned_adjectives: list[str] = [
+        clean_word(adjective).lower()
+        for adjective in adjectives
+        if clean_word(adjective) not in EXCLUDED_ADJECTIVES
+    ]
+
+    word_map = {
+        "nouns": cleaned_nouns,
+        "adverbs": cleaned_adverbs,
+        "adjectives": cleaned_adjectives,
+    }
+
+    file_path = "zerver/lib/demo_organization_words.json"
+    with open(file_path, "wb+") as f:
+        f.write(
+            orjson.dumps(
+                word_map,
+                option=orjson.OPT_APPEND_NEWLINE | orjson.OPT_INDENT_2 | orjson.OPT_SORT_KEYS,
+            )
+        )
+
+
+if __name__ == "__main__":
+    run()

tools/build-docs

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import sys
+
+ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(ZULIP_PATH)
+
+# check for the venv
+from tools.lib import sanity_check
+
+sanity_check.check_venv(__file__)
+
+from scripts.lib.zulip_tools import ENDC, WARNING, run
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-c", "--clean", action="store_true")
+    args = parser.parse_args()
+
+    path = os.path.join(ZULIP_PATH, "docs")
+    if not args.clean:
+        run(["make", "html", "-C", path])
+        print(
+            WARNING
+            + "tools/build-docs --clean is necessary for the navigation/left sidebar to update."
+            + ENDC
+        )
+        return
+
+    run(["make", "clean", "html", "-C", path])
+
+
+if __name__ == "__main__":
+    main()

tools/build-help-center

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import subprocess
+import sys
+from email.utils import parseaddr
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+from scripts.lib.setup_path import setup_path
+
+setup_path()
+
+os.environ["DJANGO_SETTINGS_MODULE"] = "zproject.settings"
+
+import django
+
+django.setup()
+from django.conf import settings
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--no-relative-links",
+    action="store_false",
+    dest="show_relative_links",
+    help="Disable relative links when using NavigationSteps component. Typically only set to true in case of the help center being hosted on zulip.com.",
+)
+
+parser.add_argument(
+    "--out-dir",
+    action="store",
+    dest="out_dir",
+    help="Output directory for the final build.",
+)
+args = parser.parse_args()
+
+
+def run() -> None:
+    env = os.environ.copy()
+    env["SUPPORT_EMAIL"] = parseaddr(settings.ZULIP_ADMINISTRATOR)[1]
+    env["CORPORATE_ENABLED"] = str(settings.CORPORATE_ENABLED).lower()
+    env["SHOW_RELATIVE_LINKS"] = str(args.show_relative_links).lower()
+    build_command = ["/usr/local/bin/corepack", "pnpm", "build"]
+    if args.out_dir:
+        build_command += ["--outDir", args.out_dir]
+
+    subprocess.check_call(
+        build_command,
+        cwd="starlight_help",
+        env=env,
+    )
+
+
+run()

tools/build-release-tarball

@@ -0,0 +1,123 @@
+#!/usr/bin/env bash
+set -eu
+
+# shellcheck source=lib/git-tools.bash
+. "$(dirname "$0")"/lib/git-tools.bash
+
+usage() {
+    echo "Usage: $0 <ZULIP_VERSION>" >&2
+    exit 1
+}
+
+args="$(getopt -o '' -l 'help' -n "$0" -- "$@")"
+eval "set -- $args"
+
+while true; do
+    case "$1" in
+        --)
+            shift
+            break
+            ;;
+        *) usage ;;
+    esac
+done
+
+if [ -z "${1:-}" ]; then
+    usage
+fi
+
+version="$1"
+prefix="zulip-server-$version"
+
+require_clean_work_tree 'build a release tarball'
+
+set -x
+GITID=$(git rev-parse HEAD)
+
+umask 022
+
+OUTPUT_DIR=${OUTPUT_DIR:-$(mktemp -d)}
+TARBALL=$OUTPUT_DIR/$prefix.tar
+BASEDIR=$(pwd)
+
+git archive -o "$TARBALL" "--prefix=$prefix/" HEAD
+
+cd "$OUTPUT_DIR"
+tar -xf "$TARBALL"
+while read -r i; do
+    rm -r --interactive=never "${OUTPUT_DIR:?}/$prefix/$i"
+done <"$OUTPUT_DIR/$prefix/tools/release-tarball-exclude.txt"
+tar -cf "$TARBALL" "$prefix"
+rm -rf "$prefix"
+
+if tar -tf "$TARBALL" | grep -q -e ^zerver/tests; then
+    set +x
+    echo "BUG: Excluded files remain in tarball!"
+    exit 1
+fi
+cd "$BASEDIR"
+
+# Check out a temporary full copy of the index to generate static files
+git checkout-index -f -a --prefix "$OUTPUT_DIR/$prefix/"
+
+# Add the Git version information file
+if [[ "$version" =~ ^[0-9]+\.[0-9]+(-[0-9a-z]+)?$ ]]; then
+    # We override the merge-base, to avoid having to push the commits before building the tarball.
+    OVERRIDE_MERGE_BASE="$version" ./tools/cache-zulip-git-version
+    generated_version=$(head -n1 zulip-git-version)
+    if [ "$generated_version" != "$version" ]; then
+        echo "Version $version looks like a release version, but is not tagged yet!  Got $generated_version"
+        exit 1
+    fi
+else
+    ./tools/cache-zulip-git-version
+fi
+mv zulip-git-version "$OUTPUT_DIR/$prefix/"
+
+cd "$OUTPUT_DIR/$prefix"
+
+env -u PYTHONDEVMODE -u PYTHONWARNINGS uv sync --frozen --no-managed-python
+
+# create var/log directory in the new temporary checkout
+mkdir -p "var/log"
+
+# Some settings need to be updated for update-prod-static to work
+#
+# TODO: Would be much better to instead run the below tools with some
+# sort of environment hack so that we don't need to create this dummy
+# secrets file.
+cat >>zproject/prod_settings_template.py <<EOF
+DEBUG = False
+EOF
+cat >>zproject/dev-secrets.conf <<EOF
+[secrets]
+local_database_password = ''
+secret_key = 'not_used_here'
+shared_secret = 'not_used_here'
+avatar_salt = 'not_used_here'
+rabbitmq_password = 'not_used_here'
+initial_password_salt = 'not_used_here'
+EOF
+
+./tools/update-prod-static
+
+# We don't need duplicate copies of emoji with hashed paths, and they would break Markdown
+find prod-static/serve/generated/emoji/images/emoji/ -regex '.*\.[0-9a-f]+\.png' -delete
+
+echo "$GITID" >build_id
+echo "$version" >version
+
+cd "$OUTPUT_DIR"
+
+# We don't include starlight_help/dist_no_relative_links since that is only useful for zulip.com
+# and zulip.com doesn't use tarballs for deployment.
+tar --append -f "$TARBALL" "$prefix/prod-static" "$prefix/build_id" "$prefix/version" "$prefix/zulip-git-version" "$prefix/locale" "$prefix/staticfiles.json" "$prefix/webpack-stats-production.json" "$prefix/starlight_help/dist"
+
+rm -rf "$prefix"
+
+gzip "$TARBALL"
+
+set +x
+TARBALL="$TARBALL.gz"
+
+echo "Generated $TARBALL"

tools/cache-zulip-git-version

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -eu
+
+cd "$(dirname "$0")/.."
+remote="$(git config zulip.zulipRemote)" || remote=upstream
+{
+    git describe --always --tags --match='[0-9]*'
+    if [ -z "${OVERRIDE_MERGE_BASE+x}" ]; then
+        branches="$(git for-each-ref --format='%(objectname)' "refs/remotes/$remote/main" "refs/remotes/$remote/*.x" "refs/remotes/$remote/*-branch")"
+        mapfile -t branches <<<"$branches"
+        if merge_base="$(git merge-base -- HEAD "${branches[@]}")"; then
+            git describe --always --tags --match='[0-9]*' -- "$merge_base"
+        fi
+    else
+        echo "$OVERRIDE_MERGE_BASE"
+    fi
+} >zulip-git-version

tools/check-capitalization

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+
+import argparse
+import json
+import os
+import re
+import subprocess
+import sys
+
+sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
+
+# check for the venv
+from tools.lib import sanity_check
+
+sanity_check.check_venv(__file__)
+
+from scripts.lib.zulip_tools import ENDC, FAIL, WARNING
+from tools.lib.capitalization import check_capitalization
+
+DJANGO_PO_REGEX = re.compile(r'msgid "(.*?)"')
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--show-ignored",
+        action="store_true",
+        help="Show strings that passed the check because they contained ignored phrases.",
+    )
+    parser.add_argument(
+        "--no-generate", action="store_true", help="Don't run makemessages command."
+    )
+    args = parser.parse_args()
+
+    if not args.no_generate:
+        subprocess.check_call(
+            ["./manage.py", "makemessages", "--locale", "en"], stderr=subprocess.STDOUT
+        )
+
+    with open("locale/en/translations.json") as f:
+        data = json.load(f)
+        frontend = check_capitalization(list(data.keys()))
+        frontend_errors, frontend_ignored, banned_errors_front = frontend
+
+    with open("locale/en/LC_MESSAGES/django.po") as f:
+        rows = [r for r in DJANGO_PO_REGEX.findall(f.read()) if r]
+        backend = check_capitalization(rows)
+        backend_errors, backend_ignored, banned_errors_back = backend
+
+    if frontend_errors:
+        print(WARNING + "Strings not properly capitalized in frontend:" + ENDC)
+        print("\n".join(frontend_errors))
+
+    if backend_errors:
+        print(WARNING + "Strings not properly capitalized in backend:" + ENDC)
+        print("\n".join(backend_errors))
+
+    if banned_errors_front:
+        print(WARNING + "Found banned words in frontend strings" + ENDC)
+        print("\n".join(banned_errors_front))
+
+    if banned_errors_back:
+        print(WARNING + "Found banned words in backend strings" + ENDC)
+        print("\n".join(banned_errors_back))
+
+    if args.show_ignored:
+        print(WARNING + "Strings which were ignored: " + ENDC)
+        print("\n".join(frontend_ignored + backend_ignored))
+
+    if frontend_errors or backend_errors or banned_errors_back or banned_errors_front:
+        # Point the user to the documentation on what the policy is.
+        docs_url = (
+            "https://zulip.readthedocs.io/en/latest/translating/translating.html#capitalization"
+        )
+        print()
+        print(WARNING + "You can usually find strings using `git grep 'String to find'`" + ENDC)
+        print(WARNING + "See also " + docs_url + ENDC)
+        print()
+        print(FAIL + "Failed!" + ENDC)
+        sys.exit(1)
+    else:
+        sys.exit(0)

tools/check-feature-level-updated

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+import os
+import sys
+from pathlib import Path
+
+
+def get_build_url_from_environment() -> str:
+    server = os.environ["GITHUB_SERVER_URL"]
+    repo = os.environ["GITHUB_REPOSITORY"]
+    run_id = os.environ["GITHUB_RUN_ID"]
+    return f"{server}/{repo}/actions/runs/{run_id}"
+
+
+if __name__ == "__main__":
+    branch = os.environ.get("GITHUB_REF", "unknown branch").split("/")[-1]
+    topic = f"{branch} failing"
+    build_url = get_build_url_from_environment()
+    github_actor = os.environ.get("GITHUB_ACTOR", "unknown user")
+
+    api_docs_folder = Path("api_docs")
+    api_docs_paths = list(api_docs_folder.glob("*.md"))
+    api_docs_paths.append(Path("zerver/openapi/zulip.yaml"))
+
+    for api_docs_path in api_docs_paths:
+        with open(api_docs_path) as file:
+            if "ZF-" in file.read():
+                content = f"[Build]({build_url}) triggered by {github_actor} on branch `{branch}` has failed: Feature level not replaced in '{api_docs_path}'. This also skips the job that posts the new-feature-level announcement to #**api documentation**. After fixing `{branch}`, post the announcement manually if needed."
+                print(f"fail=true\ntopic={topic}\ncontent={content}")
+                sys.exit(0)
+
+    print("fail=false")

tools/check-frontend-i18n

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+import argparse
+import json
+import os
+import subprocess
+import sys
+
+sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
+
+# check for the venv
+from tools.lib import sanity_check
+
+sanity_check.check_venv(__file__)
+
+from scripts.lib.zulip_tools import ENDC, FAIL, WARNING
+
+
+def find_handlebars(translatable_strings: list[str]) -> list[str]:
+    return [string for string in translatable_strings if "{{" in string]
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--no-generate", action="store_true", help="Don't run makemessages command."
+    )
+    args = parser.parse_args()
+
+    if not args.no_generate:
+        subprocess.check_call(
+            ["./manage.py", "makemessages", "--locale", "en"], stderr=subprocess.STDOUT
+        )
+
+    with open("locale/en/translations.json") as f:
+        data = json.load(f)
+
+    found = find_handlebars(list(data.keys()))
+    if not found:
+        sys.exit(0)
+
+    print(WARNING + "Translation strings contain Handlebars:" + ENDC)
+    print("\n".join(found))
+
+    print(
+        WARNING
+        + "See https://zulip.readthedocs.io/en/latest/translating/internationalization.html#web-application-translations "
+        "on how you can insert variables in the frontend translatable "
+        "strings." + ENDC
+    )
+    print(FAIL + "Failed!" + ENDC)
+    sys.exit(1)

tools/check-issue-labels

@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+import argparse
+import configparser
+import os
+import re
+import sys
+from typing import Any
+
+import requests
+
+# Scans zulip repository for issues that don't have any `area` labels.
+# GitHub API token is required as GitHub limits unauthenticated
+# requests to 60/hour. There is a good chance that this limit is
+# bypassed in consecutive attempts.
+# The API token can be generated here
+# https://github.com/settings/tokens/new?description=Zulip%20Issue%20Label%20Checker
+#
+# Copy conf.ini-template to conf.ini and populate with your API token.
+#
+# usage: python check-issue-labels
+# Pass --force as an argument to run without a token.
+
+
+def get_config() -> configparser.ConfigParser:
+    config = configparser.ConfigParser()
+    config.read(os.path.join(os.path.dirname(os.path.abspath(__file__)), "conf.ini"))
+    return config
+
+
+def area_labeled(issue: dict[str, Any]) -> bool:
+    for label in issue["labels"]:
+        label_name = str(label["name"])
+        if "area:" in label_name:
+            return True
+    return False
+
+
+def is_issue(item: dict[str, Any]) -> bool:
+    return "issues" in item["html_url"]
+
+
+def get_next_page_url(link_header: str) -> str | None:
+    matches = re.findall(r"\<(\S+)\>; rel=\"next\"", link_header)
+    try:
+        return matches[0]
+    except IndexError:
+        return None
+
+
+def check_issue_labels() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--force", action="store_true")
+    args = parser.parse_args()
+
+    if not args.force:
+        config = get_config()
+        try:
+            token = config.get("github", "api_token")
+        except configparser.Error:
+            print(
+                "Error fetching GitHub API token. Copy conf.ini-template to conf.ini and populate with "
+                "your API token. If you want to continue without using a token use --force."
+            )
+            sys.exit(1)
+
+    next_page_url: str | None = "https://api.github.com/repos/zulip/zulip/issues"
+    unlabeled_issue_urls: list[str] = []
+    while next_page_url:
+        try:
+            if args.force:
+                response = requests.get(next_page_url)
+            else:
+                response = requests.get(next_page_url, headers={"Authorization": f"token {token}"})
+            if response.status_code == 401:
+                sys.exit("Error. Please check the token.")
+            if response.status_code == 403:
+                sys.exit(
+                    "403 Error. This is generally caused when API limit is exceeded. You use an API "
+                    "token to overcome this limit."
+                )
+        except requests.exceptions.RequestException as e:
+            print(e)
+            sys.exit(1)
+
+        next_page_url = get_next_page_url(response.headers["Link"])
+        unlabeled_issue_urls.extend(
+            item["html_url"]
+            for item in response.json()
+            if is_issue(item) and not area_labeled(item)
+        )
+
+    if unlabeled_issue_urls:
+        print("The following issues don't have any area labels associated with it")
+        print("\n".join(unlabeled_issue_urls))
+    else:
+        print("No GitHub issues found with missing area labels.")
+
+
+if __name__ == "__main__":
+    check_issue_labels()

tools/check-openapi.ts

@@ -0,0 +1,194 @@
+#!/usr/bin/env node
+
+import assert from "node:assert/strict";
+import * as fs from "node:fs";
+import {parseArgs} from "node:util";
+
+import SwaggerParser from "@apidevtools/swagger-parser";
+import * as Diff from "diff";
+import * as ExampleValidator from "openapi-examples-validator";
+import {format as prettierFormat} from "prettier";
+import {CST, Composer, LineCounter, Parser, Scalar, YAMLMap, YAMLSeq, visit} from "yaml";
+
+const usage = "Usage: check-openapi.ts [--fix] <file>...";
+const {
+    values: {fix, help},
+    positionals: files,
+} = parseArgs({options: {fix: {type: "boolean"}, help: {type: "boolean"}}, allowPositionals: true});
+
+if (help) {
+    console.log(usage);
+    process.exit(0);
+}
+
+async function checkFile(file: string): Promise<void> {
+    const yaml = await fs.promises.readFile(file, "utf8");
+    const lineCounter = new LineCounter();
+    const tokens = [...new Parser(lineCounter.addNewLine).parse(yaml)];
+    const docs = [...new Composer().compose(tokens)];
+    if (docs.length !== 1) {
+        return;
+    }
+    const [doc] = docs;
+    assert.ok(doc !== undefined);
+    if (doc.errors.length > 0) {
+        for (const error of doc.errors) {
+            console.error("%s: %s", file, error.message);
+        }
+        process.exitCode = 1;
+        return;
+    }
+
+    const root = doc.contents;
+    if (!(root instanceof YAMLMap && root.has("openapi"))) {
+        return;
+    }
+
+    let ok = true;
+    const reformats = new Map<
+        number,
+        {value: string; context: Parameters<typeof CST.setScalarValue>[2]}
+    >();
+    const promises: Promise<void>[] = [];
+
+    visit(doc, {
+        Map(_key, node) {
+            if (node.has("$ref") && node.items.length !== 1) {
+                assert.ok(node.range);
+                const {line, col} = lineCounter.linePos(node.range[0]);
+                console.error("%s:%d:%d: Siblings of $ref have no effect", file, line, col);
+                ok = false;
+            }
+
+            const combinator = ["allOf", "anyOf", "oneOf"].find((combinator) =>
+                node.has(combinator),
+            );
+            if (node.has("nullable") && combinator !== undefined) {
+                assert.ok(node.range);
+                const {line, col} = lineCounter.linePos(node.range[0]);
+                console.error(
+                    `%s:%d:%d: nullable has no effect as a sibling of ${combinator}`,
+                    file,
+                    line,
+                    col,
+                );
+                ok = false;
+            }
+        },
+
+        Pair(_key, node) {
+            if (
+                node.key instanceof Scalar &&
+                node.key.value === "allOf" &&
+                node.value instanceof YAMLSeq &&
+                node.value.items.filter(
+                    (subschema) => !(subschema instanceof YAMLMap && subschema.has("$ref")),
+                ).length > 1
+            ) {
+                assert.ok(node.value.range);
+                const {line, col} = lineCounter.linePos(node.value.range[0]);
+                console.error("%s:%d:%d: Too many inline allOf subschemas", file, line, col);
+                ok = false;
+            }
+
+            if (
+                node.key instanceof Scalar &&
+                node.key.value === "description" &&
+                node.value instanceof Scalar &&
+                typeof node.value.value === "string"
+            ) {
+                const value = node.value;
+                const description = node.value.value;
+                promises.push(
+                    (async () => {
+                        let formatted = await prettierFormat(description, {
+                            parser: "markdown",
+                        });
+                        if (
+                            value.type !== Scalar.BLOCK_FOLDED &&
+                            value.type !== Scalar.BLOCK_LITERAL
+                        ) {
+                            formatted = formatted.replace(/\n$/, "");
+                        }
+                        if (formatted !== description) {
+                            assert.ok(value.range);
+                            if (fix) {
+                                reformats.set(value.range[0], {
+                                    value: formatted,
+                                    context: {afterKey: true},
+                                });
+                            } else {
+                                ok = false;
+                                const {line, col} = lineCounter.linePos(value.range[0]);
+                                console.error(
+                                    "%s:%d:%d: Format description with Prettier:",
+                                    file,
+                                    line,
+                                    col,
+                                );
+                                let diff = "";
+                                for (const part of Diff.diffLines(description, formatted)) {
+                                    const prefix = part.added
+                                        ? "\u{1B}[32m+"
+                                        : part.removed
+                                          ? "\u{1B}[31m-"
+                                          : "\u{1B}[34m ";
+                                    diff += prefix;
+                                    diff += part.value
+                                        .replace(/\n$/, "")
+                                        .replaceAll("\n", () => "\n" + prefix);
+                                    diff += "\n";
+                                }
+                                diff += "\u{1B}[0m";
+                                console.error(diff);
+                            }
+                        }
+                    })(),
+                );
+            }
+        },
+    });
+    await Promise.all(promises);
+
+    if (!ok) {
+        process.exitCode = 1;
+    }
+    if (reformats.size > 0) {
+        console.log("%s: Fixing problems", file);
+        for (const token of tokens) {
+            if (token.type === "document") {
+                CST.visit(token, ({value}) => {
+                    let reformat;
+                    if (
+                        CST.isScalar(value) &&
+                        (reformat = reformats.get(value.offset)) !== undefined
+                    ) {
+                        CST.setScalarValue(value, reformat.value, reformat.context);
+                    }
+                });
+            }
+        }
+        await fs.promises.writeFile(file, tokens.map((token) => CST.stringify(token)).join(""));
+    }
+
+    try {
+        await SwaggerParser.validate(file);
+    } catch (error) {
+        if (!(error instanceof SyntaxError)) {
+            throw error;
+        }
+        console.error("%s: %s", file, error.message);
+        process.exitCode = 1;
+    }
+    const res = await ExampleValidator.validateFile(file);
+    if (!res.valid) {
+        for (const error of res.errors) {
+            console.error(error);
+        }
+        process.exitCode = 1;
+    }
+}
+
+for (const file of files) {
+    await checkFile(file);
+}

tools/check-provision

@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+import argparse
+import os
+import sys
+
+tools_dir = os.path.dirname(os.path.abspath(__file__))
+root_dir = os.path.dirname(tools_dir)
+sys.path.insert(0, root_dir)
+
+from tools.lib.test_script import add_provision_check_override_param, assert_provisioning_status_ok
+
+
+def run() -> None:
+    parser = argparse.ArgumentParser()
+    add_provision_check_override_param(parser)
+
+    options = parser.parse_args()
+
+    assert_provisioning_status_ok(options.skip_provision_check)
+
+
+if __name__ == "__main__":
+    run()

tools/check-schemas

@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+#
+# Validates that 3 data sources agree about the structure of Zulip's events API:
+#
+# * Node fixtures for the server_events_dispatch.js tests.
+# * OpenAPI definitions in zerver/openapi/zulip.yaml
+# * The schemas defined in zerver/lib/events_schema.py used for the
+#   Zulip server's test suite.
+#
+# We compare the Python and OpenAPI schemas by converting the OpenAPI data
+# into the event_schema style of types and the diffing the schemas.
+import argparse
+import os
+import subprocess
+import sys
+from collections.abc import Callable
+from typing import Any
+
+import orjson
+
+TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, os.path.dirname(TOOLS_DIR))
+ROOT_DIR = os.path.dirname(TOOLS_DIR)
+
+EVENTS_JS = "web/tests/lib/events.cjs"
+
+# check for the venv
+from tools.lib import sanity_check
+
+sanity_check.check_venv(__file__)
+
+USAGE = """
+
+    This program reads in fixture data for our
+    node tests, and then it validates the fixture
+    data with checkers from event_schema.py (which
+    are the same Python functions we use to validate
+    events in test_events.py).
+
+    It currently takes no arguments.
+"""
+
+parser = argparse.ArgumentParser(usage=USAGE)
+parser.parse_args()
+
+# We can eliminate the django dependency in event_schema,
+# but unfortunately it"s coupled to modules like validate.py
+# and topic.py.
+import django
+
+os.environ["DJANGO_SETTINGS_MODULE"] = "zproject.test_settings"
+django.setup()
+
+from zerver.lib import event_schema
+
+make_checker = event_schema.__dict__["make_checker"]
+
+
+def get_event_checker(event: dict[str, Any]) -> Callable[[str, dict[str, Any]], None]:
+    # Follow the naming convention to find the event checker.
+    # Start by grabbing the event type.
+    name = event["type"]
+
+    # Handle things like AttachmentRemoveEvent
+    if "op" in event:
+        name += "_" + event["op"].title()
+
+    # Change to CamelCase
+    name = name.replace("_", " ").title().replace(" ", "")
+
+    # Use ModernPresenceEvent type to check "presence" events
+    if name == "Presence":
+        name = "Modern" + name
+
+    # And add the suffix.
+    name = name + "Event"
+
+    if not hasattr(event_schema, name):
+        raise ValueError(f"We could not find {name} in event_schemas.py")
+
+    return make_checker(getattr(event_schema, name))
+
+
+def check_event(name: str, event: dict[str, Any]) -> None:
+    event["id"] = 1
+    checker = get_event_checker(event)
+    try:
+        checker(name, event)
+    except AssertionError:
+        print(f"\n{EVENTS_JS} has bad data for {name}:\n\n")
+        raise
+
+
+def read_fixtures() -> dict[str, Any]:
+    cmd = [
+        "node",
+        os.path.join(TOOLS_DIR, "node_lib/dump_fixtures.js"),
+    ]
+    schema = subprocess.check_output(cmd)
+    return orjson.loads(schema)
+
+
+def verify_fixtures_are_sorted(names: list[str]) -> None:
+    for i in range(1, len(names)):
+        if names[i] < names[i - 1]:
+            raise Exception(
+                f"""
+                Please keep your fixtures in order within
+                your events.cjs file.  The following
+                key is out of order
+
+                {names[i]}
+                """
+            )
+
+
+def run() -> None:
+    fixtures = read_fixtures()
+    verify_fixtures_are_sorted(list(fixtures.keys()))
+    for name, event in fixtures.items():
+        check_event(name, event)
+    print(f"Successfully checked {len(fixtures)} fixtures. All tests passed.")
+
+
+if __name__ == "__main__":
+    run()

tools/check-templates

@@ -0,0 +1,132 @@
+#!/usr/bin/env python3
+import argparse
+import logging
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+# check for the venv
+from tools.lib import sanity_check
+
+sanity_check.check_venv(__file__)
+
+from collections.abc import Iterable
+
+from zulint import lister
+
+from tools.lib.html_branches import build_id_dict
+from tools.lib.pretty_print import validate_indent_html
+from tools.lib.template_parser import validate
+
+EXCLUDED_FILES = [
+    ## Test data Files for testing modules in tests
+    "tools/tests/test_template_data",
+    # Our parser doesn't handle the way its conditionals are layered
+    "templates/zerver/emails/missed_message.html",
+    # Previously unchecked and our parser doesn't like its indentation
+    "web/icons/template.hbs",
+    # Template checker recommends very hard to read indentation.
+    "web/templates/bookend.hbs",
+]
+
+
+def check_our_files(modified_only: bool, all_dups: bool, fix: bool, targets: list[str]) -> None:
+    by_lang = lister.list_files(
+        targets=targets,
+        modified_only=modified_only,
+        ftypes=["hbs", "html"],
+        group_by_ftype=True,
+        exclude=EXCLUDED_FILES,
+    )
+
+    check_handlebars_templates(by_lang["hbs"], fix)
+    check_html_templates(by_lang["html"], all_dups, fix)
+
+
+def check_html_templates(templates: Iterable[str], all_dups: bool, fix: bool) -> None:
+    # Our files with .html extensions are usually for Django, but we also
+    # have a few static .html files.
+    logging.basicConfig(format="%(levelname)s:%(message)s")
+    templates = sorted(fn for fn in templates)
+    # Use of lodash templates <%= %>.
+    if "templates/corporate/team.html" in templates:
+        templates.remove("templates/corporate/team.html")
+
+    def check_for_duplicate_ids(templates: list[str]) -> dict[str, list[str]]:
+        template_id_dict = build_id_dict(templates)
+        # TODO: Clean up these cases of duplicate ids in the code
+        IGNORE_IDS = [
+            "errors",
+            "email",
+            "registration",
+            "pw_strength",
+            "id_password",
+            "top_navbar",
+            "id_email",
+            "id_terms",
+            "logout_form",
+            "send_confirm",
+            "charged_amount",
+            "change-plan-status",
+        ]
+        bad_ids_dict = {
+            ids: fns
+            for ids, fns in template_id_dict.items()
+            if (ids not in IGNORE_IDS) and len(fns) > 1
+        }
+
+        if all_dups:
+            ignorable_ids_dict = {
+                ids: fns
+                for ids, fns in template_id_dict.items()
+                if ids in IGNORE_IDS and len(fns) > 1
+            }
+
+            for ids, fns in ignorable_ids_dict.items():
+                logging.warning("Duplicate ID(s) detected: ID %r present at following files:", ids)
+                for fn in fns:
+                    print(fn)
+
+        for ids, fns in bad_ids_dict.items():
+            logging.error("Duplicate ID(s) detected: ID %r present at following files:", ids)
+            for fn in fns:
+                print(fn)
+        return bad_ids_dict
+
+    bad_ids_list = list(check_for_duplicate_ids(templates).keys())
+
+    if bad_ids_list:
+        print("Exiting--please clean up all duplicates before running this again.")
+        sys.exit(1)
+
+    for fn in templates:
+        tokens = validate(fn, template_format="django")
+        if not validate_indent_html(fn, tokens, fix):
+            sys.exit(1)
+
+
+def check_handlebars_templates(templates: Iterable[str], fix: bool) -> None:
+    # Check all our Handlebars templates.
+    templates = [fn for fn in templates if fn.endswith(".hbs")]
+
+    for fn in templates:
+        tokens = validate(fn, template_format="handlebars")
+        if not validate_indent_html(fn, tokens, fix):
+            sys.exit(1)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-m", "--modified", action="store_true", help="only check modified files")
+    parser.add_argument(
+        "--all-dups",
+        action="store_true",
+        help="Run lint tool to detect duplicate ids on ignored files as well",
+    )
+    parser.add_argument(
+        "--fix", action="store_true", help="Automatically fix indentation problems."
+    )
+    parser.add_argument("targets", nargs=argparse.REMAINDER)
+    args = parser.parse_args()
+    check_our_files(args.modified, args.all_dups, args.fix, args.targets)

tools/check-thirdparty

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+"""Check the docs/THIRDPARTY file against the DEP-5 copyright format.
+
+https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+
+Disclaimer: This script is not a lawyer.  It cannot validate that the
+claimed licenses are correct.  It can only check for basic syntactic
+issues.
+"""
+
+# Copyright © 2020 Kandra Labs, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+import difflib
+import io
+import os
+import subprocess
+import sys
+
+from debian import copyright
+
+COPYRIGHT_FILENAME = "docs/THIRDPARTY"
+os.chdir(os.path.join(os.path.dirname(__file__), ".."))
+status = 0
+
+*files, empty = subprocess.check_output(
+    ["git", "ls-files", "-z"],
+    encoding="utf-8",
+).split("\0")
+assert empty == ""
+files += [
+    "static/generated/emoji/images/emoji/unicode/ignore-this-path",
+]
+
+with open(COPYRIGHT_FILENAME) as f:
+    lines = list(f)
+c = copyright.Copyright(lines)
+
+if not c.header.known_format():
+    print(f"{COPYRIGHT_FILENAME}: Unknown header format {c.header.format}")
+    status = 1
+
+defined_licenses = {
+    p.license.to_str().split("\n", 1)[0]
+    for p in c.all_license_paragraphs()
+    if "\n" in p.license.to_str()
+}
+
+for p in c.all_files_paragraphs():
+    for g in p.files:
+        if not any(map(copyright.globs_to_re([g]).fullmatch, files)):
+            print(f"{COPYRIGHT_FILENAME}: No such file {g}")
+            status = 1
+
+    if "\n" not in p.license.to_str() and p.license.to_str() not in defined_licenses:
+        print(f"{COPYRIGHT_FILENAME}: Missing license text for {p.license.to_str()}")
+        status = 1
+
+dumped = c.dump()
+if dumped != "".join(lines):
+    print(f"{COPYRIGHT_FILENAME}: Changes expected:")
+    sys.stdout.writelines(
+        difflib.unified_diff(
+            lines,
+            io.StringIO(dumped).readlines(),
+            COPYRIGHT_FILENAME,
+            COPYRIGHT_FILENAME,
+        ),
+    )
+    status = 1
+
+sys.exit(status)

tools/ci/.gitattributes

@@ -0,0 +1 @@
+success-http-headers.template.txt eol=crlf

tools/ci/Docker-prod-8.0.diff

@@ -0,0 +1,39 @@
+diff --git scripts/setup/apt-repos/zulip/custom.sh scripts/setup/apt-repos/zulip/custom.sh
+index be7c780b7e..ee363c7d65 100755
+--- scripts/setup/apt-repos/zulip/custom.sh
++++ scripts/setup/apt-repos/zulip/custom.sh
+@@ -2,7 +2,6 @@
+ set -euo pipefail
+ 
+ if [[ ! -e /usr/share/doc/groonga-apt-source/copyright ]]; then
+-    arrow_keyring=$(readlink -f "$LIST_PATH/apache-arrow-keyring.gpg")
+     pgroonga_apt_sign_key=$(readlink -f "$LIST_PATH/pgroonga-packages.groonga.org.asc")
+ 
+     remove_pgroonga_apt_tmp_dir() {
+@@ -28,16 +27,6 @@ if [[ ! -e /usr/share/doc/groonga-apt-source/copyright ]]; then
+             read -r release
+         } <<<"$os_info"
+ 
+-        if [ "$distribution" = debian ] && [ "$release" = bookworm ]; then
+-            # As of Debian 12, the Groonga repository depends on the
+-            # Apache Arrow repository.
+-            arrow_apt_source_deb="apache-arrow-apt-source-latest-$release.deb"
+-            arrow_apt_source_deb_sign="$arrow_apt_source_deb.asc"
+-            curl -fLO --retry 3 "https://apache.jfrog.io/artifactory/arrow/$distribution/$arrow_apt_source_deb"
+-            curl -fLO --retry 3 "https://apache.jfrog.io/artifactory/arrow/$distribution/$arrow_apt_source_deb_sign"
+-            gpgv --keyring="$arrow_keyring" "$arrow_apt_source_deb_sign" "$arrow_apt_source_deb"
+-        fi
+-
+         groonga_apt_source_deb="groonga-apt-source-latest-$release.deb"
+         groonga_apt_source_deb_sign="$groonga_apt_source_deb.asc.$pgroonga_apt_sign_key_fingerprint"
+         curl -fLO --retry 3 "https://packages.groonga.org/$distribution/$groonga_apt_source_deb"
+@@ -53,9 +42,6 @@ if [[ ! -e /usr/share/doc/groonga-apt-source/copyright ]]; then
+         #   accessed by user '_apt'. - pkgAcquire::Run (13: Permission denied)
+         chown _apt .
+ 
+-        if [ "$distribution" = debian ] && [ "$release" = bookworm ]; then
+-            apt-get -y install "./$arrow_apt_source_deb"
+-        fi
+         apt-get -y install "./$groonga_apt_source_deb"
+     }
+     touch "$STAMP_FILE"

tools/ci/Dockerfile

@@ -0,0 +1,65 @@
+# Dockerfile for a generic Debian/Ubuntu image with just the basics we
+# need to make it suitable for CI.  In particular:
+#  * a non-root user to run as (a pain to try to do in setup,
+#    because by then we've already cloned the repo);
+#  * Git and other basic utilities.
+
+# To rebuild from this file for a given release, say Ubuntu 22.04 jammy:
+#   docker build . --build-arg=BASE_IMAGE=ubuntu:22.04 --pull --tag=zulip/ci:jammy
+#   docker push zulip/ci:jammy
+#
+# tools/ci/build-docker-images will rebuild all images, but not push them.
+
+ARG BASE_IMAGE
+# Pin to linux/amd64 so builds on Apple Silicon (or other non-amd64
+# hosts) still produce images compatible with our GitHub Actions runners.
+FROM --platform=linux/amd64 $BASE_IMAGE
+
+RUN ln -sf /usr/share/zoneinfo/Etc/UTC /etc/localtime
+
+# Set the locale.
+ENV LC_ALL C.UTF-8
+
+# Extra packages used by Zulip.
+RUN apt-get update \
+  && apt-get -y install --no-install-recommends \
+    build-essential \
+    ca-certificates \
+    curl \
+    gettext \
+    git \
+    hunspell-en-us \
+    jq \
+    libffi-dev \
+    libfreetype6-dev \
+    libjpeg-dev \
+    libldap2-dev \
+    libpq-dev \
+    libssl-dev \
+    libxml2-dev \
+    libxslt1-dev \
+    locales \
+    memcached \
+    moreutils \
+    puppet \
+    python3-dev \
+    python3-pip \
+# We do not pre-install rabbitmq-server, as doing so fixes the
+# nodename to be the current hostname, which varies.  Letting Zulip
+# install rabbitmq allows it to fix the nodename to 'localhost'.
+    redis-server \
+    sudo \
+    supervisor \
+    unzip \
+    xvfb \
+    zlib1g-dev
+
+ARG USERNAME=github
+RUN groupadd --gid 1001 $USERNAME \
+  && useradd --uid 1001 --gid $USERNAME --shell /bin/bash --create-home $USERNAME \
+  && echo "$USERNAME ALL = (ALL) NOPASSWD: ALL" >> /etc/sudoers.d/50-$USERNAME \
+  && echo 'Defaults    env_keep += "DEBIAN_FRONTEND"' >> /etc/sudoers.d/env_keep
+
+USER $USERNAME
+
+CMD ["/bin/sh"]

tools/ci/Dockerfile.prod

@@ -0,0 +1,25 @@
+# To build these production upgrade test images, say an Debian 12 Bookworm system
+# preinstalled with Zulip 7.0:
+#   docker build --pull . -f Dockerfile.prod \
+#     --build-arg=BASE_IMAGE=zulip/ci:bookworm \
+#     --build-arg=VERSION=7.0 \
+#     --tag=zulip/ci:bookworm-7.0
+#   docker push zulip/ci:bookworm-7.0
+
+ARG BASE_IMAGE
+# See tools/ci/Dockerfile for why we pin to linux/amd64.
+FROM --platform=linux/amd64 $BASE_IMAGE
+
+# Download the release tarball, start rabbitmq server and install the server
+ARG VERSION
+ADD Docker-prod-8.0.diff /tmp
+RUN cd $(mktemp -d) \
+  && curl -fLO --retry 3 "https://download.zulip.com/server/zulip-server-$VERSION.tar.gz" \
+  && tar -xf "zulip-server-$VERSION.tar.gz" \
+  && if [ "$VERSION" = "8.0" ]; then \
+         cat /tmp/Docker-prod-8.0.diff | patch -p0 -d "zulip-server-$VERSION"; \
+     fi \
+  && sudo -s "./zulip-server-$VERSION/scripts/setup/install" --self-signed-cert --hostname 127.0.0.1 --email ci@example.com \
+  && sudo service rabbitmq-server stop
+
+CMD ["/bin/sh"]

tools/ci/activate-venv

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+ZULIP_PATH="$(dirname "${BASH_SOURCE[0]}")/../.."
+source "$ZULIP_PATH/.venv/bin/activate"
+source "$ZULIP_PATH"/tools/python-warnings.bash
+echo "Using $VIRTUAL_ENV"

tools/ci/build-docker-images

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -eux
+cd "$(dirname "${BASH_SOURCE[0]}")"
+docker build . --build-arg=BASE_IMAGE=ubuntu:22.04 --pull --tag=zulip/ci:jammy
+docker build . --build-arg=BASE_IMAGE=ubuntu:24.04 --pull --tag=zulip/ci:noble
+docker build . --build-arg=BASE_IMAGE=ubuntu:26.04 --pull --tag=zulip/ci:resolute
+docker build . --build-arg=BASE_IMAGE=debian:12 --pull --tag=zulip/ci:bookworm
+docker build . --build-arg=BASE_IMAGE=debian:13 --pull --tag=zulip/ci:trixie

tools/ci/check-executables

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -euo pipefail
+files="$(git ls-tree -r @ | sed -n 's/^100755 blob \S\+\t//p' | xargs -r grep -Lz '^#!' --)" || [ $? = 123 ]
+if [ "$files" ]; then
+    echo 'error: non-scripts should not be marked executable (fix with chmod -x):'
+    echo
+    printf '%s\n' "$files"
+    exit 1
+fi

tools/ci/generate-failure-message

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+
+import os
+
+
+def get_build_url_from_environment() -> str:
+    server = os.environ["GITHUB_SERVER_URL"]
+    repo = os.environ["GITHUB_REPOSITORY"]
+    run_id = os.environ["GITHUB_RUN_ID"]
+    return f"{server}/{repo}/actions/runs/{run_id}"
+
+
+if __name__ == "__main__":
+    branch = os.environ.get("GITHUB_REF", "unknown branch").split("/")[-1]
+    topic = f"{branch} failing"
+    build_url = get_build_url_from_environment()
+    github_actor = os.environ.get("GITHUB_ACTOR", "unknown user")
+    content = f"[Build]({build_url}) triggered by {github_actor} on branch `{branch}` has failed."
+
+    # Output in the key-value pair format GitHub Actions outputs are expected
+    # to be in.
+    print(f"topic={topic}\ncontent={content}")

tools/ci/production-build

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# In short, this provisions a Zulip development environment and then
+# builds a Zulip release tarball (the same way we build them for an
+# actual release).  The actual test job will then install that.
+
+set -e
+set -x
+
+# Provisioning may fail due to many issues but most of the times a network
+# connection issue is the reason. So we are going to retry entire provisioning
+# once again if that fixes our problem.
+tools/provision --build-release-tarball-only || {
+    ret=$?
+    if [ "$ret" = 1 ]; then
+        echo "\`provision\`: Something went wrong with the provisioning, might be a network issue, Retrying to provision..."
+        tools/provision --build-release-tarball-only
+    else
+        echo "\`provision\`: Something REALLY BAD went wrong with the provisioning, not retrying."
+        exit "$ret"
+    fi
+}
+
+source tools/ci/activate-venv
+
+if ! ./tools/build-release-tarball test; then
+    echo "Attempting to output failure logging data"
+    cat /tmp/tmp.*/update-prod-static.log || true
+    exit 1
+fi
+
+# Move all the required artifacts to /tmp/production-build
+# that will be later sent down to downstream install jobs.
+mkdir /tmp/production-build
+mv /tmp/tmp.*/zulip-server-test.tar.gz /tmp/production-build
+cp -a \
+    tools/ci/success-http-headers.template.txt \
+    tools/ci/production-install \
+    tools/ci/production-verify \
+    tools/ci/production-upgrade \
+    tools/ci/production-pgroonga \
+    tools/ci/production-upgrade-pg \
+    tools/ci/generate-failure-message \
+    package.json pnpm-lock.yaml \
+    /tmp/production-build
+
+# Check that webpack bundles use only ES2022 syntax.
+# Use the pnpm binary installed by tools/provision.
+PNPM="/usr/local/bin/pnpm"
+tar -C /tmp -xzf /tmp/production-build/zulip-server-test.tar.gz zulip-server-test/prod-static/serve/webpack-bundles
+(
+    GLOBIGNORE="/tmp/zulip-server-test/prod-static/serve/webpack-bundles/katex*.js"
+    "$PNPM" exec es-check es2022 /tmp/zulip-server-test/prod-static/serve/webpack-bundles/*.js
+)

tools/ci/production-install

@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+# This test installs a Zulip production environment (from the release
+# tarball from production-build).
+set -e
+set -x
+
+usage() {
+    cat <<'EOF'
+Usage:
+  production-install
+  production-install --test-custom-db
+  production-install --help
+
+Options:
+  --test-custom-db
+      This will instruct the install test to be run with a custom database name and user.
+
+EOF
+}
+
+# Shell option parsing.
+args="$(getopt -o '' --long help,test-custom-db -n "$0" -- "$@")"
+eval "set -- $args"
+while true; do
+    case "$1" in
+        --help)
+            usage
+            exit 0
+            ;;
+
+        --test-custom-db)
+            TEST_CUSTOM_DB=1
+            shift
+            ;;
+        --)
+            shift
+            break
+            ;;
+    esac
+done
+
+ZULIP_PATH=/root/zulip-latest
+mkdir -p "$ZULIP_PATH"
+tar -xf /tmp/zulip-server-test.tar.gz -C "$ZULIP_PATH" --strip-components=1
+
+# Disable interactive prompts from apt.
+echo 'Dpkg::Options { "--force-confdef" "--force-confold" }' >/etc/apt/apt.conf.d/noninteractive
+export DEBIAN_FRONTEND=noninteractive
+
+# Do an apt upgrade to start with an up-to-date machine
+apt-get update
+
+if ! apt-get dist-upgrade -y; then
+    echo "\`apt-get dist-upgrade\`: Failure occurred while trying to perform distribution upgrade, Retrying..."
+    apt-get dist-upgrade -y
+fi
+
+os_info="$(
+    . /etc/os-release
+    printf '%s\n' "$ID" "$VERSION_ID"
+)"
+{
+    read -r os_id
+    read -r os_version_id
+} <<<"$os_info"
+
+# Pin PostgreSQL on Ubuntu 22.04, so we can test upgrading it
+if [ "$os_id $os_version_id" = 'ubuntu 22.04' ]; then
+    export POSTGRESQL_VERSION=14
+fi
+
+# Install
+if [ -z "$TEST_CUSTOM_DB" ]; then
+    echo "Testing production install with default database name and user."
+    "$ZULIP_PATH"/scripts/setup/install --self-signed-cert --hostname 127.0.0.1 --email ci@example.com
+else
+    echo "Testing production install with custom database name and user."
+    "$ZULIP_PATH"/scripts/setup/install --self-signed-cert --hostname 127.0.0.1 --email ci@example.com --postgresql-database-user zulipcustomuser --postgresql-database-name zulipcustomdb
+fi
+
+if [ -n "${POSTGRESQL_VERSION:-}" ]; then
+    if [ "$(crudini --get /etc/zulip/zulip.conf postgresql version)" != "$POSTGRESQL_VERSION" ]; then
+        echo "Installer did not install the PostgreSQL $POSTGRESQL_VERSION that we asked for!"
+        exit 1
+    fi
+    if ! su - zulip -c "psql -tc 'select version()'" | grep -q "^ PostgreSQL $POSTGRESQL_VERSION\."; then
+        echo "Installed PostgreSQL is not actually PostgreSQL $POSTGRESQL_VERSION!"
+        exit 1
+    fi
+fi
+
+echo "Production installation complete!"
+exit 0

tools/ci/production-pgroonga

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# This tests installing the pgroonga extension
+set -e
+set -x
+
+crudini --set /etc/zulip/zulip.conf machine pgroonga enabled
+/home/zulip/deployments/current/scripts/zulip-puppet-apply -f
+echo 'USING_PGROONGA = True' >>/etc/zulip/settings.py
+su zulip -c '/home/zulip/deployments/current/manage.py migrate pgroonga'
+
+su zulip -c /home/zulip/deployments/current/scripts/restart-server
+
+echo 'Installation of pgroonga complete!'
+exit 0

tools/ci/production-upgrade

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# Given a Zulip production environment that had been installed with a
+# previous version of Zulip, upgrade it to the commit being tested.
+# This takes as input the tarball generated by production-build.
+set -e
+set -x
+
+# Start the services that would be running on a working Zulip host;
+# since this is a container, these services are not started
+# automatically.
+
+# Start the postgresql service.
+sudo service postgresql start
+
+# Starting the rabbitmq-server
+epmd -daemon
+sudo service rabbitmq-server start
+
+# Start the supervisor
+sudo service supervisor start
+
+# Disable interactive prompts from apt.
+echo 'Dpkg::Options { "--force-confdef" "--force-confold" }' >/etc/apt/apt.conf.d/noninteractive
+export DEBIAN_FRONTEND=noninteractive
+
+# Zulip releases before 2.1.8/3.5/4.4 have a bug in their
+# `upgrade-zulip` scripts, resulting in them exiting with status 0
+# unconditionally. We work around that by running
+# scripts/lib/upgrade-zulip instead.
+UPGRADE_SCRIPT=/home/zulip/deployments/current/scripts/lib/upgrade-zulip
+
+# Execute the upgrade.
+sudo "$UPGRADE_SCRIPT" /tmp/zulip-server-test.tar.gz

tools/ci/production-upgrade-pg

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+# This tests upgrading PostgreSQL
+set -e
+set -x
+
+su zulip -c /home/zulip/deployments/current/scripts/stop-server
+/home/zulip/deployments/current/scripts/setup/upgrade-postgresql
+su zulip -c /home/zulip/deployments/current/scripts/start-server
+
+echo "Upgrade of PostgreSQL complete!"
+exit 0

tools/ci/production-verify

@@ -0,0 +1,178 @@
+#!/usr/bin/env bash
+# This tests a Zulip production environment (installed via
+# production-install) with some Nagios checks and other tools to
+# verify that everything is working properly.
+set -e
+set -x
+
+usage() {
+    cat <<'EOF'
+Usage:
+  production-verify
+  production-verify --test-custom-db
+  production-verify --help
+
+Options:
+  --test-custom-db
+      Use custom database and user names.
+
+EOF
+}
+
+# Shell option parsing.
+args="$(getopt -o '' --long help,test-custom-db -n "$0" -- "$@")"
+eval "set -- $args"
+NAGIOS_USER="zulip"
+while true; do
+    case "$1" in
+        --help)
+            usage
+            exit 0
+            ;;
+
+        --test-custom-db)
+            NAGIOS_USER="zulipcustomuser"
+            shift
+            ;;
+        --)
+            shift
+            break
+            ;;
+    esac
+done
+
+. /etc/os-release
+os_info="$(
+    . /etc/os-release
+    printf '%s\n' "$ID" "$VERSION_ID"
+)"
+{
+    read -r os_id
+    read -r os_version_id
+} <<<"$os_info"
+
+cat >>/etc/zulip/settings.py <<EOF
+# CircleCI override settings above
+AUTHENTICATION_BACKENDS = ( 'zproject.backends.EmailAuthBackend', )
+NOREPLY_EMAIL_ADDRESS = 'noreply@example.com'
+ALLOWED_HOSTS = []
+EOF
+
+check_header() {
+    if ! sed "s|{nginx_version_string}|$nginx_version|g" "$success_header_file" \
+        | diff -ur - /tmp/http-headers-processed; then
+        set +x
+        echo
+        echo "FAILURE: The HTTP headers returned from loading the homepage on the server do not match the contents of tools/ci/success-http-headers.template.txt.  Typically, this means that the server threw a 500 when trying to load the homepage."
+        echo "Displaying the contents of the server's error log:"
+        echo
+        cat /var/log/zulip/errors.log
+        echo
+        echo "Displaying the contents of the main server log:"
+        echo
+        cat /var/log/zulip/server.log
+        exit 1
+    fi
+}
+
+echo
+echo "Now testing that the supervisord jobs are running properly"
+echo
+sleep 15 # Guaranteed to have a working supervisord process get an extra digit
+if supervisorctl status | grep -vq RUNNING || supervisorctl status | sed 's/^.*uptime //' | grep -q 0:00:0; then
+    set +x
+    echo
+    echo "FAILURE: Supervisor output shows daemons are crashing:"
+    echo
+    supervisorctl status
+    echo
+    echo "DEBUG: printing Zulip server's error log:"
+    cat /var/log/zulip/errors.log
+    echo
+    echo "DEBUG: printing Zulip server's workers log:"
+    cat /var/log/zulip/workers.log
+    echo
+    echo "DEBUG: printing Zulip server's tornado log:"
+    cat /var/log/zulip/tornado.log
+    echo
+    echo "DEBUG: printing process_fts_updates log:"
+    cat /var/log/zulip/fts-updates.log
+    exit 1
+fi
+
+# TODO: Ideally this would test actually logging in, but this is a start.
+echo
+echo "Now testing that the newly installed server's settings endpoint loads"
+echo
+
+curl -ILk https://localhost/api/v1/server_settings -o /tmp/http-headers
+grep -vi -e '^content-length:' -e '^date:' -e '^expires:' -e '^set-cookie:' /tmp/http-headers >/tmp/http-headers-processed
+
+nginx_version="$(nginx -v 2>&1)"
+nginx_version="${nginx_version#nginx version: }"
+case "$os_id $os_version_id" in
+    'ubuntu 22.04' | 'ubuntu 24.04')
+        ;;
+    *)
+        nginx_version="${nginx_version% (*)}"
+        ;;
+esac
+success_header_file="/tmp/success-http-headers.template.txt"
+check_header
+
+# Start the RabbitMQ queue worker related section
+echo
+echo "Now confirming all the RabbitMQ queue processors are correctly registered!"
+echo
+# These hacky shell scripts just extract the sorted list of queue processors, running and expected
+supervisorctl status | cut -f1 -dR | cut -f2- -d: | grep events | cut -f1 -d" " | cut -f3- -d_ | cut -f1 -d- | sort -u >/tmp/running_queue_processors.txt
+su zulip -c /home/zulip/deployments/current/scripts/lib/queue_workers.py | sort -u >/tmp/expected_queue_processors.txt
+if ! diff /tmp/expected_queue_processors.txt /tmp/running_queue_processors.txt >/dev/null; then
+    set +x
+    echo "FAILURE: Runnable queue processors declared in zerver/worker/queue_processors.py "
+    echo "do not match those in puppet/zulip/manifests/profile/base.pp"
+    echo "See https://zulip.readthedocs.io/en/latest/subsystems/queuing.html for details."
+    echo
+    diff -ur /tmp/expected_queue_processors.txt /tmp/running_queue_processors.txt
+    exit 1
+fi
+
+echo
+echo "Now running RabbitMQ consumer Nagios tests"
+echo
+# First run the check that usually runs in cron and populates the state files
+/home/zulip/deployments/current/scripts/nagios/check-rabbitmq-consumers
+
+# Then, compute the list of all Django queue workers to run Nagios checks against
+consumer_list=$(/home/zulip/deployments/current/scripts/lib/queue_workers.py --queue-type=consumer)
+for consumer in $consumer_list; do
+    if ! /usr/lib/nagios/plugins/zulip_app_frontend/check_cron_file "/var/lib/nagios_state/check-rabbitmq-consumers-$consumer"; then
+        set +x
+        echo
+        echo "FAILURE: Missing Nagios consumer for $consumer; displaying full consumer output:"
+        set -x
+        rabbitmqctl list_consumers
+        supervisorctl status
+        tail -n +1 /var/log/zulip/events*.log
+        exit 1
+    fi
+done
+
+# Some of the Nagios tests have been temporarily disabled to work
+# around a Travis CI infrastructure issue.
+echo
+echo "Now running additional Nagios tests"
+echo
+if ! /usr/lib/nagios/plugins/zulip_app_frontend/check_queue_worker_errors \
+    || ! su zulip -c "/usr/local/bin/process_fts_updates --nagios-check --nagios-user=$NAGIOS_USER"; then # || \
+    #   ! su zulip -c "/usr/lib/nagios/plugins/zulip_app_frontend/check_send_receive_time --site=https://127.0.0.1/api --nagios --insecure"; then
+    set +x
+    echo
+    echo "FAILURE: Nagios checks don't pass:"
+    echo
+    echo "DEBUG: printing Zulip server's error log:"
+    cat /var/log/zulip/errors.log
+    exit 1
+fi
+echo "Production installation test successful!"
+exit 0

tools/ci/setup-backend

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+set -e
+set -x
+
+ZULIP_PATH="$(dirname "${BASH_SOURCE[0]}")/../.."
+. "$ZULIP_PATH/tools/python-warnings.bash"
+
+# This is just a thin wrapper around provision.
+# Provisioning may fail due to many issues but most of the times a network
+# connection issue is the reason. So we are going to retry entire provisioning
+# once again if that fixes our problem.
+tools/provision "$@" || {
+    ret=$?
+    if [ "$ret" = 1 ]; then
+        echo "\`provision\`: Something went wrong with the provisioning, might be a network issue, Retrying to provision..."
+        tools/provision
+    else
+        echo "\`provision\`: Something REALLY BAD went wrong with the provisioning, not retrying."
+        exit "$ret"
+    fi
+}

tools/ci/setup-frontend

@@ -0,0 +1 @@
+setup-backend

tools/ci/success-http-headers.template.txt

@@ -0,0 +1,15 @@
+HTTP/2 200 
+server: {nginx_version_string}
+content-type: application/json
+vary: Accept-Encoding
+vary: Accept-Language, Cookie
+content-language: en
+strict-transport-security: max-age=31536000; includeSubdomains; preload
+x-frame-options: DENY
+x-content-type-options: nosniff
+referrer-policy: strict-origin-when-cross-origin
+cross-origin-opener-policy: same-origin
+access-control-allow-origin: *
+access-control-allow-headers: Authorization
+access-control-allow-methods: GET, POST, DELETE, PUT, PATCH, HEAD
+

tools/clean-branches

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# usage: clean-branches
+# Deletes any local branches which are ancestors of origin/main,
+# and also any branches in origin which are ancestors of
+# origin/main and are named like $USER-*.
+
+# usage: clean-branches --reviews
+# Deletes all the above mentioned branches as well as branches
+# created by the scripts like `fetch-rebase-pull-request`. Be careful
+# as this would also remove other branches with names like review-*
+
+review=0
+if [ $# -ne 0 ] && [ "$1" == "--reviews" ]; then
+    review=1
+fi
+push_args=()
+
+function is_merged() {
+    ! git rev-list -n 1 origin/main.."$1" | grep -q .
+}
+
+function clean_ref() {
+    ref="$1"
+    case "$ref" in
+        */main | */HEAD)
+            return
+            ;;
+
+        refs/heads/review-*)
+            if [ $review -ne 0 ]; then
+                echo -n "Deleting local branch ${ref#refs/heads/}"
+                echo " (was $(git rev-parse --short "$ref"))"
+                git update-ref -d "$ref"
+            fi
+            ;;
+
+        refs/heads/*)
+            if is_merged "$ref"; then
+                echo -n "Deleting local branch ${ref#refs/heads/}"
+                echo " (was $(git rev-parse --short "$ref"))"
+                git update-ref -d "$ref"
+            fi
+            ;;
+
+        refs/remotes/origin/$USER-*)
+            if is_merged "$ref"; then
+                remote_name="${ref#refs/remotes/origin/}"
+                echo -n "Deleting remote branch $remote_name"
+                echo " (was $(git rev-parse --short "$ref"))"
+                # NB: this won't handle spaces in ref names
+                push_args=("${push_args[@]}" ":$remote_name")
+            fi
+            ;;
+    esac
+}
+
+if [ "$(git symbolic-ref HEAD)" != 'refs/heads/main' ]; then
+    echo "Check out main before you run this script." >&2
+    exit 1
+fi
+
+git fetch --prune origin
+
+eval "$(git for-each-ref --shell --format='clean_ref %(refname);')"
+
+if [ "${#push_args[@]}" -ne 0 ]; then
+    git push origin "${push_args[@]}"
+fi

tools/closed-by-commits

@@ -0,0 +1,438 @@
+#!/usr/bin/env -S uv run --script --frozen --only-group release-tools  # -*-python-*-
+
+import json
+import re
+import time
+import urllib.parse
+from collections import defaultdict
+from dataclasses import dataclass, field
+from functools import cache
+from typing import Annotated, Any
+
+import requests
+import typer
+from github import Auth, Github
+from rich.console import Console
+from rich.progress import Progress
+from typing_extensions import override
+
+
+def encode_hash_component(s: str) -> str:
+    hash_replacements = {
+        "%": ".",
+        "(": ".28",
+        ")": ".29",
+        ".": ".2E",
+    }
+    encoded = urllib.parse.quote(s, safe="*")
+    return "".join(hash_replacements.get(c, c) for c in encoded)
+
+
+@cache
+def search_czo_for_number(prefix: str, number: int) -> frozenset[str]:
+    params = {
+        "anchor": "newest",
+        "num_before": "100",
+        "num_after": "0",
+        "narrow": json.dumps(
+            [
+                {"negated": False, "operator": "search", "operand": f'"#{prefix}{number}"'},
+                {"negated": False, "operator": "channels", "operand": "web-public"},
+            ]
+        ),
+    }
+
+    try:
+        response = requests.get("https://chat.zulip.org/json/messages", params=params, timeout=30)
+        response.raise_for_status()
+
+        data = response.json()
+        messages = data.get("messages", [])
+
+        # Extract unique topic URLs
+        urls = set()
+        for msg in messages:
+            stream_id = msg.get("stream_id")
+            display_recipient = msg.get("display_recipient")
+            subject = msg.get("subject")
+
+            assert stream_id
+            assert display_recipient
+
+            encoded_recipient = encode_hash_component(display_recipient.replace(" ", "-"))
+            encoded_subject = encode_hash_component(subject)
+            url = f"https://chat.zulip.org/#narrow/channel/{stream_id}-{encoded_recipient}/topic/{encoded_subject}"
+            urls.add(url)
+
+        return frozenset(urls)
+
+    except requests.exceptions.RequestException as e:
+        assert e.response
+        if e.response.status_code != 429:
+            raise
+        retry_after = int(e.response.headers["Retry-After"]) + 1
+        time.sleep(retry_after)
+        return search_czo_for_number(prefix, number)
+
+
+@dataclass
+class Issue:
+    number: int
+    title: str
+    czo_urls: set[str] = field(default_factory=set)
+    closed_by_prs: list["PullRequest"] = field(default_factory=list)
+    duplicate_issue_ids: set[int] = field(default_factory=set)
+
+    @override
+    def __hash__(self) -> int:
+        return hash(self.number)
+
+    @override
+    def __eq__(self, other: object) -> bool:
+        return isinstance(other, Issue) and self.number == other.number
+
+
+@dataclass
+class PullRequest:
+    number: int
+    title: str
+    czo_urls: set[str] = field(default_factory=set)
+
+    @override
+    def __hash__(self) -> int:
+        return hash(self.number)
+
+    @override
+    def __eq__(self, other: object) -> bool:
+        return isinstance(other, PullRequest) and self.number == other.number
+
+
+class CommitRangeAnalyzer:
+    COMMIT_PRS_QUERY = """
+    query($oid: GitObjectID!, $repo: String!) {
+      repository(owner: "zulip", name: $repo) {
+        object(oid: $oid) {
+          ... on Commit {
+            messageBody
+            associatedPullRequests(first: 10) {
+              nodes {
+                number
+                title
+                url
+                body
+                comments(first: 100) {
+                  nodes {
+                    body
+                  }
+                }
+                closingIssuesReferences(first: 50) {
+                  nodes {
+                    number
+                    title
+                    url
+                    body
+                    comments(first: 100) {
+                      nodes {
+                        body
+                      }
+                    }
+                    timelineItems(first:100, itemTypes:MARKED_AS_DUPLICATE_EVENT) {
+                      ... on IssueTimelineItemsConnection {
+                        nodes {
+                          ... on MarkedAsDuplicateEvent {
+                            duplicate {
+                              ... on Issue {
+                                number
+                              }
+                            }
+                          }
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+    """
+
+    ISSUE_QUERY = """
+    query($number: Int!, $repo: String!) {
+      repository(owner: "zulip", name: $repo) {
+        issue(number: $number) {
+          number
+          title
+          body
+          comments(first: 100) {
+            nodes {
+              body
+            }
+          }
+          timelineItems(first:100, itemTypes:MARKED_AS_DUPLICATE_EVENT) {
+            ... on IssueTimelineItemsConnection {
+              nodes {
+                ... on MarkedAsDuplicateEvent {
+                  duplicate {
+                    ... on Issue {
+                      number
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+    """
+
+    def __init__(self, token: str, reponame: str, czo_issue_prefix: str) -> None:
+        self.github = Github(auth=Auth.Token(token))
+        self.reponame = reponame
+        self.czo_issue_prefix = czo_issue_prefix
+
+    @staticmethod
+    def _extract_czo_urls(text: str | None) -> set[str]:
+        if not text:
+            return set()
+
+        matches = re.findall(r"https://chat\.zulip\.org/[^\s\)\]\>]+", text)
+
+        urls = set()
+        for url in matches:
+            # We strip off and remove /with/... and /near/... to
+            # reduce the number of unique links which are generated.
+            parsed_url = re.match(r"(.*)/topic/([^/]+)(/(near|with)/.*)?$", url)
+            if not parsed_url:
+                continue
+            urls.add(
+                parsed_url[1]
+                + "/topic/"
+                # Normalize the topic by decoding and re-encoding
+                + encode_hash_component(urllib.parse.unquote(parsed_url[2].replace(".", "%")))
+            )
+
+        return urls
+
+    @staticmethod
+    def _extract_issue_numbers(reponame: str, text: str | None) -> set[int]:
+        if not text:
+            return set()
+
+        # https://docs.github.com/en/issues/tracking-your-work-with-issues/using-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword
+        pattern = rf"\b(?:close[sd]?|fix(?:es|ed)?|resolve[sd]?):?\s+(?:zulip/{reponame})?#(\d+)\b"
+        matches = re.findall(pattern, text, re.IGNORECASE)
+        return {int(num) for num in matches}
+
+    def _issue_from_node(self, issue_node: dict[str, Any], console: Console) -> Issue:
+        issue = Issue(
+            number=issue_node["number"],
+            title=issue_node["title"].strip(),
+        )
+        issue_comments = issue_node.get("comments", {}).get("nodes", [])
+        for node in [issue_node, *issue_comments]:
+            issue.czo_urls.update(self._extract_czo_urls(node.get("body")))
+        issue.czo_urls.update(search_czo_for_number(self.czo_issue_prefix, issue.number))
+
+        issue.duplicate_issue_ids = {
+            event["duplicate"]["number"] for event in issue_node["timelineItems"]["nodes"]
+        }
+        for issue_id in issue.duplicate_issue_ids:
+            if duplicate_info := self._fetch_issue(issue_id, console):
+                issue.czo_urls.update(duplicate_info.czo_urls)
+                issue.duplicate_issue_ids.update(duplicate_info.duplicate_issue_ids)
+        return issue
+
+    def _fetch_issue(self, number: int, console: Console) -> Issue | None:
+        """Fetch issue metadata from GitHub."""
+        try:
+            _, result = self.github.requester.graphql_query(
+                self.ISSUE_QUERY, {"number": number, "repo": self.reponame}
+            )
+
+            if "errors" in result:
+                console.log(
+                    "Failed to fetch zulip/%s#%d: %s", self.reponame, number, result["errors"]
+                )
+                return None
+
+            data = result.get("data", {})
+            issue_node = data.get("repository", {}).get("issue")
+
+            if not issue_node:
+                return None
+
+            return self._issue_from_node(issue_node, console)
+
+        except Exception:
+            return None
+
+    def get_issues_for_commit(
+        self, commit_sha: str, console: Console
+    ) -> tuple[dict[Issue, set[PullRequest]], set[int]]:
+        _, result = self.github.requester.graphql_query(
+            self.COMMIT_PRS_QUERY, {"oid": commit_sha, "repo": self.reponame}
+        )
+
+        if "errors" in result:
+            error_messages = [e.get("message", str(e)) for e in result["errors"]]
+            raise RuntimeError(f"GraphQL errors: {', '.join(error_messages)}")
+
+        data = result.get("data", {})
+
+        if not data.get("repository", {}).get("object"):
+            return dict(), set()
+
+        commit_obj = data["repository"]["object"]
+        commit_message = commit_obj.get("messageBody", "")
+        pr_nodes = commit_obj.get("associatedPullRequests", {}).get("nodes", [])
+
+        results: dict[Issue, set[PullRequest]] = defaultdict(set)
+        all_seen_prs: set[int] = set()
+        for pr_node in pr_nodes:
+            pr = PullRequest(
+                number=pr_node["number"],
+                title=pr_node["title"].strip(),
+            )
+            all_seen_prs.add(pr.number)
+            pr_comments = pr_node.get("comments", {}).get("nodes", [])
+            for node in [pr_node, *pr_comments]:
+                pr.czo_urls.update(self._extract_czo_urls(node.get("body")))
+            pr.czo_urls.update(search_czo_for_number(self.czo_issue_prefix, pr.number))
+
+            # Get issues from PR metadata
+            issues_dict = {}
+            issue_nodes = pr_node.get("closingIssuesReferences", {}).get("nodes", [])
+            for issue_node in issue_nodes:
+                if issue_node is None:
+                    continue
+
+                issue = self._issue_from_node(issue_node, console)
+                issues_dict[issue.number] = issue
+
+            # Extract additional issue numbers from commit message
+            for issue_num in self._extract_issue_numbers(self.reponame, commit_message):
+                if issue_num not in issues_dict:
+                    maybe_issue = self._fetch_issue(issue_num, console)
+                    if maybe_issue is None:
+                        continue
+                    issues_dict[issue_num] = maybe_issue
+
+            for issue in issues_dict.values():
+                results[issue].add(pr)
+
+        if not pr_nodes:
+            for issue_num in self._extract_issue_numbers(self.reponame, commit_message):
+                maybe_issue = self._fetch_issue(issue_num, console)
+                if maybe_issue is None:
+                    continue
+                results[maybe_issue].update()
+
+        return results, all_seen_prs
+
+    def analyze_range(self, base: str, head: str) -> list[Issue]:
+        console = Console(stderr=True, log_path=False)
+
+        repository = self.github.get_repo(f"zulip/{self.reponame}")
+        comparison = repository.compare(base, head)
+        commit_shas = [commit.sha for commit in comparison.commits]
+        console.log(f"Found {len(commit_shas)} commits")
+
+        all_seen_prs = set()
+        issue_to_prs: dict[Issue, set[PullRequest]] = defaultdict(set)
+        with Progress(console=console) as progress:
+            task = progress.add_task("Processing commits...", total=len(commit_shas))
+            for sha in commit_shas:
+                progress.console.log(f"Processing {sha}")
+
+                this_issue_to_prs, this_seen_prs = self.get_issues_for_commit(sha, console)
+                for issue, prs in this_issue_to_prs.items():
+                    issue_to_prs[issue].update(prs)
+                all_seen_prs.update(this_seen_prs)
+                progress.advance(task)
+
+        for issue, prs in issue_to_prs.items():
+            for pr in prs:
+                issue.czo_urls.update(pr.czo_urls)
+
+            issue.closed_by_prs = sorted(prs, key=lambda p: p.number)
+
+        unique_prs = len({pr for prs in issue_to_prs.values() for pr in prs})
+        console.log(
+            f"Found {len(all_seen_prs)} unique PRs, of which {unique_prs} PRs closed {len(issue_to_prs)} issues",
+        )
+
+        return sorted(issue_to_prs.keys(), key=lambda x: x.number)
+
+
+def validate_github_token(value: str) -> str:
+    # https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/about-authentication-to-github#githubs-token-formats
+    if value.startswith("github_"):
+        return value
+    if re.match(r"gh[pousr]_", value):
+        return value
+    raise typer.BadParameter("Github access tokens start with `github_`, or `gh`")
+
+
+from enum import Enum
+
+
+class ZulipRepo(str, Enum):
+    zulip = ("zulip",)
+    flutter = "zulip-flutter"
+
+
+repo_issue_prefixes = {
+    ZulipRepo.zulip: "",
+    ZulipRepo.flutter: "F",
+}
+
+
+def main(
+    base_commit: Annotated[str, typer.Argument(help="Commit-ish, resolved on the server")],
+    head_commit: Annotated[str, typer.Argument(help="Commit-ish, resolved on the server")],
+    token: Annotated[
+        str,
+        typer.Option(
+            metavar="TOKEN",
+            envvar="GITHUB_TOKEN",
+            show_envvar=True,
+            callback=validate_github_token,
+            help=(
+                "Github access token; can be a fine-grained personal access token "
+                "with read-only access to 'Public repositories'.  "
+                "See https://github.com/settings/personal-access-tokens/new"
+            ),
+        ),
+    ],
+    repo: ZulipRepo = ZulipRepo.zulip,
+) -> int:
+    """Find issues which are closed in a commit range."""
+
+    lines = []
+    prefix = repo_issue_prefixes[repo]
+    analyzer = CommitRangeAnalyzer(token, repo.value, prefix)
+    for issue in analyzer.analyze_range(base_commit, head_commit):
+        lines.append(f"#### #{prefix}{issue.number}: {issue.title}")
+        if issue.duplicate_issue_ids:
+            lines.append(
+                " - **Duplicates:** "
+                + ", ".join(f"#{prefix}{number}" for number in issue.duplicate_issue_ids)
+            )
+        lines.extend(
+            f" - **Closed by:** #{prefix}{pr.number}: {pr.title}" for pr in issue.closed_by_prs
+        )
+        lines.extend(f" - {czo_url}" for czo_url in sorted(issue.czo_urls))
+
+        lines.append("")
+
+    print("\n".join(lines))
+
+    return 0
+
+
+if __name__ == "__main__":
+    typer.run(main)

tools/commit-message-lint

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+# Lint all commit messages that are newer than upstream/main if running
+# locally or the commits in the push or PR if in CircleCI.
+
+# The rules can be found in /.gitlint
+
+repository="zulip/zulip"
+
+if [[ "
+$(git remote -v)
+" =~ '
+'([^[:space:]]*)[[:space:]]*(https://github\.com/|ssh://git@github\.com/|git@github\.com:)"$repository"(\.git|/)?\ \(fetch\)'
+' ]]; then
+    remote=${BASH_REMATCH[1]}
+else
+    remote=upstream
+fi
+
+upstream_commits=$(git rev-parse "refs/remotes/$remote/main" --glob="refs/remotes/$remote/*.x")
+mapfile -t upstream_commits <<<"$upstream_commits"
+base=$(git merge-base HEAD "${upstream_commits[@]}")
+
+commits=$(git rev-list --count "$base..HEAD")
+if [ "$commits" -gt 0 ]; then
+    # Only run gitlint with non-empty commit lists, to avoid a printed
+    # warning.
+    gitlint --commits "$base..HEAD"
+fi

tools/commit-msg

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+# This hook runs gitlint on your commit message.
+
+# If your machine contains a provisioned Zulip development environment, the
+# linter will be invoked through `vagrant ssh`.
+
+# Do not invoke gitlint if commit message is empty
+if grep -q '^[^#]' "$1"; then
+    if
+        if [ -z "$VIRTUAL_ENV" ] && command -v vagrant >/dev/null && [ -e .vagrant ]; then
+            ! vagrant ssh -c 'cd ~/zulip && gitlint'
+        else
+            ! gitlint
+        fi <"$1"
+    then
+        echo "WARNING: Your commit message does not match Zulip's style guide."
+    fi
+fi
+
+exit 0

tools/conf.ini-template

@@ -0,0 +1,2 @@
+[github]
+api_token = API_TOKEN

tools/coveragerc

@@ -0,0 +1,39 @@
+[report]
+# Regexes for lines to exclude from consideration
+exclude_also =
+    # Re-enable the standard coverage pragma
+    nocoverage
+    # Allow excluding the entire file
+    \A(?ms:.*^# nocoverage$.*)\Z
+    # Don't complain if non-runnable code isn't run:
+    if False:
+    # Don't require coverage for base class NotImplementedErrors
+    raise NotImplementedError
+    # Don't require coverage for test suite AssertionError -- they're usually for clarity
+    raise AssertionError
+    # Don't require coverage for __str__ statements just used for printing
+    def __(repr|str)__[(]self[)] -> .*:
+    # Don't require coverage for errors about unsupported webhook event types
+    raise UnsupportedWebhookEventTypeError
+    # Don't require coverage for abstract methods; they're never called.
+    @abstractmethod
+    # Skipped unit tests
+    @skip
+
+[run]
+data_file=var/.coverage
+
+# dynamic_context=test_function, combined with using
+# html_report(..., show_contexts=True), means the HTML report can detail
+# which test(s) executed each line with coverage. This has modest
+# overhead but is very useful for finding existing tests for a code path.
+dynamic_context=test_function
+
+source =
+    analytics/
+    confirmation/
+    corporate/
+    pgroonga/
+    zerver/
+    zilencer/
+    zproject/

tools/create-api-changelog

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+import os
+import random
+import subprocess
+import sys
+from pathlib import Path
+
+if __name__ == "__main__":
+    ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    os.chdir(ZULIP_PATH)
+
+    dir_path = Path("api_docs/unmerged.d")
+    os.makedirs(dir_path, exist_ok=True)
+
+    random_hex_value = f"{random.randint(0, 0xFFFFFF):06x}"
+    file_path = f"{dir_path}/ZF-{random_hex_value}.md"
+
+    with open(file_path, "w") as f:
+        f.write("")
+
+    try:
+        subprocess.run(["git", "add", file_path], check=True)
+    except subprocess.CalledProcessError as e:
+        print(e)
+        sys.exit(1)
+
+    print(
+        f"""Created an empty API changelog file.
+If you've made changes to the API, document them here:
+{file_path}
+
+For help, see:
+    https://zulip.readthedocs.io/en/latest/documentation/api.html#step-by-step-guide
+"""
+    )

tools/deploy-branch

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+function error_out() {
+    echo -en '\e[0;31m'
+    echo "$1"
+    echo -en '\e[0m'
+    exit 1
+}
+
+[ -z "$(git status --porcelain --untracked-files=no)" ] || error_out "Working directory or index not clean"
+
+old_ref=$(git rev-list --max-count=1 HEAD)
+branch=${1-}
+[ -z "$branch" ] && error_out "You must specify a branch name to deploy"
+
+branch_ref=$(git rev-list --max-count=1 "$branch") || error_out "Unknown branch: $branch"
+
+if [ "$old_ref" == "$branch_ref" ]; then
+    new_ref=main
+else
+    if ref_name=$(git describe --all --exact "$old_ref"); then
+        new_ref=$(echo "$ref_name" | perl -pe 's{^(heads|remotes)/}{}')
+    else
+        new_ref=$old_ref
+    fi
+fi
+
+git fetch -p
+
+git rebase origin/main "$branch" || error_out "Rebase onto origin/main failed"
+
+git push . HEAD:main
+git push origin main || error_out "Push of main to origin/main failed"
+
+git checkout "$new_ref"
+git branch -D "$branch"
+git push origin ":$branch"

tools/diagnose

@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+import os
+import platform
+import shlex
+import subprocess
+import sys
+from collections.abc import Callable
+
+TOOLS_DIR = os.path.dirname(__file__)
+ROOT_DIR = os.path.dirname(TOOLS_DIR)
+sys.path.insert(0, ROOT_DIR)
+from scripts.lib.zulip_tools import get_dev_uuid_var_path
+
+UUID_VAR_PATH = get_dev_uuid_var_path()
+
+
+def run(check_func: Callable[[], bool]) -> None:
+    """
+    This decorator simply runs functions.  It makes it more
+    convenient to add new checks without a big main() function.
+    """
+    rc = check_func()
+    if not rc:
+        sys.exit(1)
+
+
+def run_command(args: list[str]) -> None:
+    print(shlex.join(args))
+    subprocess.check_call(args)
+
+
+@run
+def check_python_version() -> bool:
+    subprocess.check_call(["/usr/bin/env", "python", "-V"])
+    return True
+
+
+@run
+def pwd() -> bool:
+    print(os.getcwd())
+    return True
+
+
+@run
+def host_info() -> bool:
+    print(platform.platform())
+    return True
+
+
+@run
+def check_django() -> bool:
+    try:
+        import django
+
+        print("Django version:", django.get_version())
+        return True
+    except ImportError:
+        print(
+            f"""
+            ERROR!
+            We cannot import Django, which is usually a
+            symptom of not having your Python venv
+            set up correctly.
+
+            Make sure your shell does this at init time:
+
+            source {ROOT_DIR}/.venv/bin/activate
+
+            Or maybe you forget to run inside your VM?
+            """
+        )
+        return False
+
+
+@run
+def provision_version() -> bool:
+    fn = os.path.join(UUID_VAR_PATH, "provision_version")
+    with open(fn) as f:
+        version = tuple(map(int, f.read().strip().split(".")))
+    print("latest version provisioned:", version)
+    from version import PROVISION_VERSION
+
+    print("desired version:", PROVISION_VERSION)
+    if not (PROVISION_VERSION <= version < (PROVISION_VERSION[0] + 1,)):
+        print("You need to provision!")
+        return False
+    return True
+
+
+@run
+def node_stuff() -> bool:
+    print("node version:")
+    subprocess.check_call(["node", "--version"])
+    return True
+
+
+@run
+def test_models() -> bool:
+    settings_module = "zproject.settings"
+    os.environ["DJANGO_SETTINGS_MODULE"] = settings_module
+    import django
+
+    django.setup()
+    from zerver.models import Realm, UserProfile
+
+    print("Num realms: ", Realm.objects.count())
+    print("Num users: ", UserProfile.objects.count())
+    return True
+
+
+@run
+def check_migrations() -> bool:
+    print()
+    rc = subprocess.check_call("./tools/test-migrations")
+    return rc == 0

tools/documentation.vnufilter

@@ -0,0 +1,9 @@
+# Warnings that are probably less important.
+
+Consider using the “h1” element as a top-level heading only — or else use the “headingoffset” attribute \(otherwise, all “h1” elements are treated as top-level headings by many screen readers and other tools\)\.
+Document uses the Unicode Private Use Area\(s\), which should not be used in publicly exchanged documents\. \(Charmod C073\)
+Section lacks heading\. Consider using “h2”-“h6” elements to add identifying headings to all sections, or else use a “div” element instead for any cases where no heading is needed\.
+
+# Opinionated informational messages.
+
+Trailing slash on void elements has no effect and interacts badly with unquoted attribute values\.

tools/documentation_crawler/documentation_crawler/commands/crawl_with_status.py

@@ -0,0 +1,26 @@
+import argparse
+
+from scrapy.commands import crawl
+from scrapy.crawler import Crawler
+from scrapy.spiders import Spider
+from typing_extensions import override
+
+
+class Command(crawl.Command):
+    @override
+    def run(self, args: list[str], opts: argparse.Namespace) -> None:
+        crawlers = []
+        assert self.crawler_process is not None
+        real_create_crawler = self.crawler_process.create_crawler
+
+        def create_crawler(crawler_or_spidercls: type[Spider] | Crawler | str) -> Crawler:
+            crawler = real_create_crawler(crawler_or_spidercls)
+            crawlers.append(crawler)
+            return crawler
+
+        self.crawler_process.create_crawler = create_crawler  # type: ignore[method-assign]  # monkey patching
+        super().run(args, opts)
+        for crawler in crawlers:
+            assert crawler.stats is not None
+            if crawler.stats.get_value("log_count/ERROR"):
+                self.exitcode = 1

tools/documentation_crawler/documentation_crawler/settings.py

@@ -0,0 +1,98 @@
+# Scrapy settings for documentation_crawler project
+#
+# For simplicity, this file contains only settings considered important or
+# commonly used. You can find more settings consulting the documentation:
+#
+#     https://docs.scrapy.org/en/latest/topics/settings.html
+#     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
+#     https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+
+BOT_NAME = "documentation_crawler"
+
+SPIDER_MODULES = ["documentation_crawler.spiders"]
+NEWSPIDER_MODULE = "documentation_crawler.spiders"
+COMMANDS_MODULE = "documentation_crawler.commands"
+LOG_LEVEL = "WARNING"
+DOWNLOAD_TIMEOUT = 15
+DOWNLOAD_VERIFY_CERTIFICATES = True
+
+# Crawl responsibly by identifying yourself (and your website) on the user-agent
+USER_AGENT = (
+    "Mozilla/5.0 (X11; Linux x86_64) "
+    "AppleWebKit/537.36 (KHTML, like Gecko) "
+    "Chrome/54.0.2840.59 Safari/537.36"
+)
+
+# Obey robots.txt rules
+ROBOTSTXT_OBEY = False
+
+# Configure maximum concurrent requests performed by Scrapy (default: 16)
+# CONCURRENT_REQUESTS = 32
+
+# Configure a delay for requests for the same website (default: 0)
+# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
+# See also autothrottle settings and docs
+# DOWNLOAD_DELAY = 3
+# The download delay setting will honor only one of:
+# CONCURRENT_REQUESTS_PER_DOMAIN = 16
+# CONCURRENT_REQUESTS_PER_IP = 16
+
+# Disable cookies (enabled by default)
+# COOKIES_ENABLED = False
+
+# Disable Telnet Console (enabled by default)
+# TELNETCONSOLE_ENABLED = False
+
+# Override the default request headers:
+# DEFAULT_REQUEST_HEADERS = {
+#   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+#   'Accept-Language': 'en',
+# }
+
+# Enable or disable spider middlewares
+# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+# SPIDER_MIDDLEWARES = {
+#    'documentation_crawler.middlewares.DocumentationCrawlerSpiderMiddleware': 543,
+# }
+
+# Enable or disable downloader middlewares
+# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
+# DOWNLOADER_MIDDLEWARES = {
+#    'documentation_crawler.middlewares.DocumentationCrawlerDownloaderMiddleware': 543,
+# }
+
+# Enable or disable extensions
+# See https://docs.scrapy.org/en/latest/topics/extensions.html
+# EXTENSIONS = {
+#    'scrapy.extensions.telnet.TelnetConsole': None,
+# }
+
+# Configure item pipelines
+# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
+# ITEM_PIPELINES = {
+#    'documentation_crawler.pipelines.DocumentationCrawlerPipeline': 300,
+# }
+
+# Enable and configure the AutoThrottle extension (disabled by default)
+# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
+# AUTOTHROTTLE_ENABLED = True
+# The initial download delay
+# AUTOTHROTTLE_START_DELAY = 5
+# The maximum download delay to be set in case of high latencies
+# AUTOTHROTTLE_MAX_DELAY = 60
+# The average number of requests Scrapy should be sending in parallel to
+# each remote server
+# AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
+# Enable showing throttling stats for every response received:
+# AUTOTHROTTLE_DEBUG = False
+
+# Enable and configure HTTP caching (disabled by default)
+# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
+# HTTPCACHE_ENABLED = True
+# HTTPCACHE_EXPIRATION_SECS = 0
+# HTTPCACHE_DIR = 'httpcache'
+# HTTPCACHE_IGNORE_HTTP_CODES = []
+# HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
+
+# Set settings whose default value is deprecated to a future-proof value
+TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"

tools/documentation_crawler/documentation_crawler/spiders/__init__.py

@@ -0,0 +1,4 @@
+# This package will contain the spiders of your Scrapy project
+#
+# Please refer to the documentation for information on how to create and manage
+# your spiders.

tools/documentation_crawler/documentation_crawler/spiders/check_documentation.py

@@ -0,0 +1,22 @@
+import os
+import pathlib
+
+from .common.spiders import BaseDocumentationSpider
+
+
+def get_start_url() -> list[str]:
+    # Get index.html file as start URL and convert it to file URI
+    dir_path = os.path.dirname(os.path.realpath(__file__))
+    start_file = os.path.join(
+        dir_path, os.path.join(*[os.pardir] * 4), "docs/_build/html/index.html"
+    )
+    return [
+        pathlib.Path(os.path.abspath(start_file)).as_uri(),
+    ]
+
+
+class DocumentationSpider(BaseDocumentationSpider):
+    name = "documentation_crawler"
+    deny_domains = ["localhost:9991"]
+    deny = [r"\_sources\/.*\.txt"]
+    start_urls = get_start_url()

tools/documentation_crawler/documentation_crawler/spiders/check_help_documentation.py

@@ -0,0 +1,57 @@
+from typing_extensions import override
+
+from .common.spiders import BaseDocumentationSpider
+
+
+class HelpDocumentationSpider(BaseDocumentationSpider):
+    name = "help_documentation_crawler"
+    start_urls = ["http://localhost:9981/help"]
+    deny_domains: list[str] = []
+    deny = ["/policies/privacy"]
+
+    @override
+    def _is_external_url(self, url: str) -> bool:
+        return not f"{url}/".startswith("http://localhost:9981/help/") or self._has_extension(url)
+
+
+class APIDocumentationSpider(BaseDocumentationSpider):
+    name = "api_documentation_crawler"
+    start_urls = ["http://localhost:9981/api"]
+    deny_domains: list[str] = []
+
+    @override
+    def _is_external_url(self, url: str) -> bool:
+        return not f"{url}/".startswith("http://localhost:9981/api") or self._has_extension(url)
+
+
+class PorticoDocumentationSpider(BaseDocumentationSpider):
+    @override
+    def _is_external_url(self, url: str) -> bool:
+        return (
+            not url.startswith("http://localhost:9981")
+            or url.startswith(("http://localhost:9981/help", "http://localhost:9981/api"))
+            or self._has_extension(url)
+        )
+
+    name = "portico_documentation_crawler"
+    start_urls = [
+        "http://localhost:9981/hello/",
+        "http://localhost:9981/history/",
+        "http://localhost:9981/plans/",
+        "http://localhost:9981/team/",
+        "http://localhost:9981/apps/",
+        "http://localhost:9981/integrations/",
+        "http://localhost:9981/policies/terms",
+        "http://localhost:9981/policies/privacy",
+        "http://localhost:9981/features/",
+        "http://localhost:9981/why-zulip/",
+        "http://localhost:9981/for/open-source/",
+        "http://localhost:9981/for/business/",
+        "http://localhost:9981/for/communities/",
+        "http://localhost:9981/for/research/",
+        "http://localhost:9981/security/",
+    ]
+    deny_domains: list[str] = []
+    # Exclude /apps/download/* URLs to prevent crawler from following redirects
+    # to desktop-download.zulip.com and attempting to fetch large .dmg/.exe files
+    deny = [r".*/apps/download/.*"]

tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py

@@ -0,0 +1,255 @@
+import json
+import os
+import re
+from collections.abc import AsyncIterator, Callable, Iterator
+from urllib.parse import urlsplit
+
+import scrapy
+from scrapy.http.request import Request
+from scrapy.http.response import Response
+from scrapy.http.response.text import TextResponse
+from scrapy.linkextractors import IGNORED_EXTENSIONS
+from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor
+from scrapy.spidermiddlewares.httperror import HttpError
+from scrapy.utils.url import url_has_any_extension
+from twisted.python.failure import Failure
+from typing_extensions import override
+
+EXCLUDED_DOMAINS = [
+    # Returns 429 rate-limiting errors
+    "github.com",
+    "gist.github.com",
+    # Returns 503 errors
+    "www.amazon.com",
+    "gitlab.com",
+]
+
+EXCLUDED_URLS = [
+    # Google calendar returns 404s on HEAD requests unconditionally
+    "https://calendar.google.com/calendar/embed?src=ktiduof4eoh47lmgcl2qunnc0o@group.calendar.google.com",
+    # Returns 409 errors to HEAD requests frequently
+    "https://medium.freecodecamp.org/",
+    # Returns 404 to HEAD requests unconditionally
+    "https://www.git-tower.com/blog/command-line-cheat-sheet/",
+    "https://marketplace.visualstudio.com/items?itemName=rafaelmaiolla.remote-vscode",
+    "https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-ssh",
+    # Requires authentication
+    "https://www.linkedin.com/company/zulip-project",
+    # Returns 403 errors to HEAD requests
+    "https://giphy.com",
+    "https://giphy.com/apps/giphycapture",
+    "https://www.udemy.com/course/the-complete-react-native-and-redux-course/",
+]
+
+VNU_IGNORE = [
+    # Real errors that should be fixed.
+    r"Attribute “markdown” not allowed on element “div” at this point\.",
+    r"No “p” element in scope but a “p” end tag seen\.",
+    r"The heading “h\d” \(with computed level \d\) follows the heading “h\d” \(with computed level \d\), skipping \d heading levels?\.",
+    r"This document has heading elements but none of them has a computed heading level of 1\.",
+    # Opinionated informational messages.
+    r"Trailing slash on void elements has no effect and interacts badly with unquoted attribute values\.",
+]
+VNU_IGNORE_REGEX = re.compile(r"|".join(VNU_IGNORE))
+
+DEPLOY_ROOT = os.path.abspath(os.path.join(__file__, "../../../../../.."))
+
+ZULIP_SERVER_GITHUB_FILE_PATH_PREFIX = "/zulip/zulip/blob/main"
+ZULIP_SERVER_GITHUB_DIRECTORY_PATH_PREFIX = "/zulip/zulip/tree/main"
+
+
+class BaseDocumentationSpider(scrapy.Spider):
+    # Exclude domain address.
+    deny_domains: list[str] = []
+    start_urls: list[str] = []
+    deny: list[str] = []
+    file_extensions: list[str] = ["." + ext for ext in IGNORED_EXTENSIONS]
+    tags = ("a", "area", "img")
+    attrs = ("href", "src")
+
+    def _has_extension(self, url: str) -> bool:
+        return url_has_any_extension(url, self.file_extensions)
+
+    def _is_external_url(self, url: str) -> bool:
+        return url.startswith("http") or self._has_extension(url)
+
+    def check_existing(self, response: Response) -> None:
+        self.log(response)
+
+    def _is_external_link(self, url: str) -> bool:
+        split_url = urlsplit(url)
+        if split_url.hostname in ("chat.zulip.org", "status.zulip.com"):
+            # Since most chat.zulip.org URLs will be links to specific
+            # logged-in content that the spider cannot verify, or the
+            # homepage, there's no need to check those (which can
+            # cause errors when chat.zulip.org is being updated).
+            #
+            # status.zulip.com is externally hosted and, in a peculiar twist of
+            # cosmic irony, often itself offline.
+            return True
+        if split_url.hostname == "zulip.readthedocs.io" or f".{split_url.hostname}".endswith(
+            (".zulip.com", ".zulip.org")
+        ):
+            # We want CI to check any links to Zulip sites.
+            return False
+        if split_url.scheme == "file" or split_url.hostname == "localhost":
+            # We also want CI to check any links to built documentation.
+            return False
+        if split_url.hostname == "github.com" and f"{split_url.path}/".startswith(
+            (
+                f"{ZULIP_SERVER_GITHUB_FILE_PATH_PREFIX}/",
+                f"{ZULIP_SERVER_GITHUB_DIRECTORY_PATH_PREFIX}/",
+            )
+        ):
+            # We can verify these links directly in the local Git repo without making any requests to GitHub servers.
+            return False
+        if split_url.hostname == "github.com" and split_url.path.startswith("/zulip/"):
+            # We want to check these links but due to rate limiting from GitHub, these checks often
+            # fail in the CI. Thus, we should treat these as external links for now.
+            # TODO: Figure out how to test github.com/zulip links in CI.
+            return True
+        return True
+
+    def check_fragment(self, response: Response) -> None:
+        self.log(response)
+        xpath_template = "//*[@id='{fragment}' or @name='{fragment}']"
+        assert isinstance(response, TextResponse)
+        assert response.request is not None
+        fragment = urlsplit(response.request.url).fragment
+        # Check fragment existing on response page.
+        if not response.selector.xpath(xpath_template.format(fragment=fragment)):
+            self.logger.error(
+                "Fragment #%s is not found on page %s", fragment, response.request.url
+            )
+
+    def _vnu_callback(self, url: str) -> Callable[[Response], None]:
+        def callback(response: Response) -> None:
+            vnu_out = json.loads(response.text)
+            for message in vnu_out["messages"]:
+                if not VNU_IGNORE_REGEX.fullmatch(message["message"]):
+                    self.logger.error(
+                        '"%s":%d.%d-%d.%d: %s: %s',
+                        url,
+                        message.get("firstLine", message["lastLine"]),
+                        message.get("firstColumn", message["lastColumn"]),
+                        message["lastLine"],
+                        message["lastColumn"],
+                        message["type"],
+                        message["message"],
+                    )
+
+        return callback
+
+    def _make_requests(self, url: str) -> Iterator[Request]:
+        # These URLs are for Zulip's web app, which with recent changes
+        # can be accessible without logging into an account.  While we
+        # do crawl documentation served by the web app (e.g. /help/),
+        # we don't want to crawl the web app itself, so we exclude
+        # these.
+        split_url = urlsplit(url)
+        if split_url.netloc == "localhost:9981" and split_url.path in ["", "/"]:
+            return
+
+        # These pages have some invisible to the user anchor links like #all
+        # that are currently invisible, and thus would otherwise fail this test.
+        if url.startswith("http://localhost:9981/communities"):
+            return
+        if url.startswith("http://localhost:9981/plans"):
+            return
+
+        callback: Callable[[Response], Iterator[Request] | None] = self.parse
+        dont_filter = False
+        method = "GET"
+        if self._is_external_url(url):
+            callback = self.check_existing
+            method = "HEAD"
+
+            if split_url.hostname == "github.com" and f"{split_url.path}/".startswith(
+                f"{ZULIP_SERVER_GITHUB_FILE_PATH_PREFIX}/"
+            ):
+                file_path = DEPLOY_ROOT + split_url.path.removeprefix(
+                    ZULIP_SERVER_GITHUB_FILE_PATH_PREFIX
+                )
+                if not os.path.isfile(file_path):
+                    self.logger.error(
+                        "There is no local file associated with the GitHub URL: %s", url
+                    )
+                return
+            elif split_url.hostname == "github.com" and f"{split_url.path}/".startswith(
+                f"{ZULIP_SERVER_GITHUB_DIRECTORY_PATH_PREFIX}/"
+            ):
+                dir_path = DEPLOY_ROOT + split_url.path.removeprefix(
+                    ZULIP_SERVER_GITHUB_DIRECTORY_PATH_PREFIX
+                )
+                if not os.path.isdir(dir_path):
+                    self.logger.error(
+                        "There is no local directory associated with the GitHub URL: %s", url
+                    )
+                return
+        elif split_url.fragment != "":
+            dont_filter = True
+            callback = self.check_fragment
+        if getattr(self, "skip_external", False) and self._is_external_link(url):
+            return
+        if split_url.hostname in EXCLUDED_DOMAINS:
+            return
+        if url in EXCLUDED_URLS:
+            return
+        yield Request(
+            url,
+            method=method,
+            callback=callback,
+            dont_filter=dont_filter,
+            errback=self.error_callback,
+        )
+
+    @override
+    async def start(self) -> AsyncIterator[Request]:
+        for url in self.start_urls:
+            for request in self._make_requests(url):
+                yield request
+
+    @override
+    def parse(self, response: Response) -> Iterator[Request]:
+        self.log(response)
+
+        if getattr(self, "validate_html", False):
+            yield Request(
+                "http://127.0.0.1:9988/?out=json",
+                method="POST",
+                headers={"Content-Type": response.headers["Content-Type"]},
+                body=response.body,
+                callback=self._vnu_callback(response.url),
+                errback=self.error_callback,
+            )
+
+        assert isinstance(response, TextResponse)
+        for link in LxmlLinkExtractor(
+            deny_domains=self.deny_domains,
+            deny_extensions=["doc"],
+            tags=self.tags,
+            attrs=self.attrs,
+            deny=self.deny,
+            canonicalize=False,
+        ).extract_links(response):
+            yield from self._make_requests(link.url)
+
+    def retry_request_with_get(self, request: Request) -> Iterator[Request]:
+        request.method = "GET"
+        request.dont_filter = True
+        yield request
+
+    def error_callback(self, failure: Failure) -> Failure | Iterator[Request] | None:
+        if isinstance(failure.value, HttpError):
+            response = failure.value.response
+            # Hack: The filtering above does not catch this URL,
+            # likely due to a redirect.
+            if urlsplit(response.url).netloc == "idmsa.apple.com":
+                return None
+            assert response.request is not None
+            if response.status == 405 and response.request.method == "HEAD":
+                # Method 'HEAD' not allowed, repeat request with 'GET'
+                return self.retry_request_with_get(response.request)
+            self.logger.error("Please check link: %s", response.request.url)
+
+        return failure

tools/documentation_crawler/scrapy.cfg

@@ -0,0 +1,5 @@
+[settings]
+default = documentation_crawler.settings
+
+[deploy]
+project = documentation_crawler

tools/droplets/README.md

@@ -0,0 +1,175 @@
+# Create a remote Zulip dev server
+
+This guide is for mentors who want to help create remote Zulip dev servers
+for hackathon, GCI, or sprint participants.
+
+The machines (droplets) have been generously provided by
+[DigitalOcean](https://www.digitalocean.com/) to help Zulip contributors
+get up and running as easily as possible. Thank you DigitalOcean!
+
+The `create.py` create uses the DigitalOcean API to quickly create new virtual
+machines (droplets) with the Zulip dev server already configured.
+
+## Step 1: Join Zulip DigitalOcean team
+
+We have created a team on DigitalOcean for Zulip mentors. Ask Rishi or Tim
+to be added. You need access to the team so you can create your DigitalOcean
+API token.
+
+## Step 2: Create your DigitalOcean API token
+
+Once you've been added to the Zulip team,
+[log in](https://cloud.digitalocean.com/droplets) to the DigitalOcean control
+panel and [create your personal API token][do-create-api-token]. **Make sure
+you create your API token under the Zulip team.** (It should look something
+like [this][image-zulip-team]).
+
+Copy the API token and store it somewhere safe. You'll need it in the next
+step.
+
+## Step 3: Configure create.py
+
+In `tools/droplets/` there is a sample configuration file `conf.ini-template`.
+
+Copy this file to `conf.ini`:
+
+```
+$ cd tools/droplets/
+$ cp conf.ini-template conf.ini
+```
+
+Now edit the file and replace `APITOKEN` with the personal API token you
+generated earlier.
+
+```
+[digitalocean]
+api_token = APITOKEN
+```
+
+Now you're ready to use the script.
+
+## Usage
+
+`create.py` takes two arguments
+
+- GitHub username
+- Tags (Optional argument)
+
+```
+$ python3 create.py <username>
+$ python3 create.py <username> --tags <tag>
+$ python3 create.py <username> --tags <tag1> <tag2> <tag3>
+```
+
+Assigning tags to droplets like `GCI` can be later useful for
+listing all the droplets created during GCI.
+[Tags](https://www.digitalocean.com/community/tutorials/how-to-tag-digitalocean-droplets)
+may contain letters, numbers, colons, dashes, and underscores.
+
+You'll need to run this from the Zulip development environment (e.g., in
+Vagrant).
+
+The script will also stop if a droplet has already been created for the
+user. If you want to recreate a droplet for a user you can pass the
+`--recreate` flag.
+
+```
+$ python3 create.py <username> --recreate
+```
+
+This will destroy the old droplet and create a new droplet for
+the user.
+
+In order for the script to work, the GitHub user must have:
+
+- forked the [zulip/zulip][zulip-zulip] repository, and
+- created an SSH key pair and added it to their GitHub account.
+
+(Share [this link][how-to-request] with students if they need to do these
+steps.)
+
+The script will stop if it can't find the user's fork or SSH keys.
+
+Once the droplet is created, you will see something similar to this message:
+
+```
+Your remote Zulip dev server has been created!
+
+- Connect to your server by running
+  `ssh zulipdev@<username>.zulipdev.org` on the command line
+  (Terminal for macOS and Linux, Bash for Git on Windows).
+- There is no password; your account is configured to use your SSH keys.
+- Once you log in, you should see `(zulip-server) ~$`.
+- To start the dev server, `cd zulip` and then run `./tools/run-dev`.
+- While the dev server is running, you can see the Zulip server in your browser
+  at http://<username>.zulipdev.org:9991.
+
+See [Developing
+remotely](https://zulip.readthedocs.io/en/latest/development/remote.html) for tips on
+using the remote dev instance and [Git & GitHub
+Guide](https://zulip.readthedocs.io/en/latest/git/index.html) to learn how to
+use Git with Zulip.
+```
+
+Copy and paste this message to the user via Zulip chat. Be sure to CC the user
+so they are notified.
+
+[do-create-api-token]: https://www.digitalocean.com/community/tutorials/how-to-use-the-digitalocean-api-v2#how-to-generate-a-personal-access-token
+[image-zulip-team]: http://cdn.subfictional.com/dropshare/Screen-Shot-2016-11-28-10-53-24-X86JYrrOzu.png
+[zulip-zulip]: https://github.com/zulip/zulip
+[python-digitalocean]: https://github.com/koalalorenzo/python-digitalocean
+[how-to-request]: https://zulip.readthedocs.io/en/latest/development/request-remote.html
+
+## Updating the base image
+
+1. Switch to the Zulip organization.
+1. Create a new droplet, with:
+   - "Regular with SSD" / "2GB RAM / 1 CPU"
+   - Select your SSH key; this will not be built into the image, and
+     is only for access to debug if the build does not succeed.
+   - Check "Monitoring", "IPv6", and "User data"
+   - Paste the contents of `tools/droplets/new-droplet-image` into the
+     text box which says `Enter user data here...`
+   - Name it (e.g., `base-ubuntu-22-04.zulipdev.org`)
+1. Wait for the host to boot.
+1. Wait for the host to complete provisioning and shut down; this will take
+   about 15 minutes.
+1. Go to the Snapshots tab on the image, and "Take a Snapshot".
+1. Wait for several minutes for it to complete.
+1. "Add to region" the snapshot into `NYC3`, `SFO3`, `BLR1`, and `FRA1`.
+1. `curl -u <API_KEY>: https://api.digitalocean.com/v2/snapshots | jq .`
+1. Replace `template_id` in `create.py` in this directory with the
+   appropriate `id`.
+1. Clean up by destroying the droplet (but _leaving_ all "associated
+   resources"), and removing the DNS entry for `base.zulipdev.org`
+1. Open a PR with the updated `template_id`.
+
+## Remotely debugging a droplet
+
+To SSH into a droplet, first make sure you have a SSH key associated with your
+GitHub account, then ask the student to run the following in their
+VM:
+
+```
+$ python3 ~/zulip/tools/droplets/add_mentor.py <your username>
+```
+
+You should now be able to connect to it using:
+
+```
+$ ssh zulipdev@<their username>.zulipdev.org
+```
+
+They can remove your SSH keys by running:
+
+```
+$ python3 ~/zulip/tools/droplets/add_mentor.py <your username> --remove
+```
+
+# Creating a production droplet
+
+`create.py` can also create a production droplet quickly for testing purposes.
+
+```
+$ python3 create.py <username> --production
+```

tools/droplets/add_mentor.py

@@ -0,0 +1,73 @@
+# Allows a mentor to ssh into a DigitalOcean droplet. This is designed to be
+# executed on the target machine.
+#
+# This script takes the username of the mentor as an argument:
+#
+# $ python3 add_mentor.py <mentor's username>
+#
+# Alternatively you can pass in --remove to remove their SSH key from the
+# machine:
+#
+# $ python3 add_mentor.py --remove <mentor's username>
+import os
+import re
+import socket
+import sys
+from argparse import ArgumentParser
+
+import requests
+
+parser = ArgumentParser(description="Give a mentor ssh access to this machine.")
+parser.add_argument("username", help="GitHub username of the mentor.")
+parser.add_argument("--remove", help="Remove his/her key from the machine.", action="store_true")
+
+# Wrap keys with line comments for easier key removal.
+append_key = """\
+#<{username}>{{{{
+{key}
+#}}}}<{username}>
+"""
+
+
+def get_mentor_keys(username: str) -> list[str]:
+    url = f"https://api.github.com/users/{username}/keys"
+
+    r = requests.get(url)
+    if r.status_code != 200:
+        print("Cannot connect to GitHub...")
+        sys.exit(1)
+
+    keys = r.json()
+    if not keys:
+        print(f'Mentor "{username}" has no public key.')
+        sys.exit(1)
+
+    return [key["key"] for key in keys]
+
+
+if __name__ == "__main__":
+    args = parser.parse_args()
+    authorized_keys = os.path.expanduser("~/.ssh/authorized_keys")
+
+    if args.remove:
+        remove_re = re.compile(
+            rf"#<{args.username}>{{{{.+}}}}<{args.username}>(\n)?", re.DOTALL | re.MULTILINE
+        )
+
+        with open(authorized_keys, "r+") as f:
+            old_content = f.read()
+            new_content = re.sub(remove_re, "", old_content)
+            f.seek(0)
+            f.write(new_content)
+            f.truncate()
+
+        print(f"Successfully removed {args.username}' SSH key!")
+
+    else:
+        keys = get_mentor_keys(args.username)
+        with open(authorized_keys, "a") as f:
+            f.writelines(append_key.format(username=args.username, key=key) for key in keys)
+
+        print(f"Successfully added {args.username}'s SSH key!")
+        print("Can you let your mentor know that they can connect to this machine with:\n")
+        print(f"    $ ssh zulipdev@{socket.gethostname()}\n")

tools/droplets/cleanup.py

@@ -0,0 +1,49 @@
+import argparse
+import configparser
+import os
+
+import digitalocean
+
+
+def get_config() -> configparser.ConfigParser:
+    config = configparser.ConfigParser()
+    config.read(os.path.join(os.path.dirname(os.path.abspath(__file__)), "conf.ini"))
+    return config
+
+
+parser = argparse.ArgumentParser(description="Clean up old A / AAAA records in zulipdev.org")
+parser.add_argument("--force", action="store_true")
+
+if __name__ == "__main__":
+    args = parser.parse_args()
+
+    config = get_config()
+    api_token = config["digitalocean"]["api_token"]
+
+    seen_ips = set()
+    if not args.force:
+        print("WOULD DELETE:")
+
+    manager = digitalocean.Manager(token=api_token)
+    my_droplets = manager.get_all_droplets()
+    for droplet in my_droplets:
+        seen_ips.add(droplet.ip_address)
+        if droplet.ipv6:
+            seen_ips.update(net["ip_address"] for net in droplet.networks["v6"])
+
+    domain = digitalocean.Domain(token=api_token, name="zulipdev.org")
+    domain.load()
+    records = domain.get_records()
+
+    for record in sorted(records, key=lambda e: ".".join(reversed(e.name.split(".")))):
+        if record.type not in ("AAAA", "A"):
+            continue
+        elif record.data in seen_ips:
+            continue
+        else:
+            print(f"{record.type} {record.name} = {record.data}")
+            if args.force:
+                record.destroy()
+
+    if not args.force:
+        print("Re-run with --force to delete")

tools/droplets/conf.ini-template

@@ -0,0 +1,2 @@
+[digitalocean]
+api_token = APITOKEN

tools/droplets/create.py

@@ -0,0 +1,365 @@
+# Creates a Zulip remote development environment droplet or
+# a production droplet in DigitalOcean.
+#
+# Particularly useful for sprints/hackathons, interns, and other
+# situation where one wants to quickly onboard new contributors.
+#
+# Requires python-digitalocean library:
+# https://github.com/koalalorenzo/python-digitalocean
+#
+# Also requires DigitalOcean team membership for Zulip and API token:
+# https://cloud.digitalocean.com/settings/api/tokens
+#
+# Copy conf.ini-template to conf.ini and populate with your API token.
+import argparse
+import configparser
+import json
+import os
+import secrets
+import sys
+import time
+import urllib.error
+import urllib.request
+
+import digitalocean
+import requests
+
+parser = argparse.ArgumentParser(description="Create a Zulip development VM DigitalOcean droplet.")
+parser.add_argument(
+    "username", help="GitHub username for whom you want to create a Zulip dev droplet"
+)
+parser.add_argument("--tags", nargs="+", default=[])
+parser.add_argument("-f", "--recreate", action="store_true")
+parser.add_argument("-s", "--subdomain")
+parser.add_argument("-p", "--production", action="store_true")
+parser.add_argument("-r", "--region", choices=("nyc3", "sfo3", "blr1", "fra1"), default="nyc3")
+
+
+def get_config() -> configparser.ConfigParser:
+    config = configparser.ConfigParser()
+    config.read(os.path.join(os.path.dirname(os.path.abspath(__file__)), "conf.ini"))
+    return config
+
+
+def assert_github_user_exists(github_username: str) -> bool:
+    print(f"Checking to see if GitHub user {github_username} exists...")
+    user_api_url = f"https://api.github.com/users/{github_username}"
+    try:
+        response = urllib.request.urlopen(user_api_url)
+        json.load(response)
+        print("...user exists!")
+        return True
+    except urllib.error.HTTPError as err:
+        print(err)
+        print(f"Does the GitHub user {github_username} exist?")
+        sys.exit(1)
+
+
+def get_ssh_public_keys_from_github(github_username: str) -> list[str]:
+    print("Checking to see that GitHub user has available public keys...")
+    apiurl_keys = f"https://api.github.com/users/{github_username}/keys"
+    try:
+        response = urllib.request.urlopen(apiurl_keys)
+        userkeys = json.load(response)
+        if not userkeys:
+            print(
+                f"No keys found. Has user {github_username} added SSH keys to their GitHub account?"
+            )
+            sys.exit(1)
+        print("...public keys found!")
+        return [k["key"] for k in userkeys]
+    except urllib.error.HTTPError as err:
+        print(err)
+        print(f"Has user {github_username} added SSH keys to their GitHub account?")
+        sys.exit(1)
+
+
+def assert_user_forked_zulip_server_repo(username: str) -> bool:
+    print("Checking to see GitHub user has forked zulip/zulip...")
+    apiurl_fork = f"https://api.github.com/repos/{username}/zulip"
+    try:
+        response = urllib.request.urlopen(apiurl_fork)
+        json.load(response)
+        print("...fork found!")
+        return True
+    except urllib.error.HTTPError as err:
+        print(err)
+        print(f"Has user {username} forked zulip/zulip?")
+        sys.exit(1)
+
+
+def assert_droplet_does_not_exist(my_token: str, droplet_name: str, recreate: bool) -> None:
+    print(f"Checking to see if droplet {droplet_name} already exists...")
+    manager = digitalocean.Manager(token=my_token)
+    my_droplets = manager.get_all_droplets()
+    for droplet in my_droplets:
+        if droplet.name.lower() == droplet_name:
+            if not recreate:
+                print(
+                    f"Droplet {droplet_name} already exists. Pass --recreate if you "
+                    "need to recreate the droplet."
+                )
+                sys.exit(1)
+            else:
+                print(f"Deleting existing droplet {droplet_name}.")
+                droplet.destroy()
+                return
+    print("...No droplet found...proceeding.")
+
+
+def generate_dev_droplet_user_data(username: str, subdomain: str, public_keys: list[str]) -> str:
+    ssh_keys_string = "\n".join(public_keys)
+    setup_root_ssh_keys = f"printf '{ssh_keys_string}' > /root/.ssh/authorized_keys"
+    setup_zulipdev_ssh_keys = f"printf '{ssh_keys_string}' > /home/zulipdev/.ssh/authorized_keys"
+
+    # We pass the hostname as username.zulipdev.org to the DigitalOcean API.
+    # But some droplets (eg on 18.04) are created with hostname set to just username.
+    # So we fix the hostname using cloud-init.
+    hostname_setup = f"hostnamectl set-hostname {subdomain}.zulipdev.org"
+
+    setup_repo = (
+        "cd /home/zulipdev/{1} && "
+        "git remote add origin https://github.com/{0}/{1}.git && "
+        "git fetch origin && "
+        "git clean -f"
+    )
+
+    server_repo_setup = setup_repo.format(username, "zulip")
+    python_api_repo_setup = setup_repo.format(username, "python-zulip-api")
+
+    erlang_cookie = secrets.token_hex(16)
+    setup_erlang_cookie = (
+        f"echo '{erlang_cookie}' > /var/lib/rabbitmq/.erlang.cookie && "
+        "chown rabbitmq:rabbitmq /var/lib/rabbitmq/.erlang.cookie && "
+        "service rabbitmq-server restart"
+    )
+
+    cloudconf = f"""\
+#!/bin/bash
+
+{setup_zulipdev_ssh_keys}
+{setup_root_ssh_keys}
+{setup_erlang_cookie}
+sed -i "s/PasswordAuthentication yes/PasswordAuthentication no/g" /etc/ssh/sshd_config
+service ssh restart
+{hostname_setup}
+su -c '{server_repo_setup}' zulipdev
+su -c '{python_api_repo_setup}' zulipdev
+su -c 'git config --global core.editor nano' zulipdev
+su -c 'git config --global pull.rebase true' zulipdev
+"""
+    print("...returning cloud-config data.")
+    return cloudconf
+
+
+def generate_prod_droplet_user_data(public_keys: list[str]) -> str:
+    ssh_keys_string = "\n".join(public_keys)
+    setup_root_ssh_keys = f"printf '{ssh_keys_string}' > /root/.ssh/authorized_keys"
+
+    cloudconf = f"""\
+#!/bin/bash
+
+{setup_root_ssh_keys}
+passwd -d root
+sed -i "s/PasswordAuthentication yes/PasswordAuthentication no/g" /etc/ssh/sshd_config
+service ssh restart
+"""
+    print("...returning cloud-config data.")
+    return cloudconf
+
+
+def create_droplet(
+    my_token: str,
+    template_id: str,
+    name: str,
+    tags: list[str],
+    user_data: str,
+    region: str = "nyc3",
+) -> tuple[str, str]:
+    droplet = digitalocean.Droplet(
+        token=my_token,
+        name=name,
+        region=region,
+        image=template_id,
+        size_slug="s-2vcpu-4gb",
+        user_data=user_data,
+        tags=tags,
+        backups=False,
+        ipv6=True,
+    )
+
+    print("Initiating droplet creation...")
+    droplet.create()
+
+    incomplete = True
+    while incomplete:
+        actions = droplet.get_actions()
+        for action in actions:
+            action.load()
+            print(f"...[{action.type}]: {action.status}")
+            if action.type == "create" and action.status == "completed":
+                incomplete = False
+                break
+        if incomplete:
+            time.sleep(15)
+    print("...droplet created!")
+    droplet.load()
+    print(f"...ip address for new droplet is: {droplet.ip_address}.")
+    return (droplet.ip_address, droplet.ip_v6_address)
+
+
+def delete_existing_records(records: list[digitalocean.Record], record_name: str) -> None:
+    count = 0
+    for record in records:
+        if (
+            record.name == record_name
+            and record.domain == "zulipdev.org"
+            and record.type in ("AAAA", "A")
+        ):
+            record.destroy()
+            count += 1
+    if count:
+        print(f"Deleted {count} existing A / AAAA records for {record_name}.zulipdev.org.")
+
+
+def create_dns_record(my_token: str, record_name: str, ipv4: str, ipv6: str) -> None:
+    domain = digitalocean.Domain(token=my_token, name="zulipdev.org")
+    domain.load()
+    records = domain.get_records()
+
+    delete_existing_records(records, record_name)
+    wildcard_name = "*." + record_name
+    delete_existing_records(records, wildcard_name)
+
+    print(f"Creating new A record for {record_name}.zulipdev.org that points to {ipv4}.")
+    domain.create_new_domain_record(type="A", name=record_name, data=ipv4)
+    print(f"Creating new A record for *.{record_name}.zulipdev.org that points to {ipv4}.")
+    domain.create_new_domain_record(type="A", name=wildcard_name, data=ipv4)
+
+    print(f"Creating new AAAA record for {record_name}.zulipdev.org that points to {ipv6}.")
+    domain.create_new_domain_record(type="AAAA", name=record_name, data=ipv6)
+    print(f"Creating new AAAA record for *.{record_name}.zulipdev.org that points to {ipv6}.")
+    domain.create_new_domain_record(type="AAAA", name=wildcard_name, data=ipv6)
+
+
+def print_dev_droplet_instructions(username: str, droplet_domain_name: str) -> None:
+    print(
+        f"""
+COMPLETE! Droplet for GitHub user {username} is available at {droplet_domain_name}.
+
+Instructions for use are below. (copy and paste to the user)
+
+------
+Your remote Zulip dev server has been created!
+
+- Connect to your server by running
+  `ssh zulipdev@{droplet_domain_name}` on the command line
+  (Terminal for macOS and Linux, Bash for Git on Windows).
+- There is no password; your account is configured to use your SSH keys.
+- Once you log in, you should see `(zulip-server) ~$`.
+- To start the dev server, `cd zulip` and then run `./tools/run-dev`.
+- While the dev server is running, you can see the Zulip server in your browser at
+  http://{droplet_domain_name}:9991.
+"""
+    )
+
+    print(
+        "See [Developing remotely](https://zulip.readthedocs.io/en/latest/development/remote.html) "
+        "for tips on using the remote dev instance and "
+        "[Git & GitHub guide](https://zulip.readthedocs.io/en/latest/git/index.html) "
+        "to learn how to use Git with Zulip.\n"
+    )
+    print(
+        "Note that this droplet will automatically be deleted after a month of inactivity. "
+        "If you are leaving Zulip for more than a few weeks, we recommend pushing all of your "
+        "active branches to GitHub."
+    )
+    print("------")
+
+
+def print_production_droplet_instructions(droplet_domain_name: str) -> None:
+    print(
+        f"""
+-----
+
+Production droplet created successfully!
+
+Connect to the server by running
+
+ssh root@{droplet_domain_name}
+
+-----
+"""
+    )
+
+
+def get_zulip_oneclick_app_slug(api_token: str) -> str:
+    response = requests.get(
+        "https://api.digitalocean.com/v2/1-clicks", headers={"Authorization": f"Bearer {api_token}"}
+    ).json()
+    one_clicks = response["1_clicks"]
+
+    for one_click in one_clicks:
+        if one_click["slug"].startswith("kandralabs"):
+            return one_click["slug"]
+    raise Exception("Unable to find Zulip One-click app slug")
+
+
+if __name__ == "__main__":
+    args = parser.parse_args()
+    username = args.username.lower()
+    if args.subdomain:
+        subdomain = args.subdomain.lower()
+    elif args.production:
+        subdomain = f"{username}-prod"
+    else:
+        subdomain = username
+
+    if args.production:
+        print(f"Creating production droplet for GitHub user {username}...")
+    else:
+        print(f"Creating Zulip developer environment for GitHub user {username}...")
+
+    config = get_config()
+    api_token = config["digitalocean"]["api_token"]
+
+    assert_github_user_exists(github_username=username)
+
+    public_keys = get_ssh_public_keys_from_github(github_username=username)
+    droplet_domain_name = f"{subdomain}.zulipdev.org"
+
+    if args.production:
+        template_id = get_zulip_oneclick_app_slug(api_token)
+        user_data = generate_prod_droplet_user_data(public_keys=public_keys)
+
+    else:
+        assert_user_forked_zulip_server_repo(username=username)
+        user_data = generate_dev_droplet_user_data(
+            username=username, subdomain=subdomain, public_keys=public_keys
+        )
+
+        # define id of image to create new droplets from; see:
+        #     curl -u <API_KEY>: "https://api.digitalocean.com/v2/snapshots | jq .
+        template_id = "157280701"
+
+    assert_droplet_does_not_exist(
+        my_token=api_token, droplet_name=droplet_domain_name, recreate=args.recreate
+    )
+
+    (ipv4, ipv6) = create_droplet(
+        my_token=api_token,
+        template_id=template_id,
+        name=droplet_domain_name,
+        tags=[*args.tags, "dev"],
+        user_data=user_data,
+        region=args.region,
+    )
+
+    create_dns_record(my_token=api_token, record_name=subdomain, ipv4=ipv4, ipv6=ipv6)
+
+    if args.production:
+        print_production_droplet_instructions(droplet_domain_name=droplet_domain_name)
+    else:
+        print_dev_droplet_instructions(username=username, droplet_domain_name=droplet_domain_name)
+
+    sys.exit(1)

tools/droplets/new-droplet-image

@@ -0,0 +1,72 @@
+#!/bin/bash
+
+set -eux
+
+if [[ $(id -u) -eq 0 ]]; then
+    # First pass through -- make the zulip user
+
+    # Add the zulipdev user, let it sudo
+    useradd -U -G sudo -m zulipdev -s /bin/bash
+    echo "zulipdev ALL=(ALL) NOPASSWD:ALL" >>/etc/sudoers.d/90-cloud-init-users
+
+    # Re-run as zulip for the remainder, which is below. We disable
+    # the shellcheck because we _do_ want to read $0 as root, as it's
+    # potentially not readable as zulipdev.
+    # shellcheck disable=SC2024
+    sudo -u zulipdev bash <"$0"
+
+    # Clear out the authorized_keys; this is filled in when the image
+    # is cloned.
+    true >~/.ssh/authorized_keys
+
+    # Clear history and reboot
+    true >~/.bash_history && history -c && rm "$0" && shutdown -h now
+    exit 0
+fi
+
+# This section is run as zulipdev
+cd
+
+# Set up an empty authorized_keys with the right permissions; this is
+# filled in when the image is cloned.
+mkdir -p .ssh
+chmod 700 .ssh
+true >.ssh/authorized_keys
+chmod 600 .ssh/authorized_keys
+
+(
+    # Set up Zulip
+    git clone https://github.com/zulip/zulip
+    cd zulip
+    git remote rename origin upstream
+
+    # Provision
+    ./tools/provision
+
+    # Make sure the nodename in RabbitMQ is not host-dependent
+    sudo perl -pi -e 's/#?NODENAME=.*/NODENAME=zulip\@localhost/' /etc/rabbitmq/rabbitmq-env.conf
+    sudo service rabbitmq-server stop
+    sudo rm -rf /var/lib/rabbitmq/mnesia/
+    sudo service rabbitmq-server start
+
+    # Re-provision for the new rabbitmq nodename
+    ./tools/provision
+
+    # Make sure it's clean
+    git clean -f
+)
+
+(
+    # Set up python-zulip-api
+    git clone https://github.com/zulip/python-zulip-api
+    cd python-zulip-api
+    git remote rename origin upstream
+)
+
+# rabbitmq-server's /var/lib/rabbitmq/.erlang.cookie is a secret, and
+# should not be included in the base image
+sudo service rabbitmq-server stop
+sudo rm /var/lib/rabbitmq/.erlang.cookie
+
+# Clear our history
+true >~/.bash_history && history -c && exit

tools/droplets/zulipdev

@@ -0,0 +1,27 @@
+server {
+    listen 80;
+    listen [::]:80;
+    return 301 https://$host$request_uri;
+}
+
+server {
+    listen 443 ssl http2;
+    listen [::]:443 ssl http2;
+
+    ssl_certificate /etc/ssl/certs/zulip.combined-chain.crt;
+    ssl_certificate_key /etc/ssl/private/zulip.key;
+
+    location / {
+        proxy_set_header Connection "";
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+
+        proxy_pass          http://127.0.0.1:9991;
+        proxy_read_timeout  1200;
+
+        proxy_redirect off;
+        proxy_next_upstream off;
+    }
+}

tools/duplicate_commits.json

@@ -0,0 +1,68 @@
+{
+    "TomaszKolek": 2,
+    "zbenjamin": 91,
+    "robot-dreams": 5,
+    "lfaraone": 22,
+    "jhurwitz": 4,
+    "roed314": 2,
+    "florean": 2,
+    "sharmaeklavya2": 16,
+    "acrefoot": 4,
+    "sampritipanda": 2,
+    "rishig": 3,
+    "tukruic": 5,
+    "Bickio": 1,
+    "rwbarton": 4,
+    "Juanvulcano": 1,
+    "dariaphoebe": 1,
+    "lonerz": 3,
+    "HydraulicSheep": 3,
+    "starrvinc": 1,
+    "jajodiaraghav": 1,
+    "abhayKashyap03": 1,
+    "ImperialLlama": 1,
+    "540KJ": 6,
+    "showell": 13,
+    "umkhan": 2,
+    "anirudhjain75": 1,
+    "theairportexplorer": 3,
+    "lfranchi": 17,
+    "abhijeetkaur": 33,
+    "andersk": 3,
+    "wdaher": 16,
+    "andrewallen00": 2,
+    "jordangedney": 2,
+    "adnrs96": 20,
+    "ashishk1994": 1,
+    "maydhak": 1,
+    "enhdless": 1,
+    "neiljp": 14,
+    "reyha": 2,
+    "tommyip": 9,
+    "vabs22": 5,
+    "eeshangarg": 6,
+    "taranjeet": 1,
+    "roryk": 2,
+    "bungeye": 1,
+    "armooo": 4,
+    "fazerlicourice7": 5,
+    "jonjonsonjr": 1,
+    "joyhchen": 2,
+    "paxapy": 1,
+    "royabouhamad": 2,
+    "shane-kearns": 4,
+    "zacps": 1,
+    "AZtheAsian": 6,
+    "ParthMahawar": 1,
+    "timabbott": 500,
+    "YagoGG": 5,
+    "sahildua2305": 1,
+    "bmorg": 1,
+    "sinwar": 2,
+    "soudy": 1,
+    "aero31aero": 8,
+    "umairwaheed": 1,
+    "roberthoenig": 39,
+    "refeed": 9,
+    "dwrpayne": 1
+}

tools/fetch-contributor-data

@@ -0,0 +1,186 @@
+#!/usr/bin/env python3
+"""
+Fetch contributors data from GitHub using their API, convert it to structured
+JSON data for the /team/ page contributors section.
+"""
+
+import argparse
+import json
+import logging
+import os
+import sys
+import unicodedata
+from datetime import datetime, timezone
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+from scripts.lib.setup_path import setup_path
+
+setup_path()
+os.environ["DJANGO_SETTINGS_MODULE"] = "zproject.settings"
+
+from typing import TypedDict
+
+import django
+from django.conf import settings
+from urllib3.util import Retry
+
+django.setup()
+
+from zerver.lib.avatar_hash import gravatar_hash
+from zerver.lib.github import GithubSession
+from zproject.config import get_secret
+
+duplicate_commits_file = os.path.join(os.path.dirname(__file__), "duplicate_commits.json")
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--max-retries", type=int, default=10, help="Number of times to retry fetching data from GitHub"
+)
+args = parser.parse_args()
+
+
+class ContributorsJSON(TypedDict):
+    date: str
+    contributors: list[dict[str, int | str]]
+
+
+class Contributor(TypedDict):
+    avatar_url: str | None
+    contributions: int
+    login: str | None
+    email: str | None
+    name: str | None
+
+
+logger = logging.getLogger("zulip.fetch_contributors_json")
+
+
+def fetch_contributors(repo_name: str, max_retries: int) -> list[Contributor]:
+    contributors: list[Contributor] = []
+    page_index = 1
+
+    api_link = f"https://api.github.com/repos/zulip/{repo_name}/contributors"
+    api_data = {"anon": "1"}
+
+    headers: dict[str, str] = {}
+    personal_access_token = get_secret("github_personal_access_token")
+    if personal_access_token is not None:
+        headers = {"Authorization": f"token {personal_access_token}"}
+
+    Retry.DEFAULT_BACKOFF_MAX = 64
+    retry = Retry(
+        total=max_retries,
+        backoff_factor=2.0,
+        status_forcelist={
+            403,  # Github does unauth rate-limiting via 403's
+            429,  # The formal rate-limiting response code
+            502,  # Bad gateway
+            503,  # Service unavailable
+        },
+    )
+    session = GithubSession(max_retries=retry)
+    while True:
+        response = session.get(
+            api_link,
+            params={**api_data, "page": f"{page_index}"},
+            headers=headers,
+        )
+        response.raise_for_status()
+        data = response.json()
+        if len(data) == 0:
+            return contributors
+        contributors.extend(data)
+        page_index += 1
+
+
+def write_to_disk(json_data: ContributorsJSON, out_file: str) -> None:
+    with open(out_file, "w") as f:
+        json.dump(json_data, f, indent=2, sort_keys=True)
+        f.write("\n")
+
+
+def update_contributor_data_file() -> None:
+    # This list should hold all repositories that should be included in
+    # the total count, including those that should *not* have tabs on the team
+    # page (e.g. if they are deprecated).
+    repo_names = [
+        "docker-zulip",
+        "errbot-backend-zulip",
+        "github-actions-zulip",
+        "hubot-zulip",
+        "puppet-zulip",
+        "python-zulip-api",
+        "trello-to-zulip",
+        "swift-zulip-api",
+        "zulint",
+        "zulip",
+        "zulip-android-legacy",
+        "zulip-architecture",
+        "zulip-archive",
+        "zulip-csharp",
+        "zulip-desktop",
+        "zulip-desktop-legacy",
+        "zulip-flutter",
+        "zulip-ios-legacy",
+        "zulip-js",
+        "zulip-mobile",
+        "zulip-redmine-plugin",
+        "zulip-terminal",
+        "zulip-zapier",
+        "zulipbot",
+    ]
+
+    data: ContributorsJSON = dict(date=str(datetime.now(tz=timezone.utc).date()), contributors=[])
+    contributor_username_to_data: dict[str, dict[str, str | int]] = {}
+
+    for repo_name in repo_names:
+        contributors = fetch_contributors(repo_name, args.max_retries)
+        for contributor in contributors:
+            username = contributor.get("login") or contributor.get("email")
+            assert username is not None
+            if username in contributor_username_to_data:
+                contributor_username_to_data[username][repo_name] = contributor["contributions"]
+            else:
+                contributor_username_to_data[username] = {repo_name: contributor["contributions"]}
+
+                avatar_url = contributor.get("avatar_url")
+                if avatar_url is not None:
+                    contributor_username_to_data[username]["avatar"] = avatar_url
+
+                email = contributor.get("email")
+                if email is not None:
+                    contributor_username_to_data[username]["email"] = email
+                    hash_key = gravatar_hash(email)
+                    gravatar_url = f"https://secure.gravatar.com/avatar/{hash_key}?d=identicon"
+                    contributor_username_to_data[username]["avatar"] = gravatar_url
+
+                login = contributor.get("login")
+                if login is not None:
+                    contributor_username_to_data[username]["github_username"] = login
+
+                name = contributor.get("name")
+                if name is not None:
+                    contributor_username_to_data[username]["name"] = unicodedata.normalize(
+                        "NFC", name
+                    )
+
+    # remove duplicate contributions count
+    # find commits at the time of split and subtract from zulip-server
+    with open(duplicate_commits_file) as f:
+        duplicate_commits = json.load(f)
+        for committer in duplicate_commits:
+            if committer in contributor_username_to_data and contributor_username_to_data[
+                committer
+            ].get("zulip"):
+                total_commits = contributor_username_to_data[committer]["zulip"]
+                assert isinstance(total_commits, int)
+                duplicate_commits_count = duplicate_commits[committer]
+                original_commits = total_commits - duplicate_commits_count
+                contributor_username_to_data[committer]["zulip"] = original_commits
+
+    data["contributors"] = list(contributor_username_to_data.values())
+    write_to_disk(data, settings.CONTRIBUTOR_DATA_FILE_PATH)
+
+
+if __name__ == "__main__":
+    update_contributor_data_file()

tools/fetch-pull-request

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+this_dir=${BASH_SOURCE[0]%/*}
+# shellcheck source=lib/git-tools.bash
+. "${this_dir}"/lib/git-tools.bash
+
+require_clean_work_tree 'check out PR as branch'
+
+request_id="${1-}"
+if [ -z "$request_id" ]; then
+    echo "Usage: $0 pr_id" >&2
+    exit 1
+fi
+remote=${2:-"upstream"}
+
+set -x
+git fetch "$remote" "pull/$request_id/head"
+git checkout -B "review-original-${request_id}"
+git reset --hard FETCH_HEAD

tools/fetch-rebase-pull-request

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -e
+
+this_dir=${BASH_SOURCE[0]%/*}
+# shellcheck source=lib/git-tools.bash
+. "${this_dir}"/lib/git-tools.bash
+
+require_clean_work_tree 'check out PR as branch'
+
+request_id="$1"
+remote=${2:-"upstream"}
+
+set -x
+git fetch "$remote" "pull/$request_id/head"
+git checkout -B "review-${request_id}" "$remote/main"
+git reset --hard FETCH_HEAD
+git pull --rebase

tools/find-unused-css

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+# Hackish tool for attempting to find unused IDs / classes in our CSS
+
+for n in $(perl -lne 'print $1 while /[#.]([a-zA-Z0-9_-]+)/g' web/styles/zulip.css | sort -u); do
+    if [ "$(git grep "$n" | grep -cv '^web/styles/zulip.css')" -eq 0 ]; then
+        echo "$n"
+    fi
+done

tools/github-changes-contain-file

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+import argparse
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+
+def is_file_changed(file_path: str) -> bool:
+    event_path = os.environ.get("GITHUB_EVENT_PATH", "")
+    if not event_path:
+        sys.exit("GITHUB_EVENT_PATH environment variable not set")
+
+    with open(Path(event_path)) as f:
+        event = json.load(f)
+
+    before = event.get("before")
+    after = event.get("after")
+    if not before or not after:
+        sys.exit("Missing 'before' or 'after' commit SHAs in event data")
+
+    try:
+        result = subprocess.run(
+            ["git", "diff", "--quiet", before, after, "--", file_path],
+            check=False,
+            capture_output=True,
+        )
+        return result.returncode == 1
+    except subprocess.CalledProcessError:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "file",
+    )
+    args = parser.parse_args()
+
+    if is_file_changed(args.file):
+        sys.exit(0)
+    else:
+        sys.exit(1)

tools/i18n/sync-translations

@@ -0,0 +1,109 @@
+#!/usr/bin/env bash
+
+# See https://zulip.readthedocs.io/en/latest/translating/internationalization.html
+# for background on this subsystem.
+
+set -e
+set -x
+
+# Check we're on `main` or `\d+\.x`
+branch=$(git rev-parse --abbrev-ref HEAD)
+if [ "$branch" == "main" ]; then
+    suffix=""
+elif [[ "$branch" =~ ^[0-9]+\.x$ ]]; then
+    suffix="-${branch/./-}"
+else
+    echo "Unexpected branch name: $branch"
+    exit 1
+fi
+
+git fetch upstream
+
+local_branch="update-translations-$branch"
+if git rev-parse --verify --quiet "origin/$local_branch" >/dev/null; then
+    echo "Remote branch origin/$local_branch already exists -- delete it before continuing"
+    exit 1
+fi
+git checkout -b "$local_branch" "upstream/$branch"
+
+# Clear out local any `.mo` files, which cause locale/*/LC_MESSAGES/
+# directories to not be empty when their .po files vanish, so git
+# doesn't remove the directory.
+rm locale/*/LC_MESSAGES/*.mo || true
+
+wlc lock "zulip/frontend$suffix"
+wlc lock "zulip/django$suffix"
+trap 'wlc unlock "zulip/frontend$suffix" && wlc unlock "zulip/django$suffix"' EXIT
+
+wlc commit "zulip/frontend$suffix"
+wlc commit "zulip/django$suffix"
+
+git fetch "https://hosted.weblate.org/git/zulip/django$suffix/"
+if [ "$(git rev-list FETCH_HEAD "^upstream/$branch")" == "" ]; then
+    echo "No changes from Weblate to commit!"
+else
+    git cherry-pick FETCH_HEAD "^upstream/$branch"
+
+    git clean -dxf locale/
+
+    # Check that everything is normalized into NFC; if it isn't, we
+    # need to fix it in Weblate, or Weblate will fail to rebase the
+    # changes out after we push.
+    files="$(find locale -type f -name '*.json' -o -name '*.po')"
+    mapfile -t files <<<"$files"
+    for file in "${files[@]}"; do
+        uconv -x any-nfc "$file" | sponge -- "$file"
+    done
+
+    git add locale/
+    if ! git diff-index --quiet --cached HEAD locale/; then
+        echo "Non-NFC translations exist!  Fix them in Weblate first."
+        git diff --cached
+        exit 1
+    fi
+
+    # Update locale/*/legacy_stream_translations.json
+    ./tools/i18n/update-for-legacy-translations
+
+    # Trim out any now-empty locale directories
+    find locale/ -type d -empty -delete
+
+    git add locale/
+
+    # Double-check that they all compile
+    ./manage.py compilemessages --ignore='*'
+
+    git commit --amend -m 'i18n: Sync translations from Weblate.'
+fi
+
+./manage.py makemessages --all
+git add locale/
+
+if git diff-index --quiet --cached HEAD locale/; then
+    echo "No changes to PO files to commit!"
+else
+    git commit -m 'i18n: Updated .po files for new strings.'
+fi
+
+if [ "$(git rev-list HEAD "^upstream/$branch")" == "" ]; then
+    echo "No commits to push, aborting!"
+    git checkout "$branch"
+    git branch -D "$local_branch"
+else
+    git push origin "HEAD:$local_branch"
+
+    gh_username="$(gh api user --jq '.login')"
+    gh pr create --title "Updated translations for $branch from Weblate." --body "" --base "$branch" --head "$gh_username:$local_branch"
+    gh pr merge --rebase --auto "$gh_username:$local_branch"
+    commit=$(git rev-parse HEAD)
+
+    git checkout "$branch"
+    git branch -D "$local_branch"
+
+    echo "Waiting for PR to merge..."
+    while [ "$(gh pr list --search "$commit" --state all --json state --jq .[0].state)" != "MERGED" ]; do
+        sleep 30
+    done
+
+    git push origin --delete "$local_branch"
+fi

tools/i18n/update-for-legacy-translations

@@ -0,0 +1,258 @@
+#!/usr/bin/env python3
+import glob
+import os
+import re
+from subprocess import check_output
+
+import orjson
+
+LEGACY_STRINGS_MAP = {
+    "<p>You are searching for messages that belong to more than one channel, which is not possible.</p>": "<p>You are searching for messages that belong to more than one stream, which is not possible.</p>",
+    "<strong>{name}</strong> <i>(guest)</i> is not subscribed to this channel. They will not be notified if you mention them.": "<strong>{name}</strong> <i>(guest)</i> is not subscribed to this stream. They will not be notified if you mention them.",
+    "<strong>{name}</strong> <i>(guest)</i> is not subscribed to this channel. They will not be notified unless you subscribe them.": "<strong>{name}</strong> <i>(guest)</i> is not subscribed to this stream. They will not be notified unless you subscribe them.",
+    "<strong>{name}</strong> is not subscribed to this channel. They will not be notified if you mention them.": "<strong>{name}</strong> is not subscribed to this stream. They will not be notified if you mention them.",
+    "<strong>{name}</strong> is not subscribed to this channel. They will not be notified unless you subscribe them.": "<strong>{name}</strong> is not subscribed to this stream. They will not be notified unless you subscribe them.",
+    "<z-link>Click here</z-link> to learn about exporting private channels and direct messages.": "<z-link>Click here</z-link> to learn about exporting private streams and direct messages.",
+    "<z-user></z-user> will have the same properties as it did prior to deactivation, including role, owner and channel subscriptions.": "<z-user></z-user> will have the same properties as it did prior to deactivation, including role, owner and stream subscriptions.",
+    "<z-user></z-user> will have the same role, channel subscriptions, user group memberships, and other settings and permissions as they did prior to deactivation.": "<z-user></z-user> will have the same role, stream subscriptions, user group memberships, and other settings and permissions as they did prior to deactivation.",
+    "A channel with this name already exists.": "A stream with this name already exists.",
+    "Add default channels": "Add default streams",
+    "Add members. Use usergroup or #channelname to bulk add members.": "Add members. Use usergroup or #streamname to bulk add members.",
+    "Add channel": "Add stream",
+    "Add channels": "Add streams",
+    "Add subscribers. Use usergroup or #channelname to bulk add subscribers.": "Add subscribers. Use usergroup or #streamname to bulk add subscribers.",
+    "All messages including muted channels": "All messages including muted streams",
+    "All channels": "All streams",
+    "Allow creating web-public streams (visible to anyone on the Internet)": "Allow creating web-public streams (visible to anyone on the Internet)",
+    # "Already subscribed to {channel}": "Already subscribed to {stream}",
+    "Announce new channel in": "Announce new stream in",
+    "Archive channel": "Archive stream",
+    "Archiving channel <z-stream></z-stream> will immediately unsubscribe everyone. This action cannot be undone.": "Archiving stream <z-stream></z-stream> will immediately unsubscribe everyone. This action cannot be undone.",
+    "Archiving this channel will also disable settings that were configured to use this channel:": "Archiving this stream will also disable settings that were configured to use this stream:",
+    # "Are you sure you want to create channel ''''{channel_name}'''' and subscribe {count} users to it?": "Are you sure you want to create stream ''''{stream_name}'''' and subscribe {count} users to it?",
+    # "Are you sure you want to send @-mention notifications to the <strong>{subscriber_count}</strong> users subscribed to #{channel_name}? If not, please edit your message to remove the <strong>@{wildcard_mention}</strong> mention.": "Are you sure you want to send @-mention notifications to the <strong>{subscriber_count}</strong> users subscribed to #{stream_name}? If not, please edit your message to remove the <strong>@{stream_wildcard_mention}</strong> mention.",
+    "Automatically unmute topics in muted channels": "Automatically unmute topics in muted streams",
+    "Back to channels": "Back to streams",
+    "Because you are removing the last subscriber from a private channel, it will be automatically <z-link>archived</z-link>.": "Because you are removing the last subscriber from a private stream, it will be automatically <z-link>archived</z-link>.",
+    "Because you are the only subscriber, this channel will be automatically <z-link>archived</z-link>.": "Because you are the only subscriber, this stream will be automatically <z-link>archived</z-link>.",
+    "Browse 1 more channel": "Browse 1 more stream",
+    "Browse channels": "Browse streams",
+    "Browse {can_subscribe_stream_count} more channels": "Browse {can_subscribe_stream_count} more streams",
+    "Cannot subscribe to private channel <z-stream></z-stream>": "Cannot subscribe to private stream <z-stream></z-stream>",
+    "Cannot view channel": "Cannot view stream",
+    "Choose a name for the new channel.": "Choose a name for the new stream.",
+    "Configure how Zulip notifies you about new messages. In muted channels, channel notification settings apply only to unmuted topics.": "Configure how Zulip notifies you about new messages. In muted streams, stream notification settings apply only to unmuted topics.",
+    "Configure the default channels new users are subscribed to when joining your organization.": "Configure the default streams new users are subscribed to when joining your organization.",
+    "Consider <z-link>searching all public channels</z-link>.": "Consider <z-link>searching all public streams</z-link>.",
+    "Create a channel": "Create a stream",
+    "Create new channel": "Create new stream",
+    "Create channel": "Create stream",
+    "Creating channel...": "Creating stream...",
+    "Currently viewing the entire channel.": "Currently viewing the entire stream.",
+    "Cycle between channel views": "Cycle between stream views",
+    "Default for channel": "Default for stream",
+    "Default channels": "Default streams",
+    "Default channels for new users cannot be made private.": "Default streams for new users cannot be made private.",
+    "Default channels for this organization": "Default streams for this organization",
+    "Demote inactive channels": "Demote inactive streams",
+    # "Edit #{channel_name}": "Edit #{stream_name}",
+    "Edit channel name and description": "Edit stream name and description",
+    "Error creating channel": "Error creating stream",
+    # "Error in unsubscribing from #{channel_name}": "Error in unsubscribing from #{stream_name}",
+    # "Error removing user from #{channel_name}": "Error removing user from #{stream_name}",
+    "Error removing user from this channel.": "Error removing user from this stream.",
+    "Exports all users, settings, and all data visible in public channels.": "Exports all users, settings, and all data visible in public streams.",
+    "Failed adding one or more channels.": "Failed adding one or more streams.",
+    "Filter default channels": "Filter default streams",
+    "Filter channels": "Filter streams",
+    "First time? Read our <z-link>guidelines</z-link> for creating and naming channels.": "First time? Read our <z-link>guidelines</z-link> for creating and naming streams.",
+    "Generate channel email address": "Generate stream email address",
+    "Go to channel from topic view": "Go to stream from topic view",
+    "Go to channel settings": "Go to stream settings",
+    "However, it will no longer be subscribed to the private channels that you are not subscribed to.": "However, it will no longer be subscribed to the private streams that you are not subscribed to.",
+    "In muted channels, channel notification settings apply only to unmuted topics.": "In muted streams, stream notification settings apply only to unmuted topics.",
+    "In this channel": "In this stream",
+    "Includes muted channels and topics": "Includes muted streams and topics",
+    "Invalid channel ID": "Invalid stream ID",
+    "Let recipients see when I'm typing messages in channels": "Let recipients see when I'm typing messages in streams",
+    "Let recipients see when a user is typing channel messages": "Let recipients see when a user is typing stream messages",
+    "Log in to browse more channels": "Log in to browse more streams",
+    # "Message #{channel_name}": "Message #{stream_name}",
+    # "Message #{channel_name} > {topic_name}": "Message #{stream_name} > {topic_name}",
+    "Messages in all public channels": "Messages in all public streams",
+    "Mute channel": "Mute stream",
+    "Narrow to messages on channel <z-value></z-value>.": "Narrow to messages on stream <z-value></z-value>.",
+    "New channel announcements": "New stream announcements",
+    "New channel message": "New stream message",
+    "New channel notifications": "New stream notifications",
+    "No default channels match your current filter.": "No default streams match your current filter.",
+    "No matching channels": "No matching streams",
+    "No channel subscribers match your current filter.": "No stream subscribers match your current filter.",
+    "No channel subscriptions.": "No stream subscriptions.",
+    "No channels": "No streams",
+    "Notify channel": "Notify stream",
+    # "Now following <z-link>{channel_topic}</z-link>.": "Now following <z-link>{stream_topic}</z-link>.",
+    "Once you leave this channel, you will not be able to rejoin.": "Once you leave this stream, you will not be able to rejoin.",
+    "Only channel members can add users to a private channel.": "Only stream members can add users to a private stream.",
+    "Only subscribers can access or join private channels, so you will lose access to this channel if you convert it to a private channel while not subscribed to it.": "Only subscribers can access or join private streams, so you will lose access to this stream if you convert it to a private stream while not subscribed to it.",
+    "Only subscribers to this channel can edit channel permissions.": "Only subscribers to this stream can edit stream permissions.",
+    "Pin channel to top": "Pin stream to top",
+    "Pin channel to top of left sidebar": "Pin stream to top of left sidebar",
+    "Please select a channel.": "Please specify a stream.",
+    "Private channels cannot be default channels for new users.": "Private streams cannot be default streams for new users.",
+    "Receives new channel announcements": "Receives new stream announcements",
+    "Require topics in channel messages": "Require topics in stream messages",
+    "CHANNELS": "STREAMS",
+    "Scroll through channels": "Scroll through streams",
+    "Search all public channels in the organization.": "Search all public streams in the organization.",
+    "Select a channel": "Select a stream",
+    "Select a channel below or change topic name.": "Select a stream below or change topic name.",
+    "Select a channel to subscribe": "Select a stream to subscribe",
+    "Select channel": "Select stream",
+    "Channel": "Stream",
+    "Channel <b><z-stream></z-stream></b> created!": "Stream <b><z-stream></z-stream></b> created!",
+    "Channel ID": "Stream ID",
+    "Channel color": "Stream color",
+    "Channel created recently": "Stream created recently",
+    "Channel creation": "Stream creation",
+    "Channel description": "Stream description",
+    "Channel details": "Stream details",
+    "Channel email address:": "Stream email address:",
+    "Channel name": "Stream name",
+    "Channel permissions": "Stream permissions",
+    "Channel settings": "Stream settings",
+    "Channel successfully created!": "Stream successfully created!",
+    "Channels": "Streams",
+    "Channels they should join": "Streams they should join",
+    "Subscribe to/unsubscribe from selected channel": "Subscribe to/unsubscribe from selected stream",
+    "Subscribe {full_name} to channels": "Subscribe {full_name} to streams",
+    "Subscribed channels": "Subscribed streams",
+    # "The channel <b>#{channel_name}</b> does not exist. Manage your subscriptions <z-link>on your Channels page</z-link>.": "The stream <b>#{stream_name}</b> does not exist. Manage your subscriptions <z-link>on your Streams page</z-link>.",
+    "The channel description cannot contain newline characters.": "The stream description cannot contain newline characters.",
+    "The topic <strong>{topic_name}</strong> already exists in this channel. Are you sure you want to combine messages from these topics? This cannot be undone.": "The topic <strong>{topic_name}</strong> already exists in this stream. Are you sure you want to combine messages from these topics? This cannot be undone.",
+    "There are no default channels.": "There are no default streams.",
+    "There are no channels you can view in this organization.": "There are no streams you can view in this organization.",
+    "This change will make this channel's entire message history accessible according to the new configuration.": "This change will make this stream's entire message history accessible according to the new configuration.",
+    "This channel does not exist or is private.": "This stream does not exist or is private.",
+    "This channel does not yet have a description.": "This stream does not yet have a description.",
+    "This channel has been archived.": "This stream has been archived.",
+    "This channel has no subscribers.": "This stream has no subscribers.",
+    "This channel has {sub_count, plural, =0 {no subscribers} one {# subscriber} other {# subscribers}}.": "This stream has {sub_count, plural, =0 {no subscribers} one {# subscriber} other {# subscribers}}.",
+    "Time limit for moving messages between channels": "Time limit for moving messages between streams",
+    "Unknown channel": "Unknown stream",
+    "Unknown channel #{search_text}": "Unknown stream #{search_text}",
+    "Unmute channel": "Unmute stream",
+    # "Unmuted <z-link>{channel_topic}</z-link>.": "Unmuted <z-link>{stream_topic}</z-link>.",
+    "Unmuted channels and topics": "Unmuted streams and topics",
+    "Unpin channel from top": "Unpin stream from top",
+    "Use channel settings to unsubscribe from private channels.": "Use stream settings to unsubscribe from private streams.",
+    "Use channel settings to unsubscribe the last user from a private channel.": "Use stream settings to unsubscribe the last user from a private stream.",
+    "View all channels": "View all streams",
+    "View channel": "View stream",
+    "View channel messages": "View stream messages",
+    "View channels": "View streams",
+    # "Warning: <strong>#{channel_name}</strong> is a private channel.": "Warning: <strong>#{stream_name}</strong> is a private stream.",
+    "Which parts of the email should be included in the Zulip message sent to this channel?": "Which parts of the email should be included in the Zulip message sent to this stream?",
+    "Who can access this channel": "Who can access this stream",
+    "Who can add users to channels": "Who can add users to streams",
+    "Who can create private channels": "Who can create private streams",
+    "Who can create public channels": "Who can create public streams",
+    "Who can create web-public channels": "Who can create web-public streams",
+    "Who can move messages to another channel": "Who can move messages to another stream",
+    "Who can post to this channel": "Who can post to this stream",
+    "Who can unsubscribe others from this channel": "Who can unsubscribe others from this stream",
+    "You are not currently subscribed to this channel.": "You are not currently subscribed to this stream.",
+    "You are not subscribed to any channels.": "You are not subscribed to any streams.",
+    "You are not subscribed to channel <z-stream-name></z-stream-name>.": "You are not subscribed to stream <z-stream-name></z-stream-name>.",
+    # "You aren't subscribed to this channel and nobody has talked about that yet!": "You aren't subscribed to this stream and nobody has talked about that yet!",
+    "You can use email to send messages to Zulip channels.": "You can use email to send messages to Zulip streams.",
+    "You cannot create a channel with no subscribers.": "You cannot create a stream with no subscribers.",
+    "You do not have permission to add other users to channels in this organization.": "You do not have permission to add other users to streams in this organization.",
+    "You do not have permission to move messages to another channel in this organization.": "You do not have permission to move messages to another stream in this organization.",
+    "You do not have permission to post in this channel.": "You do not have permission to post in this stream.",
+    # "You do not have permission to use <b>@{wildcard_mention_string}</b> mentions in this channel.": "You do not have permission to use <b>@{stream_wildcard_mention}</b> mentions in this stream.",
+    "You must be an organization administrator to create a channel without subscribing.": "You must be an organization administrator to create a stream without subscribing.",
+    "You subscribed to channel <z-stream-name></z-stream-name>.": "You subscribed to stream <z-stream-name></z-stream-name>.",
+    "You unsubscribed from channel <z-stream-name></z-stream-name>.": "You unsubscribed from stream <z-stream-name></z-stream-name>.",
+    "You're not subscribed to this channel. You will not be notified if other users reply to your message.": "You're not subscribed to this stream. You will not be notified if other users reply to your message.",
+    "Your message was sent to a channel you have muted.": "Your message was sent to a stream you have muted.",
+    "back to channels": "back to streams",
+}
+
+
+def get_json_filename(locale: str) -> str:
+    return f"locale/{locale}/translations.json"
+
+
+def get_legacy_filename(locale: str) -> str:
+    return f"locale/{locale}/legacy_stream_translations.json"
+
+
+def get_locales() -> list[str]:
+    output = check_output(["git", "ls-files", "locale"], text=True)
+    tracked_files = output.split()
+    regex = re.compile(r"locale/(\w+)/LC_MESSAGES/django.po")
+    locales = []
+    for tracked_file in tracked_files:
+        matched = regex.search(tracked_file)
+        if matched and matched.group(1) != "en_GB":
+            locales.append(matched.group(1))
+
+    return locales
+
+
+def get_translations(path: str) -> dict[str, str]:
+    with open(path, "rb") as raw_resource_file:
+        translations = orjson.loads(raw_resource_file.read())
+
+    return translations
+
+
+def update_for_legacy_stream_translations(
+    current: dict[str, str], legacy: dict[str, str], path: str
+) -> None:
+    number_of_updates = 0
+    updated_translations: dict[str, str] = {}
+    for line, translation in current.items():
+        # If the string has a legacy string mapped and see if it's
+        # not currently translated (e.g. an empty string), then use
+        # the legacy translated string (which might be an empty string).
+        if (
+            line in LEGACY_STRINGS_MAP
+            and translation == ""
+            and (legacy_string := LEGACY_STRINGS_MAP[line]) in legacy
+        ):
+            updated_translations[line] = legacy[legacy_string]
+            number_of_updates += 1
+        else:
+            updated_translations[line] = translation
+
+    # Only replace file content if we've made any updates for legacy
+    # translated strings.
+    if number_of_updates > 0:
+        with open(path, "wb") as f:
+            f.write(
+                orjson.dumps(
+                    updated_translations,
+                    option=orjson.OPT_APPEND_NEWLINE | orjson.OPT_INDENT_2 | orjson.OPT_SORT_KEYS,
+                )
+            )
+        print(f"Updated {number_of_updates} strings in: {path}")
+
+
+expected_legacy_filenames = set()
+for locale in get_locales():
+    current = get_json_filename(locale)
+    legacy = get_legacy_filename(locale)
+    expected_legacy_filenames.add(legacy)
+    if os.path.exists(current) and os.path.exists(legacy):
+        print(f"Checking legacy translations for: {current}")
+        current_translations = get_translations(current)
+        legacy_translations = get_translations(legacy)
+        update_for_legacy_stream_translations(current_translations, legacy_translations, current)
+
+for extra_file in (
+    set(glob.glob("locale/*/legacy_stream_translations.json")) - expected_legacy_filenames
+):
+    print(f"Removing dangling legacy translation file {extra_file}")
+    os.unlink(extra_file)

tools/lib/capitalization.py

@@ -0,0 +1,341 @@
+import re
+from io import StringIO
+from re import Match
+
+from bs4 import BeautifulSoup
+
+# The phrases in this list will be ignored. The longest phrase is
+# tried first; this removes the chance of smaller phrases changing
+# the text before longer phrases are tried.
+# The errors shown by `tools/check-capitalization` can be added to
+# this list without any modification.
+IGNORED_PHRASES = [
+    # Proper nouns and acronyms
+    r"AI",
+    r"API",
+    r"APNS",
+    r"Apple Silicon",
+    r"Botserver",
+    r"Cookie Bot",
+    r"DevAuthBackend",
+    r"DSN",
+    r"Esc",
+    r"GCM",
+    r"GitHub",
+    r"GitLab",
+    r"Gravatar",
+    r"HTTP",
+    r"ID",
+    r"IDs",
+    r"Inbox",
+    r"Intel",
+    r"IP",
+    r"JSON",
+    r"Jitsi",
+    r"Jotform",
+    r"LinkedIn",
+    r"LDAP",
+    r"Markdown",
+    r"OAuth",
+    r"OTP",
+    r"Recent conversations",
+    r"DM",
+    r"DMs",
+    r"Slack",
+    r"Google",
+    r"Terms of Service",
+    r"TikTok",
+    r"Tuesday",
+    r"URL",
+    r"UUID",
+    r"WordPress",
+    r"YouTube",
+    r"Zoom",
+    r"Zulip",
+    r"Zulip Server",
+    r"Zulip Account Security",
+    r"Zulip Security",
+    r"Zulip Cloud",
+    r"Zulip Cloud Standard",
+    r"Zulip Cloud Plus",
+    r"Zulip Desktop",
+    r"Download Zulip for macOS \(Apple Silicon\)",
+    r"Download Zulip for macOS \(Intel\)",
+    r"BigBlueButton",
+    r"Constructor Groups",
+    r"Nextcloud Talk",
+    # Code things
+    r"\.zuliprc",
+    # BeautifulSoup will remove <z-user> which is horribly confusing,
+    # so we need more of the sentence.
+    r"<z-user></z-user> will have the same role",
+    r"<z-user></z-user> will have the same properties",
+    # Things using "I"
+    r"I understand",
+    r"I'm",
+    r"I've",
+    r"Topics I participate in",
+    r"Topics I send a message to",
+    r"Topics I start",
+    # Specific short words
+    r"beta",
+    r"and",
+    r"bot",
+    r"e\.g\.",
+    r"email",
+    r"enabled",
+    r"signups",
+    # Pasted text filename
+    r"PastedText",
+    # Placeholders
+    r"keyword",
+    r"streamname",
+    r"user@example\.com",
+    r"example\.com",
+    r"acme",
+    # Fragments of larger strings
+    r"is …",
+    r"your subscriptions on your Channels page",
+    r"Add global time<br />Everyone sees global times in their own time zone\.",
+    r"user",
+    r"an unknown operating system",
+    r"Go to Settings",
+    r"find accounts for another email address",
+    # SPECIAL CASES
+    # Because topics usually are lower-case, this would look weird if it were capitalized
+    r"show all topics",
+    # Used alone in a parenthetical where capitalized looks worse.
+    r"^deprecated$",
+    # We want the similar text in the Private Messages section to have the same capitalization.
+    r"more conversations",
+    r"back to channels",
+    # Capital 'i' looks weird in reminders popover
+    r"in 1 hour",
+    r"in 20 minutes",
+    r"in 3 hours",
+    # these are used as channel or topic names
+    r"^new channels$",
+    r"^channel events$",
+    r"^general$",
+    r"^sandbox$",
+    r"^experiments$",
+    r"^greetings$",
+    r"^moving messages$",
+    r"^start a conversation$",
+    r"^welcome to Zulip!$",
+    r"^general chat$",
+    # These are used as example short names (e.g. an uncapitalized context):
+    r"^marketing$",
+    r"^cookie$",
+    # Used to refer custom time limits
+    r"\bN\b",
+    r"minute",
+    r"minutes",
+    # Capital c feels obtrusive in clear status option
+    r"clear",
+    r"group direct messages with \{recipient\}",
+    r"direct messages with \{recipient\}",
+    r"direct messages with yourself",
+    r"GIF",
+    # Emoji name placeholder
+    r"leafy green vegetable",
+    # Subdomain placeholder
+    r"your-organization",
+    # Used in invite modal
+    r"or",
+    # Units
+    r"MB",
+    r"GB",
+    # Used in GIPHY integration setting. GIFs Rating.
+    r"rated Y",
+    r"rated G",
+    r"rated PG",
+    r"rated PG13",
+    r"rated R",
+    # Used in GIPHY popover.
+    r"GIFs",
+    r"GIPHY",
+    # Used for Tenor attributions
+    r"Search Tenor",
+    # Used for KLIPY attributions
+    r"Search KLIPY",
+    # Used in our case studies
+    r"Technical University of Munich",
+    r"University of California San Diego",
+    # Used in stream creation form
+    r"email hidden",
+    # Use in compose box.
+    r"to send",
+    r"to add a new line",
+    # Used in showing Notification Bot read receipts message
+    "Notification Bot",
+    # Used in strings around welcome bot custom messages
+    r"Welcome Bot",
+    # Used in presence_enabled setting label
+    r"invisible mode off",
+    # Typeahead suggestions for "Pronouns" custom field type.
+    r"he/him",
+    r"she/her",
+    r"they/them",
+    # Used in message-move-time-limit setting label
+    r"does not apply to moderators and administrators",
+    # Used in message-delete-time-limit setting label
+    r"does not apply to users who can delete any message",
+    # Used as indicator with names for guest users.
+    r"guest",
+    # Used as indicator with names for archived streams.
+    r"archived",
+    # Used in pills for deactivated users.
+    r"deactivated",
+    # Used in pills for resolved topics.
+    r"resolved",
+    # Used in pills for unresolved topics.
+    r"unresolved",
+    # Used in pills for followed topics.
+    r"followed",
+    # Used in pills for unfollowed topics.
+    r"unfollowed",
+    # This is a reference to a setting/secret and should be lowercase.
+    r"zulip_org_id",
+    # These are custom time unit options for modal dropdowns
+    r"minutes",
+    r"hours",
+    r"days",
+    r"weeks",
+    # Used in "Who can subscribe to this channel" label.
+    r"everyone except guests can subscribe to any public channel",
+    # Used in branch-filtering label in the integration-url-modal.
+    r"comma-separated list",
+    # Used in info_overlay.
+    r"then",
+    r"Joe Smith",
+    r"bold",
+    r"channel name",
+    r"is busy working",
+    r"italic",
+    r"strikethrough",
+    r"support team",
+    r"topic name",
+]
+
+# Sort regexes in descending order of their lengths. As a result, the
+# longer phrases will be ignored first.
+IGNORED_PHRASES.sort(key=len, reverse=True)
+
+# Compile regexes to improve performance. This also extracts the
+# text using BeautifulSoup and then removes extra whitespaces from
+# it. This step enables us to add HTML in our regexes directly.
+COMPILED_IGNORED_PHRASES = [
+    re.compile(r" ".join(BeautifulSoup(StringIO(regex), "lxml").text.split()))
+    for regex in IGNORED_PHRASES
+]
+
+SPLIT_BOUNDARY = r"?.!"  # Used to split string into sentences.
+SPLIT_BOUNDARY_REGEX = re.compile(rf"[{SPLIT_BOUNDARY}]")
+
+# Regexes which check capitalization in sentences.
+DISALLOWED = [
+    r"^[a-z](?!\})",  # Checks if the sentence starts with a lower case character.
+    r"^[A-Z][a-z]+[\sa-z0-9]+[A-Z]",  # Checks if an upper case character exists
+    # after a lower case character when the first character is in upper case.
+]
+DISALLOWED_REGEX = re.compile(r"|".join(DISALLOWED))
+
+BANNED_WORDS = {
+    "realm": "The term realm should not appear in user-facing strings. Use organization instead.",
+}
+
+
+def get_safe_phrase(phrase: str) -> str:
+    """
+    Safe phrase is in lower case and doesn't contain characters which can
+    conflict with split boundaries. All conflicting characters are replaced
+    with low dash (_).
+    """
+    phrase = SPLIT_BOUNDARY_REGEX.sub("_", phrase)
+    return phrase.lower()
+
+
+def replace_with_safe_phrase(matchobj: Match[str]) -> str:
+    """
+    The idea is to convert IGNORED_PHRASES into safe phrases, see
+    `get_safe_phrase()` function. The only exception is when the
+    IGNORED_PHRASE is at the start of the text or after a split
+    boundary; in this case, we change the first letter of the phrase
+    to upper case.
+    """
+    ignored_phrase = matchobj.group(0)
+    safe_string = get_safe_phrase(ignored_phrase)
+
+    start_index = matchobj.start()
+    complete_string = matchobj.string
+
+    is_string_start = start_index == 0
+    # We expect that there will be one space between split boundary
+    # and the next word.
+    punctuation = complete_string[max(start_index - 2, 0)]
+    is_after_split_boundary = punctuation in SPLIT_BOUNDARY
+    if is_string_start or is_after_split_boundary:
+        return safe_string.capitalize()
+
+    return safe_string
+
+
+def get_safe_text(text: str) -> str:
+    """
+    This returns text which is rendered by BeautifulSoup and is in the
+    form that can be split easily and has all IGNORED_PHRASES processed.
+    """
+    soup = BeautifulSoup(StringIO(text), "lxml")
+    text = " ".join(soup.text.split())  # Remove extra whitespaces.
+    for phrase_regex in COMPILED_IGNORED_PHRASES:
+        text = phrase_regex.sub(replace_with_safe_phrase, text)
+
+    return text
+
+
+def is_capitalized(safe_text: str) -> bool:
+    sentences = SPLIT_BOUNDARY_REGEX.split(safe_text)
+    return not any(DISALLOWED_REGEX.search(sentence.strip()) for sentence in sentences)
+
+
+def check_banned_words(text: str) -> list[str]:
+    lower_cased_text = text.lower()
+    errors = []
+    for word, reason in BANNED_WORDS.items():
+        if word in lower_cased_text:
+            # Hack: Should move this into BANNED_WORDS framework; for
+            # now, just hand-code the skips:
+            if (
+                "realm_name" in lower_cased_text
+                or "realm_uri" in lower_cased_text
+                or "realm_url" in lower_cased_text
+                or "remote_realm_host" in lower_cased_text
+                or "realm_message" in lower_cased_text
+                or "realm_move" in lower_cased_text
+            ):
+                continue
+            kwargs = dict(word=word, text=text, reason=reason)
+            msg = "{word} found in '{text}'. {reason}".format(**kwargs)
+            errors.append(msg)
+
+    return errors
+
+
+def check_capitalization(strings: list[str]) -> tuple[list[str], list[str], list[str]]:
+    errors = []
+    ignored = []
+    banned_word_errors = []
+    for text in strings:
+        text = " ".join(text.split())  # Remove extra whitespaces.
+        safe_text = get_safe_text(text)
+        has_ignored_phrase = text != safe_text
+        capitalized = is_capitalized(safe_text)
+        if not capitalized:
+            errors.append(text)
+        elif has_ignored_phrase:
+            ignored.append(text)
+
+        banned_word_errors.extend(check_banned_words(text))
+
+    return sorted(errors), sorted(ignored), sorted(banned_word_errors)

tools/lib/git-tools.bash

@@ -0,0 +1,34 @@
+# shellcheck shell=bash
+
+# Borrowed from Git's git-sh-setup.
+#
+# See git.git commit 92c62a3f4 (from 2010!); as of 2020 with Git 2.26,
+# this function has only needed one edit since then, adding localization
+# with gettext, which we can omit.
+require_clean_work_tree() {
+    local action="$1"
+
+    git rev-parse --verify HEAD >/dev/null || exit 1
+    git update-index -q --ignore-submodules --refresh
+    local err=0
+
+    if ! git diff-files --quiet --ignore-submodules; then
+        echo >&2 "Cannot $action: You have unstaged changes."
+        err=1
+    fi
+
+    if ! git diff-index --cached --quiet --ignore-submodules HEAD --; then
+        if [ $err = 0 ]; then
+            echo >&2 "Cannot $action: Your index contains uncommitted changes."
+        else
+            echo >&2 "Additionally, your index contains uncommitted changes."
+        fi
+        err=1
+    fi
+
+    if [ $err = 1 ]; then
+        git status --short
+        echo >&2 "Doing nothing to avoid losing your work."
+        exit 1
+    fi
+}

tools/lib/gitlint_rules.py

@@ -0,0 +1,108 @@
+from gitlint.git import GitCommit
+from gitlint.rules import CommitMessageTitle, LineRule, RuleViolation
+
+# Word list from https://github.com/m1foley/fit-commit
+# Copyright (c) 2015 Mike Foley
+# License: MIT
+# Ref: fit_commit/validators/tense.rb
+TENSE_DATA = [
+    (["adds", "adding", "added"], "add"),
+    (["allows", "allowing", "allowed"], "allow"),
+    (["amends", "amending", "amended"], "amend"),
+    (["bumps", "bumping", "bumped"], "bump"),
+    (["calculates", "calculating", "calculated"], "calculate"),
+    (["changes", "changing", "changed"], "change"),
+    (["cleans", "cleaning", "cleaned"], "clean"),
+    (["commits", "committing", "committed"], "commit"),
+    (["corrects", "correcting", "corrected"], "correct"),
+    (["creates", "creating", "created"], "create"),
+    (["darkens", "darkening", "darkened"], "darken"),
+    (["disables", "disabling", "disabled"], "disable"),
+    (["displays", "displaying", "displayed"], "display"),
+    (["documents", "documenting", "documented"], "document"),
+    (["drys", "drying", "dried"], "dry"),
+    (["ends", "ending", "ended"], "end"),
+    (["enforces", "enforcing", "enforced"], "enforce"),
+    (["enqueues", "enqueuing", "enqueued"], "enqueue"),
+    (["extracts", "extracting", "extracted"], "extract"),
+    (["finishes", "finishing", "finished"], "finish"),
+    (["fixes", "fixing", "fixed"], "fix"),
+    (["formats", "formatting", "formatted"], "format"),
+    (["guards", "guarding", "guarded"], "guard"),
+    (["handles", "handling", "handled"], "handle"),
+    (["hides", "hiding", "hid"], "hide"),
+    (["increases", "increasing", "increased"], "increase"),
+    (["ignores", "ignoring", "ignored"], "ignore"),
+    (["implements", "implementing", "implemented"], "implement"),
+    (["improves", "improving", "improved"], "improve"),
+    (["keeps", "keeping", "kept"], "keep"),
+    (["kills", "killing", "killed"], "kill"),
+    (["makes", "making", "made"], "make"),
+    (["merges", "merging", "merged"], "merge"),
+    (["moves", "moving", "moved"], "move"),
+    (["permits", "permitting", "permitted"], "permit"),
+    (["prevents", "preventing", "prevented"], "prevent"),
+    (["pushes", "pushing", "pushed"], "push"),
+    (["rebases", "rebasing", "rebased"], "rebase"),
+    (["refactors", "refactoring", "refactored"], "refactor"),
+    (["removes", "removing", "removed"], "remove"),
+    (["renames", "renaming", "renamed"], "rename"),
+    (["reorders", "reordering", "reordered"], "reorder"),
+    (["replaces", "replacing", "replaced"], "replace"),
+    (["requires", "requiring", "required"], "require"),
+    (["restores", "restoring", "restored"], "restore"),
+    (["sends", "sending", "sent"], "send"),
+    (["sets", "setting"], "set"),
+    (["separates", "separating", "separated"], "separate"),
+    (["shows", "showing", "showed"], "show"),
+    (["simplifies", "simplifying", "simplified"], "simplify"),
+    (["skips", "skipping", "skipped"], "skip"),
+    (["sorts", "sorting"], "sort"),
+    (["speeds", "speeding", "sped"], "speed"),
+    (["starts", "starting", "started"], "start"),
+    (["supports", "supporting", "supported"], "support"),
+    (["takes", "taking", "took"], "take"),
+    (["testing", "tested"], "test"),  # "tests" excluded to reduce false negatives
+    (["truncates", "truncating", "truncated"], "truncate"),
+    (["updates", "updating", "updated"], "update"),
+    (["uses", "using", "used"], "use"),
+]
+
+TENSE_CORRECTIONS = {word: imperative for words, imperative in TENSE_DATA for word in words}
+
+
+class ImperativeMood(LineRule):
+    """This rule will enforce that the commit message title uses imperative
+    mood. This is done by checking if the first word is in `WORD_SET`, if so
+    show the word in the correct mood."""
+
+    name = "title-imperative-mood"
+    id = "Z1"
+    target = CommitMessageTitle
+
+    error_msg = (
+        "The first word in commit title should be in imperative mood "
+        '("{word}" -> "{imperative}"): "{title}"'
+    )
+
+    def validate(self, line: str, commit: GitCommit) -> list[RuleViolation]:
+        violations = []
+
+        # Ignore the section tag (ie `<section tag>: <message body>.`)
+        words = line.split(": ", 1)[-1].split()
+        first_word = words[0].lower()
+
+        if first_word in TENSE_CORRECTIONS:
+            imperative = TENSE_CORRECTIONS[first_word]
+            violation = RuleViolation(
+                self.id,
+                self.error_msg.format(
+                    word=first_word,
+                    imperative=imperative,
+                    title=commit.message.title,
+                ),
+            )
+
+            violations.append(violation)
+
+        return violations

tools/lib/html_branches.py

@@ -0,0 +1,98 @@
+import re
+from collections import defaultdict
+
+from .template_parser import FormattedError, Token, tokenize
+
+
+class TagInfo:
+    def __init__(self, tag: str, classes: list[str], ids: list[str], token: Token) -> None:
+        self.tag = tag
+        self.classes = classes
+        self.ids = ids
+        self.token = token
+        self.words = [
+            self.tag,
+            *("." + s for s in classes),
+            *("#" + s for s in ids),
+        ]
+
+    def text(self) -> str:
+        s = self.tag
+        if self.classes:
+            s += "." + ".".join(self.classes)
+        if self.ids:
+            s += "#" + "#".join(self.ids)
+        return s
+
+
+def get_tag_info(token: Token) -> TagInfo:
+    s = token.s
+    tag = token.tag
+    classes: list[str] = []
+    ids: list[str] = []
+
+    searches = [
+        (classes, ' class="(.*?)"'),
+        (classes, " class='(.*?)'"),
+        (ids, ' id="(.*?)"'),
+        (ids, " id='(.*?)'"),
+    ]
+
+    for lst, regex in searches:
+        m = re.search(regex, s)
+        if m:
+            for g in m.groups():
+                lst += split_for_id_and_class(g)
+
+    return TagInfo(tag=tag, classes=classes, ids=ids, token=token)
+
+
+def split_for_id_and_class(element: str) -> list[str]:
+    # Here we split a given string which is expected to contain id or class
+    # attributes from HTML tags. This also takes care of template variables
+    # in string during splitting process. For eg. 'red black {{ a|b|c }}'
+    # is split as ['red', 'black', '{{ a|b|c }}']
+    outside_braces: bool = True
+    lst = []
+    s = ""
+
+    for ch in element:
+        if ch == "{":
+            outside_braces = False
+        if ch == "}":
+            outside_braces = True
+        if ch == " " and outside_braces:
+            if s != "":
+                lst.append(s)
+            s = ""
+        else:
+            s += ch
+    if s != "":
+        lst.append(s)
+
+    return lst
+
+
+def build_id_dict(templates: list[str]) -> dict[str, list[str]]:
+    template_id_dict: dict[str, list[str]] = defaultdict(list)
+
+    for fn in templates:
+        with open(fn) as f:
+            text = f.read()
+
+        try:
+            list_tags = tokenize(text)
+        except FormattedError as e:
+            raise Exception(
+                f"""
+                fn: {fn}
+                {e}"""
+            )
+
+        for tag in list_tags:
+            info = get_tag_info(tag)
+
+            for ids in info.ids:
+                template_id_dict[ids].append("Line " + str(info.token.line) + ":" + fn)
+
+    return template_id_dict

tools/lib/html_elements.py

@@ -0,0 +1,191 @@
+from collections.abc import Iterator
+
+VALID_HTML_CONTEXTS: dict[tuple[str, str], str] = {
+    # https://html.spec.whatwg.org/multipage/indices.html#elements-3
+    ("a", "phrasing"): "transparent",
+    ("abbr", "phrasing"): "phrasing",
+    ("address", "flow"): "flow",
+    ("area", "phrasing"): "void",
+    ("article", "flow"): "flow",
+    ("aside", "flow"): "flow",
+    ("audio", "phrasing"): "<audio>",
+    ("b", "phrasing"): "phrasing",
+    ("base", "<head>"): "void",
+    ("bdi", "phrasing"): "phrasing",
+    ("bdo", "phrasing"): "phrasing",
+    ("blockquote", "flow"): "flow",
+    ("body", "<html>"): "flow",
+    ("br", "phrasing"): "void",
+    ("button", "phrasing"): "phrasing",
+    ("button", "<select>"): "phrasing",
+    ("canvas", "phrasing"): "transparent",
+    ("caption", "<table>"): "flow",
+    ("center", "flow"): "flow",  # FIXME: obsolete, remove this
+    ("cite", "phrasing"): "phrasing",
+    ("code", "phrasing"): "phrasing",
+    ("col", "<colgroup>"): "void",
+    ("colgroup", "<table>"): "<colgroup>",
+    ("data", "phrasing"): "phrasing",
+    ("datalist", "phrasing"): "<datalist>",
+    ("dd", "<dl>"): "flow",
+    ("del", "phrasing"): "transparent",
+    ("details", "flow"): "<details>",
+    ("dfn", "phrasing"): "phrasing",
+    ("dialog", "flow"): "flow",
+    ("div", "flow"): "flow",
+    ("div", "<dl>"): "<dl>",
+    ("div", "<select> content"): "<select> content",
+    ("div", "<optgroup> content"): "<optgroup> content",
+    ("div", "<option> content"): "<option> content",
+    ("dl", "flow"): "<dl>",
+    ("dt", "<dl>"): "phrasing",
+    ("em", "phrasing"): "phrasing",
+    ("embed", "phrasing"): "void",
+    ("fieldset", "flow"): "<fieldset>",
+    ("figcaption", "<figure>"): "flow",
+    ("figure", "flow"): "<figure>",
+    ("footer", "flow"): "flow",
+    ("form", "flow"): "flow",
+    ("h1", "plain heading"): "phrasing",
+    ("h2", "plain heading"): "phrasing",
+    ("h3", "plain heading"): "phrasing",
+    ("h4", "plain heading"): "phrasing",
+    ("h5", "plain heading"): "phrasing",
+    ("h6", "plain heading"): "phrasing",
+    ("head", "<html>"): "<head>",
+    ("header", "flow"): "flow",
+    ("hgroup", "heading"): "<hgroup>",
+    ("hr", "flow"): "void",
+    ("hr", "<select> content"): "void",
+    ("html", "document"): "<html>",
+    ("i", "phrasing"): "phrasing",
+    ("iframe", "phrasing"): "empty",
+    ("img", "phrasing"): "void",
+    ("img", "<picture>"): "void",
+    ("input", "phrasing"): "void",
+    ("ins", "phrasing"): "transparent",
+    ("kbd", "phrasing"): "phrasing",
+    ("label", "phrasing"): "phrasing",
+    ("legend", "<fieldset>"): "phrasing/heading",
+    ("li", "list"): "flow",
+    ("link", "<head>"): "void",
+    ("link", "phrasing"): "void",
+    ("main", "flow"): "flow",
+    ("map", "phrasing"): "<map>",
+    ("mark", "phrasing"): "phrasing",
+    ("math", "phrasing"): "MathML",
+    ("menu", "flow"): "list",
+    ("meta", "<head>"): "void",
+    ("meta", "phrasing"): "void",
+    ("meter", "phrasing"): "phrasing",
+    ("nav", "flow"): "flow",
+    ("noscript", "<head>"): "transparent",
+    ("noscript", "phrasing"): "transparent",
+    ("noscript", "<select> content"): "transparent",
+    ("noscript", "<optgroup> content"): "transparent",
+    ("object", "phrasing"): "transparent",
+    ("ol", "flow"): "list",
+    ("optgroup", "<select>"): "<optgroup> content",
+    ("option", "<select>"): "<option> content",
+    ("option", "<datalist>"): "<option> content",
+    ("option", "<optgroup>"): "<option> content",
+    ("output", "phrasing"): "phrasing",
+    ("p", "flow"): "phrasing",
+    ("p", "<hgroup>"): "phrasing",
+    ("picture", "phrasing"): "<picture>",
+    ("pre", "flow"): "phrasing",
+    ("progress", "phrasing"): "phrasing",
+    ("q", "phrasing"): "phrasing",
+    ("rp", "<ruby>"): "phrasing",
+    ("rt", "<ruby>"): "phrasing",
+    ("ruby", "phrasing"): "<ruby>",
+    ("s", "phrasing"): "phrasing",
+    ("samp", "phrasing"): "phrasing",
+    ("script", "<head>"): "<script>",
+    ("script", "phrasing"): "<script>",
+    ("script", "script-supporting"): "<script>",
+    ("search", "flow"): "flow",
+    ("section", "flow"): "flow",
+    ("select", "phrasing"): "<select>",
+    ("selectedcontent", "<button>"): "empty",
+    ("slot", "phrasing"): "transparent",
+    ("small", "phrasing"): "phrasing",
+    ("source", "<picture>"): "void",
+    ("source", "<video>"): "void",
+    ("source", "<audio>"): "void",
+    ("span", "phrasing"): "phrasing",
+    ("strong", "phrasing"): "phrasing",
+    ("style", "<head>"): "<style>",
+    ("sub", "phrasing"): "phrasing",
+    ("summary", "<details>"): "phrasing/heading",
+    ("sup", "phrasing"): "phrasing",
+    ("svg", "phrasing"): "SVG",
+    ("table", "flow"): "<table>",
+    ("tbody", "<table>"): "<tbody>",
+    ("td", "<tr>"): "flow",
+    ("template", "<head>"): "unknown",
+    ("template", "phrasing"): "unknown",
+    ("template", "script-supporting"): "unknown",
+    ("template", "<colgroup>"): "unknown",
+    ("textarea", "phrasing"): "text",
+    ("tfoot", "<table>"): "<tfoot>",
+    ("th", "<tr>"): "flow",
+    ("thead", "<table>"): "<thead>",
+    ("time", "phrasing"): "phrasing",
+    ("title", "<head>"): "text",
+    ("tr", "<table>"): "<tr>",
+    ("tr", "<thead>"): "<tr>",
+    ("tr", "<tbody>"): "<tr>",
+    ("tr", "<tfoot>"): "<tr>",
+    ("track", "<audio>"): "void",
+    ("track", "<video>"): "void",
+    ("u", "phrasing"): "phrasing",
+    ("ul", "flow"): "list",
+    ("var", "phrasing"): "phrasing",
+    ("video", "phrasing"): "<video>",
+    ("wbr", "phrasing"): "void",
+    # https://html.spec.whatwg.org/multipage/embedded-content-other.html#mathml
+    ("annotation-xml", "MathML"): "flow",
+    ("mi", "MathML"): "phrasing",
+    ("mo", "MathML"): "phrasing",
+    ("mn", "MathML"): "phrasing",
+    ("ms", "MathML"): "phrasing",
+    ("mtext", "MathML"): "phrasing",
+    # https://html.spec.whatwg.org/multipage/embedded-content-other.html#svg-0
+    ("foreignObject", "SVG"): "flow",
+    ("title", "SVG"): "phrasing",
+}
+
+HTML_CONTEXT_FALLBACKS: dict[str, list[str]] = {
+    "<datalist>": ["phrasing", "script-supporting"],
+    "<details>": ["flow"],
+    "<dl>": ["script-supporting"],
+    "<fieldset>": ["flow"],
+    "<figure>": ["flow"],
+    "<hgroup>": ["plain heading", "script-supporting"],
+    "<optgroup> content": ["script-supporting"],
+    "<option> content": ["phrasing"],
+    "<option>": ["<option> content"],
+    "<picture>": ["script-supporting"],
+    "<ruby>": ["phrasing"],
+    "<select> content": ["script-supporting"],
+    "<select>": ["<select> content"],
+    "<table>": ["script-supporting"],
+    "<tbody>": ["script-supporting"],
+    "<tfoot>": ["script-supporting"],
+    "<thead>": ["script-supporting"],
+    "<tr>": ["script-supporting"],
+    "flow": ["phrasing", "heading"],
+    "heading": ["plain heading"],
+    "list": ["script-supporting"],
+    "phrasing/heading": ["phrasing", "heading"],
+    "unknown": ["document", "flow", "list", "<head>", "<select>", "<table>", "<tr>"],
+}
+
+FOREIGN_CONTEXTS = ["MathML", "SVG"]
+
+
+def html_context_fallbacks(context: str) -> Iterator[str]:
+    yield context
+    for fallback_context in HTML_CONTEXT_FALLBACKS.get(context, []):
+        yield from html_context_fallbacks(fallback_context)

tools/lib/pretty_print.py

@@ -0,0 +1,197 @@
+import subprocess
+
+from zulint.printer import BOLDRED, CYAN, ENDC, GREEN
+
+from .template_parser import Token
+
+
+def shift_indents_to_the_next_tokens(tokens: list[Token]) -> None:
+    """
+    During the parsing/validation phase, it's useful to have separate
+    tokens for "indent" chunks, but during pretty printing, we like
+    to attach an `.indent` field to the substantive node, whether
+    it's an HTML tag or template directive or whatever.
+    """
+    tokens[0].indent = ""
+
+    for i, token in enumerate(tokens[:-1]):
+        next_token = tokens[i + 1]
+
+        if token.kind == "indent":
+            next_token.indent = token.s
+            token.new_s = ""
+
+        if token.kind == "newline" and next_token.kind != "indent":
+            next_token.indent = ""
+
+
+def token_allows_children_to_skip_indents(token: Token) -> bool:
+    # To avoid excessive indentation in templates with other
+    # conditionals, we don't require extra indentation for template
+    # logic blocks don't contain further logic as direct children.
+
+    # Each blocks are excluded from this rule, since we want loops to
+    # stand out.
+    if token.tag == "each":
+        return False
+
+    return token.kind in ("django_start", "handlebars_start") or token.tag == "a"
+
+
+def adjust_block_indentation(tokens: list[Token], fn: str) -> None:
+    start_token: Token | None = None
+
+    for token in tokens:
+        if token.kind in ("indent", "whitespace", "newline"):
+            continue
+
+        if token.tag in ("code", "pre"):
+            continue
+
+        # print(token.line, repr(start_token.indent) if start_token else "?", repr(token.indent), token.s, token.end_token and "start", token.start_token and "end")
+
+        if token.tag == "else":
+            assert token.start_token
+            if token.indent is not None:
+                token.indent = token.start_token.indent
+            continue
+
+        if start_token and token.indent is not None:
+            if (
+                not start_token.indent_is_final
+                and token.indent == start_token.orig_indent
+                and token_allows_children_to_skip_indents(start_token)
+            ):
+                start_token.child_indent = start_token.indent
+            start_token.indent_is_final = True
+
+        # Detect start token by its having a end token
+        if token.end_token:
+            if token.indent is not None:
+                token.orig_indent = token.indent
+                if start_token:
+                    assert start_token.child_indent is not None
+                    token.indent = start_token.child_indent
+                else:
+                    token.indent = ""
+                token.child_indent = token.indent + "    "
+            token.parent_token = start_token
+            start_token = token
+            continue
+
+        # Detect end token by its having a start token
+        if token.start_token:
+            if start_token != token.start_token:
+                raise AssertionError(
+                    f"""
+                    {token.kind} was unexpected in {token.s}
+                    in row {token.line} of {fn}
+                    """
+                )
+
+            if token.indent is not None:
+                token.indent = start_token.indent
+            start_token = start_token.parent_token
+            continue
+
+        if token.indent is None:
+            continue
+
+        if start_token is None:
+            token.indent = ""
+            continue
+
+        if start_token.child_indent is not None:
+            token.indent = start_token.child_indent
+
+
+def fix_indents_for_multi_line_tags(tokens: list[Token]) -> None:
+    def fix(frag: str) -> str:
+        frag = frag.strip()
+        return continue_indent + frag if frag else ""
+
+    for token in tokens:
+        if token.kind == "code":
+            continue
+
+        if token.line_span == 1 or token.indent is None:
+            continue
+
+        if token.kind in ("django_comment", "handlebars_comment", "html_comment", "text"):
+            continue_indent = token.indent
+        else:
+            continue_indent = token.indent + "  "
+
+        frags = token.new_s.split("\n")
+
+        token.new_s = frags[0] + "\n" + "\n".join(fix(frag) for frag in frags[1:])
+
+
+def apply_token_indents(tokens: list[Token]) -> None:
+    for token in tokens:
+        if token.indent:
+            token.new_s = token.indent + token.new_s
+
+
+def pretty_print_html(tokens: list[Token], fn: str) -> str:
+    for token in tokens:
+        token.new_s = token.s
+
+    shift_indents_to_the_next_tokens(tokens)
+    adjust_block_indentation(tokens, fn)
+    fix_indents_for_multi_line_tags(tokens)
+    apply_token_indents(tokens)
+
+    return "".join(token.new_s for token in tokens)
+
+
+def numbered_lines(s: str) -> str:
+    return "".join(f"{i + 1: >5} {line}\n" for i, line in enumerate(s.split("\n")))
+
+
+def validate_indent_html(fn: str, tokens: list[Token], fix: bool) -> bool:
+    with open(fn) as f:
+        html = f.read()
+    phtml = pretty_print_html(tokens, fn)
+    if html.split("\n") != phtml.split("\n"):
+        if fix:
+            print(GREEN + f"Automatically fixing indentation for {fn}" + ENDC)
+            with open(fn, "w") as f:
+                f.write(phtml)
+            # Since we successfully fixed the issues, we return True.
+            return True
+        print(
+            f"""
+{BOLDRED}PROBLEM{ENDC}: formatting errors in {fn}
+
+Here is how we would like you to format
+{CYAN}{fn}{ENDC}:
+---
+{numbered_lines(phtml)}
+---
+
+Here is the diff that you should either execute in your editor
+or apply automatically with the --fix option.
+
+({CYAN}Scroll up{ENDC} to see how we would like the file formatted.)
+
+Proposed {BOLDRED}diff{ENDC} for {CYAN}{fn}{ENDC}:
+            """,
+            flush=True,
+        )
+        subprocess.run(["diff", fn, "-"], input=phtml, text=True, check=False)
+        print(
+            f"""
+---
+
+{BOLDRED}PROBLEM!!!{ENDC}
+
+    You have formatting errors in {CYAN}{fn}{ENDC}
+    (Usually these messages are related to indentation.)
+
+This problem can be fixed with the {CYAN}`--fix`{ENDC} option.
+Scroll up for more details about {BOLDRED}what you need to fix ^^^{ENDC}.
+            """
+        )
+        return False
+    return True

tools/lib/provision.py

@@ -0,0 +1,512 @@
+#!/usr/bin/env python3
+import argparse
+import contextlib
+import hashlib
+import logging
+import os
+import platform
+import subprocess
+import sys
+from typing import NoReturn
+
+os.environ["PYTHONUNBUFFERED"] = "y"
+
+ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+sys.path.append(ZULIP_PATH)
+
+from scripts.lib.node_cache import setup_node_modules
+from scripts.lib.setup_venv import get_venv_dependencies
+from scripts.lib.zulip_tools import (
+    ENDC,
+    FAIL,
+    WARNING,
+    get_dev_uuid_var_path,
+    os_families,
+    run,
+    run_as_root,
+)
+
+VAR_DIR_PATH = os.path.join(ZULIP_PATH, "var")
+
+CONTINUOUS_INTEGRATION = "GITHUB_ACTIONS" in os.environ
+
+if not os.path.exists(os.path.join(ZULIP_PATH, ".git")):
+    print(FAIL + "Error: No Zulip Git repository present!" + ENDC)
+    print("To set up the Zulip development environment, you should clone the code")
+    print("from GitHub, rather than using a Zulip production release tarball.")
+    sys.exit(1)
+
+# Check the RAM on the user's system, and throw an effort if <1.5GB.
+# This avoids users getting segfaults running `pip install` that are
+# generally more annoying to debug.
+with open("/proc/meminfo") as meminfo:
+    ram_size = meminfo.readlines()[0].strip().split(" ")[-2]
+ram_gb = float(ram_size) / 1024.0 / 1024.0
+if ram_gb < 1.5:
+    print(
+        f"You have insufficient RAM ({round(ram_gb, 2)} GB) to run the Zulip development environment."
+    )
+    print("We recommend at least 2 GB of RAM, and require at least 1.5 GB.")
+    sys.exit(1)
+
+try:
+    UUID_VAR_PATH = get_dev_uuid_var_path(create_if_missing=True)
+    os.makedirs(UUID_VAR_PATH, exist_ok=True)
+    if os.path.exists(os.path.join(VAR_DIR_PATH, "zulip-test-symlink")):
+        os.remove(os.path.join(VAR_DIR_PATH, "zulip-test-symlink"))
+    os.symlink(
+        os.path.join(ZULIP_PATH, "README.md"),
+        os.path.join(VAR_DIR_PATH, "zulip-test-symlink"),
+    )
+    os.remove(os.path.join(VAR_DIR_PATH, "zulip-test-symlink"))
+except OSError:
+    print(
+        FAIL + "Error: Unable to create symlinks. "
+        "Make sure you have permission to create symbolic links." + ENDC
+    )
+    print("See this page for more information:")
+    print(
+        "  https://zulip.readthedocs.io/en/latest/development/setup-recommended.html#os-symlink-error"
+    )
+    sys.exit(1)
+
+distro_info = platform.freedesktop_os_release()
+vendor = distro_info["ID"]
+os_version = distro_info["VERSION_ID"]
+if vendor == "debian" and os_version == "12":  # bookworm
+    POSTGRESQL_VERSION = "15"
+elif vendor == "debian" and os_version == "13":  # trixie
+    POSTGRESQL_VERSION = "17"
+elif vendor == "ubuntu" and os_version == "22.04":  # jammy
+    POSTGRESQL_VERSION = "14"
+elif vendor == "ubuntu" and os_version == "24.04":  # noble
+    POSTGRESQL_VERSION = "16"
+elif vendor == "ubuntu" and os_version == "26.04":  # resolute
+    POSTGRESQL_VERSION = "18"
+elif vendor == "fedora" and os_version in ("43", "44"):
+    POSTGRESQL_VERSION = "17"
+elif vendor == "rhel" and os_version.startswith("7."):
+    POSTGRESQL_VERSION = "10"
+elif vendor == "centos" and os_version == "7":
+    POSTGRESQL_VERSION = "10"
+else:
+    logging.critical("Unsupported platform: %s %s", vendor, os_version)
+    sys.exit(1)
+
+VENV_DEPENDENCIES = get_venv_dependencies(vendor, os_version)
+
+COMMON_DEPENDENCIES = [
+    "memcached",
+    "rabbitmq-server",
+    "supervisor",
+    "git",
+    "curl",
+    "ca-certificates",  # Explicit dependency in case e.g. curl is already installed
+    "puppet",  # Used by lint (`puppet parser validate`)
+    "gettext",  # Used by makemessages i18n
+    "curl",  # Used for testing our API documentation
+    "moreutils",  # Used for sponge command
+    "unzip",  # Needed for Slack import
+    "crudini",  # Used for shell tooling w/ zulip.conf
+    # Puppeteer dependencies from here
+    "xdg-utils",
+    # Puppeteer dependencies end here.
+]
+
+UBUNTU_COMMON_APT_DEPENDENCIES = [
+    *COMMON_DEPENDENCIES,
+    "redis-server",
+    "hunspell-en-us",
+    "puppet-lint",
+    (
+        "openjdk-17-jre-headless"
+        if vendor == "ubuntu" and os_version == "22.04"
+        else "default-jre-headless"
+    ),  # Required by vnu-jar
+    # Puppeteer dependencies from here
+    "fonts-freefont-ttf",
+    "libatk-bridge2.0-0",
+    "libgbm1",
+    "libgtk-3-0",
+    "libx11-xcb1",
+    "libxcb-dri3-0",
+    "libxss1",
+    "xvfb",
+    # Puppeteer dependencies end here.
+]
+
+COMMON_YUM_DEPENDENCIES = [
+    *COMMON_DEPENDENCIES,
+    "redis",
+    "hunspell-en-US",
+    "rubygem-puppet-lint",
+    "nmap-ncat",
+    "ccache",  # Required to build pgroonga from source.
+    # Puppeteer dependencies from here
+    "at-spi2-atk",
+    "GConf2",
+    "gtk3",
+    "libX11-xcb",
+    "libxcb",
+    "libXScrnSaver",
+    "mesa-libgbm",
+    "xorg-x11-server-Xvfb",
+    # Puppeteer dependencies end here.
+]
+
+BUILD_PGROONGA_FROM_SOURCE = False
+if (vendor == "debian" and os_version == "13") or (vendor == "ubuntu" and os_version == "26.04"):
+    # For platforms without a PGroonga release, we need to build it
+    # from source.
+    BUILD_PGROONGA_FROM_SOURCE = True
+    SYSTEM_DEPENDENCIES = [
+        *UBUNTU_COMMON_APT_DEPENDENCIES,
+        f"postgresql-{POSTGRESQL_VERSION}",
+        # Dependency for building PGroonga from source
+        "meson",
+        f"postgresql-server-dev-{POSTGRESQL_VERSION}",
+        "libgroonga-dev",
+        "libmsgpack-dev",
+        *VENV_DEPENDENCIES,
+    ]
+elif "debian" in os_families():
+    DEBIAN_DEPENDENCIES = UBUNTU_COMMON_APT_DEPENDENCIES
+
+    # If we are on an aarch64 processor, ninja will be built from source,
+    # so cmake is required
+    if platform.machine() == "aarch64":
+        DEBIAN_DEPENDENCIES.append("cmake")
+
+    SYSTEM_DEPENDENCIES = [
+        *DEBIAN_DEPENDENCIES,
+        f"postgresql-{POSTGRESQL_VERSION}",
+        f"postgresql-{POSTGRESQL_VERSION}-pgroonga",
+        *VENV_DEPENDENCIES,
+    ]
+elif "rhel" in os_families():
+    SYSTEM_DEPENDENCIES = [
+        *COMMON_YUM_DEPENDENCIES,
+        f"postgresql{POSTGRESQL_VERSION}-server",
+        f"postgresql{POSTGRESQL_VERSION}",
+        f"postgresql{POSTGRESQL_VERSION}-devel",
+        f"postgresql{POSTGRESQL_VERSION}-pgdg-pgroonga",
+        *VENV_DEPENDENCIES,
+    ]
+elif "fedora" in os_families():
+    SYSTEM_DEPENDENCIES = [
+        *COMMON_YUM_DEPENDENCIES,
+        f"postgresql{POSTGRESQL_VERSION}-server",
+        f"postgresql{POSTGRESQL_VERSION}",
+        f"postgresql{POSTGRESQL_VERSION}-devel",
+        # Needed to build PGroonga from source
+        "groonga-devel",
+        "msgpack-devel",
+        # PGroonga compiles against <xxhash.h>; on systems where
+        # `xxhash-libs` is already pulled in transitively (e.g. by
+        # blosc2 / pyarrow) but `xxhash-devel` is not, meson detects
+        # libxxhash via pkg-config and skips its vendored fallback,
+        # so we have to provide the header ourselves.
+        "xxhash-devel",
+        "meson",
+        # Provides /usr/lib/rpm/redhat/redhat-hardened-cc1 and
+        # redhat-annobin-cc1 spec files referenced by the CFLAGS that
+        # PostgreSQL's pg_config exports; without it the PGroonga
+        # compile fails with "cannot read spec file".
+        "redhat-rpm-config",
+        *VENV_DEPENDENCIES,
+    ]
+    BUILD_PGROONGA_FROM_SOURCE = True
+
+if "fedora" in os_families():
+    TSEARCH_STOPWORDS_PATH = f"/usr/pgsql-{POSTGRESQL_VERSION}/share/tsearch_data/"
+else:
+    TSEARCH_STOPWORDS_PATH = f"/usr/share/postgresql/{POSTGRESQL_VERSION}/tsearch_data/"
+REPO_STOPWORDS_PATH = os.path.join(
+    ZULIP_PATH,
+    "puppet",
+    "zulip",
+    "files",
+    "postgresql",
+    "zulip_english.stop",
+)
+
+
+def install_system_deps() -> None:
+    # By doing list -> set -> list conversion, we remove duplicates.
+    deps_to_install = sorted(set(SYSTEM_DEPENDENCIES))
+
+    if "fedora" in os_families():
+        install_yum_deps(deps_to_install)
+    elif "debian" in os_families():
+        install_apt_deps(deps_to_install)
+    else:
+        raise AssertionError("Invalid vendor")
+
+    # For some platforms, there aren't published PGroonga
+    # packages available, so we build them from source.
+    if BUILD_PGROONGA_FROM_SOURCE:
+        run_as_root(["./scripts/lib/build-pgroonga"])
+
+
+def install_apt_deps(deps_to_install: list[str]) -> None:
+    # setup-apt-repo does an `apt-get update` if the sources.list files changed.
+    run_as_root(["./scripts/lib/setup-apt-repo"])
+
+    # But we still need to do our own to make sure we have up-to-date
+    # data before installing new packages, as the system might not have
+    # done an apt update in weeks otherwise, which could result in 404s
+    # trying to download old versions that were already removed from mirrors.
+    run_as_root(["apt-get", "update"])
+    run_as_root(
+        [
+            "env",
+            "DEBIAN_FRONTEND=noninteractive",
+            "apt-get",
+            "-y",
+            "install",
+            "--allow-downgrades",
+            "--no-install-recommends",
+            *deps_to_install,
+        ]
+    )
+
+
+def install_yum_deps(deps_to_install: list[str]) -> None:
+    print(WARNING + "RedHat support is still experimental." + ENDC)
+    run_as_root(["./scripts/lib/setup-yum-repo"])
+
+    # Hack specific to unregistered RHEL system.  The moreutils
+    # package requires a perl module package, which isn't available in
+    # the unregistered RHEL repositories.
+    #
+    # Error: Package: moreutils-0.49-2.el7.x86_64 (epel)
+    #        Requires: perl(IPC::Run)
+    yum_extra_flags: list[str] = []
+    if vendor == "rhel":
+        proc = subprocess.run(
+            ["sudo", "subscription-manager", "status"],
+            stdout=subprocess.PIPE,
+            text=True,
+            check=False,
+        )
+        if proc.returncode == 1:
+            # TODO this might overkill since `subscription-manager` is already
+            # called in setup-yum-repo
+            if "Status" in proc.stdout:
+                # The output is well-formed
+                yum_extra_flags = ["--skip-broken"]
+            else:
+                print("Unrecognized output. `subscription-manager` might not be available")
+
+    run_as_root(["yum", "install", "-y", *yum_extra_flags, *deps_to_install])
+    if "rhel" in os_families():
+        # This is how a pip3 is installed to /usr/bin in CentOS/RHEL
+        # for python35 and later.
+        run_as_root(["python36", "-m", "ensurepip"])
+        # `python36` is not aliased to `python3` by default
+        run_as_root(["ln", "-nsf", "/usr/bin/python36", "/usr/bin/python3"])
+    postgresql_dir = f"pgsql-{POSTGRESQL_VERSION}"
+    for cmd in ["pg_config", "pg_isready", "psql"]:
+        # Our tooling expects these PostgreSQL scripts to be at
+        # well-known paths.  There's an argument for eventually
+        # making our tooling auto-detect, but this is simpler.
+        run_as_root(["ln", "-nsf", f"/usr/{postgresql_dir}/bin/{cmd}", f"/usr/bin/{cmd}"])
+
+    # From here, we do the first-time setup/initialization for the PostgreSQL database.
+    pg_datadir = f"/var/lib/pgsql/{POSTGRESQL_VERSION}/data"
+    pg_hba_conf = os.path.join(pg_datadir, "pg_hba.conf")
+
+    # We can't just check if the file exists with os.path, since the
+    # current user likely doesn't have permission to read the
+    # pg_datadir directory.
+    if subprocess.call(["sudo", "test", "-e", pg_hba_conf]) == 0:
+        # Skip setup if it has been applied previously
+        return
+
+    run_as_root(
+        [f"/usr/{postgresql_dir}/bin/postgresql-{POSTGRESQL_VERSION}-setup", "initdb"],
+        sudo_args=["-H"],
+    )
+    # Use vendored pg_hba.conf, which enables password authentication.
+    run_as_root(["cp", "-a", "puppet/zulip/files/postgresql/centos_pg_hba.conf", pg_hba_conf])
+    # Later steps will ensure PostgreSQL is started
+
+    # Link in tsearch data files
+    if vendor == "fedora":
+        # Since F36 dictionary files were moved away from /usr/share/myspell
+        tsearch_source_prefix = "/usr/share/hunspell"
+    else:
+        tsearch_source_prefix = "/usr/share/myspell"
+    run_as_root(
+        [
+            "ln",
+            "-nsf",
+            os.path.join(tsearch_source_prefix, "en_US.dic"),
+            f"/usr/pgsql-{POSTGRESQL_VERSION}/share/tsearch_data/en_us.dict",
+        ]
+    )
+    run_as_root(
+        [
+            "ln",
+            "-nsf",
+            os.path.join(tsearch_source_prefix, "en_US.aff"),
+            f"/usr/pgsql-{POSTGRESQL_VERSION}/share/tsearch_data/en_us.affix",
+        ]
+    )
+
+
+def main(options: argparse.Namespace) -> NoReturn:
+    # pnpm and management commands expect to be run from the root of the
+    # project.
+    os.chdir(ZULIP_PATH)
+
+    # hash the apt dependencies
+    sha_sum = hashlib.sha1()
+
+    for apt_dependency in SYSTEM_DEPENDENCIES:
+        sha_sum.update(apt_dependency.encode())
+    if "debian" in os_families():
+        with open("scripts/lib/setup-apt-repo", "rb") as fb:
+            sha_sum.update(fb.read())
+    else:
+        # hash the content of setup-yum-repo*
+        with open("scripts/lib/setup-yum-repo", "rb") as fb:
+            sha_sum.update(fb.read())
+
+    # hash the content of build-pgroonga if PGroonga is built from source
+    if BUILD_PGROONGA_FROM_SOURCE:
+        with open("scripts/lib/build-pgroonga", "rb") as fb:
+            sha_sum.update(fb.read())
+
+    new_apt_dependencies_hash = sha_sum.hexdigest()
+    last_apt_dependencies_hash = None
+    apt_hash_file_path = os.path.join(UUID_VAR_PATH, "apt_dependencies_hash")
+    with open(apt_hash_file_path, "a+") as hash_file:
+        hash_file.seek(0)
+        last_apt_dependencies_hash = hash_file.read()
+
+    if new_apt_dependencies_hash != last_apt_dependencies_hash:
+        try:
+            install_system_deps()
+        except subprocess.CalledProcessError:
+            try:
+                # Might be a failure due to network connection issues. Retrying...
+                print(WARNING + "Installing system dependencies failed; retrying..." + ENDC)
+                install_system_deps()
+            except BaseException as e:
+                # Suppress exception chaining
+                raise e from None
+        with open(apt_hash_file_path, "w") as hash_file:
+            hash_file.write(new_apt_dependencies_hash)
+    else:
+        print("No changes to apt dependencies, so skipping apt operations.")
+
+    # Here we install node.
+    proxy_env = [
+        "env",
+        "http_proxy=" + os.environ.get("http_proxy", ""),
+        "https_proxy=" + os.environ.get("https_proxy", ""),
+        "no_proxy=" + os.environ.get("no_proxy", ""),
+    ]
+    # Preserve PATH to catch mistaken extra installations of node in the user's
+    # home directory.
+    run_as_root([*proxy_env, "scripts/lib/install-node"], sudo_args=["--preserve-env=PATH"])
+
+    try:
+        setup_node_modules()
+    except subprocess.CalledProcessError:
+        print(WARNING + "`pnpm install` failed; retrying..." + ENDC)
+        try:
+            setup_node_modules()
+        except subprocess.CalledProcessError:
+            print(
+                FAIL
+                + "`pnpm install` is failing; check your network connection (and proxy settings)."
+                + ENDC
+            )
+            sys.exit(1)
+
+    # Install shellcheck.
+    run_as_root([*proxy_env, "tools/setup/install-shellcheck"], sudo_args=["--preserve-env=PATH"])
+    # Install shfmt.
+    run_as_root([*proxy_env, "tools/setup/install-shfmt"], sudo_args=["--preserve-env=PATH"])
+
+    # Install tusd
+    run_as_root([*proxy_env, "tools/setup/install-tusd"], sudo_args=["--preserve-env=PATH"])
+
+    # Install Python environment
+    run_as_root([*proxy_env, "scripts/lib/install-uv"], sudo_args=["--preserve-env=PATH"])
+    run(
+        [*proxy_env, "uv", "sync", "--frozen", "--no-managed-python"],
+        env={k: v for k, v in os.environ.items() if k not in {"PYTHONDEVMODE", "PYTHONWARNINGS"}},
+    )
+    # Clean old symlinks used before uv migration
+    with contextlib.suppress(FileNotFoundError):
+        os.unlink("zulip-py3-venv")
+    if os.path.lexists("/srv/zulip-py3-venv"):
+        run_as_root(["rm", "/srv/zulip-py3-venv"])
+
+    run_as_root(["cp", REPO_STOPWORDS_PATH, TSEARCH_STOPWORDS_PATH])
+
+    if CONTINUOUS_INTEGRATION and not options.is_build_release_tarball_only:
+        run_as_root(["service", "redis-server", "start"])
+        run_as_root(["service", "memcached", "start"])
+        run_as_root(["service", "rabbitmq-server", "start"])
+        run_as_root(["service", "postgresql", "start"])
+    elif "fedora" in os_families():
+        # These platforms don't enable and start services on
+        # installing their package, so we do that here.
+        for service in [
+            f"postgresql-{POSTGRESQL_VERSION}",
+            "rabbitmq-server",
+            "memcached",
+            "redis",
+        ]:
+            run_as_root(["systemctl", "enable", service], sudo_args=["-H"])
+            run_as_root(["systemctl", "start", service], sudo_args=["-H"])
+
+    # If we imported modules after activating the virtualenv in this
+    # Python process, they could end up mismatching with modules we’ve
+    # already imported from outside the virtualenv.  That seems like a
+    # bad idea, and empirically it can cause Python to segfault on
+    # certain cffi-related imports.  Instead, start a new Python
+    # process inside the virtualenv.
+    provision_inner = os.path.join(ZULIP_PATH, "tools", "lib", "provision_inner.py")
+    os.execvp(
+        "uv",
+        [
+            "uv",
+            "run",
+            "--no-sync",
+            provision_inner,
+            *(["--force"] if options.is_force else []),
+            *(["--build-release-tarball-only"] if options.is_build_release_tarball_only else []),
+            *(["--skip-dev-db-build"] if options.skip_dev_db_build else []),
+        ],
+    )
+
+
+if __name__ == "__main__":
+    description = "Provision script to install Zulip"
+    parser = argparse.ArgumentParser(description=description)
+    parser.add_argument(
+        "--force",
+        action="store_true",
+        dest="is_force",
+        help="Ignore all provisioning optimizations.",
+    )
+
+    parser.add_argument(
+        "--build-release-tarball-only",
+        action="store_true",
+        dest="is_build_release_tarball_only",
+        help="Provision needed to build release tarball.",
+    )
+
+    parser.add_argument(
+        "--skip-dev-db-build", action="store_true", help="Don't run migrations on dev database."
+    )
+
+    options = parser.parse_args()
+    main(options)

tools/lib/provision_inner.py

@@ -0,0 +1,451 @@
+#!/usr/bin/env python3
+
+############################## NOTE ################################
+# This script is used to provision a development environment ONLY.
+# For production, extend update-prod-static to generate new static
+# assets, and puppet to install other software.
+####################################################################
+
+import argparse
+import glob
+import os
+import pwd
+import re
+import shlex
+import shutil
+import subprocess
+import sys
+
+ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+sys.path.append(ZULIP_PATH)
+import pygments
+
+from scripts.lib import clean_unused_caches
+from scripts.lib.zulip_tools import (
+    ENDC,
+    OKBLUE,
+    get_dev_uuid_var_path,
+    get_tzdata_zi,
+    is_digest_obsolete,
+    run,
+    run_as_root,
+    write_new_digest,
+)
+from tools.setup.generate_bots_integrations_static_files import (
+    generate_pythonapi_integrations_static_files,
+    generate_zulip_bots_static_files,
+)
+from version import PROVISION_VERSION
+
+VENV_PATH = os.path.join(ZULIP_PATH, ".venv")
+UUID_VAR_PATH = get_dev_uuid_var_path()
+
+with get_tzdata_zi() as f:
+    line = f.readline()
+    assert line.startswith("# version ")
+    timezones_version = line.removeprefix("# version ")
+
+
+def create_var_directories() -> None:
+    # create var/coverage, var/log, etc.
+    var_dir = os.path.join(ZULIP_PATH, "var")
+    sub_dirs = [
+        "coverage",
+        "log",
+        "node-coverage",
+        "test_uploads",
+        "uploads",
+        "xunit-test-results",
+    ]
+    for sub_dir in sub_dirs:
+        path = os.path.join(var_dir, sub_dir)
+        os.makedirs(path, exist_ok=True)
+
+
+def build_pygments_data_paths() -> list[str]:
+    paths = [
+        "tools/setup/build_pygments_data",
+        "tools/setup/lang.json",
+    ]
+    return paths
+
+
+def build_timezones_data_paths() -> list[str]:
+    paths = [
+        "tools/setup/build_timezone_values",
+    ]
+    return paths
+
+
+def build_landing_page_images_paths() -> list[str]:
+    paths = ["tools/setup/generate_landing_page_images.py"]
+    paths += glob.glob("static/images/landing-page/hello/original/*")
+    return paths
+
+
+def compilemessages_paths() -> list[str]:
+    paths = ["zerver/management/commands/compilemessages.py"]
+    paths += glob.glob("locale/*/LC_MESSAGES/*.po")
+    paths += glob.glob("locale/*/translations.json")
+    return paths
+
+
+def configure_rabbitmq_paths() -> list[str]:
+    paths = [
+        "scripts/setup/configure-rabbitmq",
+    ]
+    return paths
+
+
+def is_wsl_instance() -> bool:
+    if "WSL_DISTRO_NAME" in os.environ:
+        return True
+
+    with open("/proc/version") as file:
+        content = file.read().lower()
+        if "microsoft" in content:
+            return True
+
+    result = subprocess.run(["uname", "-r"], capture_output=True, text=True, check=True)
+    if "microsoft" in result.stdout.lower():
+        return True
+
+    return False
+
+
+def is_vagrant_or_digitalocean_instance() -> bool:
+    user_id = os.getuid()
+    user_name = pwd.getpwuid(user_id).pw_name
+    return user_name in ["vagrant", "zulipdev"]
+
+
+def setup_shell_profile(shell_profile: str) -> None:
+    shell_profile_path = os.path.expanduser(shell_profile)
+    if os.path.exists(shell_profile_path):
+        with open(shell_profile_path) as f:
+            code = f.read()
+    else:
+        code = ""
+
+    # We want to activate the virtual environment for login shells only on virtualized systems.
+    zulip_code = ""
+    if is_vagrant_or_digitalocean_instance() or is_wsl_instance():
+        zulip_code += (
+            "if [ -L /srv/zulip-py3-venv ]; then\n"  # For development environment downgrades
+            "source /srv/zulip-py3-venv/bin/activate\n"  # Not indented so old versions recognize and avoid re-adding this
+            "else\n"
+            f"source {shlex.quote(os.path.join(VENV_PATH, 'bin', 'activate'))}\n"
+            "fi\n"
+        )
+    zulip_code += f"cd {shlex.quote(ZULIP_PATH)}\n"
+    if zulip_code:
+        zulip_code = f"\n# begin Zulip setup\n{zulip_code}# end Zulip setup\n"
+
+    def patch_code(code: str) -> str:
+        return re.sub(
+            r"\n# begin Zulip setup\n(?s:.*)# end Zulip setup\n|(?:source /srv/zulip-py3-venv/bin/activate\n|cd /srv/zulip\n)+|\Z",
+            lambda m: zulip_code,
+            code,
+            count=1,
+        )
+
+    new_code = patch_code(code)
+    if new_code != code:
+        assert patch_code(new_code) == new_code
+        with open(f"{shell_profile_path}.new", "w") as f:
+            f.write(new_code)
+        os.rename(f"{shell_profile_path}.new", shell_profile_path)
+
+
+def setup_bash_profile() -> None:
+    """Select a bash profile file to add setup code to."""
+
+    BASH_PROFILES = [
+        os.path.expanduser(p) for p in ("~/.bash_profile", "~/.bash_login", "~/.profile")
+    ]
+
+    def clear_old_profile() -> None:
+        # An earlier version of this script would output a fresh .bash_profile
+        # even though a .profile existed in the image used. As a convenience to
+        # existing developers (and, perhaps, future developers git-bisecting the
+        # provisioning scripts), check for this situation, and blow away the
+        # created .bash_profile if one is found.
+
+        BASH_PROFILE = BASH_PROFILES[0]
+        DOT_PROFILE = BASH_PROFILES[2]
+        OLD_PROFILE_TEXT = "source /srv/zulip-py3-venv/bin/activate\ncd /srv/zulip\n"
+
+        if os.path.exists(DOT_PROFILE):
+            try:
+                with open(BASH_PROFILE) as f:
+                    profile_contents = f.read()
+                if profile_contents == OLD_PROFILE_TEXT:
+                    os.unlink(BASH_PROFILE)
+            except FileNotFoundError:
+                pass
+
+    clear_old_profile()
+
+    for candidate_profile in BASH_PROFILES:
+        if os.path.exists(candidate_profile):
+            setup_shell_profile(candidate_profile)
+            break
+    else:
+        # no existing bash profile found; claim .bash_profile
+        setup_shell_profile(BASH_PROFILES[0])
+
+
+def need_to_run_build_pygments_data() -> bool:
+    if not os.path.exists("web/generated/pygments_data.json"):
+        return True
+
+    return is_digest_obsolete(
+        "build_pygments_data_hash",
+        build_pygments_data_paths(),
+        [pygments.__version__],
+    )
+
+
+def need_to_run_build_timezone_data() -> bool:
+    if not os.path.exists("web/generated/timezones.json"):
+        return True
+
+    return is_digest_obsolete(
+        "build_timezones_data_hash",
+        build_timezones_data_paths(),
+        [timezones_version],
+    )
+
+
+def need_to_regenerate_landing_page_images() -> bool:
+    if not os.path.exists("static/images/landing-page/hello/generated"):
+        return True
+
+    return is_digest_obsolete(
+        "landing_page_images_hash",
+        build_landing_page_images_paths(),
+    )
+
+
+def need_to_run_compilemessages() -> bool:
+    if not os.path.exists("locale/language_name_map.json"):
+        # User may have cleaned their Git checkout.
+        print("Need to run compilemessages due to missing language_name_map.json")
+        return True
+
+    return is_digest_obsolete(
+        "last_compilemessages_hash",
+        compilemessages_paths(),
+    )
+
+
+def need_to_run_configure_rabbitmq(settings_list: list[str]) -> bool:
+    obsolete = is_digest_obsolete(
+        "last_configure_rabbitmq_hash",
+        configure_rabbitmq_paths(),
+        settings_list,
+    )
+
+    if obsolete:
+        return True
+
+    try:
+        from zerver.lib.queue import SimpleQueueClient
+
+        SimpleQueueClient()
+        return False
+    except Exception:
+        return True
+
+
+def main(options: argparse.Namespace) -> int:
+    setup_bash_profile()
+    setup_shell_profile("~/.zprofile")
+
+    # This needs to happen before anything that imports zproject.settings.
+    run(["scripts/setup/generate_secrets.py", "--development"])
+
+    create_var_directories()
+
+    # The `build_emoji` script requires `emoji-datasource` package
+    # which we install via npm; thus this step is after installing npm
+    # packages.
+    run(["tools/setup/emoji/build_emoji"])
+
+    # copy over static files from the zulip_bots and integrations packages
+    generate_zulip_bots_static_files()
+    generate_pythonapi_integrations_static_files()
+
+    run(["node", "tools/setup/build_supported_browser_regex.ts"])
+
+    if options.is_force or need_to_run_build_pygments_data():
+        run(["tools/setup/build_pygments_data"])
+        write_new_digest(
+            "build_pygments_data_hash",
+            build_pygments_data_paths(),
+            [pygments.__version__],
+        )
+    else:
+        print("No need to run `tools/setup/build_pygments_data`.")
+
+    if options.is_force or need_to_run_build_timezone_data():
+        run(["tools/setup/build_timezone_values"])
+        write_new_digest(
+            "build_timezones_data_hash",
+            build_timezones_data_paths(),
+            [timezones_version],
+        )
+    else:
+        print("No need to run `tools/setup/build_timezone_values`.")
+
+    if options.is_force or need_to_regenerate_landing_page_images():
+        run(["tools/setup/generate_landing_page_images.py"])
+        write_new_digest(
+            "landing_page_images_hash",
+            build_landing_page_images_paths(),
+        )
+
+    if not options.is_build_release_tarball_only:
+        # The following block is skipped when we just need the development
+        # environment to build a release tarball.
+
+        # Need to set up Django before using template_status
+        os.environ.setdefault("DJANGO_SETTINGS_MODULE", "zproject.settings")
+        import django
+
+        django.setup()
+
+        from django.conf import settings
+
+        from zerver.lib.test_fixtures import (
+            DEV_DATABASE,
+            TEST_DATABASE,
+            destroy_leaked_test_databases,
+        )
+
+        if os.environ.get("CODESPACES") == "true":
+            print("Starting background services explicitly for GitHub Codespaces...")
+            run_as_root(["service", "postgresql", "start"])
+            run_as_root(["service", "rabbitmq-server", "start"])
+            run_as_root(["service", "redis-server", "start"])
+            run_as_root(["service", "memcached", "start"])
+
+        assert settings.RABBITMQ_PASSWORD is not None
+        if options.is_force or need_to_run_configure_rabbitmq([settings.RABBITMQ_PASSWORD]):
+            run_as_root(["scripts/setup/configure-rabbitmq"])
+            write_new_digest(
+                "last_configure_rabbitmq_hash",
+                configure_rabbitmq_paths(),
+                [settings.RABBITMQ_PASSWORD],
+            )
+        else:
+            print("No need to run `scripts/setup/configure-rabbitmq.")
+
+        dev_template_db_status = DEV_DATABASE.template_status()
+        if options.is_force or dev_template_db_status == "needs_rebuild":
+            run(["tools/setup/postgresql-init-dev-db"])
+            if options.skip_dev_db_build:
+                # We don't need to build the manual development
+                # database on continuous integration for running tests, so we can
+                # just leave it as a template db and save a minute.
+                #
+                # Important: We don't write a digest as that would
+                # incorrectly claim that we ran migrations.
+                pass
+            else:
+                run(["tools/rebuild-dev-database"])
+                DEV_DATABASE.write_new_db_digest()
+        elif dev_template_db_status == "run_migrations":
+            DEV_DATABASE.run_db_migrations()
+        elif dev_template_db_status == "current":
+            print("No need to regenerate the dev DB.")
+
+        test_template_db_status = TEST_DATABASE.template_status()
+        if options.is_force or test_template_db_status == "needs_rebuild":
+            run(["tools/setup/postgresql-init-test-db"])
+            run(["tools/rebuild-test-database"])
+            TEST_DATABASE.write_new_db_digest()
+        elif test_template_db_status == "run_migrations":
+            TEST_DATABASE.run_db_migrations()
+        elif test_template_db_status == "current":
+            print("No need to regenerate the test DB.")
+
+        if options.is_force or need_to_run_compilemessages():
+            run(["./manage.py", "compilemessages", "--ignore=*"])
+            write_new_digest(
+                "last_compilemessages_hash",
+                compilemessages_paths(),
+            )
+        else:
+            print("No need to run `manage.py compilemessages`.")
+
+        destroyed = destroy_leaked_test_databases()
+        if destroyed:
+            print(f"Dropped {destroyed} stale test databases!")
+
+    clean_unused_caches.main(
+        argparse.Namespace(
+            threshold_days=6,
+            # The defaults here should match parse_cache_script_args in zulip_tools.py
+            dry_run=False,
+            verbose=False,
+            no_headings=True,
+        )
+    )
+
+    # Keeping this cache file around can cause eslint to throw
+    # random TypeErrors when new/updated dependencies are added
+    if os.path.isfile(".eslintcache"):
+        # Remove this block when
+        # https://github.com/eslint/eslint/issues/11639 is fixed
+        # upstream.
+        os.remove(".eslintcache")
+
+    # Clean up the root of the `var/` directory for various
+    # testing-related files that we have migrated to
+    # `var/<uuid>/test-backend`.
+    print("Cleaning var/ directory files...")
+    var_paths = glob.glob("var/test*")
+    var_paths.append("var/bot_avatar")
+    for path in var_paths:
+        try:
+            if os.path.isdir(path):
+                shutil.rmtree(path)
+            else:
+                os.remove(path)
+        except FileNotFoundError:
+            pass
+
+    version_file = os.path.join(UUID_VAR_PATH, "provision_version")
+    print(f"writing to {version_file}\n")
+    with open(version_file, "w") as f:
+        f.write(".".join(map(str, PROVISION_VERSION)) + "\n")
+
+    print()
+    print(OKBLUE + "Zulip development environment setup succeeded!" + ENDC)
+    return 0
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--force",
+        action="store_true",
+        dest="is_force",
+        help="Ignore all provisioning optimizations.",
+    )
+
+    parser.add_argument(
+        "--build-release-tarball-only",
+        action="store_true",
+        dest="is_build_release_tarball_only",
+        help="Provision for test suite with production settings.",
+    )
+
+    parser.add_argument(
+        "--skip-dev-db-build", action="store_true", help="Don't run migrations on dev database."
+    )
+
+    options = parser.parse_args()
+    sys.exit(main(options))

tools/lib/sanity_check.py

@@ -0,0 +1,19 @@
+import os
+import pwd
+import sys
+
+
+def check_venv(filename: str) -> None:
+    BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+    venv = os.path.realpath(os.path.join(BASE_DIR, ".venv"))
+    if sys.prefix != venv:
+        print(f"You need to run {filename} inside a Zulip dev environment.")
+        user_id = os.getuid()
+        user_name = pwd.getpwuid(user_id).pw_name
+
+        print(f"You can `source {venv}/bin/activate` to enter the development environment.")
+
+        if user_name not in ("vagrant", "zulipdev"):
+            print()
+            print("If you are using Vagrant, first run `vagrant ssh` to enter the Vagrant guest.")
+        sys.exit(1)

tools/lib/template_parser.py

@@ -0,0 +1,790 @@
+from collections.abc import Callable
+
+from typing_extensions import override
+
+from .html_elements import FOREIGN_CONTEXTS, VALID_HTML_CONTEXTS, html_context_fallbacks
+
+
+class FormattedError(Exception):
+    pass
+
+
+class TemplateParserError(Exception):
+    def __init__(self, message: str) -> None:
+        self.message = message
+
+    @override
+    def __str__(self) -> str:
+        return self.message
+
+
+class TokenizationError(Exception):
+    def __init__(self, message: str, line_content: str | None = None) -> None:
+        self.message = message
+        self.line_content = line_content
+
+
+class TokenizerState:
+    def __init__(self) -> None:
+        self.i = 0
+        self.line = 1
+        self.col = 1
+
+
+class Token:
+    def __init__(self, kind: str, s: str, tag: str, line: int, col: int, line_span: int) -> None:
+        self.kind = kind
+        self.s = s
+        self.tag = tag
+        self.line = line
+        self.col = col
+        self.line_span = line_span
+
+        # These get set during the validation pass.
+        self.start_token: Token | None = None
+        self.end_token: Token | None = None
+
+        # These get set during the pretty-print phase.
+        self.new_s = ""
+        self.indent: str | None = None
+        self.orig_indent: str | None = None
+        self.child_indent: str | None = None
+        self.indent_is_final = False
+        self.parent_token: Token | None = None
+
+
+def tokenize(text: str, template_format: str | None = None) -> list[Token]:
+    in_code_block = False
+
+    def advance(n: int) -> None:
+        for _ in range(n):
+            state.i += 1
+            if state.i >= 0 and text[state.i - 1] == "\n":
+                state.line += 1
+                state.col = 1
+            else:
+                state.col += 1
+
+    def looking_at(s: str) -> bool:
+        return text[state.i : state.i + len(s)] == s
+
+    def looking_at_htmlcomment() -> bool:
+        return looking_at("<!--")
+
+    def looking_at_handlebars_comment() -> bool:
+        return looking_at("{{!") or looking_at("{{~!")
+
+    def looking_at_djangocomment() -> bool:
+        return template_format == "django" and looking_at("{#")
+
+    def looking_at_handlebars_partial() -> bool:
+        return template_format == "handlebars" and looking_at("{{>")
+
+    def looking_at_handlebars_partial_block() -> bool:
+        return template_format == "handlebars" and looking_at("{{#>")
+
+    def looking_at_handlebars_triple_stache() -> bool:
+        return template_format == "handlebars" and (looking_at("{{{") or looking_at("{{~{"))
+
+    def looking_at_html_start() -> bool:
+        return looking_at("<") and not looking_at("</")
+
+    def looking_at_html_end() -> bool:
+        return looking_at("</")
+
+    def looking_at_handlebars_start() -> bool:
+        return looking_at("{{#") or looking_at("{{^") or looking_at("{{~#")
+
+    def looking_at_handlebars_else() -> bool:
+        return template_format == "handlebars" and looking_at("{{else")
+
+    def looking_at_template_var() -> bool:
+        return looking_at("{")
+
+    def looking_at_handlebars_end() -> bool:
+        return template_format == "handlebars" and (looking_at("{{/") or looking_at("{{~/"))
+
+    def looking_at_django_start() -> bool:
+        return template_format == "django" and looking_at("{% ")
+
+    def looking_at_django_else() -> bool:
+        return template_format == "django" and (
+            looking_at("{% else")
+            or looking_at("{% elif")
+            or looking_at("{%- else")
+            or looking_at("{%- elif")
+        )
+
+    def looking_at_django_end() -> bool:
+        return template_format == "django" and looking_at("{% end")
+
+    def looking_at_jinja2_end_whitespace_stripped() -> bool:
+        return template_format == "django" and looking_at("{%- end")
+
+    def looking_at_jinja2_start_whitespace_stripped_type2() -> bool:
+        # This function detects tag like {%- if foo -%}...{% endif %}
+        return template_format == "django" and looking_at("{%-") and not looking_at("{%- end")
+
+    def looking_at_whitespace() -> bool:
+        return looking_at("\n") or looking_at(" ")
+
+    state = TokenizerState()
+    tokens: list[Token] = []
+
+    while state.i < len(text):
+        try:
+            if in_code_block:
+                in_code_block = False
+                s = get_code(text, state.i)
+                if s == "":
+                    continue
+                tag = ""
+                kind = "code"
+            elif looking_at_htmlcomment():
+                s = get_html_comment(text, state.i)
+                tag = s[4:-3]
+                kind = "html_comment"
+            elif looking_at_handlebars_comment():
+                s = get_handlebars_comment(text, state.i)
+                tag = s[3:-2]
+                kind = "handlebars_comment"
+            elif looking_at_djangocomment():
+                s = get_django_comment(text, state.i)
+                tag = s[2:-2]
+                kind = "django_comment"
+            elif looking_at_handlebars_partial():
+                s = get_handlebars_partial(text, state.i)
+                tag = s[9:-2]
+                kind = "handlebars_partial"
+            elif looking_at_handlebars_partial_block():
+                s = get_handlebars_partial(text, state.i)
+                tag = s[5:-2].split(None, 1)[0]
+                kind = "handlebars_partial_block"
+            elif looking_at_html_start():
+                s = get_html_tag(text, state.i)
+                if s.endswith("/>"):
+                    end_offset = -2
+                else:
+                    end_offset = -1
+                tag_parts = s[1:end_offset].split()
+
+                if not tag_parts:
+                    raise TemplateParserError("Tag name missing")
+
+                tag = tag_parts[0]
+
+                if tag == "!DOCTYPE":
+                    kind = "html_doctype"
+                elif s.endswith("/>"):
+                    kind = "html_singleton"
+                else:
+                    kind = "html_start"
+                if tag in ("code", "pre", "script"):
+                    in_code_block = True
+            elif looking_at_html_end():
+                s = get_html_tag(text, state.i)
+                tag = s[2:-1]
+                kind = "html_end"
+            elif looking_at_handlebars_else():
+                s = get_handlebars_tag(text, state.i)
+                tag = "else"
+                kind = "handlebars_else"
+            elif looking_at_handlebars_triple_stache():
+                s = get_handlebars_triple_stache_tag(text, state.i)
+                start_offset = end_offset = 3
+                if s.startswith("{{~{"):
+                    start_offset += 1
+                if s.endswith("}~}}"):
+                    end_offset += 1
+                tag = s[start_offset:-end_offset].strip()
+                if not tag.endswith("_html"):
+                    raise TemplateParserError(
+                        "Unescaped variables in triple staches {{{ }}} must be suffixed with `_html`"
+                    )
+                kind = "handlebars_triple_stache"
+            elif looking_at_handlebars_start():
+                s = get_handlebars_tag(text, state.i)
+                tag = s[3:-2].split()[0].strip("#").removeprefix("*")
+                kind = "handlebars_start"
+            elif looking_at_handlebars_end():
+                s = get_handlebars_tag(text, state.i)
+                tag = s[3:-2].strip("/#~")
+                kind = "handlebars_end"
+            elif looking_at_django_else():
+                s = get_django_tag(text, state.i)
+                tag = "else"
+                kind = "django_else"
+            elif looking_at_django_end():
+                s = get_django_tag(text, state.i)
+                tag = s[6:-3]
+                kind = "django_end"
+            elif looking_at_django_start():
+                # must check this after end/else
+                s = get_django_tag(text, state.i)
+                tag = s[3:-2].split()[0]
+                kind = "django_start"
+
+                if s[-3] == "-":
+                    kind = "jinja2_whitespace_stripped_start"
+            elif looking_at_jinja2_end_whitespace_stripped():
+                s = get_django_tag(text, state.i)
+                tag = s[7:-3]
+                kind = "jinja2_whitespace_stripped_end"
+            elif looking_at_jinja2_start_whitespace_stripped_type2():
+                s = get_django_tag(text, state.i, stripped=True)
+                tag = s[3:-3].split()[0]
+                kind = "jinja2_whitespace_stripped_type2_start"
+            elif looking_at_template_var():
+                # order is important here
+                s = get_template_var(text, state.i)
+                tag = "var"
+                kind = "template_var"
+            elif looking_at("\n"):
+                s = "\n"
+                tag = "newline"
+                kind = "newline"
+            elif looking_at(" "):
+                s = get_spaces(text, state.i)
+                tag = ""
+                if not tokens or tokens[-1].kind == "newline":
+                    kind = "indent"
+                else:
+                    kind = "whitespace"
+            elif text[state.i] in "{<":
+                snippet = text[state.i :][:15]
+                raise AssertionError(f"tool cannot parse {snippet}")
+            else:
+                s = get_text(text, state.i)
+                if s == "":
+                    continue
+                tag = ""
+                kind = "text"
+        except TokenizationError as e:
+            raise FormattedError(
+                f'''{e.message} at line {state.line} col {state.col}:"{e.line_content}"''',
+            )
+
+        line_span = len(s.strip("\n").split("\n"))
+        token = Token(
+            kind=kind,
+            s=s,
+            tag=tag.strip(),
+            line=state.line,
+            col=state.col,
+            line_span=line_span,
+        )
+        tokens.append(token)
+        advance(len(s))
+
+    return tokens
+
+
+# The following excludes some obscure tags that are never used
+# in Zulip code.
+HTML_INLINE_TAGS = {
+    "a",
+    "b",
+    "br",
+    "button",
+    "cite",
+    "code",
+    "em",
+    "i",
+    "img",
+    "input",
+    "kbd",
+    "label",
+    "object",
+    "script",
+    "select",
+    "small",
+    "span",
+    "strong",
+    "textarea",
+}
+
+
+def tag_flavor(token: Token) -> str | None:
+    kind = token.kind
+    tag = token.tag
+    if kind in (
+        "code",
+        "django_comment",
+        "handlebars_comment",
+        "handlebars_partial",
+        "html_comment",
+        "html_doctype",
+        "html_singleton",
+        "indent",
+        "newline",
+        "template_var",
+        "text",
+        "whitespace",
+        "handlebars_triple_stache",
+    ):
+        return None
+
+    if kind in ("handlebars_start", "handlebars_partial_block", "html_start"):
+        return "start"
+    elif kind in (
+        "django_else",
+        "django_end",
+        "handlebars_else",
+        "handlebars_end",
+        "html_end",
+        "jinja2_whitespace_stripped_end",
+    ):
+        return "end"
+    elif kind in {
+        "django_start",
+        "django_else",
+        "jinja2_whitespace_stripped_start",
+        "jinja2_whitespace_stripped_type2_start",
+    }:
+        if is_django_block_tag(tag):
+            return "start"
+        else:
+            return None
+    else:
+        raise AssertionError(f"tools programmer neglected to handle {kind} tokens")
+
+
+def validate(
+    fn: str | None = None,
+    text: str | None = None,
+    template_format: str | None = None,
+) -> list[Token]:
+    assert fn or text
+
+    if fn is None:
+        fn = "<in memory file>"
+
+    if text is None:
+        with open(fn) as f:
+            text = f.read()
+
+    lines = text.split("\n")
+
+    try:
+        tokens = tokenize(text, template_format=template_format)
+    except FormattedError as e:
+        raise TemplateParserError(
+            f"""
+            fn: {fn}
+            {e}"""
+        )
+
+    prevent_whitespace_violations(fn, tokens)
+
+    class State:
+        def __init__(self, func: Callable[[Token | None], None]) -> None:
+            self.depth = 0
+            self.matcher = func
+            self.html_context = "unknown"
+
+    def no_start_tag(token: Token | None) -> None:
+        assert token
+        raise TemplateParserError(
+            f"""
+            No start tag
+            fn: {fn}
+            end tag:
+                {token.tag}
+                line {token.line}, col {token.col}
+            """
+        )
+
+    state = State(no_start_tag)
+
+    def start_tag_matcher(start_token: Token) -> None:
+        state.depth += 1
+        start_tag = start_token.tag.strip("~")
+        start_line = start_token.line
+        start_col = start_token.col
+
+        old_matcher = state.matcher
+        old_html_context = state.html_context
+
+        def f(end_token: Token | None) -> None:
+            if end_token is None:
+                raise TemplateParserError(
+                    f"""
+
+    Problem with {fn}
+    Missing end tag for the token at row {start_line} {start_col}!
+
+{start_token.s}
+
+    It's possible you have a typo in a token that you think is
+    matching this tag.
+                    """
+                )
+
+            is_else_tag = end_token.tag == "else"
+
+            end_tag = end_token.tag.strip("~")
+            end_line = end_token.line
+            end_col = end_token.col
+
+            def report_problem() -> str | None:
+                if (start_tag == "code") and (end_line == start_line + 1):
+                    return "Code tag is split across two lines."
+
+                if is_else_tag:
+                    # We are not completely rigorous about having a sensible
+                    # order of if/elif/elif/else, but we catch obviously
+                    # mismatching else tags.
+                    if start_tag not in ("if", "else", "unless", "list_each"):
+                        return f"Unexpected else/elif tag encountered after {start_tag} tag."
+                elif start_tag != end_tag:
+                    return f"Mismatched tags: ({start_tag} != {end_tag})"
+
+                return None
+
+            problem = report_problem()
+            if problem:
+                raise TemplateParserError(
+                    f"""
+                    fn: {fn}
+                   {problem}
+                    start:
+                        {start_token.s}
+                        line {start_line}, col {start_col}
+                    end tag:
+                        {end_tag}
+                        line {end_line}, col {end_col}
+                    """
+                )
+
+            if not is_else_tag:
+                state.matcher = old_matcher
+                state.html_context = old_html_context
+                state.depth -= 1
+
+            # TODO: refine this for the else/elif use cases
+            end_token.start_token = start_token
+            start_token.end_token = end_token
+
+        state.matcher = f
+
+    for token in tokens:
+        kind = token.kind
+        tag = token.tag
+
+        flavor = tag_flavor(token)
+        if flavor == "start":
+            start_tag_matcher(token)
+        elif flavor == "end":
+            state.matcher(token)
+
+        if kind in ("html_start", "html_singleton"):
+            for context in html_context_fallbacks(state.html_context):
+                if (tag, context) in VALID_HTML_CONTEXTS:
+                    new_context = VALID_HTML_CONTEXTS[tag, context]
+                    if new_context == "transparent":
+                        new_context = state.html_context
+                    break
+            else:
+                if "-" in tag and "phrasing" in html_context_fallbacks(state.html_context):
+                    new_context = state.html_context  # custom elements
+                elif state.html_context in FOREIGN_CONTEXTS:
+                    new_context = state.html_context  # unchecked foreign elements
+                else:
+                    raise TemplateParserError(
+                        f"<{tag}> is not valid in {state.html_context} context"
+                        + (
+                            ' (consider growing HTML_CONTEXT_FALLBACKS["unknown"]?)'
+                            if state.html_context == "unknown"
+                            else ""
+                        )
+                        + f" at {fn} line {token.line}, col {token.col}"
+                    )
+
+            if new_context not in FOREIGN_CONTEXTS:
+                if kind == "html_start" and new_context == "void":
+                    raise TemplateParserError(
+                        f"Tag must be self-closing: {tag} at {fn} line {token.line}, col {token.col}"
+                    )
+                elif kind == "html_singleton" and new_context != "void":
+                    raise TemplateParserError(
+                        f"Tag must not be self-closing: {tag} at {fn} line {token.line}, col {token.col}"
+                    )
+
+            if kind == "html_start":
+                state.html_context = new_context
+
+    if state.depth != 0:
+        state.matcher(None)
+
+    ensure_matching_indentation(fn, tokens, lines)
+
+    return tokens
+
+
+def ensure_matching_indentation(fn: str, tokens: list[Token], lines: list[str]) -> None:
+    def has_bad_indentation() -> bool:
+        is_inline_tag = start_tag in HTML_INLINE_TAGS and start_token.kind == "html_start"
+
+        if end_line > start_line + 1:
+            if is_inline_tag:
+                end_row_text = lines[end_line - 1]
+                if end_row_text.lstrip().startswith(end_token.s) and end_col != start_col:
+                    return True
+            else:
+                if end_col != start_col:
+                    return True
+
+        return False
+
+    for token in tokens:
+        if token.start_token is None:
+            continue
+
+        end_token = token
+
+        start_token = token.start_token
+        start_line = start_token.line
+        start_col = start_token.col
+        start_tag = start_token.tag
+        end_tag = end_token.tag.strip("~")
+        end_line = end_token.line
+        end_col = end_token.col
+
+        if has_bad_indentation():
+            raise TemplateParserError(
+                f"""
+                fn: {fn}
+                Indentation for start/end tags does not match.
+                start tag: {start_token.s}
+
+                start:
+                    line {start_line}, col {start_col}
+                end:
+                    {end_tag}
+                    line {end_line}, col {end_col}
+                """
+            )
+
+
+def prevent_extra_newlines(fn: str, tokens: list[Token]) -> None:
+    count = 0
+
+    for token in tokens:
+        if token.kind != "newline":
+            count = 0
+            continue
+
+        count += 1
+        if count >= 4:
+            raise TemplateParserError(
+                f"""Please avoid so many blank lines near row {token.line} in {fn}."""
+            )
+
+
+def prevent_whitespace_violations(fn: str, tokens: list[Token]) -> None:
+    if tokens[0].kind in ("indent", "whitespace"):
+        raise TemplateParserError(f" Please remove the whitespace at the beginning of {fn}.")
+
+    prevent_extra_newlines(fn, tokens)
+
+    for i in range(1, len(tokens) - 1):
+        token = tokens[i]
+        next_token = tokens[i + 1]
+
+        if token.kind == "indent":
+            if next_token.kind in ("indent", "whitespace"):
+                raise AssertionError("programming error parsing indents")
+
+            if next_token.kind == "newline":
+                raise TemplateParserError(
+                    f"""Please just make row {token.line} in {fn} a truly blank line (no spaces)."""
+                )
+
+            if len(token.s) % 4 != 0:
+                raise TemplateParserError(
+                    f"""
+                        Please use 4-space indents for template files. Most of our
+                        codebase (including Python and JavaScript) uses 4-space indents,
+                        so it's worth investing in configuring your editor to use
+                        4-space indents for files like
+                        {fn}
+
+                        The line at row {token.line} is indented with {len(token.s)} spaces.
+                    """
+                )
+
+        if token.kind == "whitespace":
+            if len(token.s) > 1:
+                raise TemplateParserError(
+                    f"""
+                        We did not expect this much whitespace at row {token.line} column {token.col} in {fn}.
+                    """
+                )
+            if next_token.kind == "newline":
+                raise TemplateParserError(
+                    f"""
+                        Unexpected trailing whitespace at row {token.line} column {token.col} in {fn}.
+                    """
+                )
+
+
+def is_django_block_tag(tag: str) -> bool:
+    return tag in [
+        "autoescape",
+        "block",
+        "comment",
+        "for",
+        "if",
+        "ifequal",
+        "macro",
+        "verbatim",
+        "blocktrans",
+        "trans",
+        "raw",
+        "with",
+    ]
+
+
+def get_handlebars_tag(text: str, i: int) -> str:
+    end = i + 2
+    while end < len(text) - 2 and text[end] != "}":
+        end += 1
+    if text[end] != "}" or text[end + 1] != "}":
+        raise TokenizationError('Tag missing "}}"', text[i : end + 2])
+    s = text[i : end + 2]
+    return s
+
+
+def get_handlebars_triple_stache_tag(text: str, i: int) -> str:
+    end = i + 3
+    while end < len(text) - 3 and text[end] != "}":
+        end += 1
+    if text[end : end + 3] == "}}}":
+        return text[i : end + 3]
+    elif end + 4 <= len(text) and text[end : end + 4] == "}~}}":
+        return text[i : end + 4]
+    else:
+        raise TokenizationError('Tag missing "}}}"', text[i : end + 3])
+
+
+def get_spaces(text: str, i: int) -> str:
+    s = ""
+    while i < len(text) and text[i] == " ":
+        s += text[i]
+        i += 1
+    return s
+
+
+def get_code(text: str, i: int) -> str:
+    s = ""
+    while i < len(text) and text[i] != "<":
+        s += text[i]
+        i += 1
+    return s
+
+
+def get_text(text: str, i: int) -> str:
+    s = ""
+    while i < len(text) and text[i] not in "{<":
+        s += text[i]
+        i += 1
+    return s.strip()
+
+
+def get_django_tag(text: str, i: int, stripped: bool = False) -> str:
+    end = i + 2
+    if stripped:
+        end += 1
+    while end < len(text) - 1 and text[end] != "%":
+        end += 1
+    if text[end] != "%" or text[end + 1] != "}":
+        raise TokenizationError('Tag missing "%}"', text[i : end + 2])
+    s = text[i : end + 2]
+    return s
+
+
+def get_html_tag(text: str, i: int) -> str:
+    quote_count = 0
+    end = i + 1
+    unclosed_end = 0
+    while end < len(text) and (text[end] != ">" or (quote_count % 2 != 0 and text[end] != "<")):
+        if text[end] == '"':
+            quote_count += 1
+        if not unclosed_end and text[end] == "<":
+            unclosed_end = end
+        end += 1
+    if quote_count % 2 != 0:
+        if unclosed_end:
+            raise TokenizationError("Unbalanced quotes", text[i:unclosed_end])
+        else:
+            raise TokenizationError("Unbalanced quotes", text[i : end + 1])
+    if end == len(text) or text[end] != ">":
+        raise TokenizationError('Tag missing ">"', text[i : end + 1])
+    s = text[i : end + 1]
+    return s
+
+
+def get_html_comment(text: str, i: int) -> str:
+    end = i + 7
+    unclosed_end = 0
+    while end <= len(text):
+        if text[end - 3 : end] == "-->":
+            return text[i:end]
+        if not unclosed_end and text[end] == "<":
+            unclosed_end = end
+        end += 1
+    raise TokenizationError("Unclosed comment", text[i:unclosed_end])
+
+
+def get_handlebars_comment(text: str, i: int) -> str:
+    long = text[i : i + 5] == "{{!--" or text[i : i + 6] == "{{~!--"
+    end = i + 5
+    unclosed_end = 0
+    while end <= len(text):
+        if text[end - 2 : end] == "}}" and (
+            not long or text[end - 4 : end] == "--}}" or text[end - 5 : end] == "--~}}"
+        ):
+            return text[i:end]
+        if not unclosed_end and text[end] == "<":
+            unclosed_end = end
+        end += 1
+    raise TokenizationError("Unclosed comment", text[i:unclosed_end])
+
+
+def get_template_var(text: str, i: int) -> str:
+    end = i + 3
+    unclosed_end = 0
+    while end <= len(text):
+        if text[end - 1] == "}":
+            if end < len(text) and text[end] == "}":
+                end += 1
+            return text[i:end]
+        if not unclosed_end and text[end] == "<":
+            unclosed_end = end
+        end += 1
+    raise TokenizationError("Unclosed var", text[i:unclosed_end])
+
+
+def get_django_comment(text: str, i: int) -> str:
+    end = i + 4
+    unclosed_end = 0
+    while end <= len(text):
+        if text[end - 2 : end] == "#}":
+            return text[i:end]
+        if not unclosed_end and text[end] == "<":
+            unclosed_end = end
+        end += 1
+    raise TokenizationError("Unclosed comment", text[i:unclosed_end])
+
+
+def get_handlebars_partial(text: str, i: int) -> str:
+    """Works for both partials and partial blocks."""
+    end = i + 10
+    unclosed_end = 0
+    while end <= len(text):
+        if text[end - 2 : end] == "}}":
+            return text[i:end]
+        if not unclosed_end and text[end] == "<":
+            unclosed_end = end
+        end += 1
+    raise TokenizationError("Unclosed partial", text[i:unclosed_end])

tools/lib/test_script.py

@@ -0,0 +1,128 @@
+import glob
+import os
+import subprocess
+import sys
+from argparse import ArgumentParser
+from collections.abc import Iterable
+
+from scripts.lib.zulip_tools import get_dev_uuid_var_path
+from version import PROVISION_VERSION
+
+ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
+def get_version_file() -> str:
+    uuid_var_path = get_dev_uuid_var_path()
+    return os.path.join(uuid_var_path, "provision_version")
+
+
+PREAMBLE = """
+Provisioning state check failed! This check compares
+`var/provision_version` (currently {}) to the version in
+source control (`version.py`), which is {}, to see if you
+likely need to provision before this command can run
+properly.
+"""
+
+
+def preamble(version: tuple[int, ...]) -> str:
+    text = PREAMBLE.format(version, PROVISION_VERSION)
+    text += "\n"
+    return text
+
+
+NEED_TO_DOWNGRADE = """
+The branch you are currently on expects an older version of
+dependencies than the version you provisioned last. This may
+be ok, but it's likely that you either want to rebase your
+branch on top of upstream/main or re-provision your machine.
+
+Do this: `./tools/provision`
+"""
+
+NEED_TO_UPGRADE = """
+The branch you are currently on has added dependencies beyond
+what you last provisioned. Your command is likely to fail
+until you add dependencies by provisioning.
+
+Do this: `./tools/provision`
+"""
+
+
+def get_provisioning_status() -> tuple[bool, str | None]:
+    version_file = get_version_file()
+    if not os.path.exists(version_file):
+        # If the developer doesn't have a version_file written by
+        # a previous provision, then we don't do any safety checks
+        # here on the assumption that the developer is managing
+        # their own dependencies and not running provision.
+        return True, None
+
+    with open(version_file) as f:
+        version = tuple(map(int, f.read().strip().split(".")))
+
+    # We may be more provisioned than the branch we just moved to.  As
+    # long as the major version hasn't changed, then we should be ok.
+    if version < PROVISION_VERSION:
+        return False, preamble(version) + NEED_TO_UPGRADE
+    elif version < (PROVISION_VERSION[0] + 1,):
+        return True, None
+    else:
+        return False, preamble(version) + NEED_TO_DOWNGRADE
+
+
+def assert_provisioning_status_ok(skip_provision_check: bool) -> None:
+    if not skip_provision_check:
+        ok, msg = get_provisioning_status()
+        if not ok:
+            print(msg)
+            print(
+                "If you really know what you are doing, use --skip-provision-check to run anyway."
+            )
+            sys.exit(1)
+
+
+def add_provision_check_override_param(parser: ArgumentParser) -> None:
+    """
+    Registers --skip-provision-check argument to be used with various commands/tests in our tools.
+    """
+    parser.add_argument(
+        "--skip-provision-check",
+        action="store_true",
+        help="Skip check that provision has been run; useful to save time if you know the dependency changes are not relevant to this command and will not cause it to fail",
+    )
+
+
+def find_js_test_files(test_dir: str, files: Iterable[str]) -> list[str]:
+    test_files = []
+    for file in files:
+        file = min(
+            (
+                os.path.join(test_dir, file_name)
+                for file_name in os.listdir(test_dir)
+                if file_name.startswith(file)
+            ),
+            default=file,
+        )
+
+        if not os.path.isfile(file):
+            raise Exception(f"Cannot find a matching file for '{file}' in '{test_dir}'")
+
+        test_files.append(os.path.abspath(file))
+
+    if not test_files:
+        test_files = sorted(
+            glob.glob(os.path.join(test_dir, "*.ts")) + glob.glob(os.path.join(test_dir, "*.js"))
+        )
+
+    return test_files
+
+
+def prepare_puppeteer_run(is_firefox: bool = False) -> None:
+    os.chdir(ZULIP_PATH)
+    # This will determine if the browser will be firefox or chrome.
+    os.environ["PUPPETEER_PRODUCT"] = "firefox" if is_firefox else "chrome"
+    subprocess.check_call(["node", "install.mjs"], cwd="node_modules/puppeteer")
+    os.makedirs("var/puppeteer", exist_ok=True)
+    for f in glob.glob("var/puppeteer/failure-*.png"):
+        os.remove(f)

tools/lib/test_server.py

@@ -0,0 +1,109 @@
+import os
+import subprocess
+import sys
+import time
+from collections.abc import Iterator
+from contextlib import ExitStack, contextmanager
+
+# Verify the Zulip venv is available.
+from tools.lib import sanity_check
+
+sanity_check.check_venv(__file__)
+
+import django
+import requests
+
+MAX_SERVER_WAIT = 180
+
+TOOLS_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+if TOOLS_DIR not in sys.path:
+    sys.path.insert(0, os.path.dirname(TOOLS_DIR))
+
+from scripts.lib.zulip_tools import get_or_create_dev_uuid_var_path
+from zerver.lib.test_fixtures import update_test_databases_if_required
+
+
+def set_up_django(external_host: str) -> None:
+    os.environ["FULL_STACK_ZULIP_TEST"] = "1"
+    os.environ["TEST_EXTERNAL_HOST"] = external_host
+    os.environ["LOCAL_UPLOADS_DIR"] = get_or_create_dev_uuid_var_path("test-backend/test_uploads")
+    os.environ["DJANGO_SETTINGS_MODULE"] = "zproject.test_settings"
+    django.setup()
+    os.environ["PYTHONUNBUFFERED"] = "y"
+
+
+def assert_server_running(server: "subprocess.Popen[bytes]", log_file: str | None) -> None:
+    """Get the exit code of the server, or None if it is still running."""
+    if server.poll() is not None:
+        message = "Server died unexpectedly!"
+        if log_file:
+            message += f"\nSee {log_file}\n"
+        raise RuntimeError(message)
+
+
+def server_is_up(server: "subprocess.Popen[bytes]", log_file: str | None) -> bool:
+    assert_server_running(server, log_file)
+    try:
+        # We could get a 501 error if the reverse proxy is up but the Django app isn't.
+        # Note that zulipdev.com is mapped via DNS to 127.0.0.1.
+        return requests.get("http://zulipdev.com:9981/accounts/home").status_code == 200
+    except requests.RequestException:
+        return False
+
+
+@contextmanager
+def test_server_running(
+    skip_provision_check: bool = False,
+    external_host: str = "testserver",
+    log_file: str | None = None,
+    dots: bool = False,
+    enable_help_center: bool = False,
+) -> Iterator[None]:
+    with ExitStack() as stack:
+        log = sys.stdout
+        if log_file:
+            if os.path.exists(log_file) and os.path.getsize(log_file) < 100000:
+                log = stack.enter_context(open(log_file, "a"))
+                log.write("\n\n")
+            else:
+                log = stack.enter_context(open(log_file, "w"))
+
+        set_up_django(external_host)
+
+        update_test_databases_if_required(rebuild_test_database=True)
+
+        # Run this not through the shell, so that we have the actual PID.
+        run_dev_server_command = ["tools/run-dev", "--test", "--streamlined"]
+        if skip_provision_check:
+            run_dev_server_command.append("--skip-provision-check")
+        if enable_help_center:
+            run_dev_server_command.append("--help-center-static-build")
+        server = subprocess.Popen(run_dev_server_command, stdout=log, stderr=log)
+
+        try:
+            # Wait for the server to start up.
+            print(end="\nWaiting for test server (may take a while)")
+            if not dots:
+                print("\n", flush=True)
+            t = time.time()
+            while not server_is_up(server, log_file):
+                if dots:
+                    print(end=".", flush=True)
+                time.sleep(0.4)
+                if time.time() - t > MAX_SERVER_WAIT:
+                    raise Exception("Timeout waiting for server")
+            print("\n\n--- SERVER IS UP! ---\n", flush=True)
+
+            # DO OUR ACTUAL TESTING HERE!!!
+            yield
+
+        finally:
+            assert_server_running(server, log_file)
+            server.terminate()
+            server.wait()
+
+
+if __name__ == "__main__":
+    # The code below is for testing this module works
+    with test_server_running():
+        print("\n\n SERVER IS UP!\n\n")

tools/lint

@@ -0,0 +1,249 @@
+#!/usr/bin/env python3
+import argparse
+import os
+import sys
+
+tools_dir = os.path.dirname(os.path.abspath(__file__))
+root_dir = os.path.join(tools_dir, "..")
+sys.path.insert(0, root_dir)
+
+# check for the venv
+from tools.lib import sanity_check
+
+sanity_check.check_venv(__file__)
+
+from zulint.command import LinterConfig, add_default_linter_arguments
+
+from tools.linter_lib.custom_check import non_py_rules, python_rules
+
+
+def run() -> None:
+    from tools.lib.test_script import (
+        add_provision_check_override_param,
+        assert_provisioning_status_ok,
+    )
+    from tools.linter_lib.exclude import EXCLUDED_FILES, PUPPET_CHECK_RULES_TO_EXCLUDE
+
+    parser = argparse.ArgumentParser()
+    add_provision_check_override_param(parser)
+    parser.add_argument("--full", action="store_true", help="Check some things we typically ignore")
+    parser.add_argument("--use-mypy-daemon", action="store_true", help="Run mypy daemon instead")
+    add_default_linter_arguments(parser)
+    args = parser.parse_args()
+
+    os.chdir(root_dir)
+
+    assert_provisioning_status_ok(args.skip_provision_check)
+
+    # Invoke the appropriate lint checker for each language,
+    # and also check files for extra whitespace.
+
+    linter_config = LinterConfig(args)
+
+    by_lang = linter_config.list_files(
+        groups={
+            "backend": [
+                "bash",
+                "json",
+                "md",
+                "pp",
+                "py",
+                "pyi",
+                "sh",
+                "text",
+                "txt",
+                "yaml",
+                "yml",
+            ],
+            "frontend": [
+                "astro",
+                "cjs",
+                "css",
+                "cts",
+                "hbs",
+                "html",
+                "js",
+                "mdx",
+                "mjs",
+                "mts",
+                "svg",
+                "ts",
+            ],
+        },
+        exclude=EXCLUDED_FILES,
+    )
+
+    linter_config.external_linter(
+        "css",
+        ["node_modules/.bin/stylelint"],
+        ["css"],
+        fix_arg="--fix",
+        description="Standard CSS style and formatting linter (config: stylelint.config.js)",
+    )
+    linter_config.external_linter(
+        "eslint",
+        ["node_modules/.bin/eslint", "--max-warnings=0", "--cache"],
+        ["astro", "cjs", "cts", "js", "md", "mdx", "mjs", "mts", "ts"],
+        fix_arg="--fix",
+        description="Standard JavaScript style and formatting linter (config: eslint.config.js).",
+    )
+    linter_config.external_linter(
+        "puppet",
+        ["env", "RUBYOPT=-W0", "puppet", "parser", "validate"],
+        ["pp"],
+        description="Runs the puppet parser validator, checking for syntax errors.",
+    )
+    linter_config.external_linter(
+        "puppet-lint",
+        ["puppet-lint", "--fail-on-warnings", *PUPPET_CHECK_RULES_TO_EXCLUDE],
+        ["pp"],
+        fix_arg="--fix",
+        description="Standard puppet linter (config: tools/linter_lib/exclude.py)",
+    )
+    linter_config.external_linter(
+        "templates",
+        ["tools/check-templates"],
+        ["hbs", "html"],
+        description="Custom linter checks whitespace formatting of HTML templates",
+        fix_arg="--fix",
+    )
+    linter_config.external_linter(
+        "openapi",
+        ["node", "tools/check-openapi.ts"],
+        ["yaml"],
+        description="Validates our OpenAPI/Swagger API documentation (zerver/openapi/zulip.yaml) ",
+        fix_arg="--fix",
+    )
+    linter_config.external_linter(
+        "shellcheck",
+        ["shellcheck", "-x", "-P", "SCRIPTDIR"],
+        ["bash", "sh"],
+        description="Standard shell script linter",
+    )
+    linter_config.external_linter(
+        "shfmt",
+        ["shfmt"],
+        ["bash", "sh"],
+        check_arg="-d",
+        fix_arg="-w",
+        description="Formats shell scripts",
+    )
+    command = ["tools/run-mypy", "--quiet"]
+    if args.skip_provision_check:
+        command.append("--skip-provision-check")
+    if args.use_mypy_daemon:
+        command.append("--use-daemon")
+    linter_config.external_linter(
+        "mypy",
+        command,
+        ["py", "pyi"],
+        pass_targets=False,
+        description="Static type checker for Python (config: pyproject.toml)",
+        suppress_line=(
+            (lambda line: line.startswith("Daemon") or line == "Restarting: configuration changed")
+            if args.use_mypy_daemon
+            else lambda _: False
+        ),
+    )
+    linter_config.external_linter(
+        "ruff",
+        ["ruff", "check", "--quiet"],
+        ["py", "pyi"],
+        fix_arg="--fix",
+        description="Python linter",
+    )
+    linter_config.external_linter(
+        "tsc",
+        ["tools/run-tsc"],
+        ["ts"],
+        pass_targets=False,
+        description="TypeScript compiler (config: tsconfig.json)",
+    )
+    linter_config.external_linter(
+        "gitlint",
+        ["tools/commit-message-lint"],
+        description="Checks commit messages for common formatting errors (config: .gitlint)",
+    )
+    linter_config.external_linter(
+        "prettier",
+        ["node_modules/.bin/prettier", "--cache", "--check", "--log-level=warn"],
+        [
+            "astro",
+            "cjs",
+            "css",
+            "cts",
+            "flow",
+            "js",
+            "json",
+            "md",
+            "mjs",
+            "mts",
+            "ts",
+            "yaml",
+            "yml",
+        ],
+        fix_arg=["--write"],
+        description="Formats CSS, JavaScript, YAML",
+    )
+    linter_config.external_linter(
+        "ruff-format",
+        ["ruff", "format", "--quiet"],
+        ["py", "pyi"],
+        description="Reformats Python code",
+        check_arg=["--check"],
+    )
+
+    semgrep_command = [
+        "semgrep",
+        "scan",
+        "--scan-unknown-extensions",
+        "--error",
+        "--disable-version-check",
+        "--quiet",
+    ]
+    linter_config.external_linter(
+        "semgrep-py",
+        [*semgrep_command, "--config=./tools/semgrep-py.yml"],
+        ["py"],
+        fix_arg="--autofix",
+        description="Syntactic grep (semgrep) code search tool (config: ./tools/semgrep-py.yml)",
+    )
+
+    linter_config.external_linter(
+        "mailmap",
+        ["sh", "-c", "grep '^[^#]' .mailmap | LC_ALL=C.UTF-8 sort -cf"],
+        description="Check that .mailmap is sorted",
+    )
+    linter_config.external_linter(
+        "thirdparty",
+        ["tools/check-thirdparty"],
+        description="Check docs/THIRDPARTY copyright file syntax",
+    )
+    linter_config.external_linter(
+        "remark",
+        ["node_modules/.bin/remark", "--frail", "--quiet", "--no-stdout"],
+        ["mdx"],
+        check_arg=['--setting="checkReformatting": true', "--"],
+        fix_arg=["--output", "--"],
+        description="Lint and format MDX files",
+    )
+
+    @linter_config.lint
+    def custom_py() -> int:
+        """Runs custom checks for python files (config: tools/linter_lib/custom_check.py)"""
+        failed = python_rules.check(by_lang, verbose=args.verbose)
+        return 1 if failed else 0
+
+    @linter_config.lint
+    def custom_nonpy() -> int:
+        """Runs custom checks for non-python files (config: tools/linter_lib/custom_check.py)"""
+        failed = False
+        for rule in non_py_rules:
+            failed = failed or rule.check(by_lang, verbose=args.verbose)
+        return 1 if failed else 0
+
+    linter_config.do_lint()
+
+
+if __name__ == "__main__":
+    run()

tools/linter_lib/exclude.py

@@ -0,0 +1,10 @@
+# Exclude some directories and files from lint checking
+EXCLUDED_FILES = [
+    # Third-party code that doesn't match our style
+    "web/third",
+]
+
+PUPPET_CHECK_RULES_TO_EXCLUDE = [
+    "--no-documentation-check",
+    "--no-80chars-check",
+]

tools/merge-api-changelogs

@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+import os
+import subprocess
+import sys
+
+TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
+os.chdir(os.path.dirname(TOOLS_DIR))
+sys.path.insert(0, os.path.dirname(TOOLS_DIR))
+
+from zerver.openapi.merge_api_changelogs import (
+    get_feature_level,
+    get_unmerged_changelogs,
+    merge_changelogs,
+    remove_unmerged_changelog_files,
+    update_feature_level_in_api_docs,
+)
+
+if __name__ == "__main__":
+    ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    os.chdir(ZULIP_PATH)
+
+    changelogs = get_unmerged_changelogs()
+    if changelogs:
+        new_feature_level = get_feature_level()
+        merge_changelogs(changelogs, new_feature_level)
+        update_feature_level_in_api_docs(new_feature_level)
+        remove_unmerged_changelog_files()
+
+        commit_message = f"api: Increment API feature level to {new_feature_level}."
+
+        try:
+            subprocess.run(["git", "add", "-u"], check=True)
+            subprocess.run(["git", "commit", "-m", commit_message], check=True)
+        except subprocess.CalledProcessError as e:
+            print(e)
+            sys.exit(1)

tools/node_lib/dump_fixtures.js

@@ -0,0 +1,3 @@
+import * as events from "../../web/tests/lib/events.cjs";
+
+console.info(JSON.stringify(events.fixtures, null, 4));

tools/notify-if-api-docs-changed

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+import json
+import os
+import re
+import subprocess
+import sys
+from pathlib import Path
+from urllib.request import Request, urlopen
+
+TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
+os.chdir(os.path.dirname(TOOLS_DIR))
+sys.path.insert(0, os.path.dirname(TOOLS_DIR))
+
+from zerver.openapi.merge_api_changelogs import get_feature_level
+
+
+def get_pull_request_number_or_commit_hash() -> str:
+    github_token = os.environ["GITHUB_TOKEN"]
+    repo = os.environ["GITHUB_REPOSITORY"]
+    commit_hash = os.environ["GITHUB_SHA"]
+
+    url = f"https://api.github.com/repos/{repo}/commits/{commit_hash}/pulls"
+    headers = {
+        "Accept": "application/vnd.github.groot-preview+json",
+        "Authorization": f"token {github_token}",
+    }
+
+    try:
+        req = Request(url, headers=headers)
+        with urlopen(req) as response:
+            pull_requests = json.load(response)
+        if len(pull_requests) > 0:
+            return f"#{pull_requests[0]['number']}"
+        return commit_hash
+    except Exception:
+        return commit_hash
+
+
+def get_changelog_entries() -> str:
+    changelog_path = Path("api_docs/changelog.md")
+    feature_level_pattern = re.compile(r"\*\*Feature level \d+\*\*")
+    current_feature_level_found = False
+    changelog_entries = ""
+
+    with open(changelog_path) as file:
+        for line in file:
+            if re.fullmatch(feature_level_pattern, line.strip()):
+                if current_feature_level_found:
+                    break
+                current_feature_level_found = True
+                continue
+
+            if current_feature_level_found:
+                changelog_entries += line
+
+    return changelog_entries.replace("\n  ", " ")
+
+
+def get_feature_level_before_commit() -> int:
+    with open(os.environ["GITHUB_EVENT_PATH"]) as f:
+        event = json.load(f)
+
+    before_sha = event["before"]
+    version_file_path = "version.py"
+
+    try:
+        version_content_before_commit = subprocess.check_output(
+            ["git", "show", f"{before_sha}:{version_file_path}"], text=True
+        )
+    except subprocess.CalledProcessError:
+        sys.exit(1)
+
+    lines = version_content_before_commit.split("\n")
+    for line in lines:
+        if line.startswith("API_FEATURE_LEVEL = "):
+            match = re.search(r"\d+", line)
+            if match:
+                return int(match.group())
+    return 0
+
+
+if __name__ == "__main__":
+    pull_request = get_pull_request_number_or_commit_hash()
+
+    feature_level = get_feature_level(update_feature_level=False) - 1
+    if feature_level <= get_feature_level_before_commit():
+        sys.exit(1)
+
+    changelog_entries = get_changelog_entries()
+    topic = f"new feature level: {feature_level}"
+
+    content = f"{pull_request} has updated the API documentation for the following endpoints: \n{changelog_entries}"
+    github_output = os.environ.get("GITHUB_OUTPUT")
+    if github_output:
+        with open(github_output, "a") as f:
+            f.write(f"topic={topic}\n")
+            f.write("content<<EOF\n")
+            f.write(f"{content.strip()}\n")
+            f.write("EOF\n")

tools/oneclickapps/README.md

@@ -0,0 +1,81 @@
+# One click app release automation
+
+This directory contains scripts for automating the release of Zulip one click apps.
+
+## DigitalOcean 1-Click Application
+
+`prepare_digital_ocean_one_click_app_release.py` creates the image of DigitalOcean 1-Click
+app from the latest Zulip release (fetched from https://download.zulip.com/server/). It will
+also create a test droplet from the image and send the image and droplet
+details to a pre-configured Zulip stream. Anyone, whose key is added to the
+Zulip DigitalOcean team can SSH into the droplet for testing.
+
+### Running as GitHub action
+
+`.github/workflows/update-oneclick-apps.yml` is configured to invoke
+`prepare_digital_ocean_one_click_app_release.py` as a GitHub action during each Zulip
+server release.
+
+You also need to set the following secrets in your GitHub repository to make the action
+work correctly. These secrets are passed as environment variables to the GitHub action.
+
+- `ONE_CLICK_ACTION_DIGITALOCEAN_API_KEY` - DigitalOcean API key used for creating droplets, snapshots etc.
+- `ONE_CLICK_ACTION_ZULIP_BOT_API_KEY` - The API key of the Zulip bot used for sending messages.
+- `ONE_CLICK_ACTION_ZULIP_BOT_EMAIL` - The email of the Zulip bot.
+
+Also pass the following as environment variables in `.github/workflows/update-oneclick-apps.yml`.
+
+- `PYTHON_DIGITALOCEAN_REQUEST_TIMEOUT_SEC` - This configures the maximum number of seconds
+  to wait before the requests made by `python-digitalocean` time out. If not configured, it's
+  common for the requests to take 20+ minutes before getting timed out.
+
+### Verifying the one click app image
+
+- The action will send the image name and test droplet details to the stream configured in the
+  above steps.
+- SSH into the test droplet by following the instructions in the message. In order to do this,
+  you need to add your SSH key to the team.
+- After logging into the test droplet, exit the installer and run the following script.
+
+  https://raw.githubusercontent.com/digitalocean/marketplace-partners/master/scripts/99-img-check.sh
+
+  This script checks whether the image created is valid. It is also run by the DigitalOcean team
+  before they approve the image submission in the one click app marketplace.
+
+- If there are no errors (you can ignore most of the warnings), exit the SSH connection and
+  reconnect.
+- Populate the details asked by the installer and verify that the installer completes successfully.
+  One error is expected:
+  ```
+  User root has a populated authorized_keys file in /root/.ssh/authorized_keys
+  ```
+  The key in that file will be the SSH key you've authenticated with.
+  If there are other errors see the section below.
+- Use the link generated by the installer to create a new Zulip organization. Do some basic
+  testing like sending a bunch of messages and reloading the webpage.
+- If there are no issues, submit the image in the
+  [DigitalOcean vendor portal](https://marketplace.digitalocean.com/vendorportal). You need to be
+  added to the vendor portal team for doing this. Ask Tim to add you if required. During the submission,
+  make sure to update the blog post URL if it's a major release.
+- Keep checking the vendor portal for change in status of the submission. DigitalOcean does nominally send
+  emails when there are updates on the submission, but we have found the emails to not always arrive.
+- If there are any issues with submission, rebuild the image by manually invoking the script and
+  resubmit. The issues we have seen mostly in the past are caused by the dependencies getting outdated
+  by the time the DigitalOcean team run the checks. In that case you have to just rebuild the image
+  by invoking the script.
+- Delete the test droplet `oneclickapp-{release_version}-test` after you have completed testing
+  by going to the [DigitalOcean Zulip team account](https://cloud.digitalocean.com/droplets?i=0242e0).
+  If there are other existing test droplets with the same name format but with older release versions
+  feel free to delete them as well. These droplets are also tagged with the `github-action` and `temporary`
+  tags.
+
+**Errors**
+
+If there are any errors while setting up the one click app installer, you have three options
+
+- Include the fix in the Fabric script that setups the installer.
+  [01-initial-setup](https://raw.githubusercontent.com/zulip/marketplace-partners/master/marketplace_docs/templates/Fabric/scripts/01-initial-setup)
+  file should be a good place to include the fix. See
+  [zulip/marketplace-partners#4](https://github.com/zulip/marketplace-partners/pull/4/files) for an
+  example fix.
+- Wait for the next release to fix the error.

tools/oneclickapps/prepare_digital_ocean_one_click_app_release.py

@@ -0,0 +1,170 @@
+import os
+import subprocess
+import time
+from pathlib import Path
+
+import digitalocean
+import zulip
+from requests.adapters import HTTPAdapter
+from urllib3.util import Retry
+
+manager = digitalocean.Manager(token=os.environ["DIGITALOCEAN_API_KEY"])
+# We just temporarily create the client now, to validate that we can
+# auth to the server; reusing it after the whole install fails because
+# the connection has been half-closed in a way that breaks it.
+zulip.Client()
+TEST_DROPLET_SUBDOMAIN = "do"
+
+
+def generate_ssh_keys() -> None:
+    subprocess.check_call(
+        ["ssh-keygen", "-f", str(Path.home()) + "/.ssh/id_ed25519", "-P", "", "-t", "ed25519"]
+    )
+
+
+def get_public_ssh_key() -> str:
+    try:
+        with open(str(Path.home()) + "/.ssh/id_ed25519.pub") as f:
+            return f.read()
+    except FileNotFoundError:
+        return ""
+
+
+def sleep_until_droplet_action_is_completed(
+    droplet: digitalocean.Droplet, action_type: str
+) -> None:
+    incomplete = True
+    while incomplete:
+        for action in droplet.get_actions():
+            action.load()
+            print(f"...[{action.type}]: {action.status}")
+            if action.type == action_type and action.status == "completed":
+                incomplete = False
+                break
+        if incomplete:
+            time.sleep(5)
+
+    # Sometimes the droplet does not yet have an .ip_address value
+    # (the attribute is None) after .load()ing the droplet. We cannot
+    # proceed without the IP, so we wait in a loop until the IP is
+    # returned to us.
+    while True:
+        droplet.load()
+        if droplet.ip_address:
+            break
+        time.sleep(5)
+
+
+def set_api_request_retry_limits(api_object: digitalocean.baseapi.BaseAPI) -> None:
+    retry = Retry(connect=5, read=5, backoff_factor=0.1)
+    adapter = HTTPAdapter(max_retries=retry)
+    api_object._session.mount("https://", adapter)
+
+
+def create_droplet(
+    name: str, ssh_keys: list[str], image: str = "ubuntu-22-04-x64"
+) -> digitalocean.Droplet:
+    droplet = digitalocean.Droplet(
+        token=manager.token,
+        name=name,
+        region="nyc3",
+        size_slug="s-1vcpu-2gb",
+        image=image,
+        backups=False,
+        ssh_keys=ssh_keys,
+        tags=["github-action", "temporary"],
+    )
+    set_api_request_retry_limits(droplet)
+    droplet.create()
+    sleep_until_droplet_action_is_completed(droplet, "create")
+    return droplet
+
+
+def create_ssh_key(name: str, public_key: str) -> digitalocean.SSHKey:
+    action_public_ssh_key_object = digitalocean.SSHKey(
+        name=name, public_key=public_key, token=manager.token
+    )
+    set_api_request_retry_limits(action_public_ssh_key_object)
+    action_public_ssh_key_object.create()
+    return action_public_ssh_key_object
+
+
+def create_snapshot(droplet: digitalocean.Droplet, snapshot_name: str) -> None:
+    droplet.take_snapshot(snapshot_name, power_off=True)
+    droplet.load()
+    sleep_until_droplet_action_is_completed(droplet, "snapshot")
+
+
+def create_dns_records(droplet: digitalocean.Droplet) -> None:
+    domain = digitalocean.Domain(token=manager.token, name="oneclick.zulip.dev")
+    set_api_request_retry_limits(domain)
+    domain.load()
+
+    oneclick_test_app_record_names = [TEST_DROPLET_SUBDOMAIN, f"*.{TEST_DROPLET_SUBDOMAIN}"]
+    for record in domain.get_records():
+        if (
+            record.name in oneclick_test_app_record_names
+            and record.domain == "oneclick.zulip.dev"
+            and record.type == "A"
+        ):
+            record.destroy()
+
+    domain.load()
+    for record_name in oneclick_test_app_record_names:
+        domain.create_new_domain_record(type="A", name=record_name, data=droplet.ip_address)
+
+
+def setup_one_click_app_installer(droplet: digitalocean.Droplet) -> None:
+    subprocess.check_call(
+        [
+            "fab",
+            "build_image",
+            "-H",
+            droplet.ip_address,
+            "--keepalive",
+            "5",
+            "--connection-attempts",
+            "10",
+        ],
+        cwd="marketplace-partners/marketplace_docs/templates/Fabric",
+    )
+
+
+def send_message(content: str) -> None:
+    request = {
+        "type": "stream",
+        "to": os.environ["ONE_CLICK_ACTION_STREAM"],
+        "topic": "digitalocean installer",
+        "content": content,
+    }
+    zulip.Client().send_message(request)
+
+
+if __name__ == "__main__":
+    release_version = os.environ["RELEASE_VERSION"]
+
+    generate_ssh_keys()
+    action_public_ssh_key_object = create_ssh_key(
+        f"oneclickapp-{release_version}-image-generator-public-key", get_public_ssh_key()
+    )
+
+    image_generator_droplet = create_droplet(
+        f"oneclickapp-{release_version}-image-generator", [action_public_ssh_key_object]
+    )
+
+    setup_one_click_app_installer(image_generator_droplet)
+
+    oneclick_image_name = f"oneclickapp-{release_version}"
+    create_snapshot(image_generator_droplet, oneclick_image_name)
+    snapshot = image_generator_droplet.get_snapshots()[0]
+    send_message(f"One click app image `{oneclick_image_name}` created.")
+
+    image_generator_droplet.destroy()
+    action_public_ssh_key_object.destroy()
+
+    test_droplet_name = f"oneclickapp-{release_version}-test"
+    test_droplet = create_droplet(test_droplet_name, manager.get_all_sshkeys(), image=snapshot.id)
+    create_dns_records(test_droplet)
+    send_message(
+        f"Test droplet `{test_droplet_name}` created. SSH as root to {TEST_DROPLET_SUBDOMAIN}.oneclick.zulip.dev for testing."
+    )

tools/pre-commit

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+# This hook runs the Zulip code linter ./tools/lint and returns true
+# regardless of linter results so that your commit may continue.
+
+# Messages from the linter will be printed out to the screen.
+#
+# If you are running this one machine hosting a Vagrant guest that
+# contains your provisioned Zulip development environment, the linter
+# will automatically be run through `vagrant ssh`.
+
+changed_files=()
+while read -r -d '' f; do
+    changed_files+=("$f")
+done < <(git diff -z --cached --name-only --diff-filter=ACM)
+if [ ${#changed_files} -eq 0 ]; then
+    echo "No changed files to lint."
+    exit 0
+fi
+
+if [ -z "$VIRTUAL_ENV" ] && command -v vagrant >/dev/null && [ -e .vagrant ]; then
+    vcmd="$(dirname "$0")/lint --skip=gitlint --skip-provision-check $(printf '%q ' "${changed_files[@]}") || true"
+    echo "Running lint using vagrant..."
+    vagrant ssh -c "$vcmd"
+else
+    "$(dirname "$0")/lint" --skip=gitlint --skip-provision-check "${changed_files[@]}" || true
+fi
+exit 0

tools/provision

@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+
+# Use this script to provision dependencies for your Zulip installation.
+# This script is idempotent, so it can be restarted at any time, and it
+# will usually run fairly quickly when your dependencies are up to date.
+
+set -e
+if [ "$EUID" -eq 0 ]; then
+    echo "Error: The provision script must not be run as root" >&2
+    exit 1
+fi
+
+if [ -e "$(dirname "${BASH_SOURCE[0]}")/../.vagrant" ] && [ ! -e /home/vagrant ]; then
+    echo 'Error: provision should be run inside the Vagrant environment.' >&2
+    # shellcheck disable=SC2016
+    echo 'Try `vagrant up --provision`.' >&2
+    exit 1
+fi
+
+os="$(. /etc/os-release && echo "$ID $VERSION_ID")"
+case "$os" in
+    'ubuntu 14.04' | 'ubuntu 16.04' | 'ubuntu 18.04' | 'debian 10')
+        echo "Error: $os is no longer a supported platform for Zulip." >&2
+        if [ -e /home/vagrant ]; then
+            # shellcheck disable=SC2016
+            echo 'To upgrade, run `vagrant destroy`, and then recreate the Vagrant guest.' >&2
+            echo 'See: https://zulip.readthedocs.io/en/latest/development/setup-recommended.html' >&2
+        fi
+        exit 1
+        ;;
+esac
+
+python_version="$(python3 --version)"
+case "$python_version" in
+    Python\ 3.[0-9].*)
+        echo 'Error: Zulip requires an OS with Python 3.10 or later.' >&2
+        exit 1
+        ;;
+esac
+
+FAIL='\033[91m'
+WARNING='\033[93m'
+ENDC='\033[0m'
+
+# Make the script independent of the location from where it is executed
+PARENT_PATH=$(
+    cd "$(dirname "${BASH_SOURCE[0]}")/.."
+    pwd -P
+)
+cd "$PARENT_PATH"
+mkdir -p var/log
+LOG_PATH="var/log/provision.log"
+
+echo "PROVISIONING STARTING." >>$LOG_PATH
+
+# PYTHONUNBUFFERED is important to ensure that tracebacks don't get
+# lost far above where they should be in the output.
+export PYTHONUNBUFFERED=1
+tools/lib/provision.py "$@" 2>&1 | tee -a "$LOG_PATH"
+failed=${PIPESTATUS[0]}
+
+if [ "$failed" -ne 0 ]; then
+    echo -e "$FAIL"
+    echo "Provisioning failed (exit code $failed)!"
+    echo
+    echo "* Look at the traceback(s) above to find more about the errors."
+    echo "* Resolve the errors or get help on chat."
+    echo "* If you can fix this yourself, you can re-run tools/provision at any time."
+    echo "* Logs are here: zulip/var/log/provision.log"
+    echo -e "$ENDC"
+    exit "$failed"
+elif [ "$VIRTUAL_ENV" != "$PARENT_PATH/.venv" ] && [ -z "${SKIP_VENV_SHELL_WARNING}" ]; then
+    echo -e "$WARNING"
+    echo "WARNING: This shell does not have the Zulip Python 3 virtualenv activated."
+    echo "Zulip commands will fail until you activate the virtualenv."
+    echo
+    echo "To update the shell, run:"
+    echo "    source $PARENT_PATH/.venv/bin/activate"
+    # shellcheck disable=SC2016
+    echo 'or just close this shell and start a new one (with Vagrant, `vagrant ssh`).'
+    echo -en "$ENDC"
+fi
+exit 0

tools/push-to-pull-request

@@ -0,0 +1,170 @@
+#!/usr/bin/env bash
+set -e
+
+usage() {
+    cat <<EOF
+usage: $0 [--merge] PULL_REQUEST_ID [REMOTE]
+
+Force-push our HEAD to the given GitHub pull request branch.
+
+Useful for a maintainer to run just before pushing to main,
+after tweaking the branch and/or rebasing to latest.  This causes
+GitHub to see the subsequent push to main as representing a
+merge of the PR, rather than requiring the PR to be manually
+(and to the casual observer misleadingly) closed instead.
+
+With --merge, also go ahead and merge the PR.
+
+REMOTE defaults to the value of the Git config variable
+\`zulip.zulipRemote\` if set, else to \`upstream\`.
+
+If we have a pseudo-remote-tracking branch for the PR (as created
+by \`reset-to-pull-request\`, like \`pr/1234\`), then the tracking
+branch is updated to reflect the pushed version.
+
+See also \`reset-to-pull-request\`.
+EOF
+}
+
+merge=
+args=()
+
+while ((OPTIND <= $#)); do
+    if getopts ":-:" opt; then
+        case $opt in
+            -)
+                case "$OPTARG" in
+                    help)
+                        usage
+                        exit 0
+                        ;;
+                    merge)
+                        merge=t
+                        ;;
+                    *)
+                        echo "Invalid option: --$OPTARG" >&2
+                        exit 1
+                        ;;
+                esac
+                ;;
+            \?)
+                echo "Invalid option: -$OPTARG" >&2
+                exit 1
+                ;;
+        esac
+    else
+        args+=("${!OPTIND}")
+        ((OPTIND++))
+    fi
+done
+
+set -- "${args[@]}"
+
+remote_default="$(git config zulip.zulipRemote || echo upstream)"
+pseudo_remote="$(git config zulip.prPseudoRemote || echo)"
+
+pr_id="$1"
+remote="${2:-"$remote_default"}"
+
+if [ -z "$pr_id" ]; then
+    usage >&2
+    exit 1
+fi
+
+if ! jq --version >/dev/null 2>&1; then
+    cat >&2 <<EOF
+error: not found: jq
+
+push-to-pull-request requires the \`jq\` utility; you should install it.
+Try:
+
+  sudo apt install jq
+EOF
+    exit 1
+fi
+
+remote_url="$(git remote get-url --push "$remote")"
+repo_fq="$(echo "$remote_url" | perl -lne 'print $1 if (
+    m, ^ git\@github\.com:
+         ([^/]+ / [^/]+?)
+         (?:\.git)?
+       $ ,x )')"
+
+if [ -z "$repo_fq" ]; then
+    # We're pretty specific about what we expect the URL to look like;
+    # there are probably more cases we could legitimately cover, which
+    # we can add if/when they come up for someone.
+    echo "error: couldn't parse remote URL as GitHub repo: $remote_url" >&2
+    exit 1
+fi
+
+# See https://developer.github.com/v3/pulls/#get-a-single-pull-request .
+# This is the old REST API; the new GraphQL API does look neat, but it
+# seems to require authentication even for simple lookups of public data,
+# and that'd be a pain for a simple script like this.
+pr_url=https://api.github.com/repos/"${repo_fq}"/pulls/"${pr_id}"
+pr_details="$(curl -fLsS --retry 3 "$pr_url")"
+
+pr_jq() {
+    echo "$pr_details" | jq "$@"
+}
+
+if [ "$(pr_jq -r .message)" = "Not Found" ]; then
+    echo "error: invalid PR URL: $pr_url" >&2
+    exit 1
+fi
+
+if [ "$(pr_jq .maintainer_can_modify)" != "true" ]; then
+    # This happens when the PR has already been merged or closed, or
+    # if the contributor has turned off the (default) setting to allow
+    # maintainers of the target repo to push to their PR branch.
+    #
+    # The latter seems to be rare (in Greg's experience doing the
+    # manual equivalent of this script for many different
+    # contributors, none have ever chosen this setting), but give a
+    # decent error message if it does happen.
+    echo "error: PR already closed, or contributor has disallowed pushing to branch" >&2
+    exit 1
+fi
+
+if [ "$merge" ]; then
+    pr_base_ref="$(pr_jq -r .base.ref)"
+    git fetch -- "$remote"
+    if ! git merge-base --is-ancestor -- "$remote/$pr_base_ref" @; then
+        echo "error: You need to rebase on $remote/$pr_base_ref first" >&2
+        exit 1
+    fi
+fi
+
+pr_head_repo_fq="$(pr_jq -r .head.repo.full_name)"
+pr_head_refname="$(pr_jq -r .head.ref)"
+
+tracking_ref=
+if [ -n "$pseudo_remote" ]; then
+    tracking_ref=$(git rev-parse -q --verify --symbolic refs/remotes/"$pseudo_remote"/"$pr_id" || echo)
+fi
+
+set -x
+git push git@github.com:"$pr_head_repo_fq" +@:"$pr_head_refname"
+
+{ set +x; } 2>&-
+if [ -n "$tracking_ref" ]; then
+    set -x
+    git update-ref "$tracking_ref" @
+fi
+
+if [ "$merge" ]; then
+    tries=10
+    sha="$(git rev-parse @)"
+    while
+        out=$(git ls-remote -- "$remote" "refs/pull/$pr_id/head")
+        [ "$out" != "$sha	refs/pull/$pr_id/head" ]
+    do
+        if ! ((--tries)); then
+            echo 'error: Push was not observed in PR' >&2
+            exit 1
+        fi
+        sleep 1
+    done
+    git push -- "$remote" "@:$pr_base_ref"
+fi

tools/python-warnings.bash

@@ -0,0 +1,31 @@
+# shellcheck shell=bash
+
+export PYTHONWARNINGS=error
+
+PYTHONWARNINGS+=',ignore::ResourceWarning'
+
+# https://github.com/disqus/django-bitfield/pull/135
+PYTHONWARNINGS+=',default:Attribute s is deprecated and will be removed in Python 3.14; use value instead:DeprecationWarning:__main__'
+
+# https://github.com/mahmoud/glom/pull/258
+PYTHONWARNINGS+=',ignore:invalid escape sequence '\'\\' '\'':DeprecationWarning'
+PYTHONWARNINGS+=',ignore:invalid escape sequence '\'\\' '\'':SyntaxWarning'
+PYTHONWARNINGS+=',ignore:"\ " is an invalid escape sequence.:SyntaxWarning'
+
+# https://github.com/ipython/ipython/pull/14876
+PYTHONWARNINGS+=',ignore:'\''return'\'' in a '\''finally'\'' block:SyntaxWarning'
+
+# https://github.com/SAML-Toolkits/python3-saml/pull/420
+PYTHONWARNINGS+=',ignore:datetime.datetime.utcfromtimestamp() is deprecated and scheduled for removal in a future version.:DeprecationWarning:onelogin.saml2.utils'
+PYTHONWARNINGS+=',ignore:datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version.:DeprecationWarning:onelogin.saml2.utils'
+
+# https://github.com/seb-m/pyinotify/issues/204
+PYTHONWARNINGS+=',ignore:The asyncore module is deprecated and will be removed in Python 3.12.:DeprecationWarning:pyinotify'
+
+# Semgrep still supports Python 3.8
+PYTHONWARNINGS+=',ignore:path is deprecated.:DeprecationWarning:semgrep.semgrep_core'
+
+# https://github.com/adamchainz/time-machine/pull/486
+PYTHONWARNINGS+=',ignore:datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version.:DeprecationWarning:time_machine'
+
+export SQLALCHEMY_WARN_20=1

tools/rebuild-dev-database

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -e
+set -x
+
+PGHOST=localhost PGUSER=zulip \
+    "$(dirname "$0")/../scripts/setup/terminate-psql-sessions" zulip zulip_base
+
+psql -v ON_ERROR_STOP=1 -e -h localhost postgres zulip <<EOF
+DROP DATABASE IF EXISTS zulip;
+CREATE DATABASE zulip TEMPLATE zulip_base;
+EOF
+
+"$(dirname "$0")/../scripts/setup/flush-memcached"
+
+./manage.py purge_queue --all
+./manage.py migrate --noinput
+./manage.py get_migration_status --settings=zproject.settings --output="migration_status_dev"
+./manage.py createcachetable third_party_api_results
+./manage.py populate_db -n100 --threads=1
+# Ensure that the local user's API key is synced from ~/.zuliprc
+
+if [ -e ~/.zuliprc ]; then
+    ./manage.py sync_api_key
+fi