snapshot of https://github.com/zulip/zulip.git @ 48c69911c7dd4e03b20552eb3bd77ad8151c1357 (2026-09-15)

tools/lib/capitalization.py

@@ -0,0 +1,341 @@
+import re
+from io import StringIO
+from re import Match
+
+from bs4 import BeautifulSoup
+
+# The phrases in this list will be ignored. The longest phrase is
+# tried first; this removes the chance of smaller phrases changing
+# the text before longer phrases are tried.
+# The errors shown by `tools/check-capitalization` can be added to
+# this list without any modification.
+IGNORED_PHRASES = [
+    # Proper nouns and acronyms
+    r"AI",
+    r"API",
+    r"APNS",
+    r"Apple Silicon",
+    r"Botserver",
+    r"Cookie Bot",
+    r"DevAuthBackend",
+    r"DSN",
+    r"Esc",
+    r"GCM",
+    r"GitHub",
+    r"GitLab",
+    r"Gravatar",
+    r"HTTP",
+    r"ID",
+    r"IDs",
+    r"Inbox",
+    r"Intel",
+    r"IP",
+    r"JSON",
+    r"Jitsi",
+    r"Jotform",
+    r"LinkedIn",
+    r"LDAP",
+    r"Markdown",
+    r"OAuth",
+    r"OTP",
+    r"Recent conversations",
+    r"DM",
+    r"DMs",
+    r"Slack",
+    r"Google",
+    r"Terms of Service",
+    r"TikTok",
+    r"Tuesday",
+    r"URL",
+    r"UUID",
+    r"WordPress",
+    r"YouTube",
+    r"Zoom",
+    r"Zulip",
+    r"Zulip Server",
+    r"Zulip Account Security",
+    r"Zulip Security",
+    r"Zulip Cloud",
+    r"Zulip Cloud Standard",
+    r"Zulip Cloud Plus",
+    r"Zulip Desktop",
+    r"Download Zulip for macOS \(Apple Silicon\)",
+    r"Download Zulip for macOS \(Intel\)",
+    r"BigBlueButton",
+    r"Constructor Groups",
+    r"Nextcloud Talk",
+    # Code things
+    r"\.zuliprc",
+    # BeautifulSoup will remove <z-user> which is horribly confusing,
+    # so we need more of the sentence.
+    r"<z-user></z-user> will have the same role",
+    r"<z-user></z-user> will have the same properties",
+    # Things using "I"
+    r"I understand",
+    r"I'm",
+    r"I've",
+    r"Topics I participate in",
+    r"Topics I send a message to",
+    r"Topics I start",
+    # Specific short words
+    r"beta",
+    r"and",
+    r"bot",
+    r"e\.g\.",
+    r"email",
+    r"enabled",
+    r"signups",
+    # Pasted text filename
+    r"PastedText",
+    # Placeholders
+    r"keyword",
+    r"streamname",
+    r"user@example\.com",
+    r"example\.com",
+    r"acme",
+    # Fragments of larger strings
+    r"is …",
+    r"your subscriptions on your Channels page",
+    r"Add global time<br />Everyone sees global times in their own time zone\.",
+    r"user",
+    r"an unknown operating system",
+    r"Go to Settings",
+    r"find accounts for another email address",
+    # SPECIAL CASES
+    # Because topics usually are lower-case, this would look weird if it were capitalized
+    r"show all topics",
+    # Used alone in a parenthetical where capitalized looks worse.
+    r"^deprecated$",
+    # We want the similar text in the Private Messages section to have the same capitalization.
+    r"more conversations",
+    r"back to channels",
+    # Capital 'i' looks weird in reminders popover
+    r"in 1 hour",
+    r"in 20 minutes",
+    r"in 3 hours",
+    # these are used as channel or topic names
+    r"^new channels$",
+    r"^channel events$",
+    r"^general$",
+    r"^sandbox$",
+    r"^experiments$",
+    r"^greetings$",
+    r"^moving messages$",
+    r"^start a conversation$",
+    r"^welcome to Zulip!$",
+    r"^general chat$",
+    # These are used as example short names (e.g. an uncapitalized context):
+    r"^marketing$",
+    r"^cookie$",
+    # Used to refer custom time limits
+    r"\bN\b",
+    r"minute",
+    r"minutes",
+    # Capital c feels obtrusive in clear status option
+    r"clear",
+    r"group direct messages with \{recipient\}",
+    r"direct messages with \{recipient\}",
+    r"direct messages with yourself",
+    r"GIF",
+    # Emoji name placeholder
+    r"leafy green vegetable",
+    # Subdomain placeholder
+    r"your-organization",
+    # Used in invite modal
+    r"or",
+    # Units
+    r"MB",
+    r"GB",
+    # Used in GIPHY integration setting. GIFs Rating.
+    r"rated Y",
+    r"rated G",
+    r"rated PG",
+    r"rated PG13",
+    r"rated R",
+    # Used in GIPHY popover.
+    r"GIFs",
+    r"GIPHY",
+    # Used for Tenor attributions
+    r"Search Tenor",
+    # Used for KLIPY attributions
+    r"Search KLIPY",
+    # Used in our case studies
+    r"Technical University of Munich",
+    r"University of California San Diego",
+    # Used in stream creation form
+    r"email hidden",
+    # Use in compose box.
+    r"to send",
+    r"to add a new line",
+    # Used in showing Notification Bot read receipts message
+    "Notification Bot",
+    # Used in strings around welcome bot custom messages
+    r"Welcome Bot",
+    # Used in presence_enabled setting label
+    r"invisible mode off",
+    # Typeahead suggestions for "Pronouns" custom field type.
+    r"he/him",
+    r"she/her",
+    r"they/them",
+    # Used in message-move-time-limit setting label
+    r"does not apply to moderators and administrators",
+    # Used in message-delete-time-limit setting label
+    r"does not apply to users who can delete any message",
+    # Used as indicator with names for guest users.
+    r"guest",
+    # Used as indicator with names for archived streams.
+    r"archived",
+    # Used in pills for deactivated users.
+    r"deactivated",
+    # Used in pills for resolved topics.
+    r"resolved",
+    # Used in pills for unresolved topics.
+    r"unresolved",
+    # Used in pills for followed topics.
+    r"followed",
+    # Used in pills for unfollowed topics.
+    r"unfollowed",
+    # This is a reference to a setting/secret and should be lowercase.
+    r"zulip_org_id",
+    # These are custom time unit options for modal dropdowns
+    r"minutes",
+    r"hours",
+    r"days",
+    r"weeks",
+    # Used in "Who can subscribe to this channel" label.
+    r"everyone except guests can subscribe to any public channel",
+    # Used in branch-filtering label in the integration-url-modal.
+    r"comma-separated list",
+    # Used in info_overlay.
+    r"then",
+    r"Joe Smith",
+    r"bold",
+    r"channel name",
+    r"is busy working",
+    r"italic",
+    r"strikethrough",
+    r"support team",
+    r"topic name",
+]
+
+# Sort regexes in descending order of their lengths. As a result, the
+# longer phrases will be ignored first.
+IGNORED_PHRASES.sort(key=len, reverse=True)
+
+# Compile regexes to improve performance. This also extracts the
+# text using BeautifulSoup and then removes extra whitespaces from
+# it. This step enables us to add HTML in our regexes directly.
+COMPILED_IGNORED_PHRASES = [
+    re.compile(r" ".join(BeautifulSoup(StringIO(regex), "lxml").text.split()))
+    for regex in IGNORED_PHRASES
+]
+
+SPLIT_BOUNDARY = r"?.!"  # Used to split string into sentences.
+SPLIT_BOUNDARY_REGEX = re.compile(rf"[{SPLIT_BOUNDARY}]")
+
+# Regexes which check capitalization in sentences.
+DISALLOWED = [
+    r"^[a-z](?!\})",  # Checks if the sentence starts with a lower case character.
+    r"^[A-Z][a-z]+[\sa-z0-9]+[A-Z]",  # Checks if an upper case character exists
+    # after a lower case character when the first character is in upper case.
+]
+DISALLOWED_REGEX = re.compile(r"|".join(DISALLOWED))
+
+BANNED_WORDS = {
+    "realm": "The term realm should not appear in user-facing strings. Use organization instead.",
+}
+
+
+def get_safe_phrase(phrase: str) -> str:
+    """
+    Safe phrase is in lower case and doesn't contain characters which can
+    conflict with split boundaries. All conflicting characters are replaced
+    with low dash (_).
+    """
+    phrase = SPLIT_BOUNDARY_REGEX.sub("_", phrase)
+    return phrase.lower()
+
+
+def replace_with_safe_phrase(matchobj: Match[str]) -> str:
+    """
+    The idea is to convert IGNORED_PHRASES into safe phrases, see
+    `get_safe_phrase()` function. The only exception is when the
+    IGNORED_PHRASE is at the start of the text or after a split
+    boundary; in this case, we change the first letter of the phrase
+    to upper case.
+    """
+    ignored_phrase = matchobj.group(0)
+    safe_string = get_safe_phrase(ignored_phrase)
+
+    start_index = matchobj.start()
+    complete_string = matchobj.string
+
+    is_string_start = start_index == 0
+    # We expect that there will be one space between split boundary
+    # and the next word.
+    punctuation = complete_string[max(start_index - 2, 0)]
+    is_after_split_boundary = punctuation in SPLIT_BOUNDARY
+    if is_string_start or is_after_split_boundary:
+        return safe_string.capitalize()
+
+    return safe_string
+
+
+def get_safe_text(text: str) -> str:
+    """
+    This returns text which is rendered by BeautifulSoup and is in the
+    form that can be split easily and has all IGNORED_PHRASES processed.
+    """
+    soup = BeautifulSoup(StringIO(text), "lxml")
+    text = " ".join(soup.text.split())  # Remove extra whitespaces.
+    for phrase_regex in COMPILED_IGNORED_PHRASES:
+        text = phrase_regex.sub(replace_with_safe_phrase, text)
+
+    return text
+
+
+def is_capitalized(safe_text: str) -> bool:
+    sentences = SPLIT_BOUNDARY_REGEX.split(safe_text)
+    return not any(DISALLOWED_REGEX.search(sentence.strip()) for sentence in sentences)
+
+
+def check_banned_words(text: str) -> list[str]:
+    lower_cased_text = text.lower()
+    errors = []
+    for word, reason in BANNED_WORDS.items():
+        if word in lower_cased_text:
+            # Hack: Should move this into BANNED_WORDS framework; for
+            # now, just hand-code the skips:
+            if (
+                "realm_name" in lower_cased_text
+                or "realm_uri" in lower_cased_text
+                or "realm_url" in lower_cased_text
+                or "remote_realm_host" in lower_cased_text
+                or "realm_message" in lower_cased_text
+                or "realm_move" in lower_cased_text
+            ):
+                continue
+            kwargs = dict(word=word, text=text, reason=reason)
+            msg = "{word} found in '{text}'. {reason}".format(**kwargs)
+            errors.append(msg)
+
+    return errors
+
+
+def check_capitalization(strings: list[str]) -> tuple[list[str], list[str], list[str]]:
+    errors = []
+    ignored = []
+    banned_word_errors = []
+    for text in strings:
+        text = " ".join(text.split())  # Remove extra whitespaces.
+        safe_text = get_safe_text(text)
+        has_ignored_phrase = text != safe_text
+        capitalized = is_capitalized(safe_text)
+        if not capitalized:
+            errors.append(text)
+        elif has_ignored_phrase:
+            ignored.append(text)
+
+        banned_word_errors.extend(check_banned_words(text))
+
+    return sorted(errors), sorted(ignored), sorted(banned_word_errors)

tools/lib/git-tools.bash

@@ -0,0 +1,34 @@
+# shellcheck shell=bash
+
+# Borrowed from Git's git-sh-setup.
+#
+# See git.git commit 92c62a3f4 (from 2010!); as of 2020 with Git 2.26,
+# this function has only needed one edit since then, adding localization
+# with gettext, which we can omit.
+require_clean_work_tree() {
+    local action="$1"
+
+    git rev-parse --verify HEAD >/dev/null || exit 1
+    git update-index -q --ignore-submodules --refresh
+    local err=0
+
+    if ! git diff-files --quiet --ignore-submodules; then
+        echo >&2 "Cannot $action: You have unstaged changes."
+        err=1
+    fi
+
+    if ! git diff-index --cached --quiet --ignore-submodules HEAD --; then
+        if [ $err = 0 ]; then
+            echo >&2 "Cannot $action: Your index contains uncommitted changes."
+        else
+            echo >&2 "Additionally, your index contains uncommitted changes."
+        fi
+        err=1
+    fi
+
+    if [ $err = 1 ]; then
+        git status --short
+        echo >&2 "Doing nothing to avoid losing your work."
+        exit 1
+    fi
+}

tools/lib/gitlint_rules.py

@@ -0,0 +1,108 @@
+from gitlint.git import GitCommit
+from gitlint.rules import CommitMessageTitle, LineRule, RuleViolation
+
+# Word list from https://github.com/m1foley/fit-commit
+# Copyright (c) 2015 Mike Foley
+# License: MIT
+# Ref: fit_commit/validators/tense.rb
+TENSE_DATA = [
+    (["adds", "adding", "added"], "add"),
+    (["allows", "allowing", "allowed"], "allow"),
+    (["amends", "amending", "amended"], "amend"),
+    (["bumps", "bumping", "bumped"], "bump"),
+    (["calculates", "calculating", "calculated"], "calculate"),
+    (["changes", "changing", "changed"], "change"),
+    (["cleans", "cleaning", "cleaned"], "clean"),
+    (["commits", "committing", "committed"], "commit"),
+    (["corrects", "correcting", "corrected"], "correct"),
+    (["creates", "creating", "created"], "create"),
+    (["darkens", "darkening", "darkened"], "darken"),
+    (["disables", "disabling", "disabled"], "disable"),
+    (["displays", "displaying", "displayed"], "display"),
+    (["documents", "documenting", "documented"], "document"),
+    (["drys", "drying", "dried"], "dry"),
+    (["ends", "ending", "ended"], "end"),
+    (["enforces", "enforcing", "enforced"], "enforce"),
+    (["enqueues", "enqueuing", "enqueued"], "enqueue"),
+    (["extracts", "extracting", "extracted"], "extract"),
+    (["finishes", "finishing", "finished"], "finish"),
+    (["fixes", "fixing", "fixed"], "fix"),
+    (["formats", "formatting", "formatted"], "format"),
+    (["guards", "guarding", "guarded"], "guard"),
+    (["handles", "handling", "handled"], "handle"),
+    (["hides", "hiding", "hid"], "hide"),
+    (["increases", "increasing", "increased"], "increase"),
+    (["ignores", "ignoring", "ignored"], "ignore"),
+    (["implements", "implementing", "implemented"], "implement"),
+    (["improves", "improving", "improved"], "improve"),
+    (["keeps", "keeping", "kept"], "keep"),
+    (["kills", "killing", "killed"], "kill"),
+    (["makes", "making", "made"], "make"),
+    (["merges", "merging", "merged"], "merge"),
+    (["moves", "moving", "moved"], "move"),
+    (["permits", "permitting", "permitted"], "permit"),
+    (["prevents", "preventing", "prevented"], "prevent"),
+    (["pushes", "pushing", "pushed"], "push"),
+    (["rebases", "rebasing", "rebased"], "rebase"),
+    (["refactors", "refactoring", "refactored"], "refactor"),
+    (["removes", "removing", "removed"], "remove"),
+    (["renames", "renaming", "renamed"], "rename"),
+    (["reorders", "reordering", "reordered"], "reorder"),
+    (["replaces", "replacing", "replaced"], "replace"),
+    (["requires", "requiring", "required"], "require"),
+    (["restores", "restoring", "restored"], "restore"),
+    (["sends", "sending", "sent"], "send"),
+    (["sets", "setting"], "set"),
+    (["separates", "separating", "separated"], "separate"),
+    (["shows", "showing", "showed"], "show"),
+    (["simplifies", "simplifying", "simplified"], "simplify"),
+    (["skips", "skipping", "skipped"], "skip"),
+    (["sorts", "sorting"], "sort"),
+    (["speeds", "speeding", "sped"], "speed"),
+    (["starts", "starting", "started"], "start"),
+    (["supports", "supporting", "supported"], "support"),
+    (["takes", "taking", "took"], "take"),
+    (["testing", "tested"], "test"),  # "tests" excluded to reduce false negatives
+    (["truncates", "truncating", "truncated"], "truncate"),
+    (["updates", "updating", "updated"], "update"),
+    (["uses", "using", "used"], "use"),
+]
+
+TENSE_CORRECTIONS = {word: imperative for words, imperative in TENSE_DATA for word in words}
+
+
+class ImperativeMood(LineRule):
+    """This rule will enforce that the commit message title uses imperative
+    mood. This is done by checking if the first word is in `WORD_SET`, if so
+    show the word in the correct mood."""
+
+    name = "title-imperative-mood"
+    id = "Z1"
+    target = CommitMessageTitle
+
+    error_msg = (
+        "The first word in commit title should be in imperative mood "
+        '("{word}" -> "{imperative}"): "{title}"'
+    )
+
+    def validate(self, line: str, commit: GitCommit) -> list[RuleViolation]:
+        violations = []
+
+        # Ignore the section tag (ie `<section tag>: <message body>.`)
+        words = line.split(": ", 1)[-1].split()
+        first_word = words[0].lower()
+
+        if first_word in TENSE_CORRECTIONS:
+            imperative = TENSE_CORRECTIONS[first_word]
+            violation = RuleViolation(
+                self.id,
+                self.error_msg.format(
+                    word=first_word,
+                    imperative=imperative,
+                    title=commit.message.title,
+                ),
+            )
+
+            violations.append(violation)
+
+        return violations

tools/lib/html_branches.py

@@ -0,0 +1,98 @@
+import re
+from collections import defaultdict
+
+from .template_parser import FormattedError, Token, tokenize
+
+
+class TagInfo:
+    def __init__(self, tag: str, classes: list[str], ids: list[str], token: Token) -> None:
+        self.tag = tag
+        self.classes = classes
+        self.ids = ids
+        self.token = token
+        self.words = [
+            self.tag,
+            *("." + s for s in classes),
+            *("#" + s for s in ids),
+        ]
+
+    def text(self) -> str:
+        s = self.tag
+        if self.classes:
+            s += "." + ".".join(self.classes)
+        if self.ids:
+            s += "#" + "#".join(self.ids)
+        return s
+
+
+def get_tag_info(token: Token) -> TagInfo:
+    s = token.s
+    tag = token.tag
+    classes: list[str] = []
+    ids: list[str] = []
+
+    searches = [
+        (classes, ' class="(.*?)"'),
+        (classes, " class='(.*?)'"),
+        (ids, ' id="(.*?)"'),
+        (ids, " id='(.*?)'"),
+    ]
+
+    for lst, regex in searches:
+        m = re.search(regex, s)
+        if m:
+            for g in m.groups():
+                lst += split_for_id_and_class(g)
+
+    return TagInfo(tag=tag, classes=classes, ids=ids, token=token)
+
+
+def split_for_id_and_class(element: str) -> list[str]:
+    # Here we split a given string which is expected to contain id or class
+    # attributes from HTML tags. This also takes care of template variables
+    # in string during splitting process. For eg. 'red black {{ a|b|c }}'
+    # is split as ['red', 'black', '{{ a|b|c }}']
+    outside_braces: bool = True
+    lst = []
+    s = ""
+
+    for ch in element:
+        if ch == "{":
+            outside_braces = False
+        if ch == "}":
+            outside_braces = True
+        if ch == " " and outside_braces:
+            if s != "":
+                lst.append(s)
+            s = ""
+        else:
+            s += ch
+    if s != "":
+        lst.append(s)
+
+    return lst
+
+
+def build_id_dict(templates: list[str]) -> dict[str, list[str]]:
+    template_id_dict: dict[str, list[str]] = defaultdict(list)
+
+    for fn in templates:
+        with open(fn) as f:
+            text = f.read()
+
+        try:
+            list_tags = tokenize(text)
+        except FormattedError as e:
+            raise Exception(
+                f"""
+                fn: {fn}
+                {e}"""
+            )
+
+        for tag in list_tags:
+            info = get_tag_info(tag)
+
+            for ids in info.ids:
+                template_id_dict[ids].append("Line " + str(info.token.line) + ":" + fn)
+
+    return template_id_dict

tools/lib/html_elements.py

@@ -0,0 +1,191 @@
+from collections.abc import Iterator
+
+VALID_HTML_CONTEXTS: dict[tuple[str, str], str] = {
+    # https://html.spec.whatwg.org/multipage/indices.html#elements-3
+    ("a", "phrasing"): "transparent",
+    ("abbr", "phrasing"): "phrasing",
+    ("address", "flow"): "flow",
+    ("area", "phrasing"): "void",
+    ("article", "flow"): "flow",
+    ("aside", "flow"): "flow",
+    ("audio", "phrasing"): "<audio>",
+    ("b", "phrasing"): "phrasing",
+    ("base", "<head>"): "void",
+    ("bdi", "phrasing"): "phrasing",
+    ("bdo", "phrasing"): "phrasing",
+    ("blockquote", "flow"): "flow",
+    ("body", "<html>"): "flow",
+    ("br", "phrasing"): "void",
+    ("button", "phrasing"): "phrasing",
+    ("button", "<select>"): "phrasing",
+    ("canvas", "phrasing"): "transparent",
+    ("caption", "<table>"): "flow",
+    ("center", "flow"): "flow",  # FIXME: obsolete, remove this
+    ("cite", "phrasing"): "phrasing",
+    ("code", "phrasing"): "phrasing",
+    ("col", "<colgroup>"): "void",
+    ("colgroup", "<table>"): "<colgroup>",
+    ("data", "phrasing"): "phrasing",
+    ("datalist", "phrasing"): "<datalist>",
+    ("dd", "<dl>"): "flow",
+    ("del", "phrasing"): "transparent",
+    ("details", "flow"): "<details>",
+    ("dfn", "phrasing"): "phrasing",
+    ("dialog", "flow"): "flow",
+    ("div", "flow"): "flow",
+    ("div", "<dl>"): "<dl>",
+    ("div", "<select> content"): "<select> content",
+    ("div", "<optgroup> content"): "<optgroup> content",
+    ("div", "<option> content"): "<option> content",
+    ("dl", "flow"): "<dl>",
+    ("dt", "<dl>"): "phrasing",
+    ("em", "phrasing"): "phrasing",
+    ("embed", "phrasing"): "void",
+    ("fieldset", "flow"): "<fieldset>",
+    ("figcaption", "<figure>"): "flow",
+    ("figure", "flow"): "<figure>",
+    ("footer", "flow"): "flow",
+    ("form", "flow"): "flow",
+    ("h1", "plain heading"): "phrasing",
+    ("h2", "plain heading"): "phrasing",
+    ("h3", "plain heading"): "phrasing",
+    ("h4", "plain heading"): "phrasing",
+    ("h5", "plain heading"): "phrasing",
+    ("h6", "plain heading"): "phrasing",
+    ("head", "<html>"): "<head>",
+    ("header", "flow"): "flow",
+    ("hgroup", "heading"): "<hgroup>",
+    ("hr", "flow"): "void",
+    ("hr", "<select> content"): "void",
+    ("html", "document"): "<html>",
+    ("i", "phrasing"): "phrasing",
+    ("iframe", "phrasing"): "empty",
+    ("img", "phrasing"): "void",
+    ("img", "<picture>"): "void",
+    ("input", "phrasing"): "void",
+    ("ins", "phrasing"): "transparent",
+    ("kbd", "phrasing"): "phrasing",
+    ("label", "phrasing"): "phrasing",
+    ("legend", "<fieldset>"): "phrasing/heading",
+    ("li", "list"): "flow",
+    ("link", "<head>"): "void",
+    ("link", "phrasing"): "void",
+    ("main", "flow"): "flow",
+    ("map", "phrasing"): "<map>",
+    ("mark", "phrasing"): "phrasing",
+    ("math", "phrasing"): "MathML",
+    ("menu", "flow"): "list",
+    ("meta", "<head>"): "void",
+    ("meta", "phrasing"): "void",
+    ("meter", "phrasing"): "phrasing",
+    ("nav", "flow"): "flow",
+    ("noscript", "<head>"): "transparent",
+    ("noscript", "phrasing"): "transparent",
+    ("noscript", "<select> content"): "transparent",
+    ("noscript", "<optgroup> content"): "transparent",
+    ("object", "phrasing"): "transparent",
+    ("ol", "flow"): "list",
+    ("optgroup", "<select>"): "<optgroup> content",
+    ("option", "<select>"): "<option> content",
+    ("option", "<datalist>"): "<option> content",
+    ("option", "<optgroup>"): "<option> content",
+    ("output", "phrasing"): "phrasing",
+    ("p", "flow"): "phrasing",
+    ("p", "<hgroup>"): "phrasing",
+    ("picture", "phrasing"): "<picture>",
+    ("pre", "flow"): "phrasing",
+    ("progress", "phrasing"): "phrasing",
+    ("q", "phrasing"): "phrasing",
+    ("rp", "<ruby>"): "phrasing",
+    ("rt", "<ruby>"): "phrasing",
+    ("ruby", "phrasing"): "<ruby>",
+    ("s", "phrasing"): "phrasing",
+    ("samp", "phrasing"): "phrasing",
+    ("script", "<head>"): "<script>",
+    ("script", "phrasing"): "<script>",
+    ("script", "script-supporting"): "<script>",
+    ("search", "flow"): "flow",
+    ("section", "flow"): "flow",
+    ("select", "phrasing"): "<select>",
+    ("selectedcontent", "<button>"): "empty",
+    ("slot", "phrasing"): "transparent",
+    ("small", "phrasing"): "phrasing",
+    ("source", "<picture>"): "void",
+    ("source", "<video>"): "void",
+    ("source", "<audio>"): "void",
+    ("span", "phrasing"): "phrasing",
+    ("strong", "phrasing"): "phrasing",
+    ("style", "<head>"): "<style>",
+    ("sub", "phrasing"): "phrasing",
+    ("summary", "<details>"): "phrasing/heading",
+    ("sup", "phrasing"): "phrasing",
+    ("svg", "phrasing"): "SVG",
+    ("table", "flow"): "<table>",
+    ("tbody", "<table>"): "<tbody>",
+    ("td", "<tr>"): "flow",
+    ("template", "<head>"): "unknown",
+    ("template", "phrasing"): "unknown",
+    ("template", "script-supporting"): "unknown",
+    ("template", "<colgroup>"): "unknown",
+    ("textarea", "phrasing"): "text",
+    ("tfoot", "<table>"): "<tfoot>",
+    ("th", "<tr>"): "flow",
+    ("thead", "<table>"): "<thead>",
+    ("time", "phrasing"): "phrasing",
+    ("title", "<head>"): "text",
+    ("tr", "<table>"): "<tr>",
+    ("tr", "<thead>"): "<tr>",
+    ("tr", "<tbody>"): "<tr>",
+    ("tr", "<tfoot>"): "<tr>",
+    ("track", "<audio>"): "void",
+    ("track", "<video>"): "void",
+    ("u", "phrasing"): "phrasing",
+    ("ul", "flow"): "list",
+    ("var", "phrasing"): "phrasing",
+    ("video", "phrasing"): "<video>",
+    ("wbr", "phrasing"): "void",
+    # https://html.spec.whatwg.org/multipage/embedded-content-other.html#mathml
+    ("annotation-xml", "MathML"): "flow",
+    ("mi", "MathML"): "phrasing",
+    ("mo", "MathML"): "phrasing",
+    ("mn", "MathML"): "phrasing",
+    ("ms", "MathML"): "phrasing",
+    ("mtext", "MathML"): "phrasing",
+    # https://html.spec.whatwg.org/multipage/embedded-content-other.html#svg-0
+    ("foreignObject", "SVG"): "flow",
+    ("title", "SVG"): "phrasing",
+}
+
+HTML_CONTEXT_FALLBACKS: dict[str, list[str]] = {
+    "<datalist>": ["phrasing", "script-supporting"],
+    "<details>": ["flow"],
+    "<dl>": ["script-supporting"],
+    "<fieldset>": ["flow"],
+    "<figure>": ["flow"],
+    "<hgroup>": ["plain heading", "script-supporting"],
+    "<optgroup> content": ["script-supporting"],
+    "<option> content": ["phrasing"],
+    "<option>": ["<option> content"],
+    "<picture>": ["script-supporting"],
+    "<ruby>": ["phrasing"],
+    "<select> content": ["script-supporting"],
+    "<select>": ["<select> content"],
+    "<table>": ["script-supporting"],
+    "<tbody>": ["script-supporting"],
+    "<tfoot>": ["script-supporting"],
+    "<thead>": ["script-supporting"],
+    "<tr>": ["script-supporting"],
+    "flow": ["phrasing", "heading"],
+    "heading": ["plain heading"],
+    "list": ["script-supporting"],
+    "phrasing/heading": ["phrasing", "heading"],
+    "unknown": ["document", "flow", "list", "<head>", "<select>", "<table>", "<tr>"],
+}
+
+FOREIGN_CONTEXTS = ["MathML", "SVG"]
+
+
+def html_context_fallbacks(context: str) -> Iterator[str]:
+    yield context
+    for fallback_context in HTML_CONTEXT_FALLBACKS.get(context, []):
+        yield from html_context_fallbacks(fallback_context)

tools/lib/pretty_print.py

@@ -0,0 +1,197 @@
+import subprocess
+
+from zulint.printer import BOLDRED, CYAN, ENDC, GREEN
+
+from .template_parser import Token
+
+
+def shift_indents_to_the_next_tokens(tokens: list[Token]) -> None:
+    """
+    During the parsing/validation phase, it's useful to have separate
+    tokens for "indent" chunks, but during pretty printing, we like
+    to attach an `.indent` field to the substantive node, whether
+    it's an HTML tag or template directive or whatever.
+    """
+    tokens[0].indent = ""
+
+    for i, token in enumerate(tokens[:-1]):
+        next_token = tokens[i + 1]
+
+        if token.kind == "indent":
+            next_token.indent = token.s
+            token.new_s = ""
+
+        if token.kind == "newline" and next_token.kind != "indent":
+            next_token.indent = ""
+
+
+def token_allows_children_to_skip_indents(token: Token) -> bool:
+    # To avoid excessive indentation in templates with other
+    # conditionals, we don't require extra indentation for template
+    # logic blocks don't contain further logic as direct children.
+
+    # Each blocks are excluded from this rule, since we want loops to
+    # stand out.
+    if token.tag == "each":
+        return False
+
+    return token.kind in ("django_start", "handlebars_start") or token.tag == "a"
+
+
+def adjust_block_indentation(tokens: list[Token], fn: str) -> None:
+    start_token: Token | None = None
+
+    for token in tokens:
+        if token.kind in ("indent", "whitespace", "newline"):
+            continue
+
+        if token.tag in ("code", "pre"):
+            continue
+
+        # print(token.line, repr(start_token.indent) if start_token else "?", repr(token.indent), token.s, token.end_token and "start", token.start_token and "end")
+
+        if token.tag == "else":
+            assert token.start_token
+            if token.indent is not None:
+                token.indent = token.start_token.indent
+            continue
+
+        if start_token and token.indent is not None:
+            if (
+                not start_token.indent_is_final
+                and token.indent == start_token.orig_indent
+                and token_allows_children_to_skip_indents(start_token)
+            ):
+                start_token.child_indent = start_token.indent
+            start_token.indent_is_final = True
+
+        # Detect start token by its having a end token
+        if token.end_token:
+            if token.indent is not None:
+                token.orig_indent = token.indent
+                if start_token:
+                    assert start_token.child_indent is not None
+                    token.indent = start_token.child_indent
+                else:
+                    token.indent = ""
+                token.child_indent = token.indent + "    "
+            token.parent_token = start_token
+            start_token = token
+            continue
+
+        # Detect end token by its having a start token
+        if token.start_token:
+            if start_token != token.start_token:
+                raise AssertionError(
+                    f"""
+                    {token.kind} was unexpected in {token.s}
+                    in row {token.line} of {fn}
+                    """
+                )
+
+            if token.indent is not None:
+                token.indent = start_token.indent
+            start_token = start_token.parent_token
+            continue
+
+        if token.indent is None:
+            continue
+
+        if start_token is None:
+            token.indent = ""
+            continue
+
+        if start_token.child_indent is not None:
+            token.indent = start_token.child_indent
+
+
+def fix_indents_for_multi_line_tags(tokens: list[Token]) -> None:
+    def fix(frag: str) -> str:
+        frag = frag.strip()
+        return continue_indent + frag if frag else ""
+
+    for token in tokens:
+        if token.kind == "code":
+            continue
+
+        if token.line_span == 1 or token.indent is None:
+            continue
+
+        if token.kind in ("django_comment", "handlebars_comment", "html_comment", "text"):
+            continue_indent = token.indent
+        else:
+            continue_indent = token.indent + "  "
+
+        frags = token.new_s.split("\n")
+
+        token.new_s = frags[0] + "\n" + "\n".join(fix(frag) for frag in frags[1:])
+
+
+def apply_token_indents(tokens: list[Token]) -> None:
+    for token in tokens:
+        if token.indent:
+            token.new_s = token.indent + token.new_s
+
+
+def pretty_print_html(tokens: list[Token], fn: str) -> str:
+    for token in tokens:
+        token.new_s = token.s
+
+    shift_indents_to_the_next_tokens(tokens)
+    adjust_block_indentation(tokens, fn)
+    fix_indents_for_multi_line_tags(tokens)
+    apply_token_indents(tokens)
+
+    return "".join(token.new_s for token in tokens)
+
+
+def numbered_lines(s: str) -> str:
+    return "".join(f"{i + 1: >5} {line}\n" for i, line in enumerate(s.split("\n")))
+
+
+def validate_indent_html(fn: str, tokens: list[Token], fix: bool) -> bool:
+    with open(fn) as f:
+        html = f.read()
+    phtml = pretty_print_html(tokens, fn)
+    if html.split("\n") != phtml.split("\n"):
+        if fix:
+            print(GREEN + f"Automatically fixing indentation for {fn}" + ENDC)
+            with open(fn, "w") as f:
+                f.write(phtml)
+            # Since we successfully fixed the issues, we return True.
+            return True
+        print(
+            f"""
+{BOLDRED}PROBLEM{ENDC}: formatting errors in {fn}
+
+Here is how we would like you to format
+{CYAN}{fn}{ENDC}:
+---
+{numbered_lines(phtml)}
+---
+
+Here is the diff that you should either execute in your editor
+or apply automatically with the --fix option.
+
+({CYAN}Scroll up{ENDC} to see how we would like the file formatted.)
+
+Proposed {BOLDRED}diff{ENDC} for {CYAN}{fn}{ENDC}:
+            """,
+            flush=True,
+        )
+        subprocess.run(["diff", fn, "-"], input=phtml, text=True, check=False)
+        print(
+            f"""
+---
+
+{BOLDRED}PROBLEM!!!{ENDC}
+
+    You have formatting errors in {CYAN}{fn}{ENDC}
+    (Usually these messages are related to indentation.)
+
+This problem can be fixed with the {CYAN}`--fix`{ENDC} option.
+Scroll up for more details about {BOLDRED}what you need to fix ^^^{ENDC}.
+            """
+        )
+        return False
+    return True

tools/lib/provision.py

@@ -0,0 +1,512 @@
+#!/usr/bin/env python3
+import argparse
+import contextlib
+import hashlib
+import logging
+import os
+import platform
+import subprocess
+import sys
+from typing import NoReturn
+
+os.environ["PYTHONUNBUFFERED"] = "y"
+
+ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+sys.path.append(ZULIP_PATH)
+
+from scripts.lib.node_cache import setup_node_modules
+from scripts.lib.setup_venv import get_venv_dependencies
+from scripts.lib.zulip_tools import (
+    ENDC,
+    FAIL,
+    WARNING,
+    get_dev_uuid_var_path,
+    os_families,
+    run,
+    run_as_root,
+)
+
+VAR_DIR_PATH = os.path.join(ZULIP_PATH, "var")
+
+CONTINUOUS_INTEGRATION = "GITHUB_ACTIONS" in os.environ
+
+if not os.path.exists(os.path.join(ZULIP_PATH, ".git")):
+    print(FAIL + "Error: No Zulip Git repository present!" + ENDC)
+    print("To set up the Zulip development environment, you should clone the code")
+    print("from GitHub, rather than using a Zulip production release tarball.")
+    sys.exit(1)
+
+# Check the RAM on the user's system, and throw an effort if <1.5GB.
+# This avoids users getting segfaults running `pip install` that are
+# generally more annoying to debug.
+with open("/proc/meminfo") as meminfo:
+    ram_size = meminfo.readlines()[0].strip().split(" ")[-2]
+ram_gb = float(ram_size) / 1024.0 / 1024.0
+if ram_gb < 1.5:
+    print(
+        f"You have insufficient RAM ({round(ram_gb, 2)} GB) to run the Zulip development environment."
+    )
+    print("We recommend at least 2 GB of RAM, and require at least 1.5 GB.")
+    sys.exit(1)
+
+try:
+    UUID_VAR_PATH = get_dev_uuid_var_path(create_if_missing=True)
+    os.makedirs(UUID_VAR_PATH, exist_ok=True)
+    if os.path.exists(os.path.join(VAR_DIR_PATH, "zulip-test-symlink")):
+        os.remove(os.path.join(VAR_DIR_PATH, "zulip-test-symlink"))
+    os.symlink(
+        os.path.join(ZULIP_PATH, "README.md"),
+        os.path.join(VAR_DIR_PATH, "zulip-test-symlink"),
+    )
+    os.remove(os.path.join(VAR_DIR_PATH, "zulip-test-symlink"))
+except OSError:
+    print(
+        FAIL + "Error: Unable to create symlinks. "
+        "Make sure you have permission to create symbolic links." + ENDC
+    )
+    print("See this page for more information:")
+    print(
+        "  https://zulip.readthedocs.io/en/latest/development/setup-recommended.html#os-symlink-error"
+    )
+    sys.exit(1)
+
+distro_info = platform.freedesktop_os_release()
+vendor = distro_info["ID"]
+os_version = distro_info["VERSION_ID"]
+if vendor == "debian" and os_version == "12":  # bookworm
+    POSTGRESQL_VERSION = "15"
+elif vendor == "debian" and os_version == "13":  # trixie
+    POSTGRESQL_VERSION = "17"
+elif vendor == "ubuntu" and os_version == "22.04":  # jammy
+    POSTGRESQL_VERSION = "14"
+elif vendor == "ubuntu" and os_version == "24.04":  # noble
+    POSTGRESQL_VERSION = "16"
+elif vendor == "ubuntu" and os_version == "26.04":  # resolute
+    POSTGRESQL_VERSION = "18"
+elif vendor == "fedora" and os_version in ("43", "44"):
+    POSTGRESQL_VERSION = "17"
+elif vendor == "rhel" and os_version.startswith("7."):
+    POSTGRESQL_VERSION = "10"
+elif vendor == "centos" and os_version == "7":
+    POSTGRESQL_VERSION = "10"
+else:
+    logging.critical("Unsupported platform: %s %s", vendor, os_version)
+    sys.exit(1)
+
+VENV_DEPENDENCIES = get_venv_dependencies(vendor, os_version)
+
+COMMON_DEPENDENCIES = [
+    "memcached",
+    "rabbitmq-server",
+    "supervisor",
+    "git",
+    "curl",
+    "ca-certificates",  # Explicit dependency in case e.g. curl is already installed
+    "puppet",  # Used by lint (`puppet parser validate`)
+    "gettext",  # Used by makemessages i18n
+    "curl",  # Used for testing our API documentation
+    "moreutils",  # Used for sponge command
+    "unzip",  # Needed for Slack import
+    "crudini",  # Used for shell tooling w/ zulip.conf
+    # Puppeteer dependencies from here
+    "xdg-utils",
+    # Puppeteer dependencies end here.
+]
+
+UBUNTU_COMMON_APT_DEPENDENCIES = [
+    *COMMON_DEPENDENCIES,
+    "redis-server",
+    "hunspell-en-us",
+    "puppet-lint",
+    (
+        "openjdk-17-jre-headless"
+        if vendor == "ubuntu" and os_version == "22.04"
+        else "default-jre-headless"
+    ),  # Required by vnu-jar
+    # Puppeteer dependencies from here
+    "fonts-freefont-ttf",
+    "libatk-bridge2.0-0",
+    "libgbm1",
+    "libgtk-3-0",
+    "libx11-xcb1",
+    "libxcb-dri3-0",
+    "libxss1",
+    "xvfb",
+    # Puppeteer dependencies end here.
+]
+
+COMMON_YUM_DEPENDENCIES = [
+    *COMMON_DEPENDENCIES,
+    "redis",
+    "hunspell-en-US",
+    "rubygem-puppet-lint",
+    "nmap-ncat",
+    "ccache",  # Required to build pgroonga from source.
+    # Puppeteer dependencies from here
+    "at-spi2-atk",
+    "GConf2",
+    "gtk3",
+    "libX11-xcb",
+    "libxcb",
+    "libXScrnSaver",
+    "mesa-libgbm",
+    "xorg-x11-server-Xvfb",
+    # Puppeteer dependencies end here.
+]
+
+BUILD_PGROONGA_FROM_SOURCE = False
+if (vendor == "debian" and os_version == "13") or (vendor == "ubuntu" and os_version == "26.04"):
+    # For platforms without a PGroonga release, we need to build it
+    # from source.
+    BUILD_PGROONGA_FROM_SOURCE = True
+    SYSTEM_DEPENDENCIES = [
+        *UBUNTU_COMMON_APT_DEPENDENCIES,
+        f"postgresql-{POSTGRESQL_VERSION}",
+        # Dependency for building PGroonga from source
+        "meson",
+        f"postgresql-server-dev-{POSTGRESQL_VERSION}",
+        "libgroonga-dev",
+        "libmsgpack-dev",
+        *VENV_DEPENDENCIES,
+    ]
+elif "debian" in os_families():
+    DEBIAN_DEPENDENCIES = UBUNTU_COMMON_APT_DEPENDENCIES
+
+    # If we are on an aarch64 processor, ninja will be built from source,
+    # so cmake is required
+    if platform.machine() == "aarch64":
+        DEBIAN_DEPENDENCIES.append("cmake")
+
+    SYSTEM_DEPENDENCIES = [
+        *DEBIAN_DEPENDENCIES,
+        f"postgresql-{POSTGRESQL_VERSION}",
+        f"postgresql-{POSTGRESQL_VERSION}-pgroonga",
+        *VENV_DEPENDENCIES,
+    ]
+elif "rhel" in os_families():
+    SYSTEM_DEPENDENCIES = [
+        *COMMON_YUM_DEPENDENCIES,
+        f"postgresql{POSTGRESQL_VERSION}-server",
+        f"postgresql{POSTGRESQL_VERSION}",
+        f"postgresql{POSTGRESQL_VERSION}-devel",
+        f"postgresql{POSTGRESQL_VERSION}-pgdg-pgroonga",
+        *VENV_DEPENDENCIES,
+    ]
+elif "fedora" in os_families():
+    SYSTEM_DEPENDENCIES = [
+        *COMMON_YUM_DEPENDENCIES,
+        f"postgresql{POSTGRESQL_VERSION}-server",
+        f"postgresql{POSTGRESQL_VERSION}",
+        f"postgresql{POSTGRESQL_VERSION}-devel",
+        # Needed to build PGroonga from source
+        "groonga-devel",
+        "msgpack-devel",
+        # PGroonga compiles against <xxhash.h>; on systems where
+        # `xxhash-libs` is already pulled in transitively (e.g. by
+        # blosc2 / pyarrow) but `xxhash-devel` is not, meson detects
+        # libxxhash via pkg-config and skips its vendored fallback,
+        # so we have to provide the header ourselves.
+        "xxhash-devel",
+        "meson",
+        # Provides /usr/lib/rpm/redhat/redhat-hardened-cc1 and
+        # redhat-annobin-cc1 spec files referenced by the CFLAGS that
+        # PostgreSQL's pg_config exports; without it the PGroonga
+        # compile fails with "cannot read spec file".
+        "redhat-rpm-config",
+        *VENV_DEPENDENCIES,
+    ]
+    BUILD_PGROONGA_FROM_SOURCE = True
+
+if "fedora" in os_families():
+    TSEARCH_STOPWORDS_PATH = f"/usr/pgsql-{POSTGRESQL_VERSION}/share/tsearch_data/"
+else:
+    TSEARCH_STOPWORDS_PATH = f"/usr/share/postgresql/{POSTGRESQL_VERSION}/tsearch_data/"
+REPO_STOPWORDS_PATH = os.path.join(
+    ZULIP_PATH,
+    "puppet",
+    "zulip",
+    "files",
+    "postgresql",
+    "zulip_english.stop",
+)
+
+
+def install_system_deps() -> None:
+    # By doing list -> set -> list conversion, we remove duplicates.
+    deps_to_install = sorted(set(SYSTEM_DEPENDENCIES))
+
+    if "fedora" in os_families():
+        install_yum_deps(deps_to_install)
+    elif "debian" in os_families():
+        install_apt_deps(deps_to_install)
+    else:
+        raise AssertionError("Invalid vendor")
+
+    # For some platforms, there aren't published PGroonga
+    # packages available, so we build them from source.
+    if BUILD_PGROONGA_FROM_SOURCE:
+        run_as_root(["./scripts/lib/build-pgroonga"])
+
+
+def install_apt_deps(deps_to_install: list[str]) -> None:
+    # setup-apt-repo does an `apt-get update` if the sources.list files changed.
+    run_as_root(["./scripts/lib/setup-apt-repo"])
+
+    # But we still need to do our own to make sure we have up-to-date
+    # data before installing new packages, as the system might not have
+    # done an apt update in weeks otherwise, which could result in 404s
+    # trying to download old versions that were already removed from mirrors.
+    run_as_root(["apt-get", "update"])
+    run_as_root(
+        [
+            "env",
+            "DEBIAN_FRONTEND=noninteractive",
+            "apt-get",
+            "-y",
+            "install",
+            "--allow-downgrades",
+            "--no-install-recommends",
+            *deps_to_install,
+        ]
+    )
+
+
+def install_yum_deps(deps_to_install: list[str]) -> None:
+    print(WARNING + "RedHat support is still experimental." + ENDC)
+    run_as_root(["./scripts/lib/setup-yum-repo"])
+
+    # Hack specific to unregistered RHEL system.  The moreutils
+    # package requires a perl module package, which isn't available in
+    # the unregistered RHEL repositories.
+    #
+    # Error: Package: moreutils-0.49-2.el7.x86_64 (epel)
+    #        Requires: perl(IPC::Run)
+    yum_extra_flags: list[str] = []
+    if vendor == "rhel":
+        proc = subprocess.run(
+            ["sudo", "subscription-manager", "status"],
+            stdout=subprocess.PIPE,
+            text=True,
+            check=False,
+        )
+        if proc.returncode == 1:
+            # TODO this might overkill since `subscription-manager` is already
+            # called in setup-yum-repo
+            if "Status" in proc.stdout:
+                # The output is well-formed
+                yum_extra_flags = ["--skip-broken"]
+            else:
+                print("Unrecognized output. `subscription-manager` might not be available")
+
+    run_as_root(["yum", "install", "-y", *yum_extra_flags, *deps_to_install])
+    if "rhel" in os_families():
+        # This is how a pip3 is installed to /usr/bin in CentOS/RHEL
+        # for python35 and later.
+        run_as_root(["python36", "-m", "ensurepip"])
+        # `python36` is not aliased to `python3` by default
+        run_as_root(["ln", "-nsf", "/usr/bin/python36", "/usr/bin/python3"])
+    postgresql_dir = f"pgsql-{POSTGRESQL_VERSION}"
+    for cmd in ["pg_config", "pg_isready", "psql"]:
+        # Our tooling expects these PostgreSQL scripts to be at
+        # well-known paths.  There's an argument for eventually
+        # making our tooling auto-detect, but this is simpler.
+        run_as_root(["ln", "-nsf", f"/usr/{postgresql_dir}/bin/{cmd}", f"/usr/bin/{cmd}"])
+
+    # From here, we do the first-time setup/initialization for the PostgreSQL database.
+    pg_datadir = f"/var/lib/pgsql/{POSTGRESQL_VERSION}/data"
+    pg_hba_conf = os.path.join(pg_datadir, "pg_hba.conf")
+
+    # We can't just check if the file exists with os.path, since the
+    # current user likely doesn't have permission to read the
+    # pg_datadir directory.
+    if subprocess.call(["sudo", "test", "-e", pg_hba_conf]) == 0:
+        # Skip setup if it has been applied previously
+        return
+
+    run_as_root(
+        [f"/usr/{postgresql_dir}/bin/postgresql-{POSTGRESQL_VERSION}-setup", "initdb"],
+        sudo_args=["-H"],
+    )
+    # Use vendored pg_hba.conf, which enables password authentication.
+    run_as_root(["cp", "-a", "puppet/zulip/files/postgresql/centos_pg_hba.conf", pg_hba_conf])
+    # Later steps will ensure PostgreSQL is started
+
+    # Link in tsearch data files
+    if vendor == "fedora":
+        # Since F36 dictionary files were moved away from /usr/share/myspell
+        tsearch_source_prefix = "/usr/share/hunspell"
+    else:
+        tsearch_source_prefix = "/usr/share/myspell"
+    run_as_root(
+        [
+            "ln",
+            "-nsf",
+            os.path.join(tsearch_source_prefix, "en_US.dic"),
+            f"/usr/pgsql-{POSTGRESQL_VERSION}/share/tsearch_data/en_us.dict",
+        ]
+    )
+    run_as_root(
+        [
+            "ln",
+            "-nsf",
+            os.path.join(tsearch_source_prefix, "en_US.aff"),
+            f"/usr/pgsql-{POSTGRESQL_VERSION}/share/tsearch_data/en_us.affix",
+        ]
+    )
+
+
+def main(options: argparse.Namespace) -> NoReturn:
+    # pnpm and management commands expect to be run from the root of the
+    # project.
+    os.chdir(ZULIP_PATH)
+
+    # hash the apt dependencies
+    sha_sum = hashlib.sha1()
+
+    for apt_dependency in SYSTEM_DEPENDENCIES:
+        sha_sum.update(apt_dependency.encode())
+    if "debian" in os_families():
+        with open("scripts/lib/setup-apt-repo", "rb") as fb:
+            sha_sum.update(fb.read())
+    else:
+        # hash the content of setup-yum-repo*
+        with open("scripts/lib/setup-yum-repo", "rb") as fb:
+            sha_sum.update(fb.read())
+
+    # hash the content of build-pgroonga if PGroonga is built from source
+    if BUILD_PGROONGA_FROM_SOURCE:
+        with open("scripts/lib/build-pgroonga", "rb") as fb:
+            sha_sum.update(fb.read())
+
+    new_apt_dependencies_hash = sha_sum.hexdigest()
+    last_apt_dependencies_hash = None
+    apt_hash_file_path = os.path.join(UUID_VAR_PATH, "apt_dependencies_hash")
+    with open(apt_hash_file_path, "a+") as hash_file:
+        hash_file.seek(0)
+        last_apt_dependencies_hash = hash_file.read()
+
+    if new_apt_dependencies_hash != last_apt_dependencies_hash:
+        try:
+            install_system_deps()
+        except subprocess.CalledProcessError:
+            try:
+                # Might be a failure due to network connection issues. Retrying...
+                print(WARNING + "Installing system dependencies failed; retrying..." + ENDC)
+                install_system_deps()
+            except BaseException as e:
+                # Suppress exception chaining
+                raise e from None
+        with open(apt_hash_file_path, "w") as hash_file:
+            hash_file.write(new_apt_dependencies_hash)
+    else:
+        print("No changes to apt dependencies, so skipping apt operations.")
+
+    # Here we install node.
+    proxy_env = [
+        "env",
+        "http_proxy=" + os.environ.get("http_proxy", ""),
+        "https_proxy=" + os.environ.get("https_proxy", ""),
+        "no_proxy=" + os.environ.get("no_proxy", ""),
+    ]
+    # Preserve PATH to catch mistaken extra installations of node in the user's
+    # home directory.
+    run_as_root([*proxy_env, "scripts/lib/install-node"], sudo_args=["--preserve-env=PATH"])
+
+    try:
+        setup_node_modules()
+    except subprocess.CalledProcessError:
+        print(WARNING + "`pnpm install` failed; retrying..." + ENDC)
+        try:
+            setup_node_modules()
+        except subprocess.CalledProcessError:
+            print(
+                FAIL
+                + "`pnpm install` is failing; check your network connection (and proxy settings)."
+                + ENDC
+            )
+            sys.exit(1)
+
+    # Install shellcheck.
+    run_as_root([*proxy_env, "tools/setup/install-shellcheck"], sudo_args=["--preserve-env=PATH"])
+    # Install shfmt.
+    run_as_root([*proxy_env, "tools/setup/install-shfmt"], sudo_args=["--preserve-env=PATH"])
+
+    # Install tusd
+    run_as_root([*proxy_env, "tools/setup/install-tusd"], sudo_args=["--preserve-env=PATH"])
+
+    # Install Python environment
+    run_as_root([*proxy_env, "scripts/lib/install-uv"], sudo_args=["--preserve-env=PATH"])
+    run(
+        [*proxy_env, "uv", "sync", "--frozen", "--no-managed-python"],
+        env={k: v for k, v in os.environ.items() if k not in {"PYTHONDEVMODE", "PYTHONWARNINGS"}},
+    )
+    # Clean old symlinks used before uv migration
+    with contextlib.suppress(FileNotFoundError):
+        os.unlink("zulip-py3-venv")
+    if os.path.lexists("/srv/zulip-py3-venv"):
+        run_as_root(["rm", "/srv/zulip-py3-venv"])
+
+    run_as_root(["cp", REPO_STOPWORDS_PATH, TSEARCH_STOPWORDS_PATH])
+
+    if CONTINUOUS_INTEGRATION and not options.is_build_release_tarball_only:
+        run_as_root(["service", "redis-server", "start"])
+        run_as_root(["service", "memcached", "start"])
+        run_as_root(["service", "rabbitmq-server", "start"])
+        run_as_root(["service", "postgresql", "start"])
+    elif "fedora" in os_families():
+        # These platforms don't enable and start services on
+        # installing their package, so we do that here.
+        for service in [
+            f"postgresql-{POSTGRESQL_VERSION}",
+            "rabbitmq-server",
+            "memcached",
+            "redis",
+        ]:
+            run_as_root(["systemctl", "enable", service], sudo_args=["-H"])
+            run_as_root(["systemctl", "start", service], sudo_args=["-H"])
+
+    # If we imported modules after activating the virtualenv in this
+    # Python process, they could end up mismatching with modules we’ve
+    # already imported from outside the virtualenv.  That seems like a
+    # bad idea, and empirically it can cause Python to segfault on
+    # certain cffi-related imports.  Instead, start a new Python
+    # process inside the virtualenv.
+    provision_inner = os.path.join(ZULIP_PATH, "tools", "lib", "provision_inner.py")
+    os.execvp(
+        "uv",
+        [
+            "uv",
+            "run",
+            "--no-sync",
+            provision_inner,
+            *(["--force"] if options.is_force else []),
+            *(["--build-release-tarball-only"] if options.is_build_release_tarball_only else []),
+            *(["--skip-dev-db-build"] if options.skip_dev_db_build else []),
+        ],
+    )
+
+
+if __name__ == "__main__":
+    description = "Provision script to install Zulip"
+    parser = argparse.ArgumentParser(description=description)
+    parser.add_argument(
+        "--force",
+        action="store_true",
+        dest="is_force",
+        help="Ignore all provisioning optimizations.",
+    )
+
+    parser.add_argument(
+        "--build-release-tarball-only",
+        action="store_true",
+        dest="is_build_release_tarball_only",
+        help="Provision needed to build release tarball.",
+    )
+
+    parser.add_argument(
+        "--skip-dev-db-build", action="store_true", help="Don't run migrations on dev database."
+    )
+
+    options = parser.parse_args()
+    main(options)

tools/lib/provision_inner.py

@@ -0,0 +1,451 @@
+#!/usr/bin/env python3
+
+############################## NOTE ################################
+# This script is used to provision a development environment ONLY.
+# For production, extend update-prod-static to generate new static
+# assets, and puppet to install other software.
+####################################################################
+
+import argparse
+import glob
+import os
+import pwd
+import re
+import shlex
+import shutil
+import subprocess
+import sys
+
+ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+sys.path.append(ZULIP_PATH)
+import pygments
+
+from scripts.lib import clean_unused_caches
+from scripts.lib.zulip_tools import (
+    ENDC,
+    OKBLUE,
+    get_dev_uuid_var_path,
+    get_tzdata_zi,
+    is_digest_obsolete,
+    run,
+    run_as_root,
+    write_new_digest,
+)
+from tools.setup.generate_bots_integrations_static_files import (
+    generate_pythonapi_integrations_static_files,
+    generate_zulip_bots_static_files,
+)
+from version import PROVISION_VERSION
+
+VENV_PATH = os.path.join(ZULIP_PATH, ".venv")
+UUID_VAR_PATH = get_dev_uuid_var_path()
+
+with get_tzdata_zi() as f:
+    line = f.readline()
+    assert line.startswith("# version ")
+    timezones_version = line.removeprefix("# version ")
+
+
+def create_var_directories() -> None:
+    # create var/coverage, var/log, etc.
+    var_dir = os.path.join(ZULIP_PATH, "var")
+    sub_dirs = [
+        "coverage",
+        "log",
+        "node-coverage",
+        "test_uploads",
+        "uploads",
+        "xunit-test-results",
+    ]
+    for sub_dir in sub_dirs:
+        path = os.path.join(var_dir, sub_dir)
+        os.makedirs(path, exist_ok=True)
+
+
+def build_pygments_data_paths() -> list[str]:
+    paths = [
+        "tools/setup/build_pygments_data",
+        "tools/setup/lang.json",
+    ]
+    return paths
+
+
+def build_timezones_data_paths() -> list[str]:
+    paths = [
+        "tools/setup/build_timezone_values",
+    ]
+    return paths
+
+
+def build_landing_page_images_paths() -> list[str]:
+    paths = ["tools/setup/generate_landing_page_images.py"]
+    paths += glob.glob("static/images/landing-page/hello/original/*")
+    return paths
+
+
+def compilemessages_paths() -> list[str]:
+    paths = ["zerver/management/commands/compilemessages.py"]
+    paths += glob.glob("locale/*/LC_MESSAGES/*.po")
+    paths += glob.glob("locale/*/translations.json")
+    return paths
+
+
+def configure_rabbitmq_paths() -> list[str]:
+    paths = [
+        "scripts/setup/configure-rabbitmq",
+    ]
+    return paths
+
+
+def is_wsl_instance() -> bool:
+    if "WSL_DISTRO_NAME" in os.environ:
+        return True
+
+    with open("/proc/version") as file:
+        content = file.read().lower()
+        if "microsoft" in content:
+            return True
+
+    result = subprocess.run(["uname", "-r"], capture_output=True, text=True, check=True)
+    if "microsoft" in result.stdout.lower():
+        return True
+
+    return False
+
+
+def is_vagrant_or_digitalocean_instance() -> bool:
+    user_id = os.getuid()
+    user_name = pwd.getpwuid(user_id).pw_name
+    return user_name in ["vagrant", "zulipdev"]
+
+
+def setup_shell_profile(shell_profile: str) -> None:
+    shell_profile_path = os.path.expanduser(shell_profile)
+    if os.path.exists(shell_profile_path):
+        with open(shell_profile_path) as f:
+            code = f.read()
+    else:
+        code = ""
+
+    # We want to activate the virtual environment for login shells only on virtualized systems.
+    zulip_code = ""
+    if is_vagrant_or_digitalocean_instance() or is_wsl_instance():
+        zulip_code += (
+            "if [ -L /srv/zulip-py3-venv ]; then\n"  # For development environment downgrades
+            "source /srv/zulip-py3-venv/bin/activate\n"  # Not indented so old versions recognize and avoid re-adding this
+            "else\n"
+            f"source {shlex.quote(os.path.join(VENV_PATH, 'bin', 'activate'))}\n"
+            "fi\n"
+        )
+    zulip_code += f"cd {shlex.quote(ZULIP_PATH)}\n"
+    if zulip_code:
+        zulip_code = f"\n# begin Zulip setup\n{zulip_code}# end Zulip setup\n"
+
+    def patch_code(code: str) -> str:
+        return re.sub(
+            r"\n# begin Zulip setup\n(?s:.*)# end Zulip setup\n|(?:source /srv/zulip-py3-venv/bin/activate\n|cd /srv/zulip\n)+|\Z",
+            lambda m: zulip_code,
+            code,
+            count=1,
+        )
+
+    new_code = patch_code(code)
+    if new_code != code:
+        assert patch_code(new_code) == new_code
+        with open(f"{shell_profile_path}.new", "w") as f:
+            f.write(new_code)
+        os.rename(f"{shell_profile_path}.new", shell_profile_path)
+
+
+def setup_bash_profile() -> None:
+    """Select a bash profile file to add setup code to."""
+
+    BASH_PROFILES = [
+        os.path.expanduser(p) for p in ("~/.bash_profile", "~/.bash_login", "~/.profile")
+    ]
+
+    def clear_old_profile() -> None:
+        # An earlier version of this script would output a fresh .bash_profile
+        # even though a .profile existed in the image used. As a convenience to
+        # existing developers (and, perhaps, future developers git-bisecting the
+        # provisioning scripts), check for this situation, and blow away the
+        # created .bash_profile if one is found.
+
+        BASH_PROFILE = BASH_PROFILES[0]
+        DOT_PROFILE = BASH_PROFILES[2]
+        OLD_PROFILE_TEXT = "source /srv/zulip-py3-venv/bin/activate\ncd /srv/zulip\n"
+
+        if os.path.exists(DOT_PROFILE):
+            try:
+                with open(BASH_PROFILE) as f:
+                    profile_contents = f.read()
+                if profile_contents == OLD_PROFILE_TEXT:
+                    os.unlink(BASH_PROFILE)
+            except FileNotFoundError:
+                pass
+
+    clear_old_profile()
+
+    for candidate_profile in BASH_PROFILES:
+        if os.path.exists(candidate_profile):
+            setup_shell_profile(candidate_profile)
+            break
+    else:
+        # no existing bash profile found; claim .bash_profile
+        setup_shell_profile(BASH_PROFILES[0])
+
+
+def need_to_run_build_pygments_data() -> bool:
+    if not os.path.exists("web/generated/pygments_data.json"):
+        return True
+
+    return is_digest_obsolete(
+        "build_pygments_data_hash",
+        build_pygments_data_paths(),
+        [pygments.__version__],
+    )
+
+
+def need_to_run_build_timezone_data() -> bool:
+    if not os.path.exists("web/generated/timezones.json"):
+        return True
+
+    return is_digest_obsolete(
+        "build_timezones_data_hash",
+        build_timezones_data_paths(),
+        [timezones_version],
+    )
+
+
+def need_to_regenerate_landing_page_images() -> bool:
+    if not os.path.exists("static/images/landing-page/hello/generated"):
+        return True
+
+    return is_digest_obsolete(
+        "landing_page_images_hash",
+        build_landing_page_images_paths(),
+    )
+
+
+def need_to_run_compilemessages() -> bool:
+    if not os.path.exists("locale/language_name_map.json"):
+        # User may have cleaned their Git checkout.
+        print("Need to run compilemessages due to missing language_name_map.json")
+        return True
+
+    return is_digest_obsolete(
+        "last_compilemessages_hash",
+        compilemessages_paths(),
+    )
+
+
+def need_to_run_configure_rabbitmq(settings_list: list[str]) -> bool:
+    obsolete = is_digest_obsolete(
+        "last_configure_rabbitmq_hash",
+        configure_rabbitmq_paths(),
+        settings_list,
+    )
+
+    if obsolete:
+        return True
+
+    try:
+        from zerver.lib.queue import SimpleQueueClient
+
+        SimpleQueueClient()
+        return False
+    except Exception:
+        return True
+
+
+def main(options: argparse.Namespace) -> int:
+    setup_bash_profile()
+    setup_shell_profile("~/.zprofile")
+
+    # This needs to happen before anything that imports zproject.settings.
+    run(["scripts/setup/generate_secrets.py", "--development"])
+
+    create_var_directories()
+
+    # The `build_emoji` script requires `emoji-datasource` package
+    # which we install via npm; thus this step is after installing npm
+    # packages.
+    run(["tools/setup/emoji/build_emoji"])
+
+    # copy over static files from the zulip_bots and integrations packages
+    generate_zulip_bots_static_files()
+    generate_pythonapi_integrations_static_files()
+
+    run(["node", "tools/setup/build_supported_browser_regex.ts"])
+
+    if options.is_force or need_to_run_build_pygments_data():
+        run(["tools/setup/build_pygments_data"])
+        write_new_digest(
+            "build_pygments_data_hash",
+            build_pygments_data_paths(),
+            [pygments.__version__],
+        )
+    else:
+        print("No need to run `tools/setup/build_pygments_data`.")
+
+    if options.is_force or need_to_run_build_timezone_data():
+        run(["tools/setup/build_timezone_values"])
+        write_new_digest(
+            "build_timezones_data_hash",
+            build_timezones_data_paths(),
+            [timezones_version],
+        )
+    else:
+        print("No need to run `tools/setup/build_timezone_values`.")
+
+    if options.is_force or need_to_regenerate_landing_page_images():
+        run(["tools/setup/generate_landing_page_images.py"])
+        write_new_digest(
+            "landing_page_images_hash",
+            build_landing_page_images_paths(),
+        )
+
+    if not options.is_build_release_tarball_only:
+        # The following block is skipped when we just need the development
+        # environment to build a release tarball.
+
+        # Need to set up Django before using template_status
+        os.environ.setdefault("DJANGO_SETTINGS_MODULE", "zproject.settings")
+        import django
+
+        django.setup()
+
+        from django.conf import settings
+
+        from zerver.lib.test_fixtures import (
+            DEV_DATABASE,
+            TEST_DATABASE,
+            destroy_leaked_test_databases,
+        )
+
+        if os.environ.get("CODESPACES") == "true":
+            print("Starting background services explicitly for GitHub Codespaces...")
+            run_as_root(["service", "postgresql", "start"])
+            run_as_root(["service", "rabbitmq-server", "start"])
+            run_as_root(["service", "redis-server", "start"])
+            run_as_root(["service", "memcached", "start"])
+
+        assert settings.RABBITMQ_PASSWORD is not None
+        if options.is_force or need_to_run_configure_rabbitmq([settings.RABBITMQ_PASSWORD]):
+            run_as_root(["scripts/setup/configure-rabbitmq"])
+            write_new_digest(
+                "last_configure_rabbitmq_hash",
+                configure_rabbitmq_paths(),
+                [settings.RABBITMQ_PASSWORD],
+            )
+        else:
+            print("No need to run `scripts/setup/configure-rabbitmq.")
+
+        dev_template_db_status = DEV_DATABASE.template_status()
+        if options.is_force or dev_template_db_status == "needs_rebuild":
+            run(["tools/setup/postgresql-init-dev-db"])
+            if options.skip_dev_db_build:
+                # We don't need to build the manual development
+                # database on continuous integration for running tests, so we can
+                # just leave it as a template db and save a minute.
+                #
+                # Important: We don't write a digest as that would
+                # incorrectly claim that we ran migrations.
+                pass
+            else:
+                run(["tools/rebuild-dev-database"])
+                DEV_DATABASE.write_new_db_digest()
+        elif dev_template_db_status == "run_migrations":
+            DEV_DATABASE.run_db_migrations()
+        elif dev_template_db_status == "current":
+            print("No need to regenerate the dev DB.")
+
+        test_template_db_status = TEST_DATABASE.template_status()
+        if options.is_force or test_template_db_status == "needs_rebuild":
+            run(["tools/setup/postgresql-init-test-db"])
+            run(["tools/rebuild-test-database"])
+            TEST_DATABASE.write_new_db_digest()
+        elif test_template_db_status == "run_migrations":
+            TEST_DATABASE.run_db_migrations()
+        elif test_template_db_status == "current":
+            print("No need to regenerate the test DB.")
+
+        if options.is_force or need_to_run_compilemessages():
+            run(["./manage.py", "compilemessages", "--ignore=*"])
+            write_new_digest(
+                "last_compilemessages_hash",
+                compilemessages_paths(),
+            )
+        else:
+            print("No need to run `manage.py compilemessages`.")
+
+        destroyed = destroy_leaked_test_databases()
+        if destroyed:
+            print(f"Dropped {destroyed} stale test databases!")
+
+    clean_unused_caches.main(
+        argparse.Namespace(
+            threshold_days=6,
+            # The defaults here should match parse_cache_script_args in zulip_tools.py
+            dry_run=False,
+            verbose=False,
+            no_headings=True,
+        )
+    )
+
+    # Keeping this cache file around can cause eslint to throw
+    # random TypeErrors when new/updated dependencies are added
+    if os.path.isfile(".eslintcache"):
+        # Remove this block when
+        # https://github.com/eslint/eslint/issues/11639 is fixed
+        # upstream.
+        os.remove(".eslintcache")
+
+    # Clean up the root of the `var/` directory for various
+    # testing-related files that we have migrated to
+    # `var/<uuid>/test-backend`.
+    print("Cleaning var/ directory files...")
+    var_paths = glob.glob("var/test*")
+    var_paths.append("var/bot_avatar")
+    for path in var_paths:
+        try:
+            if os.path.isdir(path):
+                shutil.rmtree(path)
+            else:
+                os.remove(path)
+        except FileNotFoundError:
+            pass
+
+    version_file = os.path.join(UUID_VAR_PATH, "provision_version")
+    print(f"writing to {version_file}\n")
+    with open(version_file, "w") as f:
+        f.write(".".join(map(str, PROVISION_VERSION)) + "\n")
+
+    print()
+    print(OKBLUE + "Zulip development environment setup succeeded!" + ENDC)
+    return 0
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--force",
+        action="store_true",
+        dest="is_force",
+        help="Ignore all provisioning optimizations.",
+    )
+
+    parser.add_argument(
+        "--build-release-tarball-only",
+        action="store_true",
+        dest="is_build_release_tarball_only",
+        help="Provision for test suite with production settings.",
+    )
+
+    parser.add_argument(
+        "--skip-dev-db-build", action="store_true", help="Don't run migrations on dev database."
+    )
+
+    options = parser.parse_args()
+    sys.exit(main(options))

tools/lib/sanity_check.py

@@ -0,0 +1,19 @@
+import os
+import pwd
+import sys
+
+
+def check_venv(filename: str) -> None:
+    BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+    venv = os.path.realpath(os.path.join(BASE_DIR, ".venv"))
+    if sys.prefix != venv:
+        print(f"You need to run {filename} inside a Zulip dev environment.")
+        user_id = os.getuid()
+        user_name = pwd.getpwuid(user_id).pw_name
+
+        print(f"You can `source {venv}/bin/activate` to enter the development environment.")
+
+        if user_name not in ("vagrant", "zulipdev"):
+            print()
+            print("If you are using Vagrant, first run `vagrant ssh` to enter the Vagrant guest.")
+        sys.exit(1)

tools/lib/template_parser.py

@@ -0,0 +1,790 @@
+from collections.abc import Callable
+
+from typing_extensions import override
+
+from .html_elements import FOREIGN_CONTEXTS, VALID_HTML_CONTEXTS, html_context_fallbacks
+
+
+class FormattedError(Exception):
+    pass
+
+
+class TemplateParserError(Exception):
+    def __init__(self, message: str) -> None:
+        self.message = message
+
+    @override
+    def __str__(self) -> str:
+        return self.message
+
+
+class TokenizationError(Exception):
+    def __init__(self, message: str, line_content: str | None = None) -> None:
+        self.message = message
+        self.line_content = line_content
+
+
+class TokenizerState:
+    def __init__(self) -> None:
+        self.i = 0
+        self.line = 1
+        self.col = 1
+
+
+class Token:
+    def __init__(self, kind: str, s: str, tag: str, line: int, col: int, line_span: int) -> None:
+        self.kind = kind
+        self.s = s
+        self.tag = tag
+        self.line = line
+        self.col = col
+        self.line_span = line_span
+
+        # These get set during the validation pass.
+        self.start_token: Token | None = None
+        self.end_token: Token | None = None
+
+        # These get set during the pretty-print phase.
+        self.new_s = ""
+        self.indent: str | None = None
+        self.orig_indent: str | None = None
+        self.child_indent: str | None = None
+        self.indent_is_final = False
+        self.parent_token: Token | None = None
+
+
+def tokenize(text: str, template_format: str | None = None) -> list[Token]:
+    in_code_block = False
+
+    def advance(n: int) -> None:
+        for _ in range(n):
+            state.i += 1
+            if state.i >= 0 and text[state.i - 1] == "\n":
+                state.line += 1
+                state.col = 1
+            else:
+                state.col += 1
+
+    def looking_at(s: str) -> bool:
+        return text[state.i : state.i + len(s)] == s
+
+    def looking_at_htmlcomment() -> bool:
+        return looking_at("<!--")
+
+    def looking_at_handlebars_comment() -> bool:
+        return looking_at("{{!") or looking_at("{{~!")
+
+    def looking_at_djangocomment() -> bool:
+        return template_format == "django" and looking_at("{#")
+
+    def looking_at_handlebars_partial() -> bool:
+        return template_format == "handlebars" and looking_at("{{>")
+
+    def looking_at_handlebars_partial_block() -> bool:
+        return template_format == "handlebars" and looking_at("{{#>")
+
+    def looking_at_handlebars_triple_stache() -> bool:
+        return template_format == "handlebars" and (looking_at("{{{") or looking_at("{{~{"))
+
+    def looking_at_html_start() -> bool:
+        return looking_at("<") and not looking_at("</")
+
+    def looking_at_html_end() -> bool:
+        return looking_at("</")
+
+    def looking_at_handlebars_start() -> bool:
+        return looking_at("{{#") or looking_at("{{^") or looking_at("{{~#")
+
+    def looking_at_handlebars_else() -> bool:
+        return template_format == "handlebars" and looking_at("{{else")
+
+    def looking_at_template_var() -> bool:
+        return looking_at("{")
+
+    def looking_at_handlebars_end() -> bool:
+        return template_format == "handlebars" and (looking_at("{{/") or looking_at("{{~/"))
+
+    def looking_at_django_start() -> bool:
+        return template_format == "django" and looking_at("{% ")
+
+    def looking_at_django_else() -> bool:
+        return template_format == "django" and (
+            looking_at("{% else")
+            or looking_at("{% elif")
+            or looking_at("{%- else")
+            or looking_at("{%- elif")
+        )
+
+    def looking_at_django_end() -> bool:
+        return template_format == "django" and looking_at("{% end")
+
+    def looking_at_jinja2_end_whitespace_stripped() -> bool:
+        return template_format == "django" and looking_at("{%- end")
+
+    def looking_at_jinja2_start_whitespace_stripped_type2() -> bool:
+        # This function detects tag like {%- if foo -%}...{% endif %}
+        return template_format == "django" and looking_at("{%-") and not looking_at("{%- end")
+
+    def looking_at_whitespace() -> bool:
+        return looking_at("\n") or looking_at(" ")
+
+    state = TokenizerState()
+    tokens: list[Token] = []
+
+    while state.i < len(text):
+        try:
+            if in_code_block:
+                in_code_block = False
+                s = get_code(text, state.i)
+                if s == "":
+                    continue
+                tag = ""
+                kind = "code"
+            elif looking_at_htmlcomment():
+                s = get_html_comment(text, state.i)
+                tag = s[4:-3]
+                kind = "html_comment"
+            elif looking_at_handlebars_comment():
+                s = get_handlebars_comment(text, state.i)
+                tag = s[3:-2]
+                kind = "handlebars_comment"
+            elif looking_at_djangocomment():
+                s = get_django_comment(text, state.i)
+                tag = s[2:-2]
+                kind = "django_comment"
+            elif looking_at_handlebars_partial():
+                s = get_handlebars_partial(text, state.i)
+                tag = s[9:-2]
+                kind = "handlebars_partial"
+            elif looking_at_handlebars_partial_block():
+                s = get_handlebars_partial(text, state.i)
+                tag = s[5:-2].split(None, 1)[0]
+                kind = "handlebars_partial_block"
+            elif looking_at_html_start():
+                s = get_html_tag(text, state.i)
+                if s.endswith("/>"):
+                    end_offset = -2
+                else:
+                    end_offset = -1
+                tag_parts = s[1:end_offset].split()
+
+                if not tag_parts:
+                    raise TemplateParserError("Tag name missing")
+
+                tag = tag_parts[0]
+
+                if tag == "!DOCTYPE":
+                    kind = "html_doctype"
+                elif s.endswith("/>"):
+                    kind = "html_singleton"
+                else:
+                    kind = "html_start"
+                if tag in ("code", "pre", "script"):
+                    in_code_block = True
+            elif looking_at_html_end():
+                s = get_html_tag(text, state.i)
+                tag = s[2:-1]
+                kind = "html_end"
+            elif looking_at_handlebars_else():
+                s = get_handlebars_tag(text, state.i)
+                tag = "else"
+                kind = "handlebars_else"
+            elif looking_at_handlebars_triple_stache():
+                s = get_handlebars_triple_stache_tag(text, state.i)
+                start_offset = end_offset = 3
+                if s.startswith("{{~{"):
+                    start_offset += 1
+                if s.endswith("}~}}"):
+                    end_offset += 1
+                tag = s[start_offset:-end_offset].strip()
+                if not tag.endswith("_html"):
+                    raise TemplateParserError(
+                        "Unescaped variables in triple staches {{{ }}} must be suffixed with `_html`"
+                    )
+                kind = "handlebars_triple_stache"
+            elif looking_at_handlebars_start():
+                s = get_handlebars_tag(text, state.i)
+                tag = s[3:-2].split()[0].strip("#").removeprefix("*")
+                kind = "handlebars_start"
+            elif looking_at_handlebars_end():
+                s = get_handlebars_tag(text, state.i)
+                tag = s[3:-2].strip("/#~")
+                kind = "handlebars_end"
+            elif looking_at_django_else():
+                s = get_django_tag(text, state.i)
+                tag = "else"
+                kind = "django_else"
+            elif looking_at_django_end():
+                s = get_django_tag(text, state.i)
+                tag = s[6:-3]
+                kind = "django_end"
+            elif looking_at_django_start():
+                # must check this after end/else
+                s = get_django_tag(text, state.i)
+                tag = s[3:-2].split()[0]
+                kind = "django_start"
+
+                if s[-3] == "-":
+                    kind = "jinja2_whitespace_stripped_start"
+            elif looking_at_jinja2_end_whitespace_stripped():
+                s = get_django_tag(text, state.i)
+                tag = s[7:-3]
+                kind = "jinja2_whitespace_stripped_end"
+            elif looking_at_jinja2_start_whitespace_stripped_type2():
+                s = get_django_tag(text, state.i, stripped=True)
+                tag = s[3:-3].split()[0]
+                kind = "jinja2_whitespace_stripped_type2_start"
+            elif looking_at_template_var():
+                # order is important here
+                s = get_template_var(text, state.i)
+                tag = "var"
+                kind = "template_var"
+            elif looking_at("\n"):
+                s = "\n"
+                tag = "newline"
+                kind = "newline"
+            elif looking_at(" "):
+                s = get_spaces(text, state.i)
+                tag = ""
+                if not tokens or tokens[-1].kind == "newline":
+                    kind = "indent"
+                else:
+                    kind = "whitespace"
+            elif text[state.i] in "{<":
+                snippet = text[state.i :][:15]
+                raise AssertionError(f"tool cannot parse {snippet}")
+            else:
+                s = get_text(text, state.i)
+                if s == "":
+                    continue
+                tag = ""
+                kind = "text"
+        except TokenizationError as e:
+            raise FormattedError(
+                f'''{e.message} at line {state.line} col {state.col}:"{e.line_content}"''',
+            )
+
+        line_span = len(s.strip("\n").split("\n"))
+        token = Token(
+            kind=kind,
+            s=s,
+            tag=tag.strip(),
+            line=state.line,
+            col=state.col,
+            line_span=line_span,
+        )
+        tokens.append(token)
+        advance(len(s))
+
+    return tokens
+
+
+# The following excludes some obscure tags that are never used
+# in Zulip code.
+HTML_INLINE_TAGS = {
+    "a",
+    "b",
+    "br",
+    "button",
+    "cite",
+    "code",
+    "em",
+    "i",
+    "img",
+    "input",
+    "kbd",
+    "label",
+    "object",
+    "script",
+    "select",
+    "small",
+    "span",
+    "strong",
+    "textarea",
+}
+
+
+def tag_flavor(token: Token) -> str | None:
+    kind = token.kind
+    tag = token.tag
+    if kind in (
+        "code",
+        "django_comment",
+        "handlebars_comment",
+        "handlebars_partial",
+        "html_comment",
+        "html_doctype",
+        "html_singleton",
+        "indent",
+        "newline",
+        "template_var",
+        "text",
+        "whitespace",
+        "handlebars_triple_stache",
+    ):
+        return None
+
+    if kind in ("handlebars_start", "handlebars_partial_block", "html_start"):
+        return "start"
+    elif kind in (
+        "django_else",
+        "django_end",
+        "handlebars_else",
+        "handlebars_end",
+        "html_end",
+        "jinja2_whitespace_stripped_end",
+    ):
+        return "end"
+    elif kind in {
+        "django_start",
+        "django_else",
+        "jinja2_whitespace_stripped_start",
+        "jinja2_whitespace_stripped_type2_start",
+    }:
+        if is_django_block_tag(tag):
+            return "start"
+        else:
+            return None
+    else:
+        raise AssertionError(f"tools programmer neglected to handle {kind} tokens")
+
+
+def validate(
+    fn: str | None = None,
+    text: str | None = None,
+    template_format: str | None = None,
+) -> list[Token]:
+    assert fn or text
+
+    if fn is None:
+        fn = "<in memory file>"
+
+    if text is None:
+        with open(fn) as f:
+            text = f.read()
+
+    lines = text.split("\n")
+
+    try:
+        tokens = tokenize(text, template_format=template_format)
+    except FormattedError as e:
+        raise TemplateParserError(
+            f"""
+            fn: {fn}
+            {e}"""
+        )
+
+    prevent_whitespace_violations(fn, tokens)
+
+    class State:
+        def __init__(self, func: Callable[[Token | None], None]) -> None:
+            self.depth = 0
+            self.matcher = func
+            self.html_context = "unknown"
+
+    def no_start_tag(token: Token | None) -> None:
+        assert token
+        raise TemplateParserError(
+            f"""
+            No start tag
+            fn: {fn}
+            end tag:
+                {token.tag}
+                line {token.line}, col {token.col}
+            """
+        )
+
+    state = State(no_start_tag)
+
+    def start_tag_matcher(start_token: Token) -> None:
+        state.depth += 1
+        start_tag = start_token.tag.strip("~")
+        start_line = start_token.line
+        start_col = start_token.col
+
+        old_matcher = state.matcher
+        old_html_context = state.html_context
+
+        def f(end_token: Token | None) -> None:
+            if end_token is None:
+                raise TemplateParserError(
+                    f"""
+
+    Problem with {fn}
+    Missing end tag for the token at row {start_line} {start_col}!
+
+{start_token.s}
+
+    It's possible you have a typo in a token that you think is
+    matching this tag.
+                    """
+                )
+
+            is_else_tag = end_token.tag == "else"
+
+            end_tag = end_token.tag.strip("~")
+            end_line = end_token.line
+            end_col = end_token.col
+
+            def report_problem() -> str | None:
+                if (start_tag == "code") and (end_line == start_line + 1):
+                    return "Code tag is split across two lines."
+
+                if is_else_tag:
+                    # We are not completely rigorous about having a sensible
+                    # order of if/elif/elif/else, but we catch obviously
+                    # mismatching else tags.
+                    if start_tag not in ("if", "else", "unless", "list_each"):
+                        return f"Unexpected else/elif tag encountered after {start_tag} tag."
+                elif start_tag != end_tag:
+                    return f"Mismatched tags: ({start_tag} != {end_tag})"
+
+                return None
+
+            problem = report_problem()
+            if problem:
+                raise TemplateParserError(
+                    f"""
+                    fn: {fn}
+                   {problem}
+                    start:
+                        {start_token.s}
+                        line {start_line}, col {start_col}
+                    end tag:
+                        {end_tag}
+                        line {end_line}, col {end_col}
+                    """
+                )
+
+            if not is_else_tag:
+                state.matcher = old_matcher
+                state.html_context = old_html_context
+                state.depth -= 1
+
+            # TODO: refine this for the else/elif use cases
+            end_token.start_token = start_token
+            start_token.end_token = end_token
+
+        state.matcher = f
+
+    for token in tokens:
+        kind = token.kind
+        tag = token.tag
+
+        flavor = tag_flavor(token)
+        if flavor == "start":
+            start_tag_matcher(token)
+        elif flavor == "end":
+            state.matcher(token)
+
+        if kind in ("html_start", "html_singleton"):
+            for context in html_context_fallbacks(state.html_context):
+                if (tag, context) in VALID_HTML_CONTEXTS:
+                    new_context = VALID_HTML_CONTEXTS[tag, context]
+                    if new_context == "transparent":
+                        new_context = state.html_context
+                    break
+            else:
+                if "-" in tag and "phrasing" in html_context_fallbacks(state.html_context):
+                    new_context = state.html_context  # custom elements
+                elif state.html_context in FOREIGN_CONTEXTS:
+                    new_context = state.html_context  # unchecked foreign elements
+                else:
+                    raise TemplateParserError(
+                        f"<{tag}> is not valid in {state.html_context} context"
+                        + (
+                            ' (consider growing HTML_CONTEXT_FALLBACKS["unknown"]?)'
+                            if state.html_context == "unknown"
+                            else ""
+                        )
+                        + f" at {fn} line {token.line}, col {token.col}"
+                    )
+
+            if new_context not in FOREIGN_CONTEXTS:
+                if kind == "html_start" and new_context == "void":
+                    raise TemplateParserError(
+                        f"Tag must be self-closing: {tag} at {fn} line {token.line}, col {token.col}"
+                    )
+                elif kind == "html_singleton" and new_context != "void":
+                    raise TemplateParserError(
+                        f"Tag must not be self-closing: {tag} at {fn} line {token.line}, col {token.col}"
+                    )
+
+            if kind == "html_start":
+                state.html_context = new_context
+
+    if state.depth != 0:
+        state.matcher(None)
+
+    ensure_matching_indentation(fn, tokens, lines)
+
+    return tokens
+
+
+def ensure_matching_indentation(fn: str, tokens: list[Token], lines: list[str]) -> None:
+    def has_bad_indentation() -> bool:
+        is_inline_tag = start_tag in HTML_INLINE_TAGS and start_token.kind == "html_start"
+
+        if end_line > start_line + 1:
+            if is_inline_tag:
+                end_row_text = lines[end_line - 1]
+                if end_row_text.lstrip().startswith(end_token.s) and end_col != start_col:
+                    return True
+            else:
+                if end_col != start_col:
+                    return True
+
+        return False
+
+    for token in tokens:
+        if token.start_token is None:
+            continue
+
+        end_token = token
+
+        start_token = token.start_token
+        start_line = start_token.line
+        start_col = start_token.col
+        start_tag = start_token.tag
+        end_tag = end_token.tag.strip("~")
+        end_line = end_token.line
+        end_col = end_token.col
+
+        if has_bad_indentation():
+            raise TemplateParserError(
+                f"""
+                fn: {fn}
+                Indentation for start/end tags does not match.
+                start tag: {start_token.s}
+
+                start:
+                    line {start_line}, col {start_col}
+                end:
+                    {end_tag}
+                    line {end_line}, col {end_col}
+                """
+            )
+
+
+def prevent_extra_newlines(fn: str, tokens: list[Token]) -> None:
+    count = 0
+
+    for token in tokens:
+        if token.kind != "newline":
+            count = 0
+            continue
+
+        count += 1
+        if count >= 4:
+            raise TemplateParserError(
+                f"""Please avoid so many blank lines near row {token.line} in {fn}."""
+            )
+
+
+def prevent_whitespace_violations(fn: str, tokens: list[Token]) -> None:
+    if tokens[0].kind in ("indent", "whitespace"):
+        raise TemplateParserError(f" Please remove the whitespace at the beginning of {fn}.")
+
+    prevent_extra_newlines(fn, tokens)
+
+    for i in range(1, len(tokens) - 1):
+        token = tokens[i]
+        next_token = tokens[i + 1]
+
+        if token.kind == "indent":
+            if next_token.kind in ("indent", "whitespace"):
+                raise AssertionError("programming error parsing indents")
+
+            if next_token.kind == "newline":
+                raise TemplateParserError(
+                    f"""Please just make row {token.line} in {fn} a truly blank line (no spaces)."""
+                )
+
+            if len(token.s) % 4 != 0:
+                raise TemplateParserError(
+                    f"""
+                        Please use 4-space indents for template files. Most of our
+                        codebase (including Python and JavaScript) uses 4-space indents,
+                        so it's worth investing in configuring your editor to use
+                        4-space indents for files like
+                        {fn}
+
+                        The line at row {token.line} is indented with {len(token.s)} spaces.
+                    """
+                )
+
+        if token.kind == "whitespace":
+            if len(token.s) > 1:
+                raise TemplateParserError(
+                    f"""
+                        We did not expect this much whitespace at row {token.line} column {token.col} in {fn}.
+                    """
+                )
+            if next_token.kind == "newline":
+                raise TemplateParserError(
+                    f"""
+                        Unexpected trailing whitespace at row {token.line} column {token.col} in {fn}.
+                    """
+                )
+
+
+def is_django_block_tag(tag: str) -> bool:
+    return tag in [
+        "autoescape",
+        "block",
+        "comment",
+        "for",
+        "if",
+        "ifequal",
+        "macro",
+        "verbatim",
+        "blocktrans",
+        "trans",
+        "raw",
+        "with",
+    ]
+
+
+def get_handlebars_tag(text: str, i: int) -> str:
+    end = i + 2
+    while end < len(text) - 2 and text[end] != "}":
+        end += 1
+    if text[end] != "}" or text[end + 1] != "}":
+        raise TokenizationError('Tag missing "}}"', text[i : end + 2])
+    s = text[i : end + 2]
+    return s
+
+
+def get_handlebars_triple_stache_tag(text: str, i: int) -> str:
+    end = i + 3
+    while end < len(text) - 3 and text[end] != "}":
+        end += 1
+    if text[end : end + 3] == "}}}":
+        return text[i : end + 3]
+    elif end + 4 <= len(text) and text[end : end + 4] == "}~}}":
+        return text[i : end + 4]
+    else:
+        raise TokenizationError('Tag missing "}}}"', text[i : end + 3])
+
+
+def get_spaces(text: str, i: int) -> str:
+    s = ""
+    while i < len(text) and text[i] == " ":
+        s += text[i]
+        i += 1
+    return s
+
+
+def get_code(text: str, i: int) -> str:
+    s = ""
+    while i < len(text) and text[i] != "<":
+        s += text[i]
+        i += 1
+    return s
+
+
+def get_text(text: str, i: int) -> str:
+    s = ""
+    while i < len(text) and text[i] not in "{<":
+        s += text[i]
+        i += 1
+    return s.strip()
+
+
+def get_django_tag(text: str, i: int, stripped: bool = False) -> str:
+    end = i + 2
+    if stripped:
+        end += 1
+    while end < len(text) - 1 and text[end] != "%":
+        end += 1
+    if text[end] != "%" or text[end + 1] != "}":
+        raise TokenizationError('Tag missing "%}"', text[i : end + 2])
+    s = text[i : end + 2]
+    return s
+
+
+def get_html_tag(text: str, i: int) -> str:
+    quote_count = 0
+    end = i + 1
+    unclosed_end = 0
+    while end < len(text) and (text[end] != ">" or (quote_count % 2 != 0 and text[end] != "<")):
+        if text[end] == '"':
+            quote_count += 1
+        if not unclosed_end and text[end] == "<":
+            unclosed_end = end
+        end += 1
+    if quote_count % 2 != 0:
+        if unclosed_end:
+            raise TokenizationError("Unbalanced quotes", text[i:unclosed_end])
+        else:
+            raise TokenizationError("Unbalanced quotes", text[i : end + 1])
+    if end == len(text) or text[end] != ">":
+        raise TokenizationError('Tag missing ">"', text[i : end + 1])
+    s = text[i : end + 1]
+    return s
+
+
+def get_html_comment(text: str, i: int) -> str:
+    end = i + 7
+    unclosed_end = 0
+    while end <= len(text):
+        if text[end - 3 : end] == "-->":
+            return text[i:end]
+        if not unclosed_end and text[end] == "<":
+            unclosed_end = end
+        end += 1
+    raise TokenizationError("Unclosed comment", text[i:unclosed_end])
+
+
+def get_handlebars_comment(text: str, i: int) -> str:
+    long = text[i : i + 5] == "{{!--" or text[i : i + 6] == "{{~!--"
+    end = i + 5
+    unclosed_end = 0
+    while end <= len(text):
+        if text[end - 2 : end] == "}}" and (
+            not long or text[end - 4 : end] == "--}}" or text[end - 5 : end] == "--~}}"
+        ):
+            return text[i:end]
+        if not unclosed_end and text[end] == "<":
+            unclosed_end = end
+        end += 1
+    raise TokenizationError("Unclosed comment", text[i:unclosed_end])
+
+
+def get_template_var(text: str, i: int) -> str:
+    end = i + 3
+    unclosed_end = 0
+    while end <= len(text):
+        if text[end - 1] == "}":
+            if end < len(text) and text[end] == "}":
+                end += 1
+            return text[i:end]
+        if not unclosed_end and text[end] == "<":
+            unclosed_end = end
+        end += 1
+    raise TokenizationError("Unclosed var", text[i:unclosed_end])
+
+
+def get_django_comment(text: str, i: int) -> str:
+    end = i + 4
+    unclosed_end = 0
+    while end <= len(text):
+        if text[end - 2 : end] == "#}":
+            return text[i:end]
+        if not unclosed_end and text[end] == "<":
+            unclosed_end = end
+        end += 1
+    raise TokenizationError("Unclosed comment", text[i:unclosed_end])
+
+
+def get_handlebars_partial(text: str, i: int) -> str:
+    """Works for both partials and partial blocks."""
+    end = i + 10
+    unclosed_end = 0
+    while end <= len(text):
+        if text[end - 2 : end] == "}}":
+            return text[i:end]
+        if not unclosed_end and text[end] == "<":
+            unclosed_end = end
+        end += 1
+    raise TokenizationError("Unclosed partial", text[i:unclosed_end])

tools/lib/test_script.py

@@ -0,0 +1,128 @@
+import glob
+import os
+import subprocess
+import sys
+from argparse import ArgumentParser
+from collections.abc import Iterable
+
+from scripts.lib.zulip_tools import get_dev_uuid_var_path
+from version import PROVISION_VERSION
+
+ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
+def get_version_file() -> str:
+    uuid_var_path = get_dev_uuid_var_path()
+    return os.path.join(uuid_var_path, "provision_version")
+
+
+PREAMBLE = """
+Provisioning state check failed! This check compares
+`var/provision_version` (currently {}) to the version in
+source control (`version.py`), which is {}, to see if you
+likely need to provision before this command can run
+properly.
+"""
+
+
+def preamble(version: tuple[int, ...]) -> str:
+    text = PREAMBLE.format(version, PROVISION_VERSION)
+    text += "\n"
+    return text
+
+
+NEED_TO_DOWNGRADE = """
+The branch you are currently on expects an older version of
+dependencies than the version you provisioned last. This may
+be ok, but it's likely that you either want to rebase your
+branch on top of upstream/main or re-provision your machine.
+
+Do this: `./tools/provision`
+"""
+
+NEED_TO_UPGRADE = """
+The branch you are currently on has added dependencies beyond
+what you last provisioned. Your command is likely to fail
+until you add dependencies by provisioning.
+
+Do this: `./tools/provision`
+"""
+
+
+def get_provisioning_status() -> tuple[bool, str | None]:
+    version_file = get_version_file()
+    if not os.path.exists(version_file):
+        # If the developer doesn't have a version_file written by
+        # a previous provision, then we don't do any safety checks
+        # here on the assumption that the developer is managing
+        # their own dependencies and not running provision.
+        return True, None
+
+    with open(version_file) as f:
+        version = tuple(map(int, f.read().strip().split(".")))
+
+    # We may be more provisioned than the branch we just moved to.  As
+    # long as the major version hasn't changed, then we should be ok.
+    if version < PROVISION_VERSION:
+        return False, preamble(version) + NEED_TO_UPGRADE
+    elif version < (PROVISION_VERSION[0] + 1,):
+        return True, None
+    else:
+        return False, preamble(version) + NEED_TO_DOWNGRADE
+
+
+def assert_provisioning_status_ok(skip_provision_check: bool) -> None:
+    if not skip_provision_check:
+        ok, msg = get_provisioning_status()
+        if not ok:
+            print(msg)
+            print(
+                "If you really know what you are doing, use --skip-provision-check to run anyway."
+            )
+            sys.exit(1)
+
+
+def add_provision_check_override_param(parser: ArgumentParser) -> None:
+    """
+    Registers --skip-provision-check argument to be used with various commands/tests in our tools.
+    """
+    parser.add_argument(
+        "--skip-provision-check",
+        action="store_true",
+        help="Skip check that provision has been run; useful to save time if you know the dependency changes are not relevant to this command and will not cause it to fail",
+    )
+
+
+def find_js_test_files(test_dir: str, files: Iterable[str]) -> list[str]:
+    test_files = []
+    for file in files:
+        file = min(
+            (
+                os.path.join(test_dir, file_name)
+                for file_name in os.listdir(test_dir)
+                if file_name.startswith(file)
+            ),
+            default=file,
+        )
+
+        if not os.path.isfile(file):
+            raise Exception(f"Cannot find a matching file for '{file}' in '{test_dir}'")
+
+        test_files.append(os.path.abspath(file))
+
+    if not test_files:
+        test_files = sorted(
+            glob.glob(os.path.join(test_dir, "*.ts")) + glob.glob(os.path.join(test_dir, "*.js"))
+        )
+
+    return test_files
+
+
+def prepare_puppeteer_run(is_firefox: bool = False) -> None:
+    os.chdir(ZULIP_PATH)
+    # This will determine if the browser will be firefox or chrome.
+    os.environ["PUPPETEER_PRODUCT"] = "firefox" if is_firefox else "chrome"
+    subprocess.check_call(["node", "install.mjs"], cwd="node_modules/puppeteer")
+    os.makedirs("var/puppeteer", exist_ok=True)
+    for f in glob.glob("var/puppeteer/failure-*.png"):
+        os.remove(f)

tools/lib/test_server.py

@@ -0,0 +1,109 @@
+import os
+import subprocess
+import sys
+import time
+from collections.abc import Iterator
+from contextlib import ExitStack, contextmanager
+
+# Verify the Zulip venv is available.
+from tools.lib import sanity_check
+
+sanity_check.check_venv(__file__)
+
+import django
+import requests
+
+MAX_SERVER_WAIT = 180
+
+TOOLS_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+if TOOLS_DIR not in sys.path:
+    sys.path.insert(0, os.path.dirname(TOOLS_DIR))
+
+from scripts.lib.zulip_tools import get_or_create_dev_uuid_var_path
+from zerver.lib.test_fixtures import update_test_databases_if_required
+
+
+def set_up_django(external_host: str) -> None:
+    os.environ["FULL_STACK_ZULIP_TEST"] = "1"
+    os.environ["TEST_EXTERNAL_HOST"] = external_host
+    os.environ["LOCAL_UPLOADS_DIR"] = get_or_create_dev_uuid_var_path("test-backend/test_uploads")
+    os.environ["DJANGO_SETTINGS_MODULE"] = "zproject.test_settings"
+    django.setup()
+    os.environ["PYTHONUNBUFFERED"] = "y"
+
+
+def assert_server_running(server: "subprocess.Popen[bytes]", log_file: str | None) -> None:
+    """Get the exit code of the server, or None if it is still running."""
+    if server.poll() is not None:
+        message = "Server died unexpectedly!"
+        if log_file:
+            message += f"\nSee {log_file}\n"
+        raise RuntimeError(message)
+
+
+def server_is_up(server: "subprocess.Popen[bytes]", log_file: str | None) -> bool:
+    assert_server_running(server, log_file)
+    try:
+        # We could get a 501 error if the reverse proxy is up but the Django app isn't.
+        # Note that zulipdev.com is mapped via DNS to 127.0.0.1.
+        return requests.get("http://zulipdev.com:9981/accounts/home").status_code == 200
+    except requests.RequestException:
+        return False
+
+
+@contextmanager
+def test_server_running(
+    skip_provision_check: bool = False,
+    external_host: str = "testserver",
+    log_file: str | None = None,
+    dots: bool = False,
+    enable_help_center: bool = False,
+) -> Iterator[None]:
+    with ExitStack() as stack:
+        log = sys.stdout
+        if log_file:
+            if os.path.exists(log_file) and os.path.getsize(log_file) < 100000:
+                log = stack.enter_context(open(log_file, "a"))
+                log.write("\n\n")
+            else:
+                log = stack.enter_context(open(log_file, "w"))
+
+        set_up_django(external_host)
+
+        update_test_databases_if_required(rebuild_test_database=True)
+
+        # Run this not through the shell, so that we have the actual PID.
+        run_dev_server_command = ["tools/run-dev", "--test", "--streamlined"]
+        if skip_provision_check:
+            run_dev_server_command.append("--skip-provision-check")
+        if enable_help_center:
+            run_dev_server_command.append("--help-center-static-build")
+        server = subprocess.Popen(run_dev_server_command, stdout=log, stderr=log)
+
+        try:
+            # Wait for the server to start up.
+            print(end="\nWaiting for test server (may take a while)")
+            if not dots:
+                print("\n", flush=True)
+            t = time.time()
+            while not server_is_up(server, log_file):
+                if dots:
+                    print(end=".", flush=True)
+                time.sleep(0.4)
+                if time.time() - t > MAX_SERVER_WAIT:
+                    raise Exception("Timeout waiting for server")
+            print("\n\n--- SERVER IS UP! ---\n", flush=True)
+
+            # DO OUR ACTUAL TESTING HERE!!!
+            yield
+
+        finally:
+            assert_server_running(server, log_file)
+            server.terminate()
+            server.wait()
+
+
+if __name__ == "__main__":
+    # The code below is for testing this module works
+    with test_server_running():
+        print("\n\n SERVER IS UP!\n\n")