snapshot of https://github.com/zulip/zulip.git @ 48c69911c7dd4e03b20552eb3bd77ad8151c1357 (2026-09-15)

tools/setup/apns/csr.conf

@@ -0,0 +1,9 @@
+[req]
+encrypt_key = no
+
+prompt = no
+distinguished_name = req_distinguished_name
+
+[req_distinguished_name]
+CN = APNs for Zulip
+emailAddress = zulip-ops@zulip.com

tools/setup/apns/prep-cert

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+this_dir=${BASH_SOURCE[0]%/*}
+
+die() {
+    echo >&2 "$1"
+    exit 1
+}
+
+request() {
+    (($# == 2)) || die "usage: prep-cert request KEY_OUT CSR_OUT"
+    local key_out=$1
+    local csr_out=$2
+
+    openssl req -new \
+        -config "${this_dir}/csr.conf" \
+        -keyout "${key_out}" -out "${csr_out}"
+}
+
+combine() {
+    (($# == 3)) || die "usage: prep-cert combine KEY CERT OUT"
+    local key=$1
+    local cert=$2
+    local out=$3
+
+    local tmpdir
+    tmpdir=$(mktemp -d)
+    cleanup() {
+        rm -rf "${tmpdir}"
+        trap - RETURN EXIT
+    }
+    trap cleanup RETURN EXIT
+
+    local cert_pem="${tmpdir}/cert.pem"
+    local combined_p12="${tmpdir}/combined.p12"
+    openssl x509 -in "${cert}" -inform der -out "${cert_pem}"
+    openssl pkcs12 -export -passout pass: \
+        -inkey "${key}" -in "${cert_pem}" -out "${combined_p12}"
+    openssl pkcs12 -in "${combined_p12}" -passin pass: \
+        -out "${out}" -nodes
+}
+
+case "${1-}" in
+    request) shift && request "$@" ;;
+    combine) shift && combine "$@" ;;
+    *) die "usage: prep-cert {request|combine} ...ARGS" ;;
+esac

tools/setup/bootstrap-aws-installer

@@ -0,0 +1,81 @@
+#!/bin/env bash
+
+# Prepended to this automatically are the following:
+#SERVER=
+#HOSTNAME=
+#FULL_ROLES=
+#REPO_URL=
+#BRANCH=
+
+export RUNNING_IN_CLOUD_INIT=1
+if ! curl -fLs -m 5 -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 10" >/dev/null; then
+    echo "This should be run on AWS instances, not locally."
+    exit 1
+fi
+
+set -e
+set -x
+
+# Set the hostname early
+echo "$HOSTNAME" >/etc/hostname
+hostname "$HOSTNAME"
+sed -i "s/localhost$/localhost $HOSTNAME $SERVER/" /etc/hosts
+
+# Make sure root doesn't have a password
+passwd -l root
+
+# Allow root logins
+sed -i 's/disable_root: true/disable_root: false/' /etc/cloud/cloud.cfg
+
+# Ensure all apt updates (here and in the installer) are non-interactive
+export DEBIAN_FRONTEND=noninteractive
+
+# Dependencies to install AWS CLI
+(
+    apt-get -qy update
+    apt-get -qy --with-new-pkgs -o "Dpkg::Options::=--force-confdef" -o "Dpkg::Options::=--force-confold" upgrade
+    apt-get -qy install jq unzip curl
+    apt-get -qy autoclean
+)
+
+# The following line gets subbed in by a call to pack-local-script,
+# which will make $AWS_INSTALLER the path to a local copy of install-aws-cli
+AWS_INSTALLER="inline!puppet/kandra/files/install-aws-cli"
+
+# We then call it, to install the AWS CLI
+"$AWS_INSTALLER"
+
+# Set up a bare-bones AWS configuration
+mkdir -p /root/.aws
+cat >/root/.aws/config <<EOF
+[default]
+region = us-east-1
+output = text
+# Credentials are from the IAM role attached to the EC2 instance
+EOF
+
+# The following line gets replaced by pack-local-script output, which
+# smuggles the install-ssh-keys binary into this one.
+# install-ssh-keys, in turn, pulls key data from AWS' secret manager.
+INSTALL_SSH_KEYS="inline!puppet/kandra/files/install-ssh-keys"
+"$INSTALL_SSH_KEYS" root prod/ssh/keys/internal-read-only-deploy-key
+
+# Provide GitHub known_hosts setup; you can verify against fingerprints at
+# https://docs.github.com/en/github/authenticating-to-github/githubs-ssh-key-fingerprints
+# via `ssh-keygen -lf`
+GITHUB_KEYS="inline!puppet/kandra/files/github.keys"
+cat "$GITHUB_KEYS" >>/root/.ssh/known_hosts
+
+cd /root
+git clone "$REPO_URL" zulip -b "$BRANCH"
+git -C zulip checkout "$BRANCH"
+
+(
+    export APT_OPTIONS="-o Dpkg::Options::=--force-confnew"
+    /root/zulip/scripts/setup/install --puppet-classes "$FULL_ROLES" --postgresql-version=17
+)
+
+# Delete the ubuntu user
+userdel ubuntu
+
+reboot

tools/setup/build_pygments_data

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+import json
+import os
+
+from pygments.lexers import get_all_lexers
+
+ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../")
+# The current priorities data set is based on StackOverflow's 2020 survey.
+# We also prioritize text, quote, math, spoiler over others to enhance UX.
+DATA_PATH = os.path.join(ZULIP_PATH, "tools", "setup", "lang.json")
+OUT_PATH = os.path.join(ZULIP_PATH, "web", "generated", "pygments_data.json")
+
+with open(DATA_PATH) as f:
+    pygments_data = json.load(f)
+    priorities = dict(
+        **pygments_data["default"], **pygments_data["custom"], **pygments_data["aliases"]
+    )
+
+excluded_aliases = {"🔥"}
+lexers = get_all_lexers()
+langs = {
+    alias: {
+        "priority": priorities.get(alias, 0),
+        "pretty_name": longname,
+    }
+    for longname, aliases, filename_patterns, mimetypes in lexers
+    for alias in aliases
+    if alias not in excluded_aliases
+}
+
+langs |= {
+    name: {
+        "priority": priority,
+        "pretty_name": name,
+    }
+    for name, priority in priorities.items()
+    if name not in langs
+}
+
+with open(OUT_PATH, "w") as f:
+    json.dump({"langs": langs}, f)

tools/setup/build_supported_browser_regex.ts

@@ -0,0 +1,32 @@
+#!/usr/bin/env node
+
+import {writeFile} from "node:fs/promises";
+import path from "node:path";
+import {fileURLToPath} from "node:url";
+
+import {getUserAgentRegex} from "browserslist-useragent-regexp";
+
+const root_dir = fileURLToPath(new URL("../..", import.meta.url));
+const output_path = path.join(root_dir, "web", "generated", "supported_browser_regex.ts");
+// Since browserlistrc is not in the root, it will not be autodetected.
+const browserslist_config = path.join(root_dir, "web", ".browserslistrc");
+
+async function main(): Promise<void> {
+    const baselineRegex = getUserAgentRegex({
+        config: browserslist_config,
+        allowHigherVersions: true,
+    });
+    const allBrowsersRegex = getUserAgentRegex({
+        browsers: ">= 0%",
+        allowHigherVersions: true,
+    });
+    const contents =
+        `// This file is generated by tools/setup/build_supported_browser_regex.ts\n` +
+        `// Do not edit this file directly.\n` +
+        `export const baselineBrowserRegex = ${baselineRegex.toString()};\n` +
+        `export const allBrowsersRegex = ${allBrowsersRegex.toString()};\n`;
+
+    await writeFile(output_path, contents);
+}
+
+await main();

tools/setup/build_timezone_values

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+import json
+import os
+import sys
+import zoneinfo
+
+ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../")
+sys.path.insert(0, ZULIP_PATH)
+
+from zerver.lib.timezone import get_canonical_timezone_map
+
+OUT_PATH = os.path.join(ZULIP_PATH, "web", "generated", "timezones.json")
+
+with open(OUT_PATH, "w") as f:
+    json.dump(
+        {
+            "timezones": sorted(
+                zoneinfo.available_timezones()
+                - {"Factory", "localtime"}
+                - set(get_canonical_timezone_map())
+            )
+        },
+        f,
+    )

tools/setup/dev-motd

@@ -0,0 +1,11 @@
+#!/usr/bin/tail -n+2
+
+This is the Zulip development environment.  Popular commands:
+* tools/provision - Update the development environment
+* tools/run-dev - Run the development server
+* tools/lint - Run the linter (quick and catches many problems)
+* tools/test-* - Run tests (use --help to learn about options)
+
+Read https://zulip.readthedocs.io/en/latest/testing/testing.html to learn
+how to run individual test suites so that you can get a fast debug cycle.
+

tools/setup/dev-vagrant-docker/Dockerfile

@@ -0,0 +1,56 @@
+FROM ubuntu:22.04
+
+ARG UBUNTU_MIRROR
+
+# Basic packages and dependencies of docker-systemctl-replacement
+RUN echo locales locales/default_environment_locale select C.UTF-8 | debconf-set-selections \
+    && echo locales locales/locales_to_be_generated select "C.UTF-8 UTF-8" | debconf-set-selections \
+    && { [ ! "$UBUNTU_MIRROR" ] || sed -i "s|http://\(\w*\.\)*archive\.ubuntu\.com/ubuntu/\? |$UBUNTU_MIRROR |" /etc/apt/sources.list; } \
+    # This restores man pages and other documentation that have been
+    # stripped from the default Ubuntu cloud image and installs
+    # ubuntu-minimal and ubuntu-standard.
+    #
+    # This makes sense to do because we're using this image as a
+    # development environment, not a minimal production system.
+    && printf 'y\n\n' | unminimize \
+    && apt-get install --no-install-recommends -y \
+           ca-certificates \
+           curl \
+           locales \
+           openssh-server \
+           python3 \
+           sudo \
+           systemd \
+    && rm -rf /var/lib/apt/lists/*
+
+ARG VAGRANT_UID
+
+RUN \
+    # We use https://github.com/gdraheim/docker-systemctl-replacement
+    # to make services we install like PostgreSQL, Redis, etc. normally
+    # managed by systemd start within Docker, which breaks normal
+    # operation of systemd.
+    curl -fLsS --retry 3 --output-dir /bin -O 'https://raw.githubusercontent.com/gdraheim/docker-systemctl-replacement/v1.7.1097/files/docker/systemctl3.py' -O 'https://raw.githubusercontent.com/gdraheim/docker-systemctl-replacement/v1.7.1097/files/docker/journalctl3.py' \
+    && printf '5f5a47f321c7a8881dfd01f774106dc4012d664bee238eada86a7112674fea93  /bin/systemctl3.py\n16a52fe484d7c0fca464c82a56de58980440362942482c0d1fbc424e69f8f833  /bin/journalctl3.py\n' | sha256sum -c \
+    && chmod +x /bin/systemctl3.py /bin/journalctl3.py \
+    && dpkg-divert --add --rename /bin/systemctl \
+    && ln -s systemctl3.py /bin/systemctl \
+    && dpkg-divert --add --rename /bin/journalctl \
+    && ln -s journalctl3.py /bin/journalctl \
+    && ln -nsf /bin/true /usr/sbin/policy-rc.d \
+    && mkdir -p /run/sshd \
+    # docker-systemctl-replacement doesn’t work with template units yet:
+    # https://github.com/gdraheim/docker-systemctl-replacement/issues/62
+    && ln -ns /lib/systemd/system/postgresql@.service /etc/systemd/system/multi-user.target.wants/postgresql@14-main.service \
+    # Set up the vagrant user and its SSH key (globally public)
+    && useradd -ms /bin/bash -u "$VAGRANT_UID" vagrant \
+    && mkdir -m 700 ~vagrant/.ssh \
+    && curl -fLsS --retry 3 -o ~vagrant/.ssh/authorized_keys 'https://raw.githubusercontent.com/hashicorp/vagrant/v2.4.0/keys/vagrant.pub' \
+    && echo '55009a554ba2d409565018498f1ad5946854bf90fa8d13fd3fdc2faa102c1122  /home/vagrant/.ssh/authorized_keys' | sha256sum -c \
+    && chown -R vagrant: ~vagrant/.ssh \
+    && echo 'vagrant ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/vagrant
+
+CMD ["/bin/systemctl"]
+
+EXPOSE 22
+EXPOSE 9991

tools/setup/emoji/build_emoji

@@ -0,0 +1,452 @@
+#!/usr/bin/env python3
+#
+# See docs/subsystems/emoji.md for a high-level explanation of how this system
+# works.
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+from collections.abc import Iterator, Sequence
+from typing import Any
+
+import orjson
+
+ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../../")
+sys.path.append(ZULIP_PATH)
+
+from scripts.lib.zulip_tools import generate_sha1sum_emoji, run_as_root
+from tools.setup.emoji.emoji_names import EMOJI_NAME_MAPS
+from tools.setup.emoji.emoji_setup_utils import (
+    EMOTICON_CONVERSIONS,
+    REMAPPED_EMOJIS,
+    emoji_is_supported,
+    emoji_names_for_picker,
+    generate_codepoint_to_name_map,
+    generate_codepoint_to_names_map,
+    generate_emoji_catalog,
+    generate_name_to_codepoint_map,
+    get_emoji_code,
+)
+
+EMOJI_CACHE_BASE_PATH = "/srv/zulip-emoji-cache"
+EMOJI_SCRIPT_DIR_PATH = os.path.join(ZULIP_PATH, "tools", "setup", "emoji")
+NODE_MODULES_PATH = os.path.join(ZULIP_PATH, "node_modules")
+
+
+# The CSS for emoji spritesheet has somewhat tricky requirements.  One
+# is that we want to be able to use the same emoji CSS classes for
+# different display sizes of our emoji (e.g. reactions are smaller
+# than inline message emoji, which are smaller than those in the emoji
+# picker) while only downloading 1 copy of the spritesheet, having
+# good browser rendering performance, and reusing as much common CSS
+# as is possible.
+
+# Our solution to those problem is to use the `background-size` (Which
+# is e.g. 5700%) and background-position attributes to select the
+# region of the spritesheet corresponding to the target sprite and
+# display it properly scaled in an emoji span.
+SPRITE_CSS_FILE_TEMPLATE = """\
+div.emoji,
+span.emoji
+{{
+    display: inline-block;
+    background-image: url(../emoji/{emojiset}.webp);
+    background-size: {background_size};
+    background-repeat: no-repeat;
+
+    /* Hide the text. */
+    text-indent: 100%;
+    white-space: nowrap;
+    overflow: hidden;
+}}
+
+.emoji-1f419
+{{
+    background-image: url(../../../static/generated/emoji/images-google-64/1f419.png) !important;
+    background-position: 0% 0% !important;
+    background-size: contain !important;
+}}
+
+{emoji_positions}
+"""
+
+EMOJI_POS_INFO_TEMPLATE = """\
+.emoji-{codepoint} {{
+    background-position: {pos_x} {pos_y};
+}}
+"""
+
+EMOJI_OVERRIDE_TEMPLATE = """\
+.emoji-{codepoint} {{
+    background-image: url(../../../static/generated/emoji/images-google-64/{codepoint}.png) !important;
+    background-position: 0% 0% !important;
+    background-size: contain !important;
+}}
+"""
+
+# change directory
+os.chdir(EMOJI_SCRIPT_DIR_PATH)
+
+
+def main() -> None:
+    if not os.access(EMOJI_CACHE_BASE_PATH, os.W_OK):
+        # Note: In production, this block will fail, since we don't
+        # assume sudo access; but it should never run in production
+        # anyway, because EMOJI_CACHE_BASE_PATH is created by Puppet before
+        # build_emoji would be run.
+        run_as_root(["mkdir", "-p", EMOJI_CACHE_BASE_PATH])
+        run_as_root(["chown", f"{os.getuid()}:{os.getgid()}", EMOJI_CACHE_BASE_PATH])
+
+    sha1_hexdigest = generate_sha1sum_emoji(ZULIP_PATH)
+    emoji_cache_path = os.path.join(EMOJI_CACHE_BASE_PATH, sha1_hexdigest)
+    success_stamp = os.path.join(emoji_cache_path, ".success-stamp")
+
+    if not os.path.exists(success_stamp):
+        print("Dumping emojis ...")
+        dump_emojis(emoji_cache_path)
+        with open(success_stamp, "w") as f:
+            f.close()
+
+    print(f"build_emoji: Using cached emojis from {emoji_cache_path}")
+
+    # /srv/zulip-emoji-cache/*/static gets symlinked to ZULIP_PATH/static/generated/emoji
+    TARGET_STATIC_EMOJI = os.path.join(ZULIP_PATH, "static", "generated", "emoji")
+    if os.path.lexists(TARGET_STATIC_EMOJI):
+        os.remove(TARGET_STATIC_EMOJI)
+    os.symlink(os.path.join(emoji_cache_path, "static"), TARGET_STATIC_EMOJI)
+
+    # /srv/zulip-emoji-cache/*/web gets copied to ZULIP_PATH/web/generated
+    # These must not be symlinked so webpack can resolve module references.
+    for subdir in ("emoji", "emoji-styles"):
+        target_dir = os.path.join(ZULIP_PATH, "web", "generated", subdir)
+        os.makedirs(target_dir, exist_ok=True)
+        to_remove = set(os.listdir(target_dir))
+        source_emoji_dump = os.path.join(emoji_cache_path, "web", subdir)
+        for filename in os.listdir(source_emoji_dump):
+            shutil.copy2(os.path.join(source_emoji_dump, filename), target_dir)
+            to_remove.discard(filename)
+        for filename in to_remove:
+            os.remove(os.path.join(target_dir, filename))
+
+
+def percent(f: float) -> str:
+    return f"{f * 100:0.3f}%"
+
+
+def get_square_size(emoji_data: Sequence[dict[str, Any]]) -> int:
+    """
+    Spritesheets are usually NxN squares, and we have to
+    infer N from the sheet_x/sheet_y values of emojis.
+    """
+
+    def get_offsets(emoji_data: Sequence[dict[str, Any]]) -> Iterator[int]:
+        for emoji_dict in emoji_data:
+            yield emoji_dict["sheet_x"]
+            yield emoji_dict["sheet_y"]
+            if "skin_variations" in emoji_dict:
+                for img_info in emoji_dict["skin_variations"].values():
+                    yield img_info["sheet_x"]
+                    yield img_info["sheet_y"]
+
+    n = max(get_offsets(emoji_data)) + 1
+    return n
+
+
+def generate_sprite_css_files(
+    cache_path: str, emoji_data: list[dict[str, Any]], emojiset: str
+) -> None:
+    """
+    Spritesheets are usually NxN squares.
+    """
+    n = get_square_size(emoji_data)
+
+    """
+    Each single emoji is 64x64, with 1px gutters on every border.
+    We just consider the gutters to be part of the image for
+    simplicity reasons, so you can think of the spritesheet as
+    an NxN square of 66x66 pre-padded emojis.  The CSS
+    background-size parameter below says to size the background
+    element as N times the size of the element that you're drawing.
+
+    Note that we use percentages here, instead of absolute
+    pixel values, because when we render emojis as actual elements,
+    their size will vary depending on which part of the UI we're
+    in (message emojis, emoji reactions, emoji popup, emoji
+    popup showcase, etc.).
+
+    (The next step is to offset the image; that will be in the
+    upcoming loop.)
+    """
+    background_size = percent(n)
+
+    emoji_positions = ""
+    for emoji in emoji_data:
+        if emoji_is_supported(emoji):
+            """
+            For background-position we need to use percentages.
+            Absolute pixel values won't work, because the size
+            of the background sprite image is proportional to
+            the size of the element we're rendering, and we render
+            elements in multiple sizes.
+
+            The way that CSS background-position works is linear
+            interpolation.  When you tell CSS background-position
+            is "42% 37%", then in the `x` dimension it will align
+            the image such that 42% of the background image is to
+            the left of the 42% mark in the element itself.
+
+            For simplicity assume we render the emoji as 66px
+            (and everything will scale appropriately for other
+            size images as long as we use percentages).
+
+            The image size will be 66n.
+            The left offset of the x-th emoji (including its
+            padding) will be 66x.  And the element's width
+            will be 66.
+
+            So, solve this equation for `p`, where p is
+            the ratio that we'll later express as a
+            percentage:
+
+                <image offset> = <offset of p% mark of element>
+                (p * 66n) = 66x + p66
+                p * n = x + p
+                p * n - p = x
+                p * (n - 1) = x
+                p = x / (n - 1)
+
+            If you ever want to change the code so that the
+            gutters don't show up in the element, the algebra
+            will get more complicated.
+            """
+            emoji_positions += EMOJI_POS_INFO_TEMPLATE.format(
+                codepoint=get_emoji_code(emoji),
+                pos_x=percent(emoji["sheet_x"] / (n - 1)),
+                pos_y=percent(emoji["sheet_y"] / (n - 1)),
+            )
+
+    SPRITE_STYLES_DIRECTORY = os.path.join(cache_path, "web", "emoji-styles")
+    os.makedirs(SPRITE_STYLES_DIRECTORY, exist_ok=True)
+    SPRITE_CSS_PATH = os.path.join(SPRITE_STYLES_DIRECTORY, f"{emojiset}-sprite.css")
+    with open(SPRITE_CSS_PATH, "w") as f:
+        f.write(
+            SPRITE_CSS_FILE_TEMPLATE.format(
+                emojiset=emojiset,
+                emoji_positions=emoji_positions,
+                background_size=background_size,
+            ),
+        )
+
+    # The Twitter emoji team was laid off in 2022, so new emoji aren't supported.
+    # https://github.com/twitter/twemoji/issues/570#issuecomment-1303422143.
+    # The "twitter" sprite sheet we’re using does have images in those locations,
+    # but they’re fallback images that emoji-datasource fills in from the Apple
+    # sprite sheet, which has unclear licensing implications.
+    # To be able to support newer emoji, we fallback to Google Modern for any emoji
+    # not covered by Twemoji.
+    if emojiset == "twitter":
+        extra_emoji_positions = ""
+        twitter_covered_emoji_codes = {
+            get_emoji_code(emoji) for emoji in emoji_data if emoji["has_img_twitter"]
+        }
+        for emoji in emoji_data:
+            code = get_emoji_code(emoji)
+            if emoji["has_img_google"] and code not in twitter_covered_emoji_codes:
+                extra_emoji_positions += EMOJI_OVERRIDE_TEMPLATE.format(
+                    codepoint=code,
+                )
+        with open(SPRITE_CSS_PATH, "a") as f:
+            f.write(extra_emoji_positions)
+
+
+def setup_emoji_farms(cache_path: str, emoji_data: list[dict[str, Any]]) -> None:
+    def ensure_emoji_image(
+        emoji_dict: dict[str, Any], src_emoji_farm: str, target_emoji_farm: str
+    ) -> None:
+        # We use individual images from emoji farm for rendering emojis
+        # in notification messages. We have a custom emoji formatter in
+        # notifications processing code that converts `span` tags to
+        # `img` and that logic requires us to have non-qualified
+        # `emoji_code` as file name for emoji.
+        emoji_code = get_emoji_code(emoji_dict)
+        img_file_name = emoji_code + ".png"
+        src_file = os.path.join(src_emoji_farm, emoji_dict["image"])
+        dst_file = os.path.join(target_emoji_farm, img_file_name)
+        shutil.copy2(src_file, dst_file)
+
+    def setup_emoji_farm(emojiset: str, emoji_data: list[dict[str, Any]]) -> None:
+        # Copy individual emoji images from npm packages.
+        src_emoji_farm = os.path.join(
+            NODE_MODULES_PATH, "emoji-datasource-" + emojiset, "img", emojiset, "64"
+        )
+        target_emoji_farm = os.path.join(cache_path, "static", "images-" + emojiset + "-64")
+        os.makedirs(target_emoji_farm, exist_ok=True)
+        print(f"Copying individual {emojiset} image files...")
+        for emoji_dict in emoji_data:
+            if emoji_dict["has_img_" + emojiset]:
+                ensure_emoji_image(emoji_dict, src_emoji_farm, target_emoji_farm)
+                skin_variations = emoji_dict.get("skin_variations", {})
+                for img_info in skin_variations.values():
+                    if img_info["has_img_" + emojiset]:
+                        ensure_emoji_image(img_info, src_emoji_farm, target_emoji_farm)
+
+        # Copy zulip.png to the emoji farm.
+        zulip_image = os.path.join(ZULIP_PATH, "web", "images", "zulip-emoji")
+        for f in os.listdir(zulip_image):
+            shutil.copy2(os.path.join(zulip_image, f), target_emoji_farm, follow_symlinks=False)
+
+        # We hardcode octopus emoji image to Google emoji set's old
+        # "cute octopus" image. Copy it to the emoji farms.
+        input_img_file = os.path.join(EMOJI_SCRIPT_DIR_PATH, "1f419.png")
+        output_img_file = os.path.join(target_emoji_farm, "1f419.png")
+        shutil.copyfile(input_img_file, output_img_file)
+
+        generate_sprite_css_files(cache_path, emoji_data, emojiset)
+
+        print(f"Converting {emojiset} sheet to webp...")
+        TARGET_EMOJI_SHEETS = os.path.join(cache_path, "web", "emoji")
+        os.makedirs(TARGET_EMOJI_SHEETS, exist_ok=True)
+
+        sheet_src = os.path.join(
+            NODE_MODULES_PATH,
+            f"emoji-datasource-{emojiset}",
+            "img",
+            emojiset,
+            "sheets-clean",
+            "64.png",
+        )
+        sheet_dst = os.path.join(TARGET_EMOJI_SHEETS, f"{emojiset}.webp")
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            # First quantize the image to 256 colors (like sheets-256 in
+            # emoji-datasource); libvips can only do this when saving to a PNG.
+            quantized = os.path.join(tmpdir, "quantized.png")
+            subprocess.check_call(
+                ["vips", "copy", sheet_src, f"{quantized}[palette,dither=0,effort=10]"]
+            )
+
+            # From libwebp: [Q is] between 0 and 100. For lossy, 0 gives
+            #   the smallest size and 100 the largest. For lossless, this
+            #   parameter is the amount of effort put into the
+            #   compression: 0 is the fastest but gives larger files
+            #   compared to the slowest, but best, 100.
+            subprocess.check_call(["vips", "copy", quantized, f"{sheet_dst}[lossless,Q=100]"])
+
+    # Set up standard emoji sets.
+    for emojiset in ["google", "twitter"]:
+        setup_emoji_farm(emojiset, emoji_data)
+
+
+def setup_old_emoji_farm(
+    cache_path: str, emoji_map: dict[str, str], emoji_data: list[dict[str, Any]]
+) -> None:
+    # Code for setting up old emoji farm.
+    emoji_cache_path = os.path.join(cache_path, "static", "images", "emoji")
+    unicode_emoji_cache_path = os.path.join(cache_path, "static", "images", "emoji", "unicode")
+    google_emoji_cache_path = os.path.join(cache_path, "static", "images-google-64")
+    os.makedirs(emoji_cache_path, exist_ok=True)
+    os.makedirs(unicode_emoji_cache_path, exist_ok=True)
+
+    # Symlink zulip.png image file.
+    image_file_path = os.path.join(google_emoji_cache_path, "zulip.png")
+    symlink_path = os.path.join(emoji_cache_path, "zulip.png")
+    os.symlink(image_file_path, symlink_path)
+
+    unicode_symlink_path = os.path.join(unicode_emoji_cache_path, "zulip.png")
+    os.symlink(image_file_path, unicode_symlink_path)
+
+    for name, codepoint in emoji_map.items():
+        mapped_codepoint = REMAPPED_EMOJIS.get(codepoint, codepoint)
+        image_file_path = os.path.join(google_emoji_cache_path, f"{mapped_codepoint}.png")
+        symlink_path = os.path.join(emoji_cache_path, f"{name}.png")
+        os.symlink(image_file_path, symlink_path)
+        try:
+            # `emoji_map` contains duplicate entries for the same codepoint with different
+            # names. So creation of symlink for <codepoint>.png may throw `FileExistsError`.
+            unicode_symlink_path = os.path.join(unicode_emoji_cache_path, f"{codepoint}.png")
+            os.symlink(image_file_path, unicode_symlink_path)
+        except FileExistsError:
+            pass
+
+
+def generate_map_files(cache_path: str, emoji_catalog: dict[str, list[str]]) -> None:
+    # This function generates the main data files about emoji that are
+    # consumed by the web app, mobile apps, Markdown processor, etc.
+    names = emoji_names_for_picker(EMOJI_NAME_MAPS)
+    codepoint_to_name = generate_codepoint_to_name_map(EMOJI_NAME_MAPS)
+    name_to_codepoint = generate_name_to_codepoint_map(EMOJI_NAME_MAPS)
+
+    EMOJI_CODES_FILE_PATH = os.path.join(cache_path, "static", "emoji_codes.json")
+    with open(EMOJI_CODES_FILE_PATH, "wb") as emoji_codes_file:
+        emoji_codes_file.write(
+            orjson.dumps(
+                {
+                    "names": names,
+                    "name_to_codepoint": name_to_codepoint,
+                    "codepoint_to_name": codepoint_to_name,
+                    "emoji_catalog": emoji_catalog,
+                    "emoticon_conversions": EMOTICON_CONVERSIONS,
+                }
+            )
+        )
+
+    # This is the more official API for mobile to fetch data about emoji.
+    # emoji_codes.json has a lot of goo, and we're creating this new file
+    # as a cleaner data format to move towards. We could add the rest of
+    # the data into this API-described data, and then the web client could
+    # switch to that which would allow us to drop the existing file. But
+    # we'll probably instead do #18121 which will make this file obsolete.
+    # So this is a temporary solution. CZO discussion:
+    # https://chat.zulip.org/#narrow/channel/378-api-design/topic/currently.20supported.20emoji/near/1394598
+    EMOJI_API_FILE_PATH = os.path.join(cache_path, "static", "emoji_api.json")
+    with open(EMOJI_API_FILE_PATH, "wb") as emoji_api_file:
+        emoji_api_file.write(
+            orjson.dumps(
+                {
+                    "code_to_names": generate_codepoint_to_names_map(EMOJI_NAME_MAPS),
+                }
+            )
+        )
+
+
+def dump_emojis(cache_path: str) -> None:
+    with open("emoji_map.json", "rb") as emoji_map_file:
+        emoji_map = orjson.loads(emoji_map_file.read())
+
+    # `emoji.json` or any other data file can be sourced from any of the supported
+    # emoji set packages, they all contain the same data files.
+    EMOJI_DATA_FILE_PATH = os.path.join(NODE_MODULES_PATH, "emoji-datasource-google", "emoji.json")
+    with open(EMOJI_DATA_FILE_PATH, "rb") as emoji_data_file:
+        emoji_data = orjson.loads(emoji_data_file.read())
+
+    # These are the codes that we'll be able to show Google Modern
+    # emoji for. (For other emoji sets, we fall back to Google Modern.)
+    supported_codes = {get_emoji_code(emoji) for emoji in emoji_data if emoji_is_supported(emoji)}
+    # These are in the emoji dropdown so we should make sure they're supported.
+    for code in EMOJI_NAME_MAPS:
+        # If an assertion here fails, we either need to find an image for Google Modern
+        # to display for that emoji, or we need to remove that emoji as an option for users,
+        # by removing it from emoji_names.py through a change to `generate_emoji_names`.
+        assert code in supported_codes
+
+    emoji_catalog = generate_emoji_catalog(emoji_data, EMOJI_NAME_MAPS)
+
+    # Set up emoji farms.
+    if os.path.exists(cache_path):
+        shutil.rmtree(cache_path)
+    setup_emoji_farms(cache_path, emoji_data)
+    setup_old_emoji_farm(cache_path, emoji_map, emoji_data)
+
+    # This file is needed to translate emoji when importing data from Slack.
+    shutil.copyfile(
+        EMOJI_DATA_FILE_PATH,
+        os.path.join(cache_path, "static", "emoji-datasource-google-emoji.json"),
+    )
+
+    # Generate various map files.
+    generate_map_files(cache_path, emoji_catalog)
+
+
+if __name__ == "__main__":
+    main()

tools/setup/emoji/emoji_setup_utils.py

@@ -0,0 +1,127 @@
+# This file contains various helper functions used by `build_emoji` tool.
+# See docs/subsystems/emoji.md for details on how this system works.
+from collections import defaultdict
+from typing import Any
+
+from zerver.lib.emoji_utils import emoji_to_hex_codepoint, hex_codepoint_to_emoji, unqualify_emoji
+
+# Some image files in the old emoji farm had a different name than in the new emoji
+# farm. `remapped_emojis` is a map that contains a mapping of their name in the old
+# emoji farm to their name in the new emoji farm.
+REMAPPED_EMOJIS = {
+    "0023": "0023-20e3",  # Hash
+    "0030": "0030-20e3",  # Zero
+    "0031": "0031-20e3",  # One
+    "0032": "0032-20e3",  # Two
+    "0033": "0033-20e3",  # Three
+    "0034": "0034-20e3",  # Four
+    "0035": "0035-20e3",  # Five
+    "0036": "0036-20e3",  # Six
+    "0037": "0037-20e3",  # Seven
+    "0038": "0038-20e3",  # Eight
+    "0039": "0039-20e3",  # Nine
+    "1f1e8": "1f1e8-1f1f3",  # cn
+    "1f1e9": "1f1e9-1f1ea",  # de
+    "1f1ea": "1f1ea-1f1f8",  # es
+    "1f1eb": "1f1eb-1f1f7",  # fr
+    "1f1ec": "1f1ec-1f1e7",  # gb/us
+    "1f1ee": "1f1ee-1f1f9",  # it
+    "1f1ef": "1f1ef-1f1f5",  # jp
+    "1f1f0": "1f1f0-1f1f7",  # kr
+    "1f1f7": "1f1f7-1f1fa",  # ru
+    "1f1fa": "1f1fa-1f1f8",  # us
+}
+
+# Emoticons and which emoji they should become. Duplicate emoji are allowed.
+# Changes here should be mimicked in `help/configure-emoticon-translations.md`.
+EMOTICON_CONVERSIONS = {
+    ":)": ":slight_smile:",
+    "(:": ":slight_smile:",
+    ":(": ":frown:",
+    "<3": ":heart:",
+    ":|": ":neutral:",
+    ":/": ":confused:",
+    ";)": ":wink:",
+    ":D": ":smile:",
+    ":o": ":open_mouth:",
+    ":O": ":open_mouth:",
+    ":p": ":stuck_out_tongue:",
+    ":P": ":stuck_out_tongue:",
+}
+
+
+def emoji_names_for_picker(emoji_name_maps: dict[str, dict[str, Any]]) -> list[str]:
+    emoji_names: list[str] = []
+    for name_info in emoji_name_maps.values():
+        emoji_names.append(name_info["canonical_name"])
+        emoji_names.extend(name_info["aliases"])
+
+    return sorted(emoji_names)
+
+
+def get_emoji_code(emoji_dict: dict[str, Any]) -> str:
+    # There is a `non_qualified` field on `emoji_dict` but it's
+    # inconsistently present, so we'll always use the unqualified
+    # emoji by unqualifying it ourselves. This gives us more consistent
+    # behaviour between emojis, and doesn't rely on the incomplete
+    # upstream package (https://github.com/iamcal/emoji-data/pull/217).
+    return emoji_to_hex_codepoint(unqualify_emoji(hex_codepoint_to_emoji(emoji_dict["unified"])))
+
+
+# Returns a dict from categories to list of codepoints. The list of
+# codepoints are sorted according to the `sort_order` as defined in
+# `emoji_data`.
+def generate_emoji_catalog(
+    emoji_data: list[dict[str, Any]], emoji_name_maps: dict[str, dict[str, Any]]
+) -> dict[str, list[str]]:
+    sort_order: dict[str, int] = {}
+    emoji_catalog: dict[str, list[str]] = defaultdict(list)
+
+    for emoji_dict in emoji_data:
+        emoji_code = get_emoji_code(emoji_dict)
+        if not emoji_is_supported(emoji_dict) or emoji_code not in emoji_name_maps:
+            continue
+        category = emoji_dict["category"]
+        sort_order[emoji_code] = emoji_dict["sort_order"]
+        emoji_catalog[category].append(emoji_code)
+
+    # Sort the emojis according to iamcal's sort order. This sorting determines the
+    # order in which emojis will be displayed in emoji picker.
+    for emoji_codes in emoji_catalog.values():
+        emoji_codes.sort(key=lambda emoji_code: sort_order[emoji_code])
+
+    return dict(emoji_catalog)
+
+
+def generate_codepoint_to_name_map(emoji_name_maps: dict[str, dict[str, Any]]) -> dict[str, str]:
+    codepoint_to_name: dict[str, str] = {}
+    for emoji_code, name_info in emoji_name_maps.items():
+        codepoint_to_name[emoji_code] = name_info["canonical_name"]
+    return codepoint_to_name
+
+
+# We support Google Modern, and fall back to Google Modern when emoji
+# aren't supported by the other styles we use.
+def emoji_is_supported(emoji_dict: dict[str, Any]) -> bool:
+    return emoji_dict["has_img_google"]
+
+
+def generate_codepoint_to_names_map(
+    emoji_name_maps: dict[str, dict[str, Any]],
+) -> dict[str, list[str]]:
+    # The first element of the names list is always the canonical name.
+    return {
+        emoji_code: [name_info["canonical_name"], *name_info["aliases"]]
+        for emoji_code, name_info in emoji_name_maps.items()
+    }
+
+
+def generate_name_to_codepoint_map(emoji_name_maps: dict[str, dict[str, Any]]) -> dict[str, str]:
+    name_to_codepoint = {}
+    for emoji_code, name_info in emoji_name_maps.items():
+        canonical_name = name_info["canonical_name"]
+        aliases = name_info["aliases"]
+        name_to_codepoint[canonical_name] = emoji_code
+        for alias in aliases:
+            name_to_codepoint[alias] = emoji_code
+    return name_to_codepoint

tools/setup/emoji/generate_emoji_names

@@ -0,0 +1,222 @@
+#!/usr/bin/env python3
+import os
+import re
+import sys
+from collections import defaultdict
+
+import orjson
+
+ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../../")
+sys.path.append(ZULIP_PATH)
+
+from tools.setup.emoji.custom_emoji_names import CUSTOM_EMOJI_NAME_MAPS
+from tools.setup.emoji.emoji_setup_utils import get_emoji_code
+from zerver.lib.emoji_utils import hex_codepoint_to_emoji, unqualify_emoji
+
+CLDR_DATA_FILE = os.path.join(
+    ZULIP_PATH, "node_modules", "cldr-annotations-full", "annotations", "en", "annotations.json"
+)
+CLDR_DERIVED_DATA_FILE = os.path.join(
+    ZULIP_PATH,
+    "node_modules",
+    "cldr-annotations-derived-full",
+    "annotationsDerived",
+    "en",
+    "annotations.json",
+)
+EMOJI_DATA_FILE = os.path.join(ZULIP_PATH, "node_modules", "emoji-datasource-google", "emoji.json")
+OUT_EMOJI_FILE = os.path.join(ZULIP_PATH, "tools", "setup", "emoji", "emoji_names.py")
+
+with open(EMOJI_DATA_FILE, "rb") as fp:
+    EMOJI_DATA = orjson.loads(fp.read())
+with open(CLDR_DATA_FILE, "rb") as fp:
+    CLDR_DATA = orjson.loads(fp.read())["annotations"]["annotations"]
+with open(CLDR_DERIVED_DATA_FILE, "rb") as fp:
+    CLDR_DATA.update(orjson.loads(fp.read())["annotationsDerived"]["annotations"])
+
+# We don't include most clock emojis. See `custom_emoji_names` for more context.
+SKIPPED_CLOCK_EMOJI_CODES = [
+    "1f550",
+    "1f551",
+    "1f552",
+    "1f553",
+    "1f554",
+    "1f555",
+    "1f556",
+    "1f558",
+    "1f559",
+    "1f55a",
+    "1f55b",
+    "1f55c",
+    "1f55d",
+    "1f55e",
+    "1f55f",
+    "1f560",
+    "1f561",
+    "1f562",
+    "1f563",
+    "1f564",
+    "1f565",
+    "1f566",
+    "1f567",
+]
+
+# We don't include the skin tones as emojis that one can search for on their own.
+SKIN_TONE_EMOJI_CODES = [
+    "1f3fb",
+    "1f3fc",
+    "1f3fd",
+    "1f3fe",
+    "1f3ff",
+]
+
+
+def cleanup_name(name: str) -> str:
+    replacements = {
+        " ": "_",
+        "-": "_",
+        "–": "_",
+        "“": "",
+        "”": "",
+        ":": "",
+        ".": "",
+        ",": "",
+        "(": "",
+        ")": "",
+        "&": "and",
+        "‘": "",
+        "’": "",
+        "'": "",
+    }
+    for before, after in replacements.items():
+        name = name.replace(before, after)
+    name = re.sub(r"_{2,}", "_", name)
+    return name.lower()
+
+
+def convert_non_ascii_chars(name: str) -> str:
+    replacements = {
+        "è": "e",
+        "ǐ": "i",
+        "ó": "o",
+        "ā": "a",
+        "ō": "o",
+        "ñ": "n",
+        "ô": "o",
+        "ç": "c",
+        "é": "e",
+        "ã": "a",
+        "í": "i",
+        "å": "a",
+        "ü": "u",
+    }
+    for before, after in replacements.items():
+        name = name.replace(before, after)
+    assert name.isascii(), (
+        f"{name} still contains non-ascii characters. Add them to convert_non_ascii_chars."
+    )
+    return name
+
+
+def main() -> None:
+    all_emojis = {}
+    all_canonical_names = set()
+
+    alias_to_emoji_code = defaultdict(list)
+
+    # STEP 1: Generate first draft of all_emojis.
+    for emoji_dict in EMOJI_DATA:
+        emoji_code = get_emoji_code(emoji_dict)
+        if emoji_code in SKIPPED_CLOCK_EMOJI_CODES or emoji_code in SKIN_TONE_EMOJI_CODES:
+            continue
+
+        if emoji_code in CUSTOM_EMOJI_NAME_MAPS:
+            canonical_name = cleanup_name(CUSTOM_EMOJI_NAME_MAPS[emoji_code]["canonical_name"])
+            if canonical_name in all_canonical_names:
+                raise Exception(
+                    f"{canonical_name} was already added with a different codepoint. "
+                    f"Rename it in `custom_emoji_names` or add an entry for {emoji_code}."
+                )
+            all_canonical_names.add(canonical_name)
+            all_emojis[emoji_code] = CUSTOM_EMOJI_NAME_MAPS[emoji_code]
+        else:
+            # create the unicode character(s) for the emoji, since this is the key into the CLDR data
+            # We can't just use emoji_dict["non_qualified"] because of this upstream bug:
+            # https://github.com/iamcal/emoji-data/pull/217
+            emoji = unqualify_emoji(hex_codepoint_to_emoji(emoji_dict["unified"]))
+            if emoji not in CLDR_DATA:
+                print(
+                    f"{emoji} not found in custom emoji name maps, but also not found in CLDR data. Skipping."
+                )
+                continue
+            # CLDR_DATA[emoji] is of the form {'default': [...], 'tts': [...]}
+            # * "tts" is what's used for text-to-speech and always has one item, so we use that
+            #    as the canonical name.
+            # * "default" has several items in it that we use as aliases.
+            # See also: https://www.unicode.org/reports/tr35/tr35-general.html#14-annotations-and-labels
+            assert len(CLDR_DATA[emoji]["tts"]) == 1
+            canonical_name = cleanup_name(CLDR_DATA[emoji]["tts"][0].strip())
+            if canonical_name in all_canonical_names:
+                raise Exception(
+                    f"{canonical_name} was already added with a different codepoint. "
+                    f"Rename it in `custom_emoji_names` or add an entry for {emoji_code}."
+                )
+            aliases = [cleanup_name(alias.strip()) for alias in CLDR_DATA[emoji]["default"]]
+            all_emojis[emoji_code] = {"canonical_name": canonical_name, "aliases": aliases}
+            all_canonical_names.add(canonical_name)
+
+    # STEP 2: We don't support having aliases that collide with canonical names for emoji, so remove them.
+    for emoji_code, emoji_names in all_emojis.items():
+        # Copy the list to not iterate while elements are being deleted.
+        aliases = emoji_names["aliases"][:]
+        for alias in aliases:
+            if alias in all_canonical_names:
+                emoji_names["aliases"].remove(alias)
+            else:
+                alias_to_emoji_code[alias].append(emoji_code)  # This is used in STEP 3.
+
+    # STEP 3: We don't support multiple emoji sharing the same alias, but the CLDR data
+    # doesn't have that same restriction, so we have to fix this up to have unique aliases.
+    # If the alias was specifically specified in custom_emoji_names, then we can keep just
+    # that one, but otherwise there's no particular emoji that is an obvious candidate
+    # for the alias so just remove the alias for all relevant emoji.
+    for alias in alias_to_emoji_code:
+        if len(alias_to_emoji_code[alias]) > 1:
+            for emoji_code in alias_to_emoji_code[alias]:
+                if emoji_code not in CUSTOM_EMOJI_NAME_MAPS:
+                    all_emojis[emoji_code]["aliases"].remove(alias)
+
+    # STEP 4: We keep non-ascii (non-"English") characters in some emoji names if that's the correct
+    # way to spell that word, but always add an alias for an ascii-only version of the word.
+    for emoji_names in all_emojis.values():
+        for name in [emoji_names["canonical_name"]] + emoji_names["aliases"]:
+            # These are known names where we don't have an ascii-only version and there are ascii aliases
+            # that a user can still enter instead to get the same emoji.
+            if name in ["ココ", "サ", "指", "空"]:
+                assert any(alias.isascii() for alias in emoji_names["aliases"])
+                continue
+            if not name.isascii():
+                ascii_alias = convert_non_ascii_chars(name)
+                # Now no other emoji can use this alias.
+                for code in alias_to_emoji_code[ascii_alias]:
+                    all_emojis[code]["aliases"].remove(ascii_alias)
+                emoji_names["aliases"].append(ascii_alias)
+
+    # STEP 5: Write final dictionary to `emoji_names.py`.
+    with open(OUT_EMOJI_FILE, "w") as f:
+        f.write(
+            "from typing import Any\n\n"
+            "# Generated with `generate_emoji_names`.\n\n"
+            "EMOJI_NAME_MAPS: dict[str, dict[str, Any]] = {\n"
+        )
+        for key, emoji_names in all_emojis.items():
+            f.write(f"    {key!r}: {emoji_names!r},\n")
+        f.write("}\n")
+
+    print(
+        "\n\nDone! You should run the linter to format emoji_names.py with `./tools/lint --fix -m --only ruff-format`"
+    )
+
+
+if __name__ == "__main__":
+    main()

tools/setup/emoji/generate_emoji_names_table

@@ -0,0 +1,226 @@
+#!/usr/bin/env python3
+#
+# This is a debugging tool that takes as input a bunch of different
+# emoji data sources, and outputs a convenient HTML table that can be
+# used to sanity-check the differences between these different data
+# sources' decisions about what names to provide to each Unicode
+# codepoint.
+import os
+import sys
+from typing import Any
+
+import orjson
+
+TOOLS_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+ZULIP_PATH = os.path.dirname(TOOLS_DIR)
+sys.path.append(ZULIP_PATH)
+
+from tools.setup.emoji.emoji_names import EMOJI_NAME_MAPS
+from tools.setup.emoji.emoji_setup_utils import emoji_is_supported, get_emoji_code
+
+UNIFIED_REACTIONS_FILE = os.path.join(
+    ZULIP_PATH, "zerver", "management", "data", "unified_reactions.json"
+)
+EMOJI_DATA_FILE = os.path.join(ZULIP_PATH, "node_modules", "emoji-datasource-google", "emoji.json")
+EMOJI_CACHE = os.path.join(ZULIP_PATH, "static", "generated", "emoji")
+OUTPUT_FILE = os.path.join(EMOJI_CACHE, "emoji_names_table.html")
+
+# Emoji sets that we currently support.
+EMOJISETS = ["google", "twitter"]
+
+with open(EMOJI_DATA_FILE, "rb") as fp:
+    EMOJI_DATA = orjson.loads(fp.read())
+with open(UNIFIED_REACTIONS_FILE, "rb") as fp:
+    UNIFIED_REACTIONS_MAP = orjson.loads(fp.read())
+
+EMOJI_IMAGE_TEMPLATE = """
+    <img class="emoji" src="images-{emojiset}-64/{emoji_code}.png" title={emojiset}>
+"""
+
+TABLE_ROW_TEMPLATE = """
+    <tr>
+        <td class="new-sorting-info">{sorting_info}</td>
+        <td class="emoji-code">{emoji_code}</td>
+        <td class="emoji-images">{images_html}</td>
+        <td class="zulip-emoji-names">{zulip_names}</td>
+        <td class="iamcal-emoji-names">{iamcal_names}</td>
+        <td class="gemoji-emoji-names">{gemoji_names}</td>
+        <td class="unicode-name">{unicode_name}</td>
+    </tr>
+"""
+
+EMOJI_LISTING_TEMPLATE = """
+<html>
+    <head>
+        <style>
+            {table_css}
+        </style>
+        <title>Zulip emoji names</title>
+    </head>
+    <body>
+        <table>
+            <thead>
+                <tr>
+                    <th class="new-sorting-info">Category</th>
+                    <th class="emoji-code">Emoji code</th>
+                    <th class="emoji-images">Images</th>
+                    <th class="zulip-emoji-names">Zulip</th>
+                    <th class="iamcal-emoji-names">Iamcal (Slack)</th>
+                    <th class="gemoji-emoji-names">Gemoji (unordered)</th>
+                    <th class="unicode-name">Unicode</th>
+                </tr>
+            </thead>
+            <tbody>
+                {tbody}
+            </tbody>
+        </table>
+    </body>
+</html>
+"""
+
+TABLE_CSS = """
+    .emoji {
+        height: 35px;
+        width: 35px;
+        position: relative;
+        margin-top: -7px;
+        vertical-align: middle;
+        top: 3px;
+    }
+
+    .emoji-images {
+        width: 200px;
+    }
+
+    table, td, th {
+        border: 1px solid black;
+        border-collapse: collapse;
+    }
+
+    td, th {
+        height: 40px;
+        text-align: center;
+    }
+
+    .google {
+        background-image: url('sheet-google-64.png') !important;
+    }
+
+    .twitter {
+        background-image: url('sheet-twitter-64.png') !important;
+    }
+"""
+
+SORTED_CATEGORIES = [
+    "Smileys & Emotion",
+    "People & Body",
+    "Animals & Nature",
+    "Food & Drink",
+    "Activities",
+    "Travel & Places",
+    "Objects",
+    "Symbols",
+    "Flags",
+    "Component",
+]
+
+emoji_code_to_zulip_names: dict[str, str] = {}
+emoji_code_to_iamcal_names: dict[str, str] = {}
+emoji_code_to_gemoji_names: dict[str, str] = {}
+emoji_collection: dict[str, list[dict[str, Any]]] = {category: [] for category in SORTED_CATEGORIES}
+
+
+def generate_emoji_code_to_emoji_names_maps() -> None:
+    # Prepare gemoji names map.
+    reverse_unified_reactions_map: dict[str, list[str]] = {}
+    for name in UNIFIED_REACTIONS_MAP:
+        emoji_code = UNIFIED_REACTIONS_MAP[name]
+        if emoji_code in reverse_unified_reactions_map:
+            reverse_unified_reactions_map[emoji_code].append(name)
+        else:
+            reverse_unified_reactions_map[emoji_code] = [name]
+
+    for emoji_code, names in reverse_unified_reactions_map.items():
+        emoji_code_to_gemoji_names[emoji_code] = ", ".join(names)
+
+    # Prepare iamcal names map.
+    for emoji_dict in EMOJI_DATA:
+        emoji_code = get_emoji_code(emoji_dict)
+        emoji_code_to_iamcal_names[emoji_code] = ", ".join(emoji_dict["short_names"])
+
+    # Prepare zulip names map.
+    for emoji_code in EMOJI_NAME_MAPS:
+        canonical_name = EMOJI_NAME_MAPS[emoji_code]["canonical_name"]
+        aliases = EMOJI_NAME_MAPS[emoji_code]["aliases"]
+        names = [canonical_name]
+        names.extend(aliases)
+        emoji_code_to_zulip_names[emoji_code] = ", ".join(names)
+
+
+def get_sorting_info(category: str, sort_order: int) -> str:
+    return f"{category} {sort_order}"
+
+
+def get_images_html(emoji_code: str) -> str:
+    images_html = ""
+    for emojiset in EMOJISETS:
+        images_html += EMOJI_IMAGE_TEMPLATE.format(
+            emoji_code=emoji_code,
+            emojiset=emojiset,
+        )
+
+    return images_html
+
+
+def generate_emoji_collection() -> None:
+    generate_emoji_code_to_emoji_names_maps()
+    # Prepare `emoji_collection`.
+    for emoji_dict in EMOJI_DATA:
+        if not emoji_is_supported(emoji_dict):
+            continue
+        category = emoji_dict["category"]
+        emoji_code = get_emoji_code(emoji_dict)
+        sort_order = emoji_dict["sort_order"]
+        emoji_collection[category].append(
+            {
+                "category": category,
+                "emoji_code": emoji_code,
+                "images_html": get_images_html(emoji_code),
+                "gemoji_names": emoji_code_to_gemoji_names.get(emoji_code, ""),
+                "iamcal_names": emoji_code_to_iamcal_names.get(emoji_code, ""),
+                "zulip_names": emoji_code_to_zulip_names.get(emoji_code, ""),
+                "unicode_name": (emoji_dict["name"] or "").lower(),
+                "sort_order": sort_order,
+                "sorting_info": get_sorting_info(category, sort_order),
+            }
+        )
+
+    # Sort `emoji_collection`.
+    for category in SORTED_CATEGORIES:
+        emoji_collection[category].sort(key=lambda x: x["sort_order"])
+
+
+def main() -> None:
+    generate_emoji_collection()
+
+    tbody = ""
+    for category in SORTED_CATEGORIES:
+        for emoji_entry in emoji_collection[category]:
+            tbody += TABLE_ROW_TEMPLATE.format(**emoji_entry)
+
+    with open(OUTPUT_FILE, "w") as fp:
+        fp.write(
+            EMOJI_LISTING_TEMPLATE.format(
+                tbody=tbody,
+                table_css=TABLE_CSS,
+            )
+        )
+
+    print(
+        "Done! Open http://localhost:9991/static/generated/emoji/emoji_names_table.html "
+        "to view(after starting the dev server)."
+    )
+
+
+if __name__ == "__main__":
+    main()

tools/setup/generate-fixtures

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -e
+
+run() {
+    PGHOST=localhost PGUSER=zulip \
+        "$(dirname "$0")/../../scripts/setup/terminate-psql-sessions" zulip_test zulip_test_base zulip_test_template
+    psql -v ON_ERROR_STOP=1 -h localhost postgres zulip_test <<EOF
+DROP DATABASE IF EXISTS zulip_test;
+CREATE DATABASE zulip_test TEMPLATE zulip_test_template;
+EOF
+    "$(dirname "$0")/../../scripts/setup/flush-memcached"
+}
+
+run

tools/setup/generate-test-credentials

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -e
+
+cd "$(dirname "$0")"/../..
+
+email=desdemona@zulip.com
+
+mkdir -p var/puppeteer
+
+password=$(./manage.py print_initial_password "$email" | grep -F "$email" | awk '{ print $2 }')
+cat >var/puppeteer/test_credentials.json <<EOF
+{"default_user": {"username": "$email", "password": "$password"}}
+EOF

tools/setup/generate_bots_integrations_static_files.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+import glob
+import os
+import shutil
+import sys
+
+ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+if ZULIP_PATH not in sys.path:
+    sys.path.append(ZULIP_PATH)
+from scripts.lib.setup_path import setup_path
+
+setup_path()
+
+import integrations
+from zulip_bots.lib import get_bots_directory_path
+
+
+def generate_pythonapi_integrations_static_files() -> None:
+    integrations_dir = "static/generated/integrations"
+    if os.path.isdir(integrations_dir):
+        # delete old static files, they could be outdated
+        shutil.rmtree(integrations_dir)
+    os.makedirs(integrations_dir, exist_ok=True)
+
+    package_integrations_dir = os.path.dirname(os.path.abspath(integrations.__file__))
+
+    def copy_integrations_data(integration_names: list[str]) -> None:
+        # The integration name as used in zulip/python-zulip-api.
+        for name in integration_names:
+            src_dir = os.path.join(package_integrations_dir, name)
+            dst_dir = os.path.join(integrations_dir, name)
+            doc_path = os.path.join(src_dir, "doc.md")
+
+            if os.path.isfile(doc_path):
+                os.makedirs(dst_dir, exist_ok=True)
+                shutil.copyfile(doc_path, os.path.join(dst_dir, "doc.md"))
+
+    copy_integrations_data(os.listdir(package_integrations_dir))
+
+
+def generate_zulip_bots_static_files() -> None:
+    bots_dir = "static/generated/bots"
+    if os.path.isdir(bots_dir):
+        # delete old static files, they could be outdated
+        shutil.rmtree(bots_dir)
+
+    os.makedirs(bots_dir, exist_ok=True)
+
+    package_bots_dir = get_bots_directory_path()
+
+    def copy_bots_data(bot_names: list[str]) -> None:
+        for name in bot_names:
+            src_dir = os.path.join(package_bots_dir, name)
+            dst_dir = os.path.join(bots_dir, name)
+            doc_path = os.path.join(src_dir, "doc.md")
+
+            if os.path.isfile(doc_path):
+                os.makedirs(dst_dir, exist_ok=True)
+                shutil.copyfile(doc_path, os.path.join(dst_dir, "doc.md"))
+
+                logo_pattern = os.path.join(src_dir, "logo.*")
+                logos = glob.glob(logo_pattern)
+                for logo in logos:
+                    shutil.copyfile(logo, os.path.join(dst_dir, os.path.basename(logo)))
+
+                assets_path = os.path.join(src_dir, "assets")
+                if os.path.isdir(assets_path):
+                    shutil.copytree(
+                        assets_path, os.path.join(dst_dir, os.path.basename(assets_path))
+                    )
+
+    copy_bots_data(os.listdir(package_bots_dir))
+
+
+if __name__ == "__main__":
+    generate_pythonapi_integrations_static_files()
+    generate_zulip_bots_static_files()

tools/setup/generate_integration_bots_avatars.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+import argparse
+import os
+import sys
+
+ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+if ZULIP_PATH not in sys.path:
+    sys.path.append(ZULIP_PATH)
+from scripts.lib.setup_path import setup_path
+
+setup_path()
+
+os.environ["DJANGO_SETTINGS_MODULE"] = "zproject.settings"
+import django
+
+django.setup()
+
+import pyvips
+
+from zerver.lib.integrations import INTEGRATIONS
+from zerver.lib.storage import static_path
+from zerver.lib.thumbnail import DEFAULT_AVATAR_SIZE
+
+
+def create_integration_bot_avatar(logo_path: str, bot_avatar_path: str) -> None:
+    os.makedirs(os.path.dirname(bot_avatar_path), exist_ok=True)
+    avatar = pyvips.Image.thumbnail(logo_path, DEFAULT_AVATAR_SIZE, height=DEFAULT_AVATAR_SIZE)
+    if avatar.height != avatar.width:
+        avatar = avatar.gravity(
+            pyvips.CompassDirection.CENTRE, DEFAULT_AVATAR_SIZE, DEFAULT_AVATAR_SIZE
+        )
+    avatar.write_to_file(bot_avatar_path)
+
+
+def generate_integration_bots_avatars(check_missing: bool = False) -> None:
+    missing = set()
+    for integration in INTEGRATIONS.values():
+        if not integration.is_enabled_in_catalog():
+            continue
+        bot_avatar_path = integration.get_bot_avatar_path()
+        bot_avatar_path = os.path.join(ZULIP_PATH, "static", bot_avatar_path)
+        if check_missing:
+            if not os.path.isfile(bot_avatar_path):
+                missing.add(integration.name)
+        else:
+            create_integration_bot_avatar(static_path(integration.logo_path), bot_avatar_path)
+
+    if missing:
+        print(
+            "ERROR: Bot avatars are missing for these webhooks: {}.\n"
+            "ERROR: Run ./tools/setup/generate_integration_bots_avatars.py "
+            "to generate them.\nERROR: Commit the newly generated avatars to "
+            "the repository.".format(", ".join(missing))
+        )
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--check-missing", action="store_true")
+    options = parser.parse_args()
+    generate_integration_bots_avatars(options.check_missing)

tools/setup/generate_landing_page_images.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+"""Generates versions of landing page images to be served in different conditions."""
+
+import glob
+import os
+import sys
+from pathlib import Path
+
+ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+if ZULIP_PATH not in sys.path:
+    sys.path.append(ZULIP_PATH)
+
+import pyvips
+
+LANDING_IMAGES_DIR = os.path.join(ZULIP_PATH, "static", "images", "landing-page", "hello")
+ORIGINAL_IMAGES_DIR = os.path.join(LANDING_IMAGES_DIR, "original")
+GENERATED_IMAGES_DIR = os.path.join(LANDING_IMAGES_DIR, "generated")
+
+
+def get_x_size(size: int, x: int) -> int:
+    return int(x / 3.0 * size)
+
+
+def generate_landing_page_images() -> None:
+    if not os.path.exists(GENERATED_IMAGES_DIR):
+        os.mkdir(GENERATED_IMAGES_DIR)
+    else:
+        # Delete folder contents to avoid stale images between different versions of the script.
+        for file in os.listdir(GENERATED_IMAGES_DIR):
+            os.remove(os.path.join(GENERATED_IMAGES_DIR, file))
+
+    for image_file_path in glob.glob(f"{ORIGINAL_IMAGES_DIR}/*"):
+        file_name = Path(image_file_path).stem
+        image = pyvips.Image.new_from_file(image_file_path)
+        size = 2
+        scaled_width = get_x_size(image.width, size)
+        scaled_height = get_x_size(image.height, size)
+        scaled = image.thumbnail_image(scaled_width, height=scaled_height)
+        for format in ("webp[Q=60]", "jpg[Q=80,optimize-coding=true]"):
+            scaled.write_to_file(f"{GENERATED_IMAGES_DIR}/{file_name}-{size}x.{format}")
+
+
+if __name__ == "__main__":
+    generate_landing_page_images()

tools/setup/install-aws-server

@@ -0,0 +1,235 @@
+#!/usr/bin/env bash
+set -eu
+set -o pipefail
+
+usage() {
+    cat <<EOF
+USAGE: $0 [--roles=roles] [--branch=main] [--debug-key=username] server
+
+Installs an empty Ubuntu server in AWS with a Zulip server role.
+
+ * server is the local part of the hostname (e.g. postgres0)
+
+ * roles is a comma-separated list of Puppet profile names; these
+   will get 'kandra::profile::' prepended to them, and passed
+   to scripts/lib/install -- e.g. 'postgresql'
+ * branch is used to override the default branch to install from.
+ * username is the name of the AWS SSH key pair to allow logins as
+   'ubuntu' with during initial boot; this is purely for debugging the
+   bootstrapping process.
+
+Reads configuration from $HOME/.zulip-install-server.conf, which should look like:
+
+[repo]
+repo_url=git@github.com:zulip/zulip.git
+[aws]
+zone_id=Z2U988IEXAMPLE
+security_groups=sg-01234567
+instance_type=m4.large
+EOF
+}
+
+args="$(getopt -o '' --long help,branch:,roles:,debug-key: -n "$0" -- "$@")" || {
+    usage >&2
+    exit 1
+}
+eval "set -- $args"
+
+BRANCH="production"
+ROLES="base"
+DEBUG_KEY=""
+while true; do
+    case "$1" in
+        --help)
+            usage
+            exit 0
+            ;;
+        --roles)
+            shift
+            ROLES="$1"
+            shift
+            ;;
+        --branch)
+            shift
+            BRANCH="$1"
+            shift
+            ;;
+        --debug-key)
+            shift
+            DEBUG_KEY="$1"
+            shift
+            ;;
+        --)
+            shift
+            break
+            ;;
+    esac
+done
+
+if [ $# -ne 1 ]; then
+    usage >&2
+    exit 1
+fi
+
+SERVER="$1"
+
+set -x
+
+cd "$(dirname "$0")/../.."
+
+./puppet/kandra/files/install-aws-cli
+AWS=/srv/zulip-aws-tools/bin/aws
+
+zulip_install_config_file="$HOME/.zulip-install-server.conf"
+if [ ! -f "$zulip_install_config_file" ]; then
+    echo "No configuration file found in $zulip_install_config_file"
+    exit 1
+fi
+
+REPO_URL=$(crudini --get "$zulip_install_config_file" repo repo_url)
+
+for role in ${ROLES//,/ }; do
+    if ! [ -f "./puppet/kandra/manifests/profile/$role.pp" ]; then
+        echo "No such role kandra::profile::$role !"
+        exit 1
+    fi
+done
+FULL_ROLES=$(echo "$ROLES" | perl -pe '$_=join(",",map{"kandra::profile::$_"} split ",")')
+
+function lookup() {
+    KEY="$1"
+    crudini --get "$zulip_install_config_file" "aws-$ROLES" "$KEY" 2>/dev/null \
+        || crudini --get "$zulip_install_config_file" aws "$KEY"
+}
+
+AWS_ZONE_ID=$(lookup zone_id)
+SECURITY_GROUPS=$(lookup security_groups)
+read -r -a SECURITY_GROUPS <<<"$SECURITY_GROUPS"
+INSTANCE_TYPE=$(lookup instance_type)
+IAM_PROFILE=$(lookup iam_profile)
+AZ=$(lookup availability_zone)
+DISK_SIZE=$(lookup disk_size)
+
+# Determine the architecture
+ARCH=$($AWS ec2 describe-instance-types --instance-types "$INSTANCE_TYPE" --query 'InstanceTypes[*].ProcessorInfo.SupportedArchitectures[0]' --output text)
+
+# Lookup the latest 24.04 image
+AMI_ID=$($AWS ec2 describe-images --owners 099720109477 --filters 'Name=name,Values=ubuntu/images/hvm-ssd-gp3/ubuntu-*-24.04*' "Name=architecture,Values=$ARCH" --query 'sort_by(Images, &CreationDate)[-1].ImageId' --output text)
+
+# Verify it doesn't exist already
+ZONE_NAME=$($AWS route53 get-hosted-zone --id "$AWS_ZONE_ID" | jq -r '.HostedZone.Name')
+HOSTNAME="$SERVER.${ZONE_NAME%?}" # Remove trailing .
+EXISTING_RECORDS=$($AWS route53 list-resource-record-sets \
+    --hosted-zone-id "$AWS_ZONE_ID" \
+    --query "ResourceRecordSets[?Name == '$HOSTNAME.']" \
+    | jq '. | length')
+if [ "$EXISTING_RECORDS" != "0" ]; then
+    echo "$HOSTNAME already exists!"
+    exit 1
+fi
+
+DEVICES+=(
+    "DeviceName=/dev/sda1,Ebs={VolumeSize=$DISK_SIZE,VolumeType=gp3,Throughput=125,Iops=3000,Encrypted=true}"
+)
+if [[ "$ROLES" == *"frontend"* ]]; then
+    DEVICES+=(
+        "DeviceName=/dev/sdf,Ebs={VolumeSize=100,VolumeType=gp3,Throughput=125,Iops=3000,Encrypted=true,DeleteOnTermination=True}"
+    )
+fi
+
+# https://docs.aws.amazon.com/cli/latest/reference/ec2/run-instances.html
+EXTRA_ARGS+=(
+    --iam-instance-profile "Name=\"$IAM_PROFILE\""
+    --image-id "$AMI_ID"
+    --instance-type "$INSTANCE_TYPE"
+    --security-group-ids "${SECURITY_GROUPS[@]}"
+    --monitoring Enabled=true
+    --placement "AvailabilityZone=$AZ"
+    --block-device-mappings "${DEVICES[@]}"
+    --metadata-options "InstanceMetadataTags=enabled"
+)
+
+if [ -n "$DEBUG_KEY" ]; then
+    EXTRA_ARGS+=(
+        --key-name "$DEBUG_KEY"
+    )
+fi
+
+# Build up the provisioning script
+BOOTDATA=$(mktemp)
+{
+    echo "#!/bin/bash"
+    echo "SERVER=$SERVER"
+    echo "HOSTNAME=$HOSTNAME"
+    echo "FULL_ROLES=$FULL_ROLES"
+    echo "REPO_URL=$REPO_URL"
+    echo "BRANCH=$BRANCH"
+    # Replace anything which looks like FOO="inline!bar/baz" with the
+    # output of pack-local-script, which will make "$FOO" inside the
+    # $BOOTDATA be the path to that script (smuggled inline and
+    # unpacked before use).
+    perl -ple 's|^(\w+)="inline!([^"]+)"|qx(./tools/setup/pack-local-script $1 $2)|e' ./tools/setup/bootstrap-aws-installer
+} >>"$BOOTDATA"
+
+TAG_ROLE_NAMES="$ROLES"
+TAGS="[{Key=Name,Value=$SERVER},{Key=role,Value=\"$TAG_ROLE_NAMES\"}]"
+INSTANCE_DATA=$($AWS ec2 run-instances \
+    --tag-specifications "ResourceType=instance,Tags=$TAGS" \
+    "${EXTRA_ARGS[@]}" \
+    --user-data "file://$BOOTDATA")
+INSTANCEID=$(echo "$INSTANCE_DATA" | jq -r .Instances[0].InstanceId)
+
+# Wait for public IP assignment
+PUBLIC_DNS_NAME=""
+while [ -z "$PUBLIC_DNS_NAME" ]; do
+    sleep 1
+    INSTANCE_DATA=$($AWS ec2 describe-instances --instance-ids "$INSTANCEID")
+    PUBLIC_DNS_NAME=$(echo "$INSTANCE_DATA" | jq -r .Reservations[0].Instances[0].PublicDnsName)
+done
+
+# Add the hostname to the zone
+ROUTE53_CHANGES=$(mktemp)
+cat >"$ROUTE53_CHANGES" <<EOF
+{
+    "Comment": "Add the $HOSTNAME CNAME record",
+    "Changes": [
+        {
+            "Action": "CREATE",
+            "ResourceRecordSet": {
+                "Name": "$HOSTNAME",
+                "Type": "CNAME",
+                "TTL": 300,
+                "ResourceRecords": [{"Value": "$PUBLIC_DNS_NAME"}]
+            }
+        }
+    ]
+}
+EOF
+$AWS route53 change-resource-record-sets --hosted-zone-id "$AWS_ZONE_ID" --change-batch "file://$ROUTE53_CHANGES"
+rm "$ROUTE53_CHANGES"
+
+# Add the name to the root volume
+VOLUME_ID=$(
+    echo "$INSTANCE_DATA" | jq -r \
+        '.Reservations[0].Instances[0].BlockDeviceMappings[]
+         | select(.DeviceName == "/dev/sda1")
+         | .Ebs.VolumeId'
+)
+$AWS ec2 create-tags --resources "$VOLUME_ID" --tags "Key=Name,Value=$SERVER root"
+
+# Add volume name to tmp volume
+if [[ "$ROLES" == *"frontend"* ]]; then
+    VOLUME_ID=$(
+        echo "$INSTANCE_DATA" | jq -r \
+            '.Reservations[0].Instances[0].BlockDeviceMappings[]
+             | select(.DeviceName == "/dev/sdf")
+             | .Ebs.VolumeId'
+    )
+    $AWS ec2 create-tags --resources "$VOLUME_ID" --tags "Key=Name,Value=$SERVER tmp"
+fi
+
+set +x
+echo
+echo
+echo ">>> Install started successfully!  Provisioning takes 5-6min."
+echo "    sleep 360 && tsh ssh root@$HOSTNAME"

tools/setup/install-shellcheck

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -eu
+
+version=0.11.0
+arch="$(uname -m)"
+tarball="shellcheck-v$version.linux.$arch.tar.xz"
+declare -A sha256=(
+    [aarch64]=12b331c1d2db6b9eb13cfca64306b1b157a86eb69db83023e261eaa7e7c14588
+    [x86_64]=8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198
+)
+
+check_version() {
+    out="$(shellcheck --version)" && [[ "$out" = *"
+version: $version
+"* ]]
+}
+
+if ! check_version 2>/dev/null; then
+    set -x
+    tmpdir="$(mktemp -d)"
+    trap 'rm -r "$tmpdir"' EXIT
+    cd "$tmpdir"
+    curl -fLO --retry 3 "https://github.com/koalaman/shellcheck/releases/download/v$version/$tarball"
+    sha256sum -c <<<"${sha256[$arch]} $tarball"
+    tar -xJf "$tarball" --no-same-owner --strip-components=1 -C /usr/local/bin "shellcheck-v$version/shellcheck"
+    check_version
+fi

tools/setup/install-shfmt

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+set -eu
+
+version=3.13.1
+arch="$(uname -m)"
+
+case $arch in
+    "x86_64")
+        binary="shfmt_v${version}_linux_amd64"
+        sha256=fb096c5d1ac6beabbdbaa2874d025badb03ee07929f0c9ff67563ce8c75398b1
+        ;;
+
+    "aarch64")
+        binary="shfmt_v${version}_linux_arm64"
+        sha256=32d92acaa5cd8abb29fc49dac123dc412442d5713967819d8af2c29f1b3857c7
+        ;;
+esac
+
+check_version() {
+    out="$(shfmt --version)" && [ "$out" = "v$version" ]
+}
+
+if ! check_version 2>/dev/null; then
+    set -x
+    tmpdir="$(mktemp -d)"
+    trap 'rm -r "$tmpdir"' EXIT
+    cd "$tmpdir"
+    curl -fLO --retry 3 "https://github.com/mvdan/sh/releases/download/v$version/$binary"
+    sha256sum -c <<<"$sha256 $binary"
+    chmod +x "$binary"
+    mv "$binary" /usr/local/bin/shfmt
+    check_version
+fi

tools/setup/install-tusd

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+set -eu
+
+# This should be kept in sync with puppet/zulip/manifests/common.pp
+version=2.9.1
+arch="$(uname -m)"
+
+case $arch in
+    x86_64)
+        tarball="tusd_linux_amd64"
+        sha256=140066be543e40493abd1fda1d1e33ab7fa0e8b9a61d247130f6777e64bf28f6
+        ;;
+
+    aarch64)
+        tarball="tusd_linux_arm64"
+        sha256=c9c46eba6e46b8062f863af6a5423f6f72b5157939d967087cd9b6d7c6bd19cf
+        ;;
+esac
+
+check_version() {
+    out="$(tusd --version)" && [[ "$out" = "Version: v$version
+"* ]]
+}
+
+if ! check_version 2>/dev/null; then
+    set -x
+    tmpdir="$(mktemp -d)"
+    trap 'rm -r "$tmpdir"' EXIT
+    cd "$tmpdir"
+    curl_opts=(-fLO --retry 3)
+    curl "${curl_opts[@]}" "https://github.com/tus/tusd/releases/download/v${version}/${tarball}.tar.gz"
+    sha256sum -c <<<"${sha256} ${tarball}.tar.gz"
+    tar -xzf "${tarball}.tar.gz" --no-same-owner "${tarball}/tusd"
+    install -Dm755 "${tarball}/tusd" /usr/local/bin/tusd
+    check_version
+fi

tools/setup/lang.json

@@ -0,0 +1,45 @@
+{
+    "default": {
+        "text": 31,
+        "javascript": 27,
+        "python": 26,
+        "java": 25,
+        "go": 24,
+        "rust": 23,
+        "html": 22,
+        "css": 21,
+        "sql": 20,
+        "bash": 19,
+        "powershell": 18,
+        "c#": 17,
+        "php": 16,
+        "typescript": 15,
+        "c++": 14,
+        "c": 13,
+        "kotlin": 12,
+        "ruby": 11,
+        "asm": 10,
+        "vb.net": 9,
+        "swift": 8,
+        "r": 7,
+        "objective-c": 6,
+        "dart": 5,
+        "scala": 4,
+        "perl": 3,
+        "haskell": 2,
+        "julia": 1
+    },
+    "custom": {
+        "latex": 10,
+        "math": 28,
+        "quote": 30,
+        "spoiler": 29
+    },
+    "aliases": {
+        "js": 27,
+        "csharp": 17,
+        "cpp": 14,
+        "tex": 10,
+        "vbnet": 9
+    }
+}

tools/setup/optimize-svg

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+set -e
+
+usage() {
+    cat <<'EOF'
+Usage:
+  optimize-svg [--check] [filename]
+  optimize-svg --help
+
+Options:
+  --check
+      This will check for unoptimized SVG files rather than automatically optimizing them.
+      This allows us to run the script in CI.
+  [filename]
+      The filename of the SVG file to optimize, located in static/images/integrations/logos.
+      If not provided, all files in the directory will be optimized.
+
+EOF
+}
+
+# Shell option parsing.  Over time, we'll want to move some of the
+# environment variables below into this self-documenting system.
+args="$(getopt -o '' --long help,check -n "$0" -- "$@")"
+eval "set -- $args"
+while true; do
+    case "$1" in
+        --help)
+            usage
+            exit 0
+            ;;
+        --check)
+            CHECK_UNOPTIMIZED=1
+            shift
+            ;;
+        --)
+            shift
+            break
+            ;;
+    esac
+done
+
+if [ "$#" -gt 1 ]; then
+    usage >&2
+    exit 1
+fi
+
+ZULIP_PATH="$(readlink -f "$(dirname "$0")"/../..)"
+PNPM="/usr/local/bin/pnpm"
+BASE_PATH="static/images/integrations/logos"
+
+if [ "$#" -eq 0 ]; then
+    FILE_PATH="$BASE_PATH"
+    SVGO_FLAG="-f"
+else
+    FILE_PATH="$BASE_PATH/$1"
+    SVGO_FLAG=""
+fi
+
+if [ -n "$CHECK_UNOPTIMIZED" ]; then
+    TEMP_PATH=$(mktemp -d svgo_temp_XXXXXX)
+    RESULT=$("$PNPM" exec svgo -o "$TEMP_PATH" $SVGO_FLAG "$FILE_PATH" | grep -o '\.[0-9]% = ' | wc -l)
+    rm -rf "$TEMP_PATH"
+    if [ "$RESULT" -ge 1 ]; then
+        echo "ERROR: svgo detected unoptimized SVG file(s)." 1>&2
+        echo "Please run tools/setup/optimize-svg and commit the file changes to optimize them."
+        exit 1
+    else
+        echo "SUCCESS: SVG file(s) are optimized!"
+    fi
+else
+    "$PNPM" exec svgo -q $SVGO_FLAG "$FILE_PATH"
+    if [ "$#" -eq 0 ]; then
+        "$ZULIP_PATH"/tools/setup/generate_integration_bots_avatars.py
+    fi
+fi

tools/setup/pack-local-script

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+# This tool generates code in shell which is meant to be inline'd into
+# a larger script; called with a variable name and a path, it produces
+# a script which will result in that variable being set to the path to
+# the contents of that path.
+#
+# This is used in bootstrap-aws-installer to bundle local files into
+# the EC2 user data, so that those canonical versions can be used to
+# bootstrap the host.
+set -eu
+
+var="$1"
+file="$2"
+
+mode="$(stat -c "%a" "$file")"
+
+encoded="$(gzip --stdout "$file" | base64)"
+cat <<embedded-shell-output
+$var="\$(mktemp)"
+chmod "$mode" "\$$var"
+base64 -d <<"encoded-shell-script" | gzip -d > "\$$var"
+$encoded
+encoded-shell-script
+embedded-shell-output

tools/setup/postgresql-init-dev-db

@@ -0,0 +1,128 @@
+#!/usr/bin/env bash
+set -e
+
+if [[ $# == 0 ]]; then
+    MODE=dev
+elif [[ $# == 1 ]]; then
+    MODE=$1
+else
+    echo "Too many arguments"
+    exit 1
+fi
+
+if [[ "$MODE" == "dev" ]]; then
+    USERNAME=zulip
+    DBNAME=zulip
+    STATUS_FILE_NAME=migration_status_dev
+elif [[ "$MODE" == "test" ]]; then
+    USERNAME=zulip_test
+    DBNAME=zulip_test
+    STATUS_FILE_NAME=migration_status_test
+else
+    echo "Usage instructions"
+    echo "Run with either no arguments (sets up devel db for local deploy--zulip with user zulip)"
+    echo "or specify 'test'"
+    exit 1
+fi
+
+set -x
+
+POSTGRES_USER="postgres"
+if [ "$(uname)" = "OpenBSD" ]; then
+    POSTGRES_USER="_postgresql"
+fi
+
+ROOT_POSTGRES=(sudo -i -u "$POSTGRES_USER" psql)
+DEFAULT_DB=""
+if [ "$(uname)" = "Darwin" ]; then
+    ROOT_POSTGRES=(psql)
+    DEFAULT_DB="postgres"
+fi
+
+if [ "$(uname)" = "OpenBSD" ]; then
+    DEFAULT_DB="postgres"
+fi
+
+VAGRANTUSERNAME=$(whoami)
+
+PASSWORD=$("$(dirname "$0")/../../scripts/get-django-setting" LOCAL_DATABASE_PASSWORD)
+DBNAME_BASE=${DBNAME}_base
+
+if ! pg_isready -U "$POSTGRES_USER" -q; then
+    set +x
+    echo
+    echo 'ERROR: PostgreSQL server is not running! Ensure the service is enabled.'
+    # shellcheck disable=SC2016
+    echo 'ERROR: Try `sudo service postgresql start`?'
+    echo "ERROR: You can easily test if you fixed it using: pg_isready -U \$POSTGRES_USER"
+    exit 1
+fi
+
+# We need to remove the stamp file indicating that the database
+# is properly provisioned with migrations.
+uuid_var_path=$("$(readlink -f "$(dirname "$0")/../../scripts/lib/zulip_tools.py")" get_dev_uuid)
+rm -f "$uuid_var_path/$STATUS_FILE_NAME"
+
+"${ROOT_POSTGRES[@]}" -v ON_ERROR_STOP=1 -e "$DEFAULT_DB" <<EOF
+DO \$\$BEGIN
+    CREATE USER $USERNAME;
+EXCEPTION WHEN duplicate_object THEN
+    RAISE NOTICE '$USERNAME user already exists';
+END\$\$;
+ALTER USER $USERNAME PASSWORD '$PASSWORD';
+ALTER USER $USERNAME CREATEDB;
+EOF
+
+if [ "$USERNAME" != "$VAGRANTUSERNAME" ]; then
+    "${ROOT_POSTGRES[@]}" -v ON_ERROR_STOP=1 -e "$DEFAULT_DB" <<EOF
+    DO \$\$BEGIN
+        CREATE USER "$VAGRANTUSERNAME"; 
+    EXCEPTION WHEN duplicate_object THEN
+        RAISE NOTICE '$VAGRANTUSERNAME user already exists';
+    END\$\$;
+    GRANT $USERNAME TO "$VAGRANTUSERNAME";
+EOF
+fi
+
+"${ROOT_POSTGRES[@]}" -v ON_ERROR_STOP=1 -e "$DEFAULT_DB" <<EOF
+ALTER ROLE "$VAGRANTUSERNAME" SET search_path TO zulip,public;
+EOF
+
+umask go-rw
+PGPASS_PREFIX="*:*:*:$USERNAME:"
+PGPASS_ESCAPED_PREFIX="*:\\*:\\*:$USERNAME:"
+if ! grep -q "$PGPASS_ESCAPED_PREFIX" ~/.pgpass; then
+    echo "$PGPASS_PREFIX$PASSWORD" >>~/.pgpass
+else
+    sed -i "s/$PGPASS_ESCAPED_PREFIX.*\$/$PGPASS_PREFIX$PASSWORD/" ~/.pgpass
+fi
+chmod go-rw ~/.pgpass
+
+PGHOST=localhost PGUSER="$USERNAME" \
+    "$(dirname "$0")/../../scripts/setup/terminate-psql-sessions" "$DBNAME" "$DBNAME_BASE"
+
+psql -v ON_ERROR_STOP=1 -e -h localhost postgres "$USERNAME" <<EOF
+DROP DATABASE IF EXISTS $DBNAME;
+DROP DATABASE IF EXISTS $DBNAME_BASE;
+CREATE DATABASE $DBNAME_BASE
+EOF
+
+"${ROOT_POSTGRES[@]}" -v ON_ERROR_STOP=1 -e "$DBNAME_BASE" <<EOF
+CREATE EXTENSION pgroonga;
+
+-- Work around https://github.com/pgroonga/pgroonga/issues/335
+SELECT unnest(
+    CASE WHEN current_setting('server_version_num')::integer >= 150000
+        AND current_setting('pgroonga.enable_wal', TRUE) = 'on' IS NOT TRUE THEN
+        ARRAY['ALTER SYSTEM SET pgroonga.enable_wal = ''on''', 'SELECT pg_reload_conf()']
+    END
+) \gexec
+EOF
+
+psql -v ON_ERROR_STOP=1 -e -h localhost postgres "$USERNAME" <<EOF
+CREATE DATABASE $DBNAME TEMPLATE $DBNAME_BASE;
+EOF
+
+"$(dirname "$0")/../../scripts/setup/flush-memcached"
+
+echo "Database created"

tools/setup/postgresql-init-test-db

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -x
+set -e
+
+"$(dirname "$0")/postgresql-init-dev-db" test

tools/setup/provision-codespace

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# 1) Give passwordless sudo access to the 'vscode' user for postgres.
+sudo sh -c "echo 'vscode ALL=(postgres) NOPASSWD: ALL' > /etc/sudoers.d/vscode_postgres"
+
+# 2) Add 127.0.0.1 \\$(hostname) to /etc/hosts.
+if ! grep -qE "^127\.0\.0\.1[[:space:]]+$(hostname)([[:space:]]|$)" /etc/hosts; then
+    echo "127.0.0.1 $(hostname)" | sudo tee -a /etc/hosts >/dev/null
+fi
+
+# 3) Unset variables to prevent PyICU conflicts.
+unset VIRTUAL_ENV
+unset CONDA_PREFIX
+unset CONDA_DEFAULT_ENV
+unset CONDA_PROMPT_MODIFIER
+unset CONDA_EXE
+unset CE_CONDA
+
+# 4) Finally, execute Zulip provisioning.
+./tools/provision

tools/setup/vagrant-provision

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+# Code should go here, rather than tools/provision, only if it is
+# something that we don't want to happen when running provision in a
+# development environment not using Vagrant.
+
+if [ -e /srv/zulip ]; then
+    # Backwards compatibility
+    ZULIP_PATH=/srv/zulip
+fi
+
+# Set the Ubuntu mirror
+[ ! "$UBUNTU_MIRROR" ] || sudo sed -i 's|http://\(\w*\.\)*archive\.ubuntu\.com/ubuntu/\? |'"$UBUNTU_MIRROR"' |' /etc/apt/sources.list
+
+# Set the MOTD on the system to have Zulip instructions
+sudo ln -nsf "$ZULIP_PATH/tools/setup/dev-motd" /etc/update-motd.d/99-zulip-dev
+sudo rm -f /etc/update-motd.d/10-help-text /etc/update-motd.d/99-bento
+sudo dpkg --purge landscape-common
+sudo dpkg-divert --add --rename /etc/default/motd-news
+sudo sh -c 'echo ENABLED=0 > /etc/default/motd-news'
+
+# Set default locale, this prevents errors if the user has another locale set.
+if ! grep -q 'LC_ALL=C.UTF-8' /etc/default/locale; then
+    echo "LC_ALL=C.UTF-8" | sudo tee -a /etc/default/locale
+fi
+
+# Set an environment variable, so that we won't print the virtualenv
+# shell warning (it'll be wrong, since the shell is dying anyway)
+export SKIP_VENV_SHELL_WARNING=1
+
+# End `set -x`, so that the end of provision doesn't look like an error
+# message after a successful run.
+set +x
+
+# Check if the zulip directory is writable
+if [ ! -w "$ZULIP_PATH" ]; then
+    echo "The vagrant user is unable to write to the zulip directory."
+    echo "To fix this, run the following commands on the host machine:"
+    # sudo is required since our uid is not 1000
+    echo '    vagrant halt -f'
+    echo '    rm -rf /PATH/TO/ZULIP/CLONE/.vagrant'
+    # shellcheck disable=SC2016
+    echo '    sudo chown -R 1000:$(id -g) /PATH/TO/ZULIP/CLONE'
+    echo "Replace /PATH/TO/ZULIP/CLONE with the path to where zulip code is cloned."
+    echo "You can resume setting up your vagrant environment by running:"
+    echo "    vagrant up"
+    exit 1
+fi
+# Provision the development environment
+ln -nsf "$ZULIP_PATH" ~/zulip
+"$ZULIP_PATH/tools/provision"
+
+# Run any custom provision hooks the user has configured
+if [ -f "$ZULIP_PATH/tools/custom_provision" ]; then
+    chmod +x "$ZULIP_PATH/tools/custom_provision"
+    "$ZULIP_PATH/tools/custom_provision"
+fi