snapshot of https://github.com/zulip/zulip.git @ 48c69911c7dd4e03b20552eb3bd77ad8151c1357 (2026-09-15)

tools/setup/emoji/build_emoji

@@ -0,0 +1,452 @@
+#!/usr/bin/env python3
+#
+# See docs/subsystems/emoji.md for a high-level explanation of how this system
+# works.
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+from collections.abc import Iterator, Sequence
+from typing import Any
+
+import orjson
+
+ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../../")
+sys.path.append(ZULIP_PATH)
+
+from scripts.lib.zulip_tools import generate_sha1sum_emoji, run_as_root
+from tools.setup.emoji.emoji_names import EMOJI_NAME_MAPS
+from tools.setup.emoji.emoji_setup_utils import (
+    EMOTICON_CONVERSIONS,
+    REMAPPED_EMOJIS,
+    emoji_is_supported,
+    emoji_names_for_picker,
+    generate_codepoint_to_name_map,
+    generate_codepoint_to_names_map,
+    generate_emoji_catalog,
+    generate_name_to_codepoint_map,
+    get_emoji_code,
+)
+
+EMOJI_CACHE_BASE_PATH = "/srv/zulip-emoji-cache"
+EMOJI_SCRIPT_DIR_PATH = os.path.join(ZULIP_PATH, "tools", "setup", "emoji")
+NODE_MODULES_PATH = os.path.join(ZULIP_PATH, "node_modules")
+
+
+# The CSS for emoji spritesheet has somewhat tricky requirements.  One
+# is that we want to be able to use the same emoji CSS classes for
+# different display sizes of our emoji (e.g. reactions are smaller
+# than inline message emoji, which are smaller than those in the emoji
+# picker) while only downloading 1 copy of the spritesheet, having
+# good browser rendering performance, and reusing as much common CSS
+# as is possible.
+
+# Our solution to those problem is to use the `background-size` (Which
+# is e.g. 5700%) and background-position attributes to select the
+# region of the spritesheet corresponding to the target sprite and
+# display it properly scaled in an emoji span.
+SPRITE_CSS_FILE_TEMPLATE = """\
+div.emoji,
+span.emoji
+{{
+    display: inline-block;
+    background-image: url(../emoji/{emojiset}.webp);
+    background-size: {background_size};
+    background-repeat: no-repeat;
+
+    /* Hide the text. */
+    text-indent: 100%;
+    white-space: nowrap;
+    overflow: hidden;
+}}
+
+.emoji-1f419
+{{
+    background-image: url(../../../static/generated/emoji/images-google-64/1f419.png) !important;
+    background-position: 0% 0% !important;
+    background-size: contain !important;
+}}
+
+{emoji_positions}
+"""
+
+EMOJI_POS_INFO_TEMPLATE = """\
+.emoji-{codepoint} {{
+    background-position: {pos_x} {pos_y};
+}}
+"""
+
+EMOJI_OVERRIDE_TEMPLATE = """\
+.emoji-{codepoint} {{
+    background-image: url(../../../static/generated/emoji/images-google-64/{codepoint}.png) !important;
+    background-position: 0% 0% !important;
+    background-size: contain !important;
+}}
+"""
+
+# change directory
+os.chdir(EMOJI_SCRIPT_DIR_PATH)
+
+
+def main() -> None:
+    if not os.access(EMOJI_CACHE_BASE_PATH, os.W_OK):
+        # Note: In production, this block will fail, since we don't
+        # assume sudo access; but it should never run in production
+        # anyway, because EMOJI_CACHE_BASE_PATH is created by Puppet before
+        # build_emoji would be run.
+        run_as_root(["mkdir", "-p", EMOJI_CACHE_BASE_PATH])
+        run_as_root(["chown", f"{os.getuid()}:{os.getgid()}", EMOJI_CACHE_BASE_PATH])
+
+    sha1_hexdigest = generate_sha1sum_emoji(ZULIP_PATH)
+    emoji_cache_path = os.path.join(EMOJI_CACHE_BASE_PATH, sha1_hexdigest)
+    success_stamp = os.path.join(emoji_cache_path, ".success-stamp")
+
+    if not os.path.exists(success_stamp):
+        print("Dumping emojis ...")
+        dump_emojis(emoji_cache_path)
+        with open(success_stamp, "w") as f:
+            f.close()
+
+    print(f"build_emoji: Using cached emojis from {emoji_cache_path}")
+
+    # /srv/zulip-emoji-cache/*/static gets symlinked to ZULIP_PATH/static/generated/emoji
+    TARGET_STATIC_EMOJI = os.path.join(ZULIP_PATH, "static", "generated", "emoji")
+    if os.path.lexists(TARGET_STATIC_EMOJI):
+        os.remove(TARGET_STATIC_EMOJI)
+    os.symlink(os.path.join(emoji_cache_path, "static"), TARGET_STATIC_EMOJI)
+
+    # /srv/zulip-emoji-cache/*/web gets copied to ZULIP_PATH/web/generated
+    # These must not be symlinked so webpack can resolve module references.
+    for subdir in ("emoji", "emoji-styles"):
+        target_dir = os.path.join(ZULIP_PATH, "web", "generated", subdir)
+        os.makedirs(target_dir, exist_ok=True)
+        to_remove = set(os.listdir(target_dir))
+        source_emoji_dump = os.path.join(emoji_cache_path, "web", subdir)
+        for filename in os.listdir(source_emoji_dump):
+            shutil.copy2(os.path.join(source_emoji_dump, filename), target_dir)
+            to_remove.discard(filename)
+        for filename in to_remove:
+            os.remove(os.path.join(target_dir, filename))
+
+
+def percent(f: float) -> str:
+    return f"{f * 100:0.3f}%"
+
+
+def get_square_size(emoji_data: Sequence[dict[str, Any]]) -> int:
+    """
+    Spritesheets are usually NxN squares, and we have to
+    infer N from the sheet_x/sheet_y values of emojis.
+    """
+
+    def get_offsets(emoji_data: Sequence[dict[str, Any]]) -> Iterator[int]:
+        for emoji_dict in emoji_data:
+            yield emoji_dict["sheet_x"]
+            yield emoji_dict["sheet_y"]
+            if "skin_variations" in emoji_dict:
+                for img_info in emoji_dict["skin_variations"].values():
+                    yield img_info["sheet_x"]
+                    yield img_info["sheet_y"]
+
+    n = max(get_offsets(emoji_data)) + 1
+    return n
+
+
+def generate_sprite_css_files(
+    cache_path: str, emoji_data: list[dict[str, Any]], emojiset: str
+) -> None:
+    """
+    Spritesheets are usually NxN squares.
+    """
+    n = get_square_size(emoji_data)
+
+    """
+    Each single emoji is 64x64, with 1px gutters on every border.
+    We just consider the gutters to be part of the image for
+    simplicity reasons, so you can think of the spritesheet as
+    an NxN square of 66x66 pre-padded emojis.  The CSS
+    background-size parameter below says to size the background
+    element as N times the size of the element that you're drawing.
+
+    Note that we use percentages here, instead of absolute
+    pixel values, because when we render emojis as actual elements,
+    their size will vary depending on which part of the UI we're
+    in (message emojis, emoji reactions, emoji popup, emoji
+    popup showcase, etc.).
+
+    (The next step is to offset the image; that will be in the
+    upcoming loop.)
+    """
+    background_size = percent(n)
+
+    emoji_positions = ""
+    for emoji in emoji_data:
+        if emoji_is_supported(emoji):
+            """
+            For background-position we need to use percentages.
+            Absolute pixel values won't work, because the size
+            of the background sprite image is proportional to
+            the size of the element we're rendering, and we render
+            elements in multiple sizes.
+
+            The way that CSS background-position works is linear
+            interpolation.  When you tell CSS background-position
+            is "42% 37%", then in the `x` dimension it will align
+            the image such that 42% of the background image is to
+            the left of the 42% mark in the element itself.
+
+            For simplicity assume we render the emoji as 66px
+            (and everything will scale appropriately for other
+            size images as long as we use percentages).
+
+            The image size will be 66n.
+            The left offset of the x-th emoji (including its
+            padding) will be 66x.  And the element's width
+            will be 66.
+
+            So, solve this equation for `p`, where p is
+            the ratio that we'll later express as a
+            percentage:
+
+                <image offset> = <offset of p% mark of element>
+                (p * 66n) = 66x + p66
+                p * n = x + p
+                p * n - p = x
+                p * (n - 1) = x
+                p = x / (n - 1)
+
+            If you ever want to change the code so that the
+            gutters don't show up in the element, the algebra
+            will get more complicated.
+            """
+            emoji_positions += EMOJI_POS_INFO_TEMPLATE.format(
+                codepoint=get_emoji_code(emoji),
+                pos_x=percent(emoji["sheet_x"] / (n - 1)),
+                pos_y=percent(emoji["sheet_y"] / (n - 1)),
+            )
+
+    SPRITE_STYLES_DIRECTORY = os.path.join(cache_path, "web", "emoji-styles")
+    os.makedirs(SPRITE_STYLES_DIRECTORY, exist_ok=True)
+    SPRITE_CSS_PATH = os.path.join(SPRITE_STYLES_DIRECTORY, f"{emojiset}-sprite.css")
+    with open(SPRITE_CSS_PATH, "w") as f:
+        f.write(
+            SPRITE_CSS_FILE_TEMPLATE.format(
+                emojiset=emojiset,
+                emoji_positions=emoji_positions,
+                background_size=background_size,
+            ),
+        )
+
+    # The Twitter emoji team was laid off in 2022, so new emoji aren't supported.
+    # https://github.com/twitter/twemoji/issues/570#issuecomment-1303422143.
+    # The "twitter" sprite sheet we’re using does have images in those locations,
+    # but they’re fallback images that emoji-datasource fills in from the Apple
+    # sprite sheet, which has unclear licensing implications.
+    # To be able to support newer emoji, we fallback to Google Modern for any emoji
+    # not covered by Twemoji.
+    if emojiset == "twitter":
+        extra_emoji_positions = ""
+        twitter_covered_emoji_codes = {
+            get_emoji_code(emoji) for emoji in emoji_data if emoji["has_img_twitter"]
+        }
+        for emoji in emoji_data:
+            code = get_emoji_code(emoji)
+            if emoji["has_img_google"] and code not in twitter_covered_emoji_codes:
+                extra_emoji_positions += EMOJI_OVERRIDE_TEMPLATE.format(
+                    codepoint=code,
+                )
+        with open(SPRITE_CSS_PATH, "a") as f:
+            f.write(extra_emoji_positions)
+
+
+def setup_emoji_farms(cache_path: str, emoji_data: list[dict[str, Any]]) -> None:
+    def ensure_emoji_image(
+        emoji_dict: dict[str, Any], src_emoji_farm: str, target_emoji_farm: str
+    ) -> None:
+        # We use individual images from emoji farm for rendering emojis
+        # in notification messages. We have a custom emoji formatter in
+        # notifications processing code that converts `span` tags to
+        # `img` and that logic requires us to have non-qualified
+        # `emoji_code` as file name for emoji.
+        emoji_code = get_emoji_code(emoji_dict)
+        img_file_name = emoji_code + ".png"
+        src_file = os.path.join(src_emoji_farm, emoji_dict["image"])
+        dst_file = os.path.join(target_emoji_farm, img_file_name)
+        shutil.copy2(src_file, dst_file)
+
+    def setup_emoji_farm(emojiset: str, emoji_data: list[dict[str, Any]]) -> None:
+        # Copy individual emoji images from npm packages.
+        src_emoji_farm = os.path.join(
+            NODE_MODULES_PATH, "emoji-datasource-" + emojiset, "img", emojiset, "64"
+        )
+        target_emoji_farm = os.path.join(cache_path, "static", "images-" + emojiset + "-64")
+        os.makedirs(target_emoji_farm, exist_ok=True)
+        print(f"Copying individual {emojiset} image files...")
+        for emoji_dict in emoji_data:
+            if emoji_dict["has_img_" + emojiset]:
+                ensure_emoji_image(emoji_dict, src_emoji_farm, target_emoji_farm)
+                skin_variations = emoji_dict.get("skin_variations", {})
+                for img_info in skin_variations.values():
+                    if img_info["has_img_" + emojiset]:
+                        ensure_emoji_image(img_info, src_emoji_farm, target_emoji_farm)
+
+        # Copy zulip.png to the emoji farm.
+        zulip_image = os.path.join(ZULIP_PATH, "web", "images", "zulip-emoji")
+        for f in os.listdir(zulip_image):
+            shutil.copy2(os.path.join(zulip_image, f), target_emoji_farm, follow_symlinks=False)
+
+        # We hardcode octopus emoji image to Google emoji set's old
+        # "cute octopus" image. Copy it to the emoji farms.
+        input_img_file = os.path.join(EMOJI_SCRIPT_DIR_PATH, "1f419.png")
+        output_img_file = os.path.join(target_emoji_farm, "1f419.png")
+        shutil.copyfile(input_img_file, output_img_file)
+
+        generate_sprite_css_files(cache_path, emoji_data, emojiset)
+
+        print(f"Converting {emojiset} sheet to webp...")
+        TARGET_EMOJI_SHEETS = os.path.join(cache_path, "web", "emoji")
+        os.makedirs(TARGET_EMOJI_SHEETS, exist_ok=True)
+
+        sheet_src = os.path.join(
+            NODE_MODULES_PATH,
+            f"emoji-datasource-{emojiset}",
+            "img",
+            emojiset,
+            "sheets-clean",
+            "64.png",
+        )
+        sheet_dst = os.path.join(TARGET_EMOJI_SHEETS, f"{emojiset}.webp")
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            # First quantize the image to 256 colors (like sheets-256 in
+            # emoji-datasource); libvips can only do this when saving to a PNG.
+            quantized = os.path.join(tmpdir, "quantized.png")
+            subprocess.check_call(
+                ["vips", "copy", sheet_src, f"{quantized}[palette,dither=0,effort=10]"]
+            )
+
+            # From libwebp: [Q is] between 0 and 100. For lossy, 0 gives
+            #   the smallest size and 100 the largest. For lossless, this
+            #   parameter is the amount of effort put into the
+            #   compression: 0 is the fastest but gives larger files
+            #   compared to the slowest, but best, 100.
+            subprocess.check_call(["vips", "copy", quantized, f"{sheet_dst}[lossless,Q=100]"])
+
+    # Set up standard emoji sets.
+    for emojiset in ["google", "twitter"]:
+        setup_emoji_farm(emojiset, emoji_data)
+
+
+def setup_old_emoji_farm(
+    cache_path: str, emoji_map: dict[str, str], emoji_data: list[dict[str, Any]]
+) -> None:
+    # Code for setting up old emoji farm.
+    emoji_cache_path = os.path.join(cache_path, "static", "images", "emoji")
+    unicode_emoji_cache_path = os.path.join(cache_path, "static", "images", "emoji", "unicode")
+    google_emoji_cache_path = os.path.join(cache_path, "static", "images-google-64")
+    os.makedirs(emoji_cache_path, exist_ok=True)
+    os.makedirs(unicode_emoji_cache_path, exist_ok=True)
+
+    # Symlink zulip.png image file.
+    image_file_path = os.path.join(google_emoji_cache_path, "zulip.png")
+    symlink_path = os.path.join(emoji_cache_path, "zulip.png")
+    os.symlink(image_file_path, symlink_path)
+
+    unicode_symlink_path = os.path.join(unicode_emoji_cache_path, "zulip.png")
+    os.symlink(image_file_path, unicode_symlink_path)
+
+    for name, codepoint in emoji_map.items():
+        mapped_codepoint = REMAPPED_EMOJIS.get(codepoint, codepoint)
+        image_file_path = os.path.join(google_emoji_cache_path, f"{mapped_codepoint}.png")
+        symlink_path = os.path.join(emoji_cache_path, f"{name}.png")
+        os.symlink(image_file_path, symlink_path)
+        try:
+            # `emoji_map` contains duplicate entries for the same codepoint with different
+            # names. So creation of symlink for <codepoint>.png may throw `FileExistsError`.
+            unicode_symlink_path = os.path.join(unicode_emoji_cache_path, f"{codepoint}.png")
+            os.symlink(image_file_path, unicode_symlink_path)
+        except FileExistsError:
+            pass
+
+
+def generate_map_files(cache_path: str, emoji_catalog: dict[str, list[str]]) -> None:
+    # This function generates the main data files about emoji that are
+    # consumed by the web app, mobile apps, Markdown processor, etc.
+    names = emoji_names_for_picker(EMOJI_NAME_MAPS)
+    codepoint_to_name = generate_codepoint_to_name_map(EMOJI_NAME_MAPS)
+    name_to_codepoint = generate_name_to_codepoint_map(EMOJI_NAME_MAPS)
+
+    EMOJI_CODES_FILE_PATH = os.path.join(cache_path, "static", "emoji_codes.json")
+    with open(EMOJI_CODES_FILE_PATH, "wb") as emoji_codes_file:
+        emoji_codes_file.write(
+            orjson.dumps(
+                {
+                    "names": names,
+                    "name_to_codepoint": name_to_codepoint,
+                    "codepoint_to_name": codepoint_to_name,
+                    "emoji_catalog": emoji_catalog,
+                    "emoticon_conversions": EMOTICON_CONVERSIONS,
+                }
+            )
+        )
+
+    # This is the more official API for mobile to fetch data about emoji.
+    # emoji_codes.json has a lot of goo, and we're creating this new file
+    # as a cleaner data format to move towards. We could add the rest of
+    # the data into this API-described data, and then the web client could
+    # switch to that which would allow us to drop the existing file. But
+    # we'll probably instead do #18121 which will make this file obsolete.
+    # So this is a temporary solution. CZO discussion:
+    # https://chat.zulip.org/#narrow/channel/378-api-design/topic/currently.20supported.20emoji/near/1394598
+    EMOJI_API_FILE_PATH = os.path.join(cache_path, "static", "emoji_api.json")
+    with open(EMOJI_API_FILE_PATH, "wb") as emoji_api_file:
+        emoji_api_file.write(
+            orjson.dumps(
+                {
+                    "code_to_names": generate_codepoint_to_names_map(EMOJI_NAME_MAPS),
+                }
+            )
+        )
+
+
+def dump_emojis(cache_path: str) -> None:
+    with open("emoji_map.json", "rb") as emoji_map_file:
+        emoji_map = orjson.loads(emoji_map_file.read())
+
+    # `emoji.json` or any other data file can be sourced from any of the supported
+    # emoji set packages, they all contain the same data files.
+    EMOJI_DATA_FILE_PATH = os.path.join(NODE_MODULES_PATH, "emoji-datasource-google", "emoji.json")
+    with open(EMOJI_DATA_FILE_PATH, "rb") as emoji_data_file:
+        emoji_data = orjson.loads(emoji_data_file.read())
+
+    # These are the codes that we'll be able to show Google Modern
+    # emoji for. (For other emoji sets, we fall back to Google Modern.)
+    supported_codes = {get_emoji_code(emoji) for emoji in emoji_data if emoji_is_supported(emoji)}
+    # These are in the emoji dropdown so we should make sure they're supported.
+    for code in EMOJI_NAME_MAPS:
+        # If an assertion here fails, we either need to find an image for Google Modern
+        # to display for that emoji, or we need to remove that emoji as an option for users,
+        # by removing it from emoji_names.py through a change to `generate_emoji_names`.
+        assert code in supported_codes
+
+    emoji_catalog = generate_emoji_catalog(emoji_data, EMOJI_NAME_MAPS)
+
+    # Set up emoji farms.
+    if os.path.exists(cache_path):
+        shutil.rmtree(cache_path)
+    setup_emoji_farms(cache_path, emoji_data)
+    setup_old_emoji_farm(cache_path, emoji_map, emoji_data)
+
+    # This file is needed to translate emoji when importing data from Slack.
+    shutil.copyfile(
+        EMOJI_DATA_FILE_PATH,
+        os.path.join(cache_path, "static", "emoji-datasource-google-emoji.json"),
+    )
+
+    # Generate various map files.
+    generate_map_files(cache_path, emoji_catalog)
+
+
+if __name__ == "__main__":
+    main()

tools/setup/emoji/emoji_setup_utils.py

@@ -0,0 +1,127 @@
+# This file contains various helper functions used by `build_emoji` tool.
+# See docs/subsystems/emoji.md for details on how this system works.
+from collections import defaultdict
+from typing import Any
+
+from zerver.lib.emoji_utils import emoji_to_hex_codepoint, hex_codepoint_to_emoji, unqualify_emoji
+
+# Some image files in the old emoji farm had a different name than in the new emoji
+# farm. `remapped_emojis` is a map that contains a mapping of their name in the old
+# emoji farm to their name in the new emoji farm.
+REMAPPED_EMOJIS = {
+    "0023": "0023-20e3",  # Hash
+    "0030": "0030-20e3",  # Zero
+    "0031": "0031-20e3",  # One
+    "0032": "0032-20e3",  # Two
+    "0033": "0033-20e3",  # Three
+    "0034": "0034-20e3",  # Four
+    "0035": "0035-20e3",  # Five
+    "0036": "0036-20e3",  # Six
+    "0037": "0037-20e3",  # Seven
+    "0038": "0038-20e3",  # Eight
+    "0039": "0039-20e3",  # Nine
+    "1f1e8": "1f1e8-1f1f3",  # cn
+    "1f1e9": "1f1e9-1f1ea",  # de
+    "1f1ea": "1f1ea-1f1f8",  # es
+    "1f1eb": "1f1eb-1f1f7",  # fr
+    "1f1ec": "1f1ec-1f1e7",  # gb/us
+    "1f1ee": "1f1ee-1f1f9",  # it
+    "1f1ef": "1f1ef-1f1f5",  # jp
+    "1f1f0": "1f1f0-1f1f7",  # kr
+    "1f1f7": "1f1f7-1f1fa",  # ru
+    "1f1fa": "1f1fa-1f1f8",  # us
+}
+
+# Emoticons and which emoji they should become. Duplicate emoji are allowed.
+# Changes here should be mimicked in `help/configure-emoticon-translations.md`.
+EMOTICON_CONVERSIONS = {
+    ":)": ":slight_smile:",
+    "(:": ":slight_smile:",
+    ":(": ":frown:",
+    "<3": ":heart:",
+    ":|": ":neutral:",
+    ":/": ":confused:",
+    ";)": ":wink:",
+    ":D": ":smile:",
+    ":o": ":open_mouth:",
+    ":O": ":open_mouth:",
+    ":p": ":stuck_out_tongue:",
+    ":P": ":stuck_out_tongue:",
+}
+
+
+def emoji_names_for_picker(emoji_name_maps: dict[str, dict[str, Any]]) -> list[str]:
+    emoji_names: list[str] = []
+    for name_info in emoji_name_maps.values():
+        emoji_names.append(name_info["canonical_name"])
+        emoji_names.extend(name_info["aliases"])
+
+    return sorted(emoji_names)
+
+
+def get_emoji_code(emoji_dict: dict[str, Any]) -> str:
+    # There is a `non_qualified` field on `emoji_dict` but it's
+    # inconsistently present, so we'll always use the unqualified
+    # emoji by unqualifying it ourselves. This gives us more consistent
+    # behaviour between emojis, and doesn't rely on the incomplete
+    # upstream package (https://github.com/iamcal/emoji-data/pull/217).
+    return emoji_to_hex_codepoint(unqualify_emoji(hex_codepoint_to_emoji(emoji_dict["unified"])))
+
+
+# Returns a dict from categories to list of codepoints. The list of
+# codepoints are sorted according to the `sort_order` as defined in
+# `emoji_data`.
+def generate_emoji_catalog(
+    emoji_data: list[dict[str, Any]], emoji_name_maps: dict[str, dict[str, Any]]
+) -> dict[str, list[str]]:
+    sort_order: dict[str, int] = {}
+    emoji_catalog: dict[str, list[str]] = defaultdict(list)
+
+    for emoji_dict in emoji_data:
+        emoji_code = get_emoji_code(emoji_dict)
+        if not emoji_is_supported(emoji_dict) or emoji_code not in emoji_name_maps:
+            continue
+        category = emoji_dict["category"]
+        sort_order[emoji_code] = emoji_dict["sort_order"]
+        emoji_catalog[category].append(emoji_code)
+
+    # Sort the emojis according to iamcal's sort order. This sorting determines the
+    # order in which emojis will be displayed in emoji picker.
+    for emoji_codes in emoji_catalog.values():
+        emoji_codes.sort(key=lambda emoji_code: sort_order[emoji_code])
+
+    return dict(emoji_catalog)
+
+
+def generate_codepoint_to_name_map(emoji_name_maps: dict[str, dict[str, Any]]) -> dict[str, str]:
+    codepoint_to_name: dict[str, str] = {}
+    for emoji_code, name_info in emoji_name_maps.items():
+        codepoint_to_name[emoji_code] = name_info["canonical_name"]
+    return codepoint_to_name
+
+
+# We support Google Modern, and fall back to Google Modern when emoji
+# aren't supported by the other styles we use.
+def emoji_is_supported(emoji_dict: dict[str, Any]) -> bool:
+    return emoji_dict["has_img_google"]
+
+
+def generate_codepoint_to_names_map(
+    emoji_name_maps: dict[str, dict[str, Any]],
+) -> dict[str, list[str]]:
+    # The first element of the names list is always the canonical name.
+    return {
+        emoji_code: [name_info["canonical_name"], *name_info["aliases"]]
+        for emoji_code, name_info in emoji_name_maps.items()
+    }
+
+
+def generate_name_to_codepoint_map(emoji_name_maps: dict[str, dict[str, Any]]) -> dict[str, str]:
+    name_to_codepoint = {}
+    for emoji_code, name_info in emoji_name_maps.items():
+        canonical_name = name_info["canonical_name"]
+        aliases = name_info["aliases"]
+        name_to_codepoint[canonical_name] = emoji_code
+        for alias in aliases:
+            name_to_codepoint[alias] = emoji_code
+    return name_to_codepoint

tools/setup/emoji/generate_emoji_names

@@ -0,0 +1,222 @@
+#!/usr/bin/env python3
+import os
+import re
+import sys
+from collections import defaultdict
+
+import orjson
+
+ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../../")
+sys.path.append(ZULIP_PATH)
+
+from tools.setup.emoji.custom_emoji_names import CUSTOM_EMOJI_NAME_MAPS
+from tools.setup.emoji.emoji_setup_utils import get_emoji_code
+from zerver.lib.emoji_utils import hex_codepoint_to_emoji, unqualify_emoji
+
+CLDR_DATA_FILE = os.path.join(
+    ZULIP_PATH, "node_modules", "cldr-annotations-full", "annotations", "en", "annotations.json"
+)
+CLDR_DERIVED_DATA_FILE = os.path.join(
+    ZULIP_PATH,
+    "node_modules",
+    "cldr-annotations-derived-full",
+    "annotationsDerived",
+    "en",
+    "annotations.json",
+)
+EMOJI_DATA_FILE = os.path.join(ZULIP_PATH, "node_modules", "emoji-datasource-google", "emoji.json")
+OUT_EMOJI_FILE = os.path.join(ZULIP_PATH, "tools", "setup", "emoji", "emoji_names.py")
+
+with open(EMOJI_DATA_FILE, "rb") as fp:
+    EMOJI_DATA = orjson.loads(fp.read())
+with open(CLDR_DATA_FILE, "rb") as fp:
+    CLDR_DATA = orjson.loads(fp.read())["annotations"]["annotations"]
+with open(CLDR_DERIVED_DATA_FILE, "rb") as fp:
+    CLDR_DATA.update(orjson.loads(fp.read())["annotationsDerived"]["annotations"])
+
+# We don't include most clock emojis. See `custom_emoji_names` for more context.
+SKIPPED_CLOCK_EMOJI_CODES = [
+    "1f550",
+    "1f551",
+    "1f552",
+    "1f553",
+    "1f554",
+    "1f555",
+    "1f556",
+    "1f558",
+    "1f559",
+    "1f55a",
+    "1f55b",
+    "1f55c",
+    "1f55d",
+    "1f55e",
+    "1f55f",
+    "1f560",
+    "1f561",
+    "1f562",
+    "1f563",
+    "1f564",
+    "1f565",
+    "1f566",
+    "1f567",
+]
+
+# We don't include the skin tones as emojis that one can search for on their own.
+SKIN_TONE_EMOJI_CODES = [
+    "1f3fb",
+    "1f3fc",
+    "1f3fd",
+    "1f3fe",
+    "1f3ff",
+]
+
+
+def cleanup_name(name: str) -> str:
+    replacements = {
+        " ": "_",
+        "-": "_",
+        "–": "_",
+        "“": "",
+        "”": "",
+        ":": "",
+        ".": "",
+        ",": "",
+        "(": "",
+        ")": "",
+        "&": "and",
+        "‘": "",
+        "’": "",
+        "'": "",
+    }
+    for before, after in replacements.items():
+        name = name.replace(before, after)
+    name = re.sub(r"_{2,}", "_", name)
+    return name.lower()
+
+
+def convert_non_ascii_chars(name: str) -> str:
+    replacements = {
+        "è": "e",
+        "ǐ": "i",
+        "ó": "o",
+        "ā": "a",
+        "ō": "o",
+        "ñ": "n",
+        "ô": "o",
+        "ç": "c",
+        "é": "e",
+        "ã": "a",
+        "í": "i",
+        "å": "a",
+        "ü": "u",
+    }
+    for before, after in replacements.items():
+        name = name.replace(before, after)
+    assert name.isascii(), (
+        f"{name} still contains non-ascii characters. Add them to convert_non_ascii_chars."
+    )
+    return name
+
+
+def main() -> None:
+    all_emojis = {}
+    all_canonical_names = set()
+
+    alias_to_emoji_code = defaultdict(list)
+
+    # STEP 1: Generate first draft of all_emojis.
+    for emoji_dict in EMOJI_DATA:
+        emoji_code = get_emoji_code(emoji_dict)
+        if emoji_code in SKIPPED_CLOCK_EMOJI_CODES or emoji_code in SKIN_TONE_EMOJI_CODES:
+            continue
+
+        if emoji_code in CUSTOM_EMOJI_NAME_MAPS:
+            canonical_name = cleanup_name(CUSTOM_EMOJI_NAME_MAPS[emoji_code]["canonical_name"])
+            if canonical_name in all_canonical_names:
+                raise Exception(
+                    f"{canonical_name} was already added with a different codepoint. "
+                    f"Rename it in `custom_emoji_names` or add an entry for {emoji_code}."
+                )
+            all_canonical_names.add(canonical_name)
+            all_emojis[emoji_code] = CUSTOM_EMOJI_NAME_MAPS[emoji_code]
+        else:
+            # create the unicode character(s) for the emoji, since this is the key into the CLDR data
+            # We can't just use emoji_dict["non_qualified"] because of this upstream bug:
+            # https://github.com/iamcal/emoji-data/pull/217
+            emoji = unqualify_emoji(hex_codepoint_to_emoji(emoji_dict["unified"]))
+            if emoji not in CLDR_DATA:
+                print(
+                    f"{emoji} not found in custom emoji name maps, but also not found in CLDR data. Skipping."
+                )
+                continue
+            # CLDR_DATA[emoji] is of the form {'default': [...], 'tts': [...]}
+            # * "tts" is what's used for text-to-speech and always has one item, so we use that
+            #    as the canonical name.
+            # * "default" has several items in it that we use as aliases.
+            # See also: https://www.unicode.org/reports/tr35/tr35-general.html#14-annotations-and-labels
+            assert len(CLDR_DATA[emoji]["tts"]) == 1
+            canonical_name = cleanup_name(CLDR_DATA[emoji]["tts"][0].strip())
+            if canonical_name in all_canonical_names:
+                raise Exception(
+                    f"{canonical_name} was already added with a different codepoint. "
+                    f"Rename it in `custom_emoji_names` or add an entry for {emoji_code}."
+                )
+            aliases = [cleanup_name(alias.strip()) for alias in CLDR_DATA[emoji]["default"]]
+            all_emojis[emoji_code] = {"canonical_name": canonical_name, "aliases": aliases}
+            all_canonical_names.add(canonical_name)
+
+    # STEP 2: We don't support having aliases that collide with canonical names for emoji, so remove them.
+    for emoji_code, emoji_names in all_emojis.items():
+        # Copy the list to not iterate while elements are being deleted.
+        aliases = emoji_names["aliases"][:]
+        for alias in aliases:
+            if alias in all_canonical_names:
+                emoji_names["aliases"].remove(alias)
+            else:
+                alias_to_emoji_code[alias].append(emoji_code)  # This is used in STEP 3.
+
+    # STEP 3: We don't support multiple emoji sharing the same alias, but the CLDR data
+    # doesn't have that same restriction, so we have to fix this up to have unique aliases.
+    # If the alias was specifically specified in custom_emoji_names, then we can keep just
+    # that one, but otherwise there's no particular emoji that is an obvious candidate
+    # for the alias so just remove the alias for all relevant emoji.
+    for alias in alias_to_emoji_code:
+        if len(alias_to_emoji_code[alias]) > 1:
+            for emoji_code in alias_to_emoji_code[alias]:
+                if emoji_code not in CUSTOM_EMOJI_NAME_MAPS:
+                    all_emojis[emoji_code]["aliases"].remove(alias)
+
+    # STEP 4: We keep non-ascii (non-"English") characters in some emoji names if that's the correct
+    # way to spell that word, but always add an alias for an ascii-only version of the word.
+    for emoji_names in all_emojis.values():
+        for name in [emoji_names["canonical_name"]] + emoji_names["aliases"]:
+            # These are known names where we don't have an ascii-only version and there are ascii aliases
+            # that a user can still enter instead to get the same emoji.
+            if name in ["ココ", "サ", "指", "空"]:
+                assert any(alias.isascii() for alias in emoji_names["aliases"])
+                continue
+            if not name.isascii():
+                ascii_alias = convert_non_ascii_chars(name)
+                # Now no other emoji can use this alias.
+                for code in alias_to_emoji_code[ascii_alias]:
+                    all_emojis[code]["aliases"].remove(ascii_alias)
+                emoji_names["aliases"].append(ascii_alias)
+
+    # STEP 5: Write final dictionary to `emoji_names.py`.
+    with open(OUT_EMOJI_FILE, "w") as f:
+        f.write(
+            "from typing import Any\n\n"
+            "# Generated with `generate_emoji_names`.\n\n"
+            "EMOJI_NAME_MAPS: dict[str, dict[str, Any]] = {\n"
+        )
+        for key, emoji_names in all_emojis.items():
+            f.write(f"    {key!r}: {emoji_names!r},\n")
+        f.write("}\n")
+
+    print(
+        "\n\nDone! You should run the linter to format emoji_names.py with `./tools/lint --fix -m --only ruff-format`"
+    )
+
+
+if __name__ == "__main__":
+    main()

tools/setup/emoji/generate_emoji_names_table

@@ -0,0 +1,226 @@
+#!/usr/bin/env python3
+#
+# This is a debugging tool that takes as input a bunch of different
+# emoji data sources, and outputs a convenient HTML table that can be
+# used to sanity-check the differences between these different data
+# sources' decisions about what names to provide to each Unicode
+# codepoint.
+import os
+import sys
+from typing import Any
+
+import orjson
+
+TOOLS_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+ZULIP_PATH = os.path.dirname(TOOLS_DIR)
+sys.path.append(ZULIP_PATH)
+
+from tools.setup.emoji.emoji_names import EMOJI_NAME_MAPS
+from tools.setup.emoji.emoji_setup_utils import emoji_is_supported, get_emoji_code
+
+UNIFIED_REACTIONS_FILE = os.path.join(
+    ZULIP_PATH, "zerver", "management", "data", "unified_reactions.json"
+)
+EMOJI_DATA_FILE = os.path.join(ZULIP_PATH, "node_modules", "emoji-datasource-google", "emoji.json")
+EMOJI_CACHE = os.path.join(ZULIP_PATH, "static", "generated", "emoji")
+OUTPUT_FILE = os.path.join(EMOJI_CACHE, "emoji_names_table.html")
+
+# Emoji sets that we currently support.
+EMOJISETS = ["google", "twitter"]
+
+with open(EMOJI_DATA_FILE, "rb") as fp:
+    EMOJI_DATA = orjson.loads(fp.read())
+with open(UNIFIED_REACTIONS_FILE, "rb") as fp:
+    UNIFIED_REACTIONS_MAP = orjson.loads(fp.read())
+
+EMOJI_IMAGE_TEMPLATE = """
+    <img class="emoji" src="images-{emojiset}-64/{emoji_code}.png" title={emojiset}>
+"""
+
+TABLE_ROW_TEMPLATE = """
+    <tr>
+        <td class="new-sorting-info">{sorting_info}</td>
+        <td class="emoji-code">{emoji_code}</td>
+        <td class="emoji-images">{images_html}</td>
+        <td class="zulip-emoji-names">{zulip_names}</td>
+        <td class="iamcal-emoji-names">{iamcal_names}</td>
+        <td class="gemoji-emoji-names">{gemoji_names}</td>
+        <td class="unicode-name">{unicode_name}</td>
+    </tr>
+"""
+
+EMOJI_LISTING_TEMPLATE = """
+<html>
+    <head>
+        <style>
+            {table_css}
+        </style>
+        <title>Zulip emoji names</title>
+    </head>
+    <body>
+        <table>
+            <thead>
+                <tr>
+                    <th class="new-sorting-info">Category</th>
+                    <th class="emoji-code">Emoji code</th>
+                    <th class="emoji-images">Images</th>
+                    <th class="zulip-emoji-names">Zulip</th>
+                    <th class="iamcal-emoji-names">Iamcal (Slack)</th>
+                    <th class="gemoji-emoji-names">Gemoji (unordered)</th>
+                    <th class="unicode-name">Unicode</th>
+                </tr>
+            </thead>
+            <tbody>
+                {tbody}
+            </tbody>
+        </table>
+    </body>
+</html>
+"""
+
+TABLE_CSS = """
+    .emoji {
+        height: 35px;
+        width: 35px;
+        position: relative;
+        margin-top: -7px;
+        vertical-align: middle;
+        top: 3px;
+    }
+
+    .emoji-images {
+        width: 200px;
+    }
+
+    table, td, th {
+        border: 1px solid black;
+        border-collapse: collapse;
+    }
+
+    td, th {
+        height: 40px;
+        text-align: center;
+    }
+
+    .google {
+        background-image: url('sheet-google-64.png') !important;
+    }
+
+    .twitter {
+        background-image: url('sheet-twitter-64.png') !important;
+    }
+"""
+
+SORTED_CATEGORIES = [
+    "Smileys & Emotion",
+    "People & Body",
+    "Animals & Nature",
+    "Food & Drink",
+    "Activities",
+    "Travel & Places",
+    "Objects",
+    "Symbols",
+    "Flags",
+    "Component",
+]
+
+emoji_code_to_zulip_names: dict[str, str] = {}
+emoji_code_to_iamcal_names: dict[str, str] = {}
+emoji_code_to_gemoji_names: dict[str, str] = {}
+emoji_collection: dict[str, list[dict[str, Any]]] = {category: [] for category in SORTED_CATEGORIES}
+
+
+def generate_emoji_code_to_emoji_names_maps() -> None:
+    # Prepare gemoji names map.
+    reverse_unified_reactions_map: dict[str, list[str]] = {}
+    for name in UNIFIED_REACTIONS_MAP:
+        emoji_code = UNIFIED_REACTIONS_MAP[name]
+        if emoji_code in reverse_unified_reactions_map:
+            reverse_unified_reactions_map[emoji_code].append(name)
+        else:
+            reverse_unified_reactions_map[emoji_code] = [name]
+
+    for emoji_code, names in reverse_unified_reactions_map.items():
+        emoji_code_to_gemoji_names[emoji_code] = ", ".join(names)
+
+    # Prepare iamcal names map.
+    for emoji_dict in EMOJI_DATA:
+        emoji_code = get_emoji_code(emoji_dict)
+        emoji_code_to_iamcal_names[emoji_code] = ", ".join(emoji_dict["short_names"])
+
+    # Prepare zulip names map.
+    for emoji_code in EMOJI_NAME_MAPS:
+        canonical_name = EMOJI_NAME_MAPS[emoji_code]["canonical_name"]
+        aliases = EMOJI_NAME_MAPS[emoji_code]["aliases"]
+        names = [canonical_name]
+        names.extend(aliases)
+        emoji_code_to_zulip_names[emoji_code] = ", ".join(names)
+
+
+def get_sorting_info(category: str, sort_order: int) -> str:
+    return f"{category} {sort_order}"
+
+
+def get_images_html(emoji_code: str) -> str:
+    images_html = ""
+    for emojiset in EMOJISETS:
+        images_html += EMOJI_IMAGE_TEMPLATE.format(
+            emoji_code=emoji_code,
+            emojiset=emojiset,
+        )
+
+    return images_html
+
+
+def generate_emoji_collection() -> None:
+    generate_emoji_code_to_emoji_names_maps()
+    # Prepare `emoji_collection`.
+    for emoji_dict in EMOJI_DATA:
+        if not emoji_is_supported(emoji_dict):
+            continue
+        category = emoji_dict["category"]
+        emoji_code = get_emoji_code(emoji_dict)
+        sort_order = emoji_dict["sort_order"]
+        emoji_collection[category].append(
+            {
+                "category": category,
+                "emoji_code": emoji_code,
+                "images_html": get_images_html(emoji_code),
+                "gemoji_names": emoji_code_to_gemoji_names.get(emoji_code, ""),
+                "iamcal_names": emoji_code_to_iamcal_names.get(emoji_code, ""),
+                "zulip_names": emoji_code_to_zulip_names.get(emoji_code, ""),
+                "unicode_name": (emoji_dict["name"] or "").lower(),
+                "sort_order": sort_order,
+                "sorting_info": get_sorting_info(category, sort_order),
+            }
+        )
+
+    # Sort `emoji_collection`.
+    for category in SORTED_CATEGORIES:
+        emoji_collection[category].sort(key=lambda x: x["sort_order"])
+
+
+def main() -> None:
+    generate_emoji_collection()
+
+    tbody = ""
+    for category in SORTED_CATEGORIES:
+        for emoji_entry in emoji_collection[category]:
+            tbody += TABLE_ROW_TEMPLATE.format(**emoji_entry)
+
+    with open(OUTPUT_FILE, "w") as fp:
+        fp.write(
+            EMOJI_LISTING_TEMPLATE.format(
+                tbody=tbody,
+                table_css=TABLE_CSS,
+            )
+        )
+
+    print(
+        "Done! Open http://localhost:9991/static/generated/emoji/emoji_names_table.html "
+        "to view(after starting the dev server)."
+    )
+
+
+if __name__ == "__main__":
+    main()