snapshot of https://github.com/zulip/zulip.git @ 48c69911c7dd4e03b20552eb3bd77ad8151c1357 (2026-09-15)

zerver/management/commands/compilemessages.py

@@ -0,0 +1,160 @@
+import os
+import re
+import unicodedata
+from subprocess import CalledProcessError, check_output
+from typing import Any
+
+import orjson
+import polib
+from django.conf import settings
+from django.conf.locale import LANG_INFO
+from django.core.management.base import CommandParser
+from django.core.management.commands import compilemessages
+from django.utils.translation import gettext as _
+from django.utils.translation import override as override_language
+from django.utils.translation import to_language
+from pyuca import Collator
+from typing_extensions import override
+
+
+class Command(compilemessages.Command):
+    @override
+    def add_arguments(self, parser: CommandParser) -> None:
+        super().add_arguments(parser)
+
+        parser.add_argument(
+            "--strict", "-s", action="store_true", help="Stop execution in case of errors."
+        )
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        super().handle(*args, **options)
+        self.strict = options["strict"]
+        self.extract_language_options()
+        self.create_language_name_map()
+
+    def create_language_name_map(self) -> None:
+        join = os.path.join
+        deploy_root = settings.DEPLOY_ROOT
+        path = join(deploy_root, "locale", "language_options.json")
+        output_path = join(deploy_root, "locale", "language_name_map.json")
+
+        with open(path, "rb") as reader:
+            languages = orjson.loads(reader.read())
+            lang_list = []
+            for lang_info in languages["languages"]:
+                lang_info["name"] = lang_info["name_local"]
+                del lang_info["name_local"]
+                lang_list.append(lang_info)
+
+            collator = Collator()
+            lang_list.sort(key=lambda lang: collator.sort_key(lang["name"]))
+
+        with open(output_path, "wb") as output_file:
+            output_file.write(
+                orjson.dumps(
+                    {"name_map": lang_list},
+                    option=orjson.OPT_APPEND_NEWLINE | orjson.OPT_INDENT_2 | orjson.OPT_SORT_KEYS,
+                )
+            )
+
+    def get_po_filename(self, locale_path: str, locale: str) -> str:
+        po_template = "{}/{}/LC_MESSAGES/django.po"
+        return po_template.format(locale_path, locale)
+
+    def get_json_filename(self, locale_path: str, locale: str) -> str:
+        return f"{locale_path}/{locale}/translations.json"
+
+    def get_name_from_po_file(self, po_filename: str, locale: str) -> str:
+        try:
+            team = polib.pofile(po_filename).metadata["Language-Team"]
+            return team[: team.rindex(" <")]
+        except (KeyError, ValueError):
+            raise Exception(f"Unknown language {locale}")
+
+    def get_locales(self) -> list[str]:
+        output = check_output(["git", "ls-files", "locale"], text=True)
+        tracked_files = output.split()
+        regex = re.compile(r"locale/(\w+)/LC_MESSAGES/django.po")
+        locales = ["en"]
+        for tracked_file in tracked_files:
+            matched = regex.search(tracked_file)
+            if matched:
+                locales.append(matched.group(1))
+
+        return locales
+
+    def extract_language_options(self) -> None:
+        locale_path = f"{settings.DEPLOY_ROOT}/locale"
+        output_path = f"{locale_path}/language_options.json"
+
+        data: dict[str, list[dict[str, Any]]] = {"languages": []}
+
+        try:
+            locales = self.get_locales()
+        except CalledProcessError:
+            # In case we are not under a Git repo, fallback to getting the
+            # locales using listdir().
+            locales = os.listdir(locale_path)
+            locales.append("en")
+            locales = list(set(locales))
+
+        for locale in sorted(locales):
+            if locale == "en":
+                data["languages"].append(
+                    {
+                        "name": "English",
+                        "name_local": "English",
+                        "code": "en",
+                        "locale": "en",
+                    }
+                )
+                continue
+
+            lc_messages_path = os.path.join(locale_path, locale, "LC_MESSAGES")
+            if not os.path.exists(lc_messages_path):
+                # Not a locale.
+                continue
+
+            info: dict[str, Any] = {}
+            code = to_language(locale)
+            percentage = self.get_translation_percentage(locale_path, locale)
+            try:
+                name = LANG_INFO[code]["name"]
+                name_local = LANG_INFO[code]["name_local"]
+            except KeyError:
+                # Fallback to getting the name from PO file.
+                filename = self.get_po_filename(locale_path, locale)
+                name = self.get_name_from_po_file(filename, locale)
+                with override_language(code):
+                    name_local = _(name)
+
+            info["name"] = unicodedata.normalize("NFC", name)
+            info["name_local"] = unicodedata.normalize("NFC", name_local)
+            info["code"] = code
+            info["locale"] = locale
+            info["percent_translated"] = percentage
+            data["languages"].append(info)
+
+        with open(output_path, "wb") as writer:
+            writer.write(
+                orjson.dumps(
+                    data,
+                    option=orjson.OPT_APPEND_NEWLINE | orjson.OPT_INDENT_2 | orjson.OPT_SORT_KEYS,
+                )
+            )
+
+    def get_translation_percentage(self, locale_path: str, locale: str) -> int:
+        # backend stats
+        po = polib.pofile(self.get_po_filename(locale_path, locale))
+        not_translated = len(po.untranslated_entries())
+        total = len(po.translated_entries()) + not_translated
+
+        # frontend stats
+        with open(self.get_json_filename(locale_path, locale), "rb") as reader:
+            for value in orjson.loads(reader.read()).values():
+                total += 1
+                if value == "":
+                    not_translated += 1
+
+        return (total - not_translated) * 100 // total