snapshot of https://github.com/zulip/zulip.git @ 48c69911c7dd4e03b20552eb3bd77ad8151c1357 (2026-09-15)

zerver/lib/addressee.py

@@ -0,0 +1,225 @@
+from collections.abc import Iterable, Sequence
+from typing import cast
+
+from django.utils.translation import gettext as _
+
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.string_validation import check_stream_topic
+from zerver.lib.topic import (
+    maybe_rename_general_chat_to_empty_topic,
+    maybe_rename_no_topic_to_empty_topic,
+)
+from zerver.models import Realm, Stream, UserProfile
+from zerver.models.users import (
+    get_user_by_id_in_realm_including_cross_realm,
+    get_user_including_cross_realm,
+)
+
+
+def get_user_profiles(emails: Iterable[str], realm: Realm) -> list[UserProfile]:
+    user_profiles: list[UserProfile] = []
+    for email in emails:
+        try:
+            user_profile = get_user_including_cross_realm(email, realm)
+        except UserProfile.DoesNotExist:
+            raise JsonableError(_("Invalid email '{email}'").format(email=email))
+        user_profiles.append(user_profile)
+    return user_profiles
+
+
+def get_user_profiles_by_ids(user_ids: Iterable[int], realm: Realm) -> list[UserProfile]:
+    user_profiles: list[UserProfile] = []
+    for user_id in user_ids:
+        try:
+            user_profile = get_user_by_id_in_realm_including_cross_realm(user_id, realm)
+        except UserProfile.DoesNotExist:
+            raise JsonableError(_("Invalid user ID {user_id}").format(user_id=user_id))
+        user_profiles.append(user_profile)
+    return user_profiles
+
+
+class Addressee:
+    # This is really just a holder for vars that tended to be passed
+    # around in a non-type-safe way before this class was introduced.
+    #
+    # It also avoids some nonsense where you have to think about whether
+    # topic should be None or '' for a direct message, or you have to
+    # make an array of one stream.
+    #
+    # Eventually we can use this to cache Stream and UserProfile objects
+    # in memory.
+    #
+    # This should be treated as an immutable class.
+    def __init__(
+        self,
+        msg_type: str,
+        user_profiles: Sequence[UserProfile] | None = None,
+        stream: Stream | None = None,
+        stream_name: str | None = None,
+        stream_id: int | None = None,
+        topic_name: str | None = None,
+    ) -> None:
+        assert msg_type in ["stream", "private"]
+        if msg_type == "stream" and topic_name is None:
+            raise JsonableError(_("Missing topic"))
+        self._msg_type = msg_type
+        self._user_profiles = user_profiles
+        self._stream = stream
+        self._stream_name = stream_name
+        self._stream_id = stream_id
+        self._topic_name = topic_name
+
+    def is_stream(self) -> bool:
+        return self._msg_type == "stream"
+
+    def is_private(self) -> bool:
+        return self._msg_type == "private"
+
+    def user_profiles(self) -> Sequence[UserProfile]:
+        assert self.is_private()
+        assert self._user_profiles is not None
+        return self._user_profiles
+
+    def stream(self) -> Stream | None:
+        assert self.is_stream()
+        return self._stream
+
+    def stream_name(self) -> str | None:
+        assert self.is_stream()
+        return self._stream_name
+
+    def stream_id(self) -> int | None:
+        assert self.is_stream()
+        return self._stream_id
+
+    def topic_name(self) -> str:
+        assert self.is_stream()
+        assert self._topic_name is not None
+        return self._topic_name
+
+    def is_message_to_self(self, sender: UserProfile) -> bool:
+        return (
+            self.is_private()
+            and len(self.user_profiles()) == 1
+            and self.user_profiles()[0].id == sender.id
+        )
+
+    @staticmethod
+    def legacy_build(
+        sender: UserProfile,
+        recipient_type_name: str,
+        message_to: Sequence[int] | Sequence[str],
+        topic_name: str | None,
+        realm: Realm | None = None,
+    ) -> "Addressee":
+        # For legacy reason message_to used to be either a list of
+        # emails or a list of streams.  We haven't fixed all of our
+        # callers yet.
+        if realm is None:
+            realm = sender.realm
+
+        if recipient_type_name == "stream":
+            if len(message_to) > 1:
+                raise JsonableError(_("Cannot send to multiple channels"))
+
+            if message_to:
+                stream_name_or_id = message_to[0]
+            else:
+                # This is a hack to deal with the fact that we still support
+                # default streams (and the None will be converted later in the
+                # call path).
+                if sender.default_sending_stream_id:
+                    # Use the user's default stream
+                    stream_name_or_id = sender.default_sending_stream_id
+                else:
+                    raise JsonableError(_("Missing channel"))
+
+            if topic_name is None:
+                raise JsonableError(_("Missing topic"))
+
+            if isinstance(stream_name_or_id, int):
+                return Addressee.for_stream_id(stream_name_or_id, topic_name)
+
+            return Addressee.for_stream_name(stream_name_or_id, topic_name)
+        elif recipient_type_name == "private":
+            if not message_to:
+                raise JsonableError(_("Message must have recipients"))
+
+            if isinstance(message_to[0], str):
+                emails = cast(Sequence[str], message_to)
+                return Addressee.for_private(emails, realm)
+            elif isinstance(message_to[0], int):
+                user_ids = cast(Sequence[int], message_to)
+                return Addressee.for_user_ids(user_ids=user_ids, realm=realm)
+        else:
+            raise JsonableError(_("Invalid message type"))
+
+    @staticmethod
+    def for_stream(stream: Stream, topic_name: str) -> "Addressee":
+        topic_name = topic_name.strip()
+        topic_name = maybe_rename_general_chat_to_empty_topic(topic_name)
+        topic_name = maybe_rename_no_topic_to_empty_topic(topic_name)
+        check_stream_topic(topic_name)
+        return Addressee(
+            msg_type="stream",
+            stream=stream,
+            topic_name=topic_name,
+        )
+
+    @staticmethod
+    def for_stream_name(stream_name: str, topic_name: str) -> "Addressee":
+        topic_name = topic_name.strip()
+        topic_name = maybe_rename_general_chat_to_empty_topic(topic_name)
+        topic_name = maybe_rename_no_topic_to_empty_topic(topic_name)
+        check_stream_topic(topic_name)
+        return Addressee(
+            msg_type="stream",
+            stream_name=stream_name,
+            topic_name=topic_name,
+        )
+
+    @staticmethod
+    def for_stream_id(stream_id: int, topic_name: str) -> "Addressee":
+        topic_name = topic_name.strip()
+        topic_name = maybe_rename_general_chat_to_empty_topic(topic_name)
+        topic_name = maybe_rename_no_topic_to_empty_topic(topic_name)
+        check_stream_topic(topic_name)
+        return Addressee(
+            msg_type="stream",
+            stream_id=stream_id,
+            topic_name=topic_name,
+        )
+
+    @staticmethod
+    def for_private(emails: Sequence[str], realm: Realm) -> "Addressee":
+        assert len(emails) > 0
+        user_profiles = get_user_profiles(emails, realm)
+        return Addressee(
+            msg_type="private",
+            user_profiles=user_profiles,
+        )
+
+    @staticmethod
+    def for_user_ids(user_ids: Sequence[int], realm: Realm) -> "Addressee":
+        assert len(user_ids) > 0
+        user_profiles = get_user_profiles_by_ids(user_ids, realm)
+        return Addressee(
+            msg_type="private",
+            user_profiles=user_profiles,
+        )
+
+    @staticmethod
+    def for_user_profile(user_profile: UserProfile) -> "Addressee":
+        user_profiles = [user_profile]
+        return Addressee(
+            msg_type="private",
+            user_profiles=user_profiles,
+        )
+
+    @staticmethod
+    def for_user_profiles(user_profiles: Sequence[UserProfile]) -> "Addressee":
+        assert len(user_profiles) > 0
+        return Addressee(
+            msg_type="private",
+            user_profiles=user_profiles,
+        )

zerver/lib/alert_words.py

@@ -0,0 +1,84 @@
+from collections.abc import Iterable
+
+import ahocorasick
+from django.db import transaction
+from django.db.models.functions import Lower
+
+from zerver.lib.cache import (
+    cache_with_key,
+    realm_alert_words_automaton_cache_key,
+    realm_alert_words_cache_key,
+)
+from zerver.models import AlertWord, Realm, UserProfile
+from zerver.models.alert_words import flush_realm_alert_words
+
+
+@cache_with_key(lambda realm: realm_alert_words_cache_key(realm.id), timeout=3600 * 24)
+def alert_words_in_realm(realm: Realm) -> dict[int, list[str]]:
+    user_ids_and_words = AlertWord.objects.filter(realm=realm, user_profile__is_active=True).values(
+        "user_profile_id", "word"
+    )
+    user_ids_with_words: dict[int, list[str]] = {}
+    for id_and_word in user_ids_and_words:
+        user_ids_with_words.setdefault(id_and_word["user_profile_id"], [])
+        user_ids_with_words[id_and_word["user_profile_id"]].append(id_and_word["word"])
+    return user_ids_with_words
+
+
+@cache_with_key(lambda realm: realm_alert_words_automaton_cache_key(realm.id), timeout=3600 * 24)
+def get_alert_word_automaton(realm: Realm) -> ahocorasick.Automaton:
+    user_id_with_words = alert_words_in_realm(realm)
+    alert_word_automaton = ahocorasick.Automaton()
+    for user_id, alert_words in user_id_with_words.items():
+        for alert_word in alert_words:
+            alert_word_lower = alert_word.lower()
+            if alert_word_automaton.exists(alert_word_lower):
+                (_key, user_ids_for_alert_word) = alert_word_automaton.get(alert_word_lower)
+                user_ids_for_alert_word.add(user_id)
+            else:
+                alert_word_automaton.add_word(alert_word_lower, (alert_word_lower, {user_id}))
+    alert_word_automaton.make_automaton()
+    # If the kind is not AHOCORASICK after calling make_automaton, it means there is no key present
+    # and hence we cannot call items on the automaton yet. To avoid it we return None for such cases
+    # where there is no alert-words in the realm.
+    # https://pyahocorasick.readthedocs.io/en/latest/#make-automaton
+    if alert_word_automaton.kind != ahocorasick.AHOCORASICK:
+        return None
+    return alert_word_automaton
+
+
+def user_alert_words(user_profile: UserProfile) -> list[str]:
+    return list(AlertWord.objects.filter(user_profile=user_profile).values_list("word", flat=True))
+
+
+@transaction.atomic(savepoint=False)
+def add_user_alert_words(user_profile: UserProfile, new_words: Iterable[str]) -> list[str]:
+    existing_words_lower = {word.lower() for word in user_alert_words(user_profile)}
+
+    # Keeping the case, use a dictionary to get the set of
+    # case-insensitive distinct, new alert words
+    word_dict: dict[str, str] = {}
+    for word in new_words:
+        if word.lower() in existing_words_lower:
+            continue
+        word_dict[word.lower()] = word
+
+    AlertWord.objects.bulk_create(
+        AlertWord(user_profile=user_profile, word=word, realm=user_profile.realm)
+        for word in word_dict.values()
+    )
+    # Django bulk_create operations don't flush caches, so we need to do this ourselves.
+    flush_realm_alert_words(user_profile.realm_id)
+
+    return user_alert_words(user_profile)
+
+
+@transaction.atomic(savepoint=False)
+def remove_user_alert_words(user_profile: UserProfile, delete_words: Iterable[str]) -> list[str]:
+    delete_words_lower = [word.lower() for word in delete_words]
+    AlertWord.objects.annotate(word_lower=Lower("word")).filter(
+        user_profile=user_profile,
+        word_lower__in=delete_words_lower,
+    ).delete()
+    flush_realm_alert_words(user_profile.realm_id)
+    return user_alert_words(user_profile)

zerver/lib/async_utils.py

@@ -0,0 +1,23 @@
+# nocoverage
+
+import asyncio
+import sys
+
+from typing_extensions import override
+
+if sys.version_info < (3, 14):
+
+    class NoAutoCreateEventLoopPolicy(asyncio.DefaultEventLoopPolicy):
+        """
+        By default asyncio.get_event_loop() automatically creates an event
+        loop for the main thread if one isn't currently installed.  Since
+        Django intentionally uninstalls the event loop within
+        sync_to_async, that autocreation proliferates confusing extra
+        event loops that will never be run.  It is also deprecated in
+        Python 3.10.  This policy disables it so we don't rely on it by
+        accident.
+        """
+
+        @override
+        def get_event_loop(self) -> asyncio.AbstractEventLoop:
+            return asyncio.get_running_loop()

zerver/lib/attachments.py

@@ -0,0 +1,280 @@
+from datetime import timedelta
+from typing import Any
+
+from django.conf import settings
+from django.contrib.auth.models import AnonymousUser
+from django.db.models import Exists, OuterRef, QuerySet
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+
+from zerver.lib.exceptions import JsonableError, RateLimitedError
+from zerver.lib.streams import is_user_in_groups_granting_content_access
+from zerver.lib.upload import delete_message_attachment
+from zerver.lib.user_groups import get_recursive_membership_groups
+from zerver.models import (
+    ArchivedAttachment,
+    Attachment,
+    ImageAttachment,
+    Message,
+    Realm,
+    Recipient,
+    Stream,
+    Subscription,
+    UserMessage,
+    UserProfile,
+)
+
+
+def user_attachments(user_profile: UserProfile) -> list[dict[str, Any]]:
+    attachments = Attachment.objects.filter(owner=user_profile).prefetch_related("messages")
+    return [a.to_dict() for a in attachments]
+
+
+def access_attachment_by_id(
+    user_profile: UserProfile, attachment_id: int, needs_owner: bool = False
+) -> Attachment:
+    query = Attachment.objects.filter(id=attachment_id)
+    if needs_owner:
+        query = query.filter(owner=user_profile)
+
+    attachment = query.first()
+    if attachment is None:
+        raise JsonableError(_("Invalid attachment"))
+    return attachment
+
+
+def remove_attachment(user_profile: UserProfile, attachment: Attachment) -> None:
+    try:
+        delete_message_attachment(attachment.path_id)
+    except Exception:
+        raise JsonableError(
+            _("An error occurred while deleting the attachment. Please try again later.")
+        )
+    ImageAttachment.objects.filter(path_id=attachment.path_id).delete()
+    attachment.delete()
+
+
+def validate_attachment_request_for_spectator_access(
+    realm: Realm, attachment: Attachment
+) -> bool | None:
+    """Returns whether the spectator is authorized to access the
+    attachment, or None if the attachment was deleted concurrently
+    with the request.
+    """
+    if attachment.realm != realm:
+        return False
+
+    if not realm.web_public_streams_enabled():
+        return False
+
+    # Update cached is_web_public property, if necessary.
+    if attachment.is_web_public is None:
+        # Fill the cache in a single query. This is important to avoid
+        # a potential race condition between checking and setting,
+        # where the attachment could have been moved again.
+        Attachment.objects.filter(id=attachment.id, is_web_public__isnull=True).update(
+            is_web_public=Exists(
+                Message.objects.filter(
+                    # Uses index: zerver_attachment_messages_attachment_id_message_id_key
+                    realm_id=realm.id,
+                    attachment=OuterRef("id"),
+                    recipient__stream__invite_only=False,
+                    recipient__stream__is_web_public=True,
+                ),
+            ),
+        )
+
+        try:
+            attachment.refresh_from_db()
+        except Attachment.DoesNotExist:
+            return None
+
+    if not attachment.is_web_public:
+        return False
+
+    if settings.RATE_LIMITING:
+        try:
+            from zerver.lib.rate_limiter import rate_limit_spectator_attachment_access_by_file
+
+            rate_limit_spectator_attachment_access_by_file(attachment.path_id)
+        except RateLimitedError:
+            return False
+
+    return True
+
+
+def validate_attachment_request(
+    maybe_user_profile: UserProfile | AnonymousUser,
+    path_id: str,
+    realm: Realm | None = None,
+) -> tuple[bool, Attachment | None]:
+    try:
+        attachment = Attachment.objects.get(path_id=path_id)
+    except Attachment.DoesNotExist:
+        return False, None
+
+    if isinstance(maybe_user_profile, AnonymousUser):
+        assert realm is not None
+        is_authorized = validate_attachment_request_for_spectator_access(realm, attachment)
+        if is_authorized is None:
+            # The attachment was deleted concurrently with this
+            # request; report it as not existing, just as if we
+            # had observed that initially.
+            return False, None
+        return is_authorized, attachment
+
+    user_profile = maybe_user_profile
+    assert isinstance(user_profile, UserProfile)
+
+    # Update cached is_realm_public property, if necessary.
+    if attachment.is_realm_public is None:
+        # Fill the cache in a single query. This is important to avoid
+        # a potential race condition between checking and setting,
+        # where the attachment could have been moved again.
+        Attachment.objects.filter(id=attachment.id, is_realm_public__isnull=True).update(
+            is_realm_public=Exists(
+                Message.objects.filter(
+                    # Uses index: zerver_attachment_messages_attachment_id_message_id_key
+                    realm_id=user_profile.realm_id,
+                    attachment=OuterRef("id"),
+                    recipient__stream__invite_only=False,
+                ),
+            ),
+        )
+        try:
+            attachment.refresh_from_db()
+        except Attachment.DoesNotExist:
+            # The attachment was deleted concurrently with this
+            # request; report it as not existing, just as if we
+            # had observed that initially.
+            return False, None
+
+    if user_profile.id == attachment.owner_id:
+        # If you own the file, you can access it.
+        return True, attachment
+    if (
+        attachment.is_realm_public
+        and attachment.realm == user_profile.realm
+        and user_profile.can_access_public_streams()
+    ):
+        # Any user in the realm can access realm-public files
+        return True, attachment
+
+    messages = attachment.messages.all().select_related("recipient")
+
+    usermessages_channel_ids = set()
+    usermessage_rows = UserMessage.objects.filter(
+        user_profile=user_profile, message__in=messages
+    ).select_related("message", "message__recipient")
+    for um in usermessage_rows:
+        if not um.message.is_channel_message:
+            # If the attachment was sent in a direct message or group direct
+            # message then anyone who received that message can access it.
+            return True, attachment
+        else:
+            usermessages_channel_ids.add(um.message.recipient.type_id)
+
+    # These are subscriptions to a channel one of the messages was sent to
+    subscribed_channel_ids = Subscription.objects.filter(
+        user_profile=user_profile,
+        active=True,
+        recipient__type=Recipient.STREAM,
+        recipient__in=[m.recipient_id for m in messages],
+    ).values_list("recipient__type_id", flat=True)
+
+    if usermessages_channel_ids.intersection(subscribed_channel_ids):
+        # If the attachment was sent in a channel with public
+        # or protected history and the user is still subscribed
+        # to the channel then anyone who received that message
+        # can access it.
+        return True, attachment
+
+    message_channel_ids = set()
+    for message in messages:
+        if message.is_channel_message:
+            message_channel_ids.add(message.recipient.type_id)
+
+    if len(message_channel_ids) == 0:
+        # If only DMs are relevant, return early here.
+        return False, attachment
+
+    # The remaining code path is slow but should only be relevant
+    # rarely: Users trying to view an attachment that was shared with
+    # at least one channel, but the user is not subscribed to any such
+    # channel. So it's not important that the accurate check for this
+    # corner case is somewhat more expensive to check groups-based
+    # permissions; we're no longer in a hot code path.
+    message_channels = Stream.objects.filter(id__in=message_channel_ids)
+    for channel in message_channels:
+        # The user didn't receive any of the messages that included
+        # this attachment. But they might still have access to it,
+        # if it was sent to a stream they are subscribed to where
+        # history is public to subscribers.
+        if channel.id in subscribed_channel_ids and channel.is_history_public_to_subscribers():
+            return True, attachment
+
+    user_recursive_group_ids = set(
+        get_recursive_membership_groups(user_profile).values_list("id", flat=True)
+    )
+    for channel in message_channels:
+        if is_user_in_groups_granting_content_access(channel, user_recursive_group_ids):
+            if channel.is_history_public_to_subscribers():
+                return True, attachment
+            # If the user had received the message at one point of
+            # time, but they are no longer subscribed to a stream
+            # with protected history. They can still access that
+            # message and it's attachment
+            elif channel.id in usermessages_channel_ids:
+                return True, attachment
+
+    return False, attachment
+
+
+def get_old_unclaimed_attachments(
+    weeks_ago: int,
+) -> tuple[QuerySet[Attachment], QuerySet[ArchivedAttachment]]:
+    """
+    The logic in this function is fairly tricky. The essence is that
+    a file should be cleaned up if and only if it not referenced by any
+    Message, ScheduledMessage or ArchivedMessage. The way to find that out is through the
+    Attachment and ArchivedAttachment tables.
+    The queries are complicated by the fact that an uploaded file
+    may have either only an Attachment row, only an ArchivedAttachment row,
+    or both - depending on whether some, all or none of the messages
+    linking to it have been archived.
+    """
+    delta_weeks_ago = timezone_now() - timedelta(weeks=weeks_ago)
+
+    # The Attachment vs ArchivedAttachment queries are asymmetric because only
+    # Attachment has the scheduled_messages relation.
+    old_attachments = (
+        Attachment.objects.alias(
+            has_other_messages=Exists(
+                ArchivedAttachment.objects.filter(id=OuterRef("id")).exclude(messages=None)
+            )
+        )
+        .filter(
+            messages=None,
+            scheduled_messages=None,
+            create_time__lt=delta_weeks_ago,
+            has_other_messages=False,
+        )
+        .order_by("id")
+    )
+    old_archived_attachments = (
+        ArchivedAttachment.objects.alias(
+            has_other_messages=Exists(
+                Attachment.objects.filter(id=OuterRef("id")).exclude(
+                    messages=None, scheduled_messages=None
+                )
+            )
+        )
+        .filter(
+            messages=None,
+            create_time__lt=delta_weeks_ago,
+            has_other_messages=False,
+        )
+        .order_by("id")
+    )
+
+    return old_attachments, old_archived_attachments

zerver/lib/avatar.py

@@ -0,0 +1,241 @@
+import logging
+import os
+import subprocess
+from urllib.parse import urljoin
+
+from django.conf import settings
+from django.contrib.staticfiles.storage import staticfiles_storage
+
+from zerver.lib.avatar_hash import (
+    gravatar_hash,
+    user_avatar_base_path_from_ids,
+    user_avatar_content_hash,
+    user_avatar_path,
+)
+from zerver.lib.thumbnail import DEFAULT_AVATAR_SIZE, MEDIUM_AVATAR_SIZE
+from zerver.lib.upload import get_avatar_url, write_jdenticon_avatars
+from zerver.lib.url_encoding import append_url_query_string
+from zerver.models import UserProfile
+from zerver.models.users import is_cross_realm_bot_email
+
+STATIC_AVATARS_DIR = "images/static_avatars/"
+
+DEFAULT_AVATAR_FILE = "images/default-avatar.png"
+
+logger = logging.getLogger(__name__)
+
+
+def avatar_url(
+    user_profile: UserProfile, medium: bool = False, client_gravatar: bool = False
+) -> str | None:
+    return get_avatar_field(
+        user_id=user_profile.id,
+        realm_id=user_profile.realm_id,
+        email=user_profile.delivery_email,
+        avatar_source=user_profile.avatar_source,
+        avatar_version=user_profile.avatar_version,
+        medium=medium,
+        client_gravatar=client_gravatar,
+    )
+
+
+def get_system_bots_avatar_file_name(email: str) -> str:
+    system_bot_avatar_name_map = {
+        settings.WELCOME_BOT: "welcome-bot",
+        settings.NOTIFICATION_BOT: "notification-bot",
+        settings.EMAIL_GATEWAY_BOT: "emailgateway",
+    }
+    return urljoin(STATIC_AVATARS_DIR, system_bot_avatar_name_map.get(email, "unknown"))
+
+
+def get_static_avatar_url(email: str, medium: bool) -> str:
+    avatar_file_name = get_system_bots_avatar_file_name(email)
+    avatar_file_name += "-medium.png" if medium else ".png"
+
+    if settings.DEBUG:
+        # This find call may not be cheap, so we only do it in the
+        # development environment to do an assertion.
+        from django.contrib.staticfiles.finders import find
+
+        if not find(avatar_file_name):
+            raise AssertionError(f"Unknown avatar file for: {email}")
+    elif settings.STATIC_ROOT and not staticfiles_storage.exists(avatar_file_name):
+        # Fallback for the case where no avatar exists; this should
+        # never happen in practice. This logic cannot be executed
+        # while STATIC_ROOT is not defined, so the above STATIC_ROOT
+        # check is important.
+        return DEFAULT_AVATAR_FILE
+
+    return staticfiles_storage.url(avatar_file_name)
+
+
+def get_avatar_field(
+    user_id: int,
+    realm_id: int,
+    email: str,
+    avatar_source: str,
+    avatar_version: int,
+    medium: bool,
+    client_gravatar: bool,
+) -> str | None:
+    """
+    Most of the parameters to this function map to fields
+    by the same name in UserProfile (avatar_source, realm_id,
+    email, etc.).
+
+    Then there are these:
+
+        medium - This means we want a medium-sized avatar. This can
+            affect the "s" parameter for gravatar avatars, or it
+            can give us something like foo-medium.png for
+            user-uploaded avatars.
+
+        client_gravatar - If the client can compute their own
+            gravatars, this will be set to True, and we'll avoid
+            computing them on the server (mostly to save bandwidth).
+    """
+
+    # System bots have hardcoded avatars
+    if is_cross_realm_bot_email(email):
+        return get_static_avatar_url(email, medium)
+
+    """
+    If our client knows how to calculate gravatar hashes, we
+    will return None and let the client compute the gravatar
+    url.
+    """
+    if (
+        client_gravatar
+        and settings.ENABLE_GRAVATAR
+        and avatar_source == UserProfile.AVATAR_FROM_GRAVATAR
+    ):
+        return None
+
+    """
+    If we get this far, we'll compute an avatar URL based on the
+    avatar source, and then we'll add version info to try to avoid
+    stale caches.
+    """
+    if avatar_source in [UserProfile.AVATAR_FROM_USER, UserProfile.AVATAR_FROM_JDENTICON]:
+        hash_key = user_avatar_base_path_from_ids(user_id, avatar_version, realm_id)
+        return get_avatar_url(hash_key, medium=medium)
+
+    return get_gravatar_url(
+        email=email,
+        avatar_version=avatar_version,
+        realm_id=realm_id,
+        medium=medium,
+    )
+
+
+def get_gravatar_url(email: str, avatar_version: int, realm_id: int, medium: bool = False) -> str:
+    url = _get_unversioned_gravatar_url(email, medium, realm_id)
+    return append_url_query_string(url, f"version={avatar_version:d}")
+
+
+def _get_unversioned_gravatar_url(email: str, medium: bool, realm_id: int) -> str:
+    use_gravatar = settings.ENABLE_GRAVATAR
+    if realm_id in settings.GRAVATAR_REALM_OVERRIDE:
+        use_gravatar = settings.GRAVATAR_REALM_OVERRIDE[realm_id]
+
+    if use_gravatar:
+        gravitar_query_suffix = f"&s={MEDIUM_AVATAR_SIZE}" if medium else ""
+        hash_key = gravatar_hash(email)
+        return f"https://secure.gravatar.com/avatar/{hash_key}?d=identicon{gravitar_query_suffix}"
+    elif settings.DEFAULT_AVATAR_URI is not None:
+        return settings.DEFAULT_AVATAR_URI
+    else:
+        return staticfiles_storage.url("images/default-avatar.png")
+
+
+def absolute_avatar_url(
+    user_profile: UserProfile,
+    # Pass `realm_url` to avoid a DB query when `user_profile.realm` isn't
+    # already loaded but the caller has realm available from another source.
+    realm_url: str | None = None,
+) -> str:
+    """
+    Absolute URLs are used to simplify logic for applications that
+    won't be served by browsers, such as rendering GCM notifications.
+    """
+    avatar = avatar_url(user_profile)
+    # avatar_url can return None if client_gravatar=True, however here we use the default value of False
+    assert avatar is not None
+    if realm_url is None:
+        realm_url = user_profile.realm.url
+    return urljoin(realm_url, avatar)
+
+
+def is_avatar_new(ldap_avatar: bytes, user_profile: UserProfile) -> bool:
+    new_avatar_hash = user_avatar_content_hash(ldap_avatar)
+
+    if user_profile.avatar_hash and user_profile.avatar_hash == new_avatar_hash:
+        # If an avatar exists and is the same as the new avatar,
+        # then, no need to change the avatar.
+        return False
+
+    return True
+
+
+def get_avatar_for_inaccessible_user() -> str:
+    return staticfiles_storage.url("images/unknown-user-avatar.png")
+
+
+def generate_avatar_jdenticon(input: str, medium: bool) -> bytes:
+    from zerver.lib.storage import static_path
+
+    jdenticon_path = (
+        static_path("webpack-bundles/jdenticon.js")
+        if settings.PRODUCTION
+        else os.path.join(settings.DEPLOY_ROOT, "node_modules/jdenticon/bin/jdenticon.js")
+    )
+    size = str(MEDIUM_AVATAR_SIZE if medium else DEFAULT_AVATAR_SIZE)
+    command = [
+        "node",
+        jdenticon_path,
+        input,
+        "-s",
+        size,
+        "-p",
+        "0",
+        "--lightness-color",
+        "0.3,0.7",
+        "--lightness-grayscale",
+        "0.3,0.7",
+    ]
+    try:
+        stdout = subprocess.check_output(command)
+        return stdout
+    except subprocess.CalledProcessError:  # nocoverage
+        logger.exception("Jdenticon generation failed for user_id: %s", input)
+        raise
+
+
+def generate_and_upload_jdenticon_avatar(
+    user_profile: UserProfile,
+    realm_uuid: str,
+    future: bool,
+) -> None:
+    # We use a combination of user ID and realm_uuid (salt) as the key
+    # for Jdenticon generation, so that clients that prefer to use computation
+    # instead of network to provide default avatars for users can do that
+    # in the future.
+    #
+    # Using only user ID (no salt) can result in situations where a person is
+    # part of multiple zulip servers, and they find same avatar for different
+    # people in different servers.
+    #
+    # Note: The key effectively changes when user IDs are renumbered when
+    # migrating between Zulip servers,
+    jdenticon_key = f"{realm_uuid}:{user_profile.id}"
+    image_data = generate_avatar_jdenticon(jdenticon_key, medium=False)
+    image_data_medium = generate_avatar_jdenticon(jdenticon_key, medium=True)
+    file_path = user_avatar_path(user_profile, future)
+
+    write_jdenticon_avatars(
+        file_path,
+        user_profile,
+        image_data=image_data,
+        image_data_medium=image_data_medium,
+        future=future,
+    )

zerver/lib/avatar_hash.py

@@ -0,0 +1,41 @@
+import hashlib
+
+from django.conf import settings
+
+from zerver.models import UserProfile
+
+
+def gravatar_hash(email: str) -> str:
+    """Compute the Gravatar hash for an email address."""
+    # Non-ASCII characters aren't permitted by the currently active e-mail
+    # RFCs. However, the IETF has published https://tools.ietf.org/html/rfc4952,
+    # outlining internationalization of email addresses, and regardless if we
+    # typo an address or someone manages to give us a non-ASCII address, let's
+    # not error out on it.
+    return hashlib.md5(email.lower().encode()).hexdigest()
+
+
+def user_avatar_hash(uid: str, version: str) -> str:
+    # WARNING: If this method is changed, you may need to do a migration
+    # similar to zerver/migrations/0060_move_avatars_to_be_uid_based.py .
+
+    # The salt prevents unauthenticated clients from enumerating the
+    # avatars of all users.
+    user_key = uid + ":" + version + ":" + settings.AVATAR_SALT
+    return hashlib.sha256(user_key.encode()).hexdigest()[:40]
+
+
+def user_avatar_path(user_profile: UserProfile, future: bool = False) -> str:
+    # 'future' is if this is for the current avatar version, or the next one.
+    return user_avatar_base_path_from_ids(
+        user_profile.id, user_profile.avatar_version + (1 if future else 0), user_profile.realm_id
+    )
+
+
+def user_avatar_base_path_from_ids(user_profile_id: int, version: int, realm_id: int) -> str:
+    user_id_hash = user_avatar_hash(str(user_profile_id), str(version))
+    return f"{realm_id}/{user_id_hash}"
+
+
+def user_avatar_content_hash(ldap_avatar: bytes) -> str:
+    return hashlib.sha256(ldap_avatar).hexdigest()

zerver/lib/bot_config.py

@@ -0,0 +1,80 @@
+import configparser
+import importlib
+import os
+from collections import defaultdict
+
+from django.conf import settings
+from django.db.models import F, Sum
+from django.db.models.functions import Length
+
+from zerver.models import BotConfigData, UserProfile
+
+
+class ConfigError(Exception):
+    pass
+
+
+def get_bot_config(bot_profile: UserProfile) -> dict[str, str]:
+    entries = BotConfigData.objects.filter(bot_profile=bot_profile)
+    if not entries:
+        raise ConfigError("No config data available.")
+    return {entry.key: entry.value for entry in entries}
+
+
+def get_bot_configs(bot_profile_ids: list[int]) -> dict[int, dict[str, str]]:
+    if not bot_profile_ids:
+        return {}
+    entries = BotConfigData.objects.filter(bot_profile_id__in=bot_profile_ids)
+    entries_by_uid: dict[int, dict[str, str]] = defaultdict(dict)
+    for entry in entries:
+        entries_by_uid[entry.bot_profile_id].update({entry.key: entry.value})
+    return entries_by_uid
+
+
+def get_bot_config_size(bot_profile: UserProfile, key: str | None = None) -> int:
+    if key is None:
+        return (
+            BotConfigData.objects.filter(bot_profile=bot_profile)
+            .annotate(key_size=Length("key"), value_size=Length("value"))
+            .aggregate(sum=Sum(F("key_size") + F("value_size")))["sum"]
+            or 0
+        )
+    else:
+        try:
+            return len(key) + len(BotConfigData.objects.get(bot_profile=bot_profile, key=key).value)
+        except BotConfigData.DoesNotExist:
+            return 0
+
+
+def set_bot_config(bot_profile: UserProfile, key: str, value: str) -> None:
+    config_size_limit = settings.BOT_CONFIG_SIZE_LIMIT
+    old_entry_size = get_bot_config_size(bot_profile, key)
+    new_entry_size = len(key) + len(value)
+    old_config_size = get_bot_config_size(bot_profile)
+    new_config_size = old_config_size + (new_entry_size - old_entry_size)
+    if new_config_size > config_size_limit:
+        raise ConfigError(
+            f"Cannot store configuration. Request would require {new_config_size} characters. "
+            f"The current configuration size limit is {config_size_limit} characters."
+        )
+    obj, created = BotConfigData.objects.get_or_create(
+        bot_profile=bot_profile, key=key, defaults={"value": value}
+    )
+    if not created:
+        obj.value = value
+        obj.save()
+
+
+def load_bot_config_template(bot: str) -> dict[str, str]:
+    bot_module_name = f"zulip_bots.bots.{bot}"
+    bot_module = importlib.import_module(bot_module_name)
+    assert bot_module.__file__ is not None
+    bot_module_path = os.path.dirname(bot_module.__file__)
+    config_path = os.path.join(bot_module_path, f"{bot}.conf")
+    if os.path.isfile(config_path):
+        config = configparser.ConfigParser()
+        with open(config_path) as conf:
+            config.read_file(conf)
+        return dict(config.items(bot))
+    else:
+        return {}

zerver/lib/bot_lib.py

@@ -0,0 +1,166 @@
+import importlib
+import json
+from collections.abc import Callable
+from typing import Any
+
+from django.conf import settings
+from django.utils.translation import gettext as _
+from zulip_bots.lib import BotIdentity, RateLimit
+
+from zerver.actions.message_flags import do_update_message_flags
+from zerver.actions.message_send import (
+    internal_send_group_direct_message,
+    internal_send_private_message,
+    internal_send_stream_message_by_name,
+)
+from zerver.lib.bot_config import ConfigError, get_bot_config
+from zerver.lib.bot_storage import (
+    get_bot_storage,
+    is_key_in_bot_storage,
+    remove_bot_storage,
+    set_bot_storage,
+)
+from zerver.lib.integrations import EMBEDDED_BOTS
+from zerver.lib.topic import get_topic_from_message_info
+from zerver.models import UserProfile
+from zerver.models.users import get_active_user
+
+
+def get_bot_handler(service_name: str) -> Any:
+    # Check that this service is present in EMBEDDED_BOTS, add exception handling.
+    configured_service = ""
+    for embedded_bot_service in EMBEDDED_BOTS:
+        if service_name == embedded_bot_service.name:
+            configured_service = embedded_bot_service.name
+    if not configured_service:
+        return None
+    bot_module_name = f"zulip_bots.bots.{configured_service}.{configured_service}"
+    bot_module: Any = importlib.import_module(bot_module_name)
+    return bot_module.handler_class()
+
+
+class StateHandler:
+    storage_size_limit: int = settings.USER_STATE_SIZE_LIMIT
+
+    def __init__(self, user_profile: UserProfile) -> None:
+        self.user_profile = user_profile
+        self.marshal: Callable[[object], str] = json.dumps
+        self.demarshal: Callable[[str], object] = json.loads
+
+    def get(self, key: str) -> object:
+        return self.demarshal(get_bot_storage(self.user_profile, key))
+
+    def put(self, key: str, value: object) -> None:
+        set_bot_storage(self.user_profile, [(key, self.marshal(value))])
+
+    def remove(self, key: str) -> None:
+        remove_bot_storage(self.user_profile, [key])
+
+    def contains(self, key: str) -> bool:
+        return is_key_in_bot_storage(self.user_profile, key)
+
+
+class EmbeddedBotQuitError(Exception):
+    pass
+
+
+class EmbeddedBotEmptyRecipientsListError(Exception):
+    pass
+
+
+class EmbeddedBotHandler:
+    def __init__(self, user_profile: UserProfile) -> None:
+        # Only expose a subset of our UserProfile's functionality
+        self.user_profile = user_profile
+        self._rate_limit = RateLimit(20, 5)
+        self.full_name = user_profile.full_name
+        self.email = user_profile.email
+        self.storage = StateHandler(user_profile)
+        self.user_id = user_profile.id
+
+    def identity(self) -> BotIdentity:
+        return BotIdentity(self.full_name, self.email)
+
+    def react(self, message: dict[str, Any], emoji_name: str) -> dict[str, Any]:
+        return {}  # Not implemented
+
+    def send_message(self, message: dict[str, Any]) -> dict[str, Any]:
+        if not self._rate_limit.is_legal():
+            self._rate_limit.show_error_and_exit()
+
+        if message["type"] == "stream":
+            message_id = internal_send_stream_message_by_name(
+                self.user_profile.realm,
+                self.user_profile,
+                message["to"],
+                message["topic"],
+                message["content"],
+            )
+            return {"id": message_id}
+
+        assert message["type"] == "private"
+        # Ensure that it's a comma-separated list, even though the
+        # usual 'to' field could be either a List[str] or a str.
+        recipients = ",".join(message["to"]).split(",")
+
+        if len(message["to"]) == 0:
+            raise EmbeddedBotEmptyRecipientsListError(_("Message must have recipients!"))
+        elif len(message["to"]) == 1:
+            recipient_user = get_active_user(recipients[0], self.user_profile.realm)
+            message_id = internal_send_private_message(
+                self.user_profile, recipient_user, message["content"]
+            )
+        else:
+            message_id = internal_send_group_direct_message(
+                self.user_profile.realm, self.user_profile, message["content"], emails=recipients
+            )
+        return {"id": message_id}
+
+    def send_reply(
+        self, message: dict[str, Any], response: str, widget_content: str | None = None
+    ) -> dict[str, Any]:
+        if message["type"] == "private":
+            result = self.send_message(
+                dict(
+                    type="private",
+                    to=[x["email"] for x in message["display_recipient"]],
+                    content=response,
+                    sender_email=message["sender_email"],
+                )
+            )
+        else:
+            result = self.send_message(
+                dict(
+                    type="stream",
+                    to=message["display_recipient"],
+                    topic=get_topic_from_message_info(message),
+                    content=response,
+                    sender_email=message["sender_email"],
+                )
+            )
+        return {"id": result["id"]}
+
+    def update_message(self, message: dict[str, Any]) -> None:
+        pass  # Not implemented
+
+    # The bot_name argument exists only to comply with ExternalBotHandler.get_config_info().
+    def get_config_info(self, bot_name: str, optional: bool = False) -> dict[str, str]:
+        try:
+            return get_bot_config(self.user_profile)
+        except ConfigError:
+            if optional:
+                return {}
+            raise
+
+    def quit(self, message: str = "") -> None:
+        raise EmbeddedBotQuitError(message)
+
+
+def do_flag_message_triggered_bots_messages_as_processed(
+    bot_profile: UserProfile, message_ids: list[int]
+) -> None:
+    assert (
+        bot_profile.is_bot is True
+        and bot_profile.bot_type in UserProfile.MESSAGE_TRIGGERED_BOT_TYPES
+    )
+    do_update_message_flags(bot_profile, "add", "read", message_ids)

zerver/lib/bot_storage.py

@@ -0,0 +1,69 @@
+from django.conf import settings
+from django.db.models import F, Sum
+from django.db.models.functions import Length
+
+from zerver.models import BotStorageData, UserProfile
+
+
+class StateError(Exception):
+    pass
+
+
+def get_bot_storage(bot_profile: UserProfile, key: str) -> str:
+    try:
+        return BotStorageData.objects.get(bot_profile=bot_profile, key=key).value
+    except BotStorageData.DoesNotExist:
+        raise StateError("Key does not exist.")
+
+
+def get_bot_storage_size(bot_profile: UserProfile, key: str | None = None) -> int:
+    if key is None:
+        return (
+            BotStorageData.objects.filter(bot_profile=bot_profile)
+            .annotate(key_size=Length("key"), value_size=Length("value"))
+            .aggregate(sum=Sum(F("key_size") + F("value_size")))["sum"]
+            or 0
+        )
+    else:
+        try:
+            return len(key) + len(
+                BotStorageData.objects.get(bot_profile=bot_profile, key=key).value
+            )
+        except BotStorageData.DoesNotExist:
+            return 0
+
+
+def set_bot_storage(bot_profile: UserProfile, entries: list[tuple[str, str]]) -> None:
+    storage_size_limit = settings.USER_STATE_SIZE_LIMIT
+    storage_size_difference = 0
+    for key, value in entries:
+        assert isinstance(key, str), "Key type should be str."
+        assert isinstance(value, str), "Value type should be str."
+        storage_size_difference += (len(key) + len(value)) - get_bot_storage_size(bot_profile, key)
+    new_storage_size = get_bot_storage_size(bot_profile) + storage_size_difference
+    if new_storage_size > storage_size_limit:
+        raise StateError(
+            f"Request exceeds storage limit by {new_storage_size - storage_size_limit} characters. The limit is {storage_size_limit} characters."
+        )
+    else:
+        for key, value in entries:
+            BotStorageData.objects.update_or_create(
+                bot_profile=bot_profile, key=key, defaults={"value": value}
+            )
+
+
+def remove_bot_storage(bot_profile: UserProfile, keys: list[str]) -> None:
+    queryset = BotStorageData.objects.filter(bot_profile=bot_profile, key__in=keys)
+    if queryset.count() < len(keys):
+        raise StateError("Key does not exist.")
+    queryset.delete()
+
+
+def is_key_in_bot_storage(bot_profile: UserProfile, key: str) -> bool:
+    return BotStorageData.objects.filter(bot_profile=bot_profile, key=key).exists()
+
+
+def get_keys_in_bot_storage(bot_profile: UserProfile) -> list[str]:
+    return list(
+        BotStorageData.objects.filter(bot_profile=bot_profile).values_list("key", flat=True)
+    )

zerver/lib/bulk_create.py

@@ -0,0 +1,218 @@
+from collections.abc import Collection, Iterable
+from typing import Any
+
+from django.db.models import QuerySet
+from django.utils.timezone import now as timezone_now
+
+from zerver.lib.avatar import generate_and_upload_jdenticon_avatar
+from zerver.lib.create_user import create_user_profile, get_display_email_address
+from zerver.lib.initial_password import initial_password
+from zerver.lib.streams import (
+    get_default_values_for_stream_permission_group_settings,
+    render_stream_description,
+)
+from zerver.models import (
+    NamedUserGroup,
+    Realm,
+    RealmAuditLog,
+    RealmUserDefault,
+    Recipient,
+    Stream,
+    UserGroupMembership,
+    UserProfile,
+)
+from zerver.models.groups import SystemGroups
+from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.models.streams import StreamTopicsPolicyEnum
+
+
+def bulk_create_users(
+    realm: Realm,
+    users_raw: set[tuple[str, str, bool]],
+    bot_type: int | None = None,
+    bot_owner: UserProfile | None = None,
+    tos_version: str | None = None,
+    timezone: str = "",
+    is_imported_stub: bool = False,
+) -> None:
+    """
+    Creates and saves a UserProfile with the given email.
+    Has some code based off of UserManage.create_user, but doesn't .save()
+    """
+    existing_users = frozenset(
+        UserProfile.objects.filter(realm=realm).values_list("email", flat=True)
+    )
+    users = sorted(user_raw for user_raw in users_raw if user_raw[0] not in existing_users)
+
+    realm_user_default = RealmUserDefault.objects.get(realm=realm)
+    if bot_type is None:
+        email_address_visibility = realm_user_default.email_address_visibility
+    else:
+        # There is no privacy motivation for limiting access to bot email addresses,
+        # so we hardcode them to EMAIL_ADDRESS_VISIBILITY_EVERYONE.
+        email_address_visibility = UserProfile.EMAIL_ADDRESS_VISIBILITY_EVERYONE
+
+    # Now create user_profiles
+    profiles_to_create: list[UserProfile] = []
+    for email, full_name, active in users:
+        profile = create_user_profile(
+            realm,
+            email,
+            initial_password(email),
+            active,
+            bot_type,
+            full_name,
+            bot_owner,
+            False,
+            tos_version,
+            timezone,
+            is_imported_stub=is_imported_stub,
+            default_language=realm.default_language,
+            email_address_visibility=email_address_visibility,
+        )
+
+        if bot_type is None:
+            # This block simulates copy_default_settings from
+            # zerver/lib/create_user.py.
+            #
+            # We cannot use 'copy_default_settings' directly here
+            # because it calls '.save' after copying the settings, and
+            # we are bulk creating the objects here instead.
+            for settings_name in RealmUserDefault.property_types:
+                if settings_name in ["default_language", "enable_login_emails"]:
+                    continue
+                value = getattr(realm_user_default, settings_name)
+                setattr(profile, settings_name, value)
+        profiles_to_create.append(profile)
+
+    if email_address_visibility == UserProfile.EMAIL_ADDRESS_VISIBILITY_EVERYONE:
+        UserProfile.objects.bulk_create(profiles_to_create)
+    else:
+        for user_profile in profiles_to_create:
+            user_profile.email = user_profile.delivery_email
+
+        UserProfile.objects.bulk_create(profiles_to_create)
+
+        for user_profile in profiles_to_create:
+            user_profile.email = get_display_email_address(user_profile)
+        UserProfile.objects.bulk_update(profiles_to_create, ["email"])
+
+    for user_profile in profiles_to_create:
+        generate_and_upload_jdenticon_avatar(user_profile, str(realm.uuid), future=False)
+
+    RealmAuditLog.objects.bulk_create(
+        RealmAuditLog(
+            realm=realm,
+            modified_user=profile_,
+            event_type=AuditLogEventType.USER_CREATED,
+            event_time=profile_.date_joined,
+        )
+        for profile_ in profiles_to_create
+    )
+
+    full_members_system_group = NamedUserGroup.objects.get(
+        name=SystemGroups.FULL_MEMBERS, realm_for_sharding=realm, is_system_group=True
+    )
+    members_system_group = NamedUserGroup.objects.get(
+        name=SystemGroups.MEMBERS, realm_for_sharding=realm, is_system_group=True
+    )
+    group_memberships_to_create: list[UserGroupMembership] = []
+    for user_profile in profiles_to_create:
+        # All users are members since this function is only used to create bots
+        # and test and development environment users.
+        assert user_profile.role == UserProfile.ROLE_MEMBER
+        group_memberships_to_create.append(
+            UserGroupMembership(user_profile=user_profile, user_group=members_system_group)
+        )
+        if not user_profile.is_provisional_member:
+            group_memberships_to_create.append(
+                UserGroupMembership(user_profile=user_profile, user_group=full_members_system_group)
+            )
+
+    UserGroupMembership.objects.bulk_create(group_memberships_to_create)
+    now = timezone_now()
+    RealmAuditLog.objects.bulk_create(
+        RealmAuditLog(
+            realm=realm,
+            modified_user=membership.user_profile,
+            modified_user_group=membership.user_group.named_user_group,
+            event_type=AuditLogEventType.USER_GROUP_DIRECT_USER_MEMBERSHIP_ADDED,
+            event_time=now,
+            acting_user=None,
+        )
+        for membership in group_memberships_to_create
+    )
+
+
+def bulk_set_stream_recipient_fields(
+    streams: Collection[Stream] | QuerySet[Stream],
+    recipients: Iterable[Recipient] | None = None,
+) -> None:
+    if recipients is None:
+        stream_ids = [stream.id for stream in streams]
+        recipients = Recipient.objects.filter(type=Recipient.STREAM, type_id__in=stream_ids)
+
+    streams_dict = {stream.id: stream for stream in streams}
+
+    streams_to_update = set()
+    for recipient in recipients:
+        assert recipient.type == Recipient.STREAM
+        stream = streams_dict.get(recipient.type_id)
+        if stream is not None:
+            stream.recipient = recipient
+            streams_to_update.add(stream)
+    Stream.objects.bulk_update(streams_to_update, ["recipient"])
+
+
+# This is only sed in populate_db, so doesn't really need tests
+def bulk_create_streams(realm: Realm, stream_dict: dict[str, dict[str, Any]]) -> None:  # nocoverage
+    existing_streams = {
+        name.lower() for name in Stream.objects.filter(realm=realm).values_list("name", flat=True)
+    }
+    streams_to_create: list[Stream] = []
+    for name, options in stream_dict.items():
+        creator = options.get("creator", None)
+        if name.lower() not in existing_streams:
+            stream = Stream(
+                realm=realm,
+                name=name,
+                description=options["description"],
+                rendered_description=render_stream_description(options["description"], realm),
+                invite_only=options.get("invite_only", False),
+                history_public_to_subscribers=options.get("history_public_to_subscribers", True),
+                is_web_public=options.get("is_web_public", False),
+                creator=options.get("creator", None),
+                folder_id=options.get("folder_id", None),
+                topics_policy=options.get("topics_policy", StreamTopicsPolicyEnum.inherit.value),
+                **get_default_values_for_stream_permission_group_settings(realm, creator),
+            )
+            if "can_send_message_group" in options:
+                stream.can_send_message_group = options["can_send_message_group"]
+
+            streams_to_create.append(stream)
+    # Sort streams by name before creating them so that we can have a
+    # reliable ordering of `stream_id` across different python versions.
+    # This is required for test fixtures which contain `stream_id`. Prior
+    # to python 3.3 hashes were not randomized but after a security fix
+    # hash randomization was enabled in python 3.3 which made iteration
+    # of dictionaries and sets completely unpredictable. Here the order
+    # of elements while iterating `stream_dict` will be completely random
+    # for python 3.3 and later versions.
+    streams_to_create.sort(key=lambda x: x.name)
+    Stream.objects.bulk_create(streams_to_create)
+
+    recipients_to_create = [
+        Recipient(type_id=stream["id"], type=Recipient.STREAM)
+        for stream in Stream.objects.filter(realm=realm).values("id", "name")
+        if stream["name"].lower() not in existing_streams
+    ]
+    Recipient.objects.bulk_create(recipients_to_create)
+
+    bulk_set_stream_recipient_fields(streams_to_create, recipients_to_create)
+
+
+def create_users(
+    realm: Realm, name_list: Iterable[tuple[str, str]], bot_type: int | None = None
+) -> None:
+    user_set = {(email, full_name, True) for full_name, email in name_list}
+    bulk_create_users(realm, user_set, bot_type)

zerver/lib/cache.py

@@ -0,0 +1,894 @@
+# See https://zulip.readthedocs.io/en/latest/subsystems/caching.html for docs
+import hashlib
+import logging
+import os
+import re
+import secrets
+import sys
+import time
+import traceback
+from collections.abc import Callable, Iterable, Iterator, Sequence
+from functools import _lru_cache_wrapper, lru_cache, wraps
+from itertools import islice, product
+from typing import TYPE_CHECKING, Any, Generic, TypeVar
+
+from bmemcached.exceptions import MemcachedException
+from django.conf import settings
+from django.core.cache import caches
+from django.core.cache.backends.base import BaseCache
+from django.db.models import Q, QuerySet
+from typing_extensions import ParamSpec
+
+from scripts.lib.zulip_tools import DEPLOYMENTS_DIR, get_recent_deployments
+
+if TYPE_CHECKING:
+    # These modules have to be imported for type annotations but
+    # they cannot be imported at runtime due to cyclic dependency.
+    from zerver.models import Attachment, Message, MutedUser, Realm, Stream, SubMessage, UserProfile
+
+MEMCACHED_MAX_KEY_LENGTH = 250
+
+ParamT = ParamSpec("ParamT")
+ReturnT = TypeVar("ReturnT")
+
+logger = logging.getLogger()
+
+remote_cache_time_start = 0.0
+remote_cache_total_time = 0.0
+remote_cache_total_requests = 0
+
+
+def get_remote_cache_time() -> float:
+    return remote_cache_total_time
+
+
+def get_remote_cache_requests() -> int:
+    return remote_cache_total_requests
+
+
+def remote_cache_stats_start() -> None:
+    global remote_cache_time_start
+    remote_cache_time_start = time.time()
+
+
+def remote_cache_stats_finish() -> None:
+    global remote_cache_total_time, remote_cache_total_requests
+    remote_cache_total_requests += 1
+    remote_cache_total_time += time.time() - remote_cache_time_start
+
+
+def update_cached_cache_key_prefixes() -> list[str]:
+    # Clearing cache keys happens for all cache prefixes at once.
+    # Because the list of cache prefixes can only be derived from
+    # reading disk, we cache the list of cache prefixes, itself, in
+    # the cache.
+    found_prefixes: set[str] = set()
+    for deploy_dir in get_recent_deployments(None):
+        filename = os.path.join(deploy_dir, "var", "remote_cache_prefix")
+        if not os.path.exists(filename):
+            continue
+        with open(filename) as f:
+            found_prefixes.add(f.readline().removesuffix("\n"))
+
+    # We reach into the bmemcached client directly to do this set
+    # *without* compression, so that changes in our choice of
+    # bmemcached compression algorithm are always
+    # backwards-compatible.
+    caches["default"]._cache.set(  # type: ignore[attr-defined] # not in stubs
+        caches["default"].make_key("cache_key_prefixes"),
+        list(found_prefixes),
+        60 * 60 * 24,  # 24h
+        compress_level=0,
+    )
+    return list(found_prefixes)
+
+
+def get_or_create_key_prefix() -> str:
+    if settings.PUPPETEER_TESTS:
+        # This sets the prefix for the benefit of the Puppeteer tests.
+        #
+        # Having a fixed key is OK since we don't support running
+        # multiple copies of the Puppeteer tests at the same time anyway.
+        return "puppeteer_tests:"
+    elif settings.TEST_SUITE:
+        # The Python tests overwrite KEY_PREFIX on each test, but use
+        # this codepath as well, just to save running the more complex
+        # code below for reading the normal key prefix.
+        return "django_tests_unused:"
+
+    # directory `var` should exist in production
+    os.makedirs(os.path.join(settings.DEPLOY_ROOT, "var"), exist_ok=True)
+
+    filename = os.path.join(settings.DEPLOY_ROOT, "var", "remote_cache_prefix")
+    try:
+        with open(filename, "x") as f:
+            prefix = secrets.token_hex(16) + ":"
+            f.write(prefix + "\n")
+    except FileExistsError:
+        tries = 1
+        while tries < 10:
+            with open(filename) as f:
+                prefix = f.readline().removesuffix("\n")
+            if len(prefix) == 33:
+                break
+            tries += 1
+            prefix = ""
+            time.sleep(0.5)
+
+    if not prefix:
+        print("Could not read remote cache key prefix file")
+        sys.exit(1)
+
+    update_cached_cache_key_prefixes()
+    return prefix
+
+
+KEY_PREFIX: str = get_or_create_key_prefix()
+
+
+def get_all_cache_key_prefixes() -> list[str]:
+    if not settings.PRODUCTION or not os.path.exists(DEPLOYMENTS_DIR):
+        return [KEY_PREFIX]
+    return get_all_deployment_cache_key_prefixes()
+
+
+def get_all_deployment_cache_key_prefixes() -> list[str]:
+    stored_prefixes = caches["default"].get("cache_key_prefixes")
+    if stored_prefixes:
+        return stored_prefixes
+
+    return update_cached_cache_key_prefixes()
+
+
+def bounce_key_prefix_for_testing(test_name: str) -> None:
+    global KEY_PREFIX
+    KEY_PREFIX = test_name + ":" + str(os.getpid()) + ":"
+    # We are taking the hash of the KEY_PREFIX to decrease the size of the key.
+    # Memcached keys should have a length of less than 250.
+    KEY_PREFIX = hashlib.sha1(KEY_PREFIX.encode()).hexdigest() + ":"
+
+
+def get_cache_backend(cache_name: str | None) -> BaseCache:
+    if cache_name is None:
+        cache_name = "default"
+    return caches[cache_name]
+
+
+def cache_with_key(
+    keyfunc: Callable[ParamT, str],
+    cache_name: str | None = None,
+    timeout: int | None = None,
+    pickled_tupled: bool = True,
+) -> Callable[[Callable[ParamT, ReturnT]], Callable[ParamT, ReturnT]]:
+    """Decorator which applies Django caching to a function.
+
+    Decorator argument is a function which computes a cache key
+    from the original function's arguments.  You are responsible
+    for avoiding collisions with other uses of this decorator or
+    other uses of caching."""
+
+    def decorator(func: Callable[ParamT, ReturnT]) -> Callable[ParamT, ReturnT]:
+        @wraps(func)
+        def func_with_caching(*args: ParamT.args, **kwargs: ParamT.kwargs) -> ReturnT:
+            key = keyfunc(*args, **kwargs)
+
+            try:
+                val = cache_get(key, cache_name=cache_name)
+            except InvalidCacheKeyError:
+                stack_trace = traceback.format_exc()
+                log_invalid_cache_keys(stack_trace, [key])
+                return func(*args, **kwargs)
+
+            # Values are singleton tuples so that we can distinguish a
+            # result of None from a missing key.  Setting
+            # pickled_tupled=False avoids pickling the result (if it's
+            # a raw string or bytes) at the cost of losing this
+            # distinction.
+            if val is not None and pickled_tupled:
+                return val[0]
+
+            val = func(*args, **kwargs)
+            if isinstance(val, QuerySet):
+                logging.error(
+                    "cache_with_key attempted to store a full QuerySet object -- declining to cache",
+                    stack_info=True,
+                )
+            else:
+                cache_set(
+                    key, val, cache_name=cache_name, timeout=timeout, pickled_tupled=pickled_tupled
+                )
+
+            return val
+
+        return func_with_caching
+
+    return decorator
+
+
+class InvalidCacheKeyError(Exception):
+    pass
+
+
+def log_invalid_cache_keys(stack_trace: str, key: list[str]) -> None:
+    logger.warning(
+        "Invalid cache key used: %s\nStack trace: %s\n",
+        key,
+        stack_trace,
+    )
+
+
+def validate_cache_key(key: str, auto_prepend_prefix: bool = True) -> None:
+    if auto_prepend_prefix and not key.startswith(KEY_PREFIX):
+        key = KEY_PREFIX + key
+
+    # Theoretically memcached can handle non-ascii characters
+    # and only "control" characters are strictly disallowed, see:
+    # https://github.com/memcached/memcached/blob/master/doc/protocol.txt
+    # However, limiting the characters we allow in keys simiplifies things,
+    # and anyway we use a hash function when forming some keys to ensure
+    # the resulting keys fit the regex below.
+    # The regex checks "all characters between ! and ~ in the ascii table",
+    # which happens to be the set of all "nice" ascii characters.
+    if not bool(re.fullmatch(r"([!-~])+", key)):
+        raise InvalidCacheKeyError("Invalid characters in the cache key: " + key)
+    if len(key) > MEMCACHED_MAX_KEY_LENGTH:
+        raise InvalidCacheKeyError(f"Cache key too long: {key} Length: {len(key)}")
+
+
+def cache_set(
+    key: str,
+    val: Any,
+    cache_name: str | None = None,
+    timeout: int | None = None,
+    pickled_tupled: bool = True,
+) -> None:
+    final_key = KEY_PREFIX + key
+    validate_cache_key(final_key)
+
+    remote_cache_stats_start()
+    cache_backend = get_cache_backend(cache_name)
+    if pickled_tupled:
+        val = (val,)
+    try:
+        cache_backend.set(final_key, val, timeout=timeout)
+    except MemcachedException:
+        logger.exception("Error while storing to cache")
+    remote_cache_stats_finish()
+
+
+def cache_get(key: str, cache_name: str | None = None) -> Any:
+    final_key = KEY_PREFIX + key
+    validate_cache_key(final_key)
+
+    remote_cache_stats_start()
+    cache_backend = get_cache_backend(cache_name)
+    ret = cache_backend.get(final_key)
+    remote_cache_stats_finish()
+    return ret
+
+
+def cache_get_many(keys: list[str], cache_name: str | None = None) -> dict[str, Any]:
+    keys = [KEY_PREFIX + key for key in keys]
+    for key in keys:
+        validate_cache_key(key)
+    remote_cache_stats_start()
+    ret = get_cache_backend(cache_name).get_many(keys)
+    remote_cache_stats_finish()
+    return {key.removeprefix(KEY_PREFIX): value for key, value in ret.items()}
+
+
+def safe_cache_get_many(keys: list[str], cache_name: str | None = None) -> dict[str, Any]:
+    """Variant of cache_get_many that drops any keys that fail
+    validation, rather than throwing an exception visible to the
+    caller."""
+    try:
+        # Almost always the keys will all be correct, so we just try
+        # to do normal cache_get_many to avoid the overhead of
+        # validating all the keys here.
+        return cache_get_many(keys, cache_name)
+    except InvalidCacheKeyError:
+        stack_trace = traceback.format_exc()
+        good_keys, bad_keys = filter_good_and_bad_keys(keys)
+
+        log_invalid_cache_keys(stack_trace, bad_keys)
+        return cache_get_many(good_keys, cache_name)
+
+
+def cache_set_many(
+    items: dict[str, Any], cache_name: str | None = None, timeout: int | None = None
+) -> None:
+    new_items = {}
+    for key, item in items.items():
+        new_key = KEY_PREFIX + key
+        validate_cache_key(new_key)
+        new_items[new_key] = item
+    items = new_items
+    remote_cache_stats_start()
+    try:
+        get_cache_backend(cache_name).set_many(items, timeout=timeout)
+    except MemcachedException:
+        logger.exception("Error while storing to cache")
+    remote_cache_stats_finish()
+
+
+def safe_cache_set_many(
+    items: dict[str, Any], cache_name: str | None = None, timeout: int | None = None
+) -> None:
+    """Variant of cache_set_many that drops saving any keys that fail
+    validation, rather than throwing an exception visible to the
+    caller."""
+    try:
+        # Almost always the keys will all be correct, so we just try
+        # to do normal cache_set_many to avoid the overhead of
+        # validating all the keys here.
+        return cache_set_many(items, cache_name, timeout)
+    except InvalidCacheKeyError:
+        stack_trace = traceback.format_exc()
+
+        good_keys, bad_keys = filter_good_and_bad_keys(list(items.keys()))
+        log_invalid_cache_keys(stack_trace, bad_keys)
+
+        good_items = {key: items[key] for key in good_keys}
+        return cache_set_many(good_items, cache_name, timeout)
+
+
+def cache_delete(key: str, cache_name: str | None = None) -> None:
+    cache_delete_many([key], cache_name)
+
+
+def cache_delete_many(items: Iterable[str], cache_name: str | None = None) -> None:
+    remote_cache_stats_start()
+    keys = iter(e[0] + e[1] for e in product(get_all_cache_key_prefixes(), items))
+    while True:
+        batch = tuple(islice(keys, 10000))
+        if not batch:
+            break
+        for key in batch:
+            validate_cache_key(key, auto_prepend_prefix=False)
+        get_cache_backend(cache_name).delete_many(batch)
+    remote_cache_stats_finish()
+
+
+def filter_good_and_bad_keys(keys: list[str]) -> tuple[list[str], list[str]]:
+    good_keys = []
+    bad_keys = []
+    for key in keys:
+        try:
+            validate_cache_key(key)
+            good_keys.append(key)
+        except InvalidCacheKeyError:
+            bad_keys.append(key)
+
+    return good_keys, bad_keys
+
+
+# Generic_bulk_cached fetch and its helpers.  We start with declaring
+# a few type variables that help define its interface.
+
+# Type for the cache's keys; will typically be int or str.
+ObjKT = TypeVar("ObjKT")
+
+# Type for items to be fetched from the database (e.g. a Django model object)
+ItemT = TypeVar("ItemT")
+
+# Type for items to be stored in the cache (e.g. a dictionary serialization).
+# Will equal ItemT unless a cache_transformer is specified.
+CacheItemT = TypeVar("CacheItemT")
+
+# Type for compressed items for storage in the cache.  For
+# serializable objects, will be the object; if encoded, bytes.
+CompressedItemT = TypeVar("CompressedItemT")
+
+
+# Required arguments are as follows:
+# * object_ids: The list of object ids to look up
+# * cache_key_function: object_id => cache key
+# * query_function: [object_ids] => [objects from database]
+# * setter: Function to call before storing items to cache (e.g. compression)
+# * extractor: Function to call on items returned from cache
+#   (e.g. decompression).  Should be the inverse of the setter
+#   function.
+# * id_fetcher: Function mapping an object from database => object_id
+#   (in case we're using a key more complex than obj.id)
+# * cache_transformer: Function mapping an object from database =>
+#   value for cache (in case the values that we're caching are some
+#   function of the objects, not the objects themselves)
+def generic_bulk_cached_fetch(
+    cache_key_function: Callable[[ObjKT], str],
+    query_function: Callable[[list[ObjKT]], Iterable[ItemT]],
+    object_ids: Sequence[ObjKT],
+    *,
+    extractor: Callable[[CompressedItemT], CacheItemT],
+    setter: Callable[[CacheItemT], CompressedItemT],
+    id_fetcher: Callable[[ItemT], ObjKT],
+    cache_transformer: Callable[[ItemT], CacheItemT],
+    pickled_tupled: bool = True,
+) -> dict[ObjKT, CacheItemT]:
+    if len(object_ids) == 0:
+        # Nothing to fetch.
+        return {}
+
+    if pickled_tupled:
+        return generic_bulk_cached_fetch(
+            cache_key_function,
+            query_function,
+            object_ids,
+            extractor=lambda val: extractor(val[0]),
+            setter=lambda val: (setter(val),),
+            id_fetcher=id_fetcher,
+            cache_transformer=cache_transformer,
+            pickled_tupled=False,
+        )
+
+    cache_keys: dict[ObjKT, str] = {}
+    for object_id in object_ids:
+        cache_keys[object_id] = cache_key_function(object_id)
+
+    cached_objects_compressed: dict[str, CompressedItemT] = safe_cache_get_many(
+        [cache_keys[object_id] for object_id in object_ids],
+    )
+
+    cached_objects = {key: extractor(val) for key, val in cached_objects_compressed.items()}
+    needed_ids = [
+        object_id for object_id in object_ids if cache_keys[object_id] not in cached_objects
+    ]
+
+    # Only call query_function if there are some ids to fetch from the database:
+    if len(needed_ids) > 0:
+        db_objects = query_function(needed_ids)
+    else:
+        db_objects = []
+
+    items_for_remote_cache: dict[str, CompressedItemT] = {}
+    for obj in db_objects:
+        key = cache_keys[id_fetcher(obj)]
+        item = cache_transformer(obj)
+        items_for_remote_cache[key] = setter(item)
+        cached_objects[key] = item
+    if len(items_for_remote_cache) > 0:
+        safe_cache_set_many(items_for_remote_cache)
+    return {
+        object_id: cached_objects[cache_keys[object_id]]
+        for object_id in object_ids
+        if cache_keys[object_id] in cached_objects
+    }
+
+
+def bulk_cached_fetch(
+    cache_key_function: Callable[[ObjKT], str],
+    query_function: Callable[[list[ObjKT]], Iterable[ItemT]],
+    object_ids: Sequence[ObjKT],
+    *,
+    id_fetcher: Callable[[ItemT], ObjKT],
+) -> dict[ObjKT, ItemT]:
+    return generic_bulk_cached_fetch(
+        cache_key_function,
+        query_function,
+        object_ids,
+        id_fetcher=id_fetcher,
+        extractor=lambda obj: obj,
+        setter=lambda obj: obj,
+        cache_transformer=lambda obj: obj,
+    )
+
+
+def preview_url_cache_key(url: str) -> str:
+    return f"preview_url:{hashlib.sha1(url.encode()).hexdigest()}"
+
+
+def display_recipient_cache_key(recipient_id: int) -> str:
+    return f"display_recipient_dict:{recipient_id}"
+
+
+def single_user_display_recipient_cache_key(user_id: int) -> str:
+    return f"single_user_display_recipient:{user_id}"
+
+
+def user_profile_by_email_realm_id_cache_key(email: str, realm_id: int) -> str:
+    return f"user_profile:{hashlib.sha1(email.strip().encode()).hexdigest()}:{realm_id}"
+
+
+def user_profile_by_email_realm_cache_key(email: str, realm: "Realm") -> str:
+    return user_profile_by_email_realm_id_cache_key(email, realm.id)
+
+
+def user_profile_delivery_email_cache_key(delivery_email: str, realm_id: int) -> str:
+    return f"user_profile_by_delivery_email:{hashlib.sha1(delivery_email.strip().encode()).hexdigest()}:{realm_id}"
+
+
+def bot_profile_cache_key(email: str, realm_id: int) -> str:
+    return f"bot_profile:{hashlib.sha1(email.strip().encode()).hexdigest()}"
+
+
+def user_profile_by_id_cache_key(user_profile_id: int) -> str:
+    return f"user_profile_by_id:{user_profile_id}"
+
+
+def user_profile_narrow_by_id_cache_key(user_profile_id: int) -> str:
+    return f"user_profile_narrow_by_id:{user_profile_id}"
+
+
+def user_profile_by_api_key_cache_key(api_key: str) -> str:
+    return f"user_profile_by_api_key:{api_key}"
+
+
+def get_cross_realm_dicts_key() -> str:
+    emails = list(settings.CROSS_REALM_BOT_EMAILS)
+    raw_key = ",".join(sorted(emails))
+    digest = hashlib.sha1(raw_key.encode()).hexdigest()
+    return f"get_cross_realm_dicts:{digest}"
+
+
+realm_user_dict_fields: list[str] = [
+    "id",
+    "full_name",
+    "email",
+    "avatar_source",
+    "avatar_version",
+    "is_active",
+    "role",
+    "is_bot",
+    "timezone",
+    "date_joined",
+    "bot_owner_id",
+    "delivery_email",
+    "bot_type",
+    "long_term_idle",
+    "email_address_visibility",
+    "is_imported_stub",
+    "is_deleted",
+]
+
+
+def realm_user_dicts_cache_key(realm_id: int) -> str:
+    return f"realm_user_dicts:{realm_id}"
+
+
+def get_muting_users_cache_key(muted_user_id: int) -> str:
+    return f"muting_users_list:{muted_user_id}"
+
+
+def get_realm_used_upload_space_cache_key(realm_id: int) -> str:
+    return f"realm_used_upload_space:{realm_id}"
+
+
+def get_realm_seat_count_cache_key(realm_id: int) -> str:
+    return f"realm_seat_count:{realm_id}"
+
+
+def active_user_ids_cache_key(realm_id: int) -> str:
+    return f"active_user_ids:{realm_id}"
+
+
+def active_non_guest_user_ids_cache_key(realm_id: int) -> str:
+    return f"active_non_guest_user_ids:{realm_id}"
+
+
+def active_guest_user_ids_cache_key(realm_id: int) -> str:
+    return f"active_guest_user_ids:{realm_id}"
+
+
+def get_realm_system_groups_cache_key(realm_id: int) -> str:
+    return f"realm_system_groups:{realm_id}"
+
+
+bot_dict_fields: list[str] = [
+    "api_key",
+    "avatar_source",
+    "avatar_version",
+    "bot_owner_id",
+    "bot_type",
+    "default_all_public_streams",
+    "default_events_register_stream__name",
+    "default_sending_stream__name",
+    "email",
+    "full_name",
+    "id",
+    "is_active",
+    "realm_id",
+]
+
+
+def bot_dicts_in_realm_cache_key(realm_id: int) -> str:
+    return f"bot_dicts_in_realm:{realm_id}"
+
+
+def delete_user_profile_caches(user_profiles: Iterable["UserProfile"], realm_id: int) -> None:
+    # Imported here to avoid cyclic dependency.
+    from zerver.models.users import is_cross_realm_bot_email
+
+    def user_profile_key_iterator() -> Iterator[str]:
+        for user_profile in user_profiles:
+            yield user_profile_by_id_cache_key(user_profile.id)
+            yield user_profile_narrow_by_id_cache_key(user_profile.id)
+            yield user_profile_by_api_key_cache_key(user_profile.api_key)
+            yield user_profile_by_email_realm_id_cache_key(user_profile.email, realm_id)
+            yield user_profile_delivery_email_cache_key(user_profile.delivery_email, realm_id)
+            if user_profile.is_bot and is_cross_realm_bot_email(user_profile.email):
+                # Handle clearing system bots from their special cache.
+                yield bot_profile_cache_key(user_profile.email, realm_id)
+                yield get_cross_realm_dicts_key()
+
+    cache_delete_many(user_profile_key_iterator())
+
+
+def delete_display_recipient_cache(user_profiles: list["UserProfile"]) -> None:
+    from zerver.models import Subscription  # We need to import here to avoid cyclic dependency.
+
+    recipient_ids = Subscription.objects.filter(user_profile__in=user_profiles).values_list(
+        "recipient_id", flat=True
+    )
+    keys = [display_recipient_cache_key(rid) for rid in recipient_ids]
+    keys.extend(
+        [single_user_display_recipient_cache_key(user_profile.id) for user_profile in user_profiles]
+    )
+    cache_delete_many(keys)
+
+
+def changed(update_fields: Sequence[str] | None, fields: list[str]) -> bool:
+    if update_fields is None:
+        # adds/deletes should invalidate the cache
+        return True
+
+    update_fields_set = set(update_fields)
+    return any(f in update_fields_set for f in fields)
+
+
+def bulk_flush_users(
+    *,
+    user_profiles: list["UserProfile"],
+    realm: "Realm",
+    update_fields: Sequence[str] | None = None,
+) -> None:
+    delete_user_profile_caches(user_profiles, realm.id)
+
+    cache_keys_to_delete = set()
+    if changed(update_fields, realm_user_dict_fields):
+        cache_keys_to_delete.add(realm_user_dicts_cache_key(realm.id))
+
+    if changed(update_fields, ["is_active"]):
+        cache_keys_to_delete.add(active_user_ids_cache_key(realm.id))
+        cache_keys_to_delete.add(active_non_guest_user_ids_cache_key(realm.id))
+        cache_keys_to_delete.add(active_guest_user_ids_cache_key(realm.id))
+
+    if changed(update_fields, ["role"]):
+        cache_keys_to_delete.add(active_non_guest_user_ids_cache_key(realm.id))
+        cache_keys_to_delete.add(active_guest_user_ids_cache_key(realm.id))
+
+    # Invalidate our bots_in_realm info dict if any bot has
+    # changed the fields in the dict or become (in)active
+    if changed(update_fields, bot_dict_fields):
+        for user_profile in user_profiles:
+            if user_profile.is_bot:
+                cache_keys_to_delete.add(bot_dicts_in_realm_cache_key(realm.id))
+
+    cache_delete_many(list(cache_keys_to_delete))
+
+    if changed(update_fields, ["email", "full_name", "id", "is_mirror_dummy"]):
+        delete_display_recipient_cache(user_profiles)
+
+
+# Called by models/users.py to flush the user_profile cache whenever we save
+# a user_profile object
+def flush_user_profile(
+    *,
+    instance: "UserProfile",
+    update_fields: Sequence[str] | None = None,
+    **kwargs: object,
+) -> None:
+    user_profile = instance
+    bulk_flush_users(
+        user_profiles=[user_profile], realm=user_profile.realm, update_fields=update_fields
+    )
+
+
+def flush_muting_users_cache(*, instance: "MutedUser", **kwargs: object) -> None:
+    mute_object = instance
+    cache_delete(get_muting_users_cache_key(mute_object.muted_user_id))
+
+
+# Called by models/realms.py to flush various caches whenever we save
+# a Realm object.  The main tricky thing here is that Realm info is
+# generally cached indirectly through user_profile objects.
+def flush_realm(
+    *,
+    instance: "Realm",
+    update_fields: Sequence[str] | None = None,
+    from_deletion: bool = False,
+    **kwargs: object,
+) -> None:
+    realm = instance
+    users = realm.get_active_users()
+    delete_user_profile_caches(users, realm.id)
+
+    if (
+        from_deletion
+        or realm.deactivated
+        or (update_fields is not None and "string_id" in update_fields)
+    ):
+        cache_delete(realm_user_dicts_cache_key(realm.id))
+        cache_delete(active_user_ids_cache_key(realm.id))
+        cache_delete(bot_dicts_in_realm_cache_key(realm.id))
+        cache_delete(realm_alert_words_cache_key(realm.id))
+        cache_delete(realm_alert_words_automaton_cache_key(realm.id))
+        cache_delete(active_non_guest_user_ids_cache_key(realm.id))
+        cache_delete(active_guest_user_ids_cache_key(realm.id))
+        cache_delete(realm_rendered_description_cache_key(realm))
+        cache_delete(realm_text_description_cache_key(realm))
+    elif changed(update_fields, ["description"]):
+        cache_delete(realm_rendered_description_cache_key(realm))
+        cache_delete(realm_text_description_cache_key(realm))
+
+
+def realm_alert_words_cache_key(realm_id: int) -> str:
+    return f"realm_alert_words:{realm_id}"
+
+
+def realm_alert_words_automaton_cache_key(realm_id: int) -> str:
+    return f"realm_alert_words_automaton:{realm_id}"
+
+
+def realm_rendered_description_cache_key(realm: "Realm") -> str:
+    return f"realm_rendered_description:{realm.string_id}"
+
+
+def realm_text_description_cache_key(realm: "Realm") -> str:
+    return f"realm_text_description:{realm.string_id}"
+
+
+# Called by models/streams.py to flush the stream cache whenever we save a stream
+# object.
+def flush_stream(
+    *,
+    instance: "Stream",
+    update_fields: Sequence[str] | None = None,
+    **kwargs: object,
+) -> None:
+    from zerver.models import UserProfile
+
+    stream = instance
+
+    if update_fields is None or (
+        "name" in update_fields
+        and UserProfile.objects.filter(
+            Q(default_sending_stream=stream) | Q(default_events_register_stream=stream)
+        ).exists()
+    ):
+        cache_delete(bot_dicts_in_realm_cache_key(stream.realm_id))
+
+
+def flush_used_upload_space_cache(
+    *,
+    instance: "Attachment",
+    created: bool = True,
+    **kwargs: object,
+) -> None:
+    attachment = instance
+
+    if created:
+        cache_delete(get_realm_used_upload_space_cache_key(attachment.owner.realm_id))
+
+
+def to_dict_cache_key_id(message_id: int) -> str:
+    return f"message_dict:{message_id}"
+
+
+def to_dict_cache_key(message: "Message", realm_id: int | None = None) -> str:
+    return to_dict_cache_key_id(message.id)
+
+
+def open_graph_description_cache_key(content: bytes, request_url: str) -> str:
+    return f"open_graph_description_path:{hashlib.sha1(request_url.encode()).hexdigest()}"
+
+
+def zoom_server_access_token_cache_key(account_id: str) -> str:
+    return f"zoom_server_to_server_access_token:{account_id}"
+
+
+def flush_zoom_server_access_token_cache(account_id: str) -> None:
+    cache_delete(zoom_server_access_token_cache_key(account_id))
+
+
+def flush_message(*, instance: "Message", **kwargs: object) -> None:
+    message = instance
+    cache_delete(to_dict_cache_key_id(message.id))
+
+
+def flush_submessage(*, instance: "SubMessage", **kwargs: object) -> None:
+    submessage = instance
+    # submessages are not cached directly, they are part of their
+    # parent messages
+    message_id = submessage.message_id
+    cache_delete(to_dict_cache_key_id(message_id))
+
+
+class IgnoreUnhashableLruCacheWrapper(Generic[ParamT, ReturnT]):
+    def __init__(
+        self, function: Callable[ParamT, ReturnT], cached_function: "_lru_cache_wrapper[ReturnT]"
+    ) -> None:
+        self.key_prefix = KEY_PREFIX
+        self.function = function
+        self.cached_function = cached_function
+        self.cache_info = cached_function.cache_info
+        self.cache_clear = cached_function.cache_clear
+
+    def __call__(self, *args: ParamT.args, **kwargs: ParamT.kwargs) -> ReturnT:
+        if settings.DEVELOPMENT and not settings.TEST_SUITE:  # nocoverage
+            # In the development environment, we want every file
+            # change to refresh the source files from disk.
+            return self.function(*args, **kwargs)
+
+        if self.key_prefix != KEY_PREFIX:
+            # Clear cache when cache.KEY_PREFIX changes. This is used in
+            # tests.
+            self.cache_clear()
+            self.key_prefix = KEY_PREFIX
+
+        try:
+            return self.cached_function(
+                *args,
+                **kwargs,  # type: ignore[arg-type] # might be unhashable
+            )
+        except TypeError:
+            # args or kwargs contains an element which is unhashable. In
+            # this case we don't cache the result.
+            pass
+
+        # Deliberately calling this function from outside of exception
+        # handler to get a more descriptive traceback. Otherwise traceback
+        # can include the exception from cached_function as well.
+        return self.function(*args, **kwargs)
+
+
+def ignore_unhashable_lru_cache(
+    maxsize: int = 128, typed: bool = False
+) -> Callable[[Callable[ParamT, ReturnT]], IgnoreUnhashableLruCacheWrapper[ParamT, ReturnT]]:
+    """
+    This is a wrapper over lru_cache function. It adds following features on
+    top of lru_cache:
+
+        * It will not cache result of functions with unhashable arguments.
+        * It will clear cache whenever zerver.lib.cache.KEY_PREFIX changes.
+    """
+    internal_decorator = lru_cache(maxsize=maxsize, typed=typed)
+
+    def decorator(
+        user_function: Callable[ParamT, ReturnT],
+    ) -> IgnoreUnhashableLruCacheWrapper[ParamT, ReturnT]:
+        return IgnoreUnhashableLruCacheWrapper(user_function, internal_decorator(user_function))
+
+    return decorator
+
+
+def dict_to_items_tuple(user_function: Callable[..., Any]) -> Callable[..., Any]:
+    """Wrapper that converts any dict args to dict item tuples."""
+
+    def dict_to_tuple(arg: Any) -> Any:
+        if isinstance(arg, dict):
+            return tuple(sorted(arg.items()))
+        return arg
+
+    def wrapper(*args: Any, **kwargs: Any) -> Any:
+        new_args = (dict_to_tuple(arg) for arg in args)
+        return user_function(*new_args, **kwargs)
+
+    return wrapper
+
+
+def items_tuple_to_dict(user_function: Callable[..., Any]) -> Callable[..., Any]:
+    """Wrapper that converts any dict items tuple args to dicts."""
+
+    def dict_items_to_dict(arg: Any) -> Any:
+        if isinstance(arg, tuple):
+            try:
+                return dict(arg)
+            except TypeError:
+                pass
+        return arg
+
+    def wrapper(*args: Any, **kwargs: Any) -> Any:
+        new_args = (dict_items_to_dict(arg) for arg in args)
+        new_kwargs = {key: dict_items_to_dict(val) for key, val in kwargs.items()}
+        return user_function(*new_args, **new_kwargs)
+
+    return wrapper

zerver/lib/cache_helpers.py

@@ -0,0 +1,146 @@
+# See https://zulip.readthedocs.io/en/latest/subsystems/caching.html for docs
+import logging
+from collections.abc import Callable, Iterable, Iterator
+from datetime import timedelta
+from typing import Any
+
+from django.conf import settings
+from django.contrib.sessions.models import Session
+from django.db import connection
+from django.db.models import QuerySet
+from django.utils.timezone import now as timezone_now
+
+# This file needs to be different from cache.py because cache.py
+# cannot import anything from zerver.models or we'd have an import
+# loop
+from analytics.models import RealmCount
+from zerver.lib.cache import (
+    cache_set_many,
+    get_remote_cache_requests,
+    get_remote_cache_time,
+    user_profile_narrow_by_id_cache_key,
+)
+from zerver.lib.safe_session_cached_db import SessionStore
+from zerver.lib.sessions import session_engine
+from zerver.models import Client, UserProfile
+from zerver.models.clients import get_client_cache_key
+from zerver.models.users import base_get_user_narrow_queryset
+
+
+def get_narrow_users() -> Iterator[UserProfile]:
+    return (
+        base_get_user_narrow_queryset()
+        .filter(long_term_idle=False, realm__in=get_active_realm_ids())
+        .iterator()
+    )
+
+
+def user_narrow_cache_items(
+    items_for_remote_cache: dict[str, tuple[UserProfile]], user_profile: UserProfile
+) -> None:
+    items_for_remote_cache[user_profile_narrow_by_id_cache_key(user_profile.id)] = (user_profile,)
+
+
+def client_cache_items(items_for_remote_cache: dict[str, tuple[Client]], client: Client) -> None:
+    items_for_remote_cache[get_client_cache_key(client.name)] = (client,)
+
+
+def session_cache_items(
+    items_for_remote_cache: dict[str, dict[str, object]], session: Session
+) -> None:
+    if settings.SESSION_ENGINE != "zerver.lib.safe_session_cached_db":
+        # If we're not using the cached_db session engine, we there
+        # will be no store.cache_key attribute, and in any case we
+        # don't need to fill the cache, since it won't exist.
+        return
+    store = session_engine.SessionStore(session_key=session.session_key)
+    assert isinstance(store, SessionStore)
+    items_for_remote_cache[store.cache_key] = store.decode(session.session_data)
+
+
+def get_active_realm_ids() -> QuerySet[RealmCount, int]:
+    """For installations like Zulip Cloud hosting a lot of realms, it only makes
+    sense to do cache-filling work for realms that have any currently
+    active users/clients.  Otherwise, we end up with every single-user
+    trial organization that has ever been created costing us N streams
+    worth of cache work (where N is the number of default streams for
+    a new organization).
+    """
+    date = timezone_now() - timedelta(days=2)
+    return (
+        RealmCount.objects.filter(
+            end_time__gte=date,
+            property="1day_actives::day",
+            # Filtering on subgroup is important to ensure we use the good indexes.
+            subgroup=None,
+            value__gt=0,
+        )
+        .distinct("realm_id")
+        .values_list("realm_id", flat=True)
+    )
+
+
+# Format is (objects query, items filler function, timeout, batch size)
+#
+# The objects queries are put inside lambdas to prevent Django from
+# doing any setup for things we're unlikely to use (without the lambda
+# wrapper the below adds an extra 3ms or so to startup time for
+# anything importing this file).
+cache_fillers: dict[
+    str, tuple[Callable[[], Iterable[Any]], Callable[[dict[str, Any], Any], None], int, int]
+] = {
+    "user_narrow": (get_narrow_users, user_narrow_cache_items, 3600 * 24 * 7, 10000),
+    "client": (
+        lambda: Client.objects.all().iterator(),
+        client_cache_items,
+        3600 * 24 * 7,
+        10000,
+    ),
+    "session": (
+        lambda: Session.objects.all().iterator(),
+        session_cache_items,
+        3600 * 24 * 7,
+        10000,
+    ),
+}
+
+
+class SQLQueryCounter:
+    def __init__(self) -> None:
+        self.count = 0
+
+    def __call__(
+        self,
+        execute: Callable[[str, Any, bool, dict[str, Any]], Any],
+        sql: str,
+        params: Any,
+        many: bool,
+        context: dict[str, Any],
+    ) -> Any:
+        self.count += 1
+        return execute(sql, params, many, context)
+
+
+def fill_remote_cache(cache: str) -> None:
+    remote_cache_time_start = get_remote_cache_time()
+    remote_cache_requests_start = get_remote_cache_requests()
+    items_for_remote_cache: dict[str, Any] = {}
+    (objects, items_filler, timeout, batch_size) = cache_fillers[cache]
+    count = 0
+    db_query_counter = SQLQueryCounter()
+    with connection.execute_wrapper(db_query_counter):
+        for obj in objects():
+            items_filler(items_for_remote_cache, obj)
+            count += 1
+            if count % batch_size == 0:
+                cache_set_many(items_for_remote_cache, timeout=timeout)
+                items_for_remote_cache = {}
+        cache_set_many(items_for_remote_cache, timeout=timeout)
+    logging.info(
+        "Successfully populated %s cache: %d items, %d DB queries, %d memcached sets, %.2f seconds",
+        cache,
+        count,
+        db_query_counter.count,
+        get_remote_cache_requests() - remote_cache_requests_start,
+        get_remote_cache_time() - remote_cache_time_start,
+    )

zerver/lib/camo.py

@@ -0,0 +1,28 @@
+import hashlib
+import hmac
+
+from django.conf import settings
+from django.utils.crypto import constant_time_compare
+
+
+def generate_camo_url(url: str) -> str:
+    encoded_url = url.encode()
+    assert settings.CAMO_KEY is not None
+    encoded_camo_key = settings.CAMO_KEY.encode()
+    digest = hmac.new(encoded_camo_key, encoded_url, hashlib.sha1).hexdigest()
+    return f"{digest}/{encoded_url.hex()}"
+
+
+# Encodes the provided URL using the same algorithm used by the camo
+# caching https image proxy
+def get_camo_url(url: str) -> str:
+    # Only encode the URL if Camo is enabled
+    if settings.CAMO_URI == "":
+        return url
+    return f"{settings.CAMO_URI}{generate_camo_url(url)}"
+
+
+def is_camo_url_valid(digest: str, url: str) -> bool:
+    camo_url = generate_camo_url(url)
+    camo_url_digest = camo_url.split("/")[0]
+    return constant_time_compare(camo_url_digest, digest)

zerver/lib/captcha.py

@@ -0,0 +1,105 @@
+import base64
+import logging
+from typing import Any
+
+import orjson
+from altcha.v1 import verify_solution
+from django import forms
+from django.conf import settings
+from django.forms.renderers import BaseRenderer
+from django.http import HttpRequest
+from django.utils.html import format_html
+from django.utils.safestring import SafeString
+from django.utils.translation import gettext as _
+from django.utils.translation import gettext_lazy
+from typing_extensions import override
+
+
+def captcha_enabled() -> bool:
+    return bool(settings.USING_CAPTCHA and settings.ALTCHA_HMAC_KEY)
+
+
+class AltchaWidget(forms.TextInput):
+    @override
+    def render(
+        self,
+        name: str,
+        value: Any,
+        attrs: dict[str, Any] | None = None,
+        renderer: BaseRenderer | None = None,
+    ) -> SafeString:
+        return format_html(
+            (
+                "<altcha-widget"
+                '  name="captcha"'
+                '  challengeurl="/json/antispam_challenge"'
+                "  hidelogo"
+                "  hidefooter"
+                "  refetchonexpire"
+                '  style="{}"'
+                '  strings="{}"'
+                ">"
+            ),
+            "--altcha-max-width: 300px;",
+            orjson.dumps(
+                {
+                    "verified": _("Verified that you're a human user!"),
+                    "verifying": _("Verifying that you're not a bot…"),
+                }
+            ).decode(),
+        )
+
+
+class CaptchaFormMixin(forms.Form):
+    captcha = forms.CharField(
+        widget=AltchaWidget,
+        error_messages={"required": gettext_lazy("Validation failed, please try again.")},
+    )
+
+    def __init__(self, *args: Any, request: HttpRequest, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self.request = request
+        if not captcha_enabled():
+            del self.fields["captcha"]
+
+    @override
+    def clean(self) -> None:
+        super().clean()
+        # Validating the payload consumes the solved challenge, to
+        # prevent replay; defer it until the rest of the form is
+        # valid, so that a failure in some other field does not
+        # invalidate a successful solve.  The captcha field does not
+        # exist when the captcha is not enabled.
+        if "captcha" in self.fields and not self.errors:
+            try:
+                validate_captcha_payload(self.request, self.cleaned_data["captcha"])
+            except forms.ValidationError as error:
+                self.add_error("captcha", error)
+
+
+def validate_captcha_payload(request: HttpRequest, captcha_payload: str) -> None:
+    if not captcha_enabled():  # nocoverage
+        raise forms.ValidationError(_("Challenges are not enabled."))
+
+    try:
+        ok, err = verify_solution(captcha_payload, settings.ALTCHA_HMAC_KEY, check_expires=True)
+        if not ok:
+            logging.warning("Invalid altcha solution: %s", err)
+            raise forms.ValidationError(_("Validation failed, please try again."))
+    except forms.ValidationError:
+        raise
+    except Exception:
+        logging.exception("Error while validating altcha solution")
+        raise forms.ValidationError(_("Validation failed, please try again."))
+
+    captcha_data = orjson.loads(base64.b64decode(captcha_payload))
+    challenge = captcha_data["challenge"]
+    session_challenges = [e[0] for e in request.session.get("altcha_challenges", [])]
+    if challenge not in session_challenges:
+        logging.warning("Expired or replayed altcha solution")
+        raise forms.ValidationError(_("Validation failed, please try again."))
+
+    # Remove the successful solve from the session, to prevent replay
+    request.session["altcha_challenges"] = [
+        e for e in request.session.get("altcha_challenges", []) if e[0] != challenge
+    ]

zerver/lib/channel_folders.py

@@ -0,0 +1,92 @@
+from dataclasses import dataclass
+
+from django.utils.translation import gettext as _
+
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.markdown import markdown_convert
+from zerver.lib.streams import get_web_public_streams_queryset
+from zerver.lib.string_validation import check_string_is_printable
+from zerver.lib.timestamp import datetime_to_timestamp
+from zerver.models import ChannelFolder, Realm, Stream, UserProfile
+
+
+@dataclass
+class ChannelFolderData:
+    id: int
+    name: str
+    description: str
+    rendered_description: str
+    order: int
+    creator_id: int | None
+    date_created: int
+    is_archived: bool
+
+
+def check_channel_folder_name(name: str, realm: Realm) -> None:
+    if name.strip() == "":
+        raise JsonableError(_("Channel folder name can't be empty."))
+
+    invalid_character_pos = check_string_is_printable(name)
+    if invalid_character_pos is not None:
+        raise JsonableError(
+            _("Invalid character in channel folder name, at position {position}.").format(
+                position=invalid_character_pos
+            )
+        )
+
+    if ChannelFolder.objects.filter(name__iexact=name, realm=realm, is_archived=False).exists():
+        raise JsonableError(_("Channel folder name already in use"))
+
+
+def render_channel_folder_description(text: str, realm: Realm, *, acting_user: UserProfile) -> str:
+    return markdown_convert(
+        text, message_realm=realm, no_previews=True, acting_user=acting_user
+    ).rendered_content
+
+
+def get_channel_folder_data(channel_folder: ChannelFolder) -> ChannelFolderData:
+    date_created = datetime_to_timestamp(channel_folder.date_created)
+    return ChannelFolderData(
+        id=channel_folder.id,
+        name=channel_folder.name,
+        description=channel_folder.description,
+        rendered_description=channel_folder.rendered_description,
+        order=channel_folder.order,
+        date_created=date_created,
+        creator_id=channel_folder.creator_id,
+        is_archived=channel_folder.is_archived,
+    )
+
+
+def get_channel_folders_in_realm(
+    realm: Realm, include_archived: bool = False
+) -> list[ChannelFolderData]:
+    folders = ChannelFolder.objects.filter(realm=realm)
+    if not include_archived:
+        folders = folders.exclude(is_archived=True)
+
+    channel_folders = [get_channel_folder_data(channel_folder) for channel_folder in folders]
+    return sorted(channel_folders, key=lambda folder: folder.order)
+
+
+def get_channel_folder_by_id(channel_folder_id: int, realm: Realm) -> ChannelFolder:
+    try:
+        channel_folder = ChannelFolder.objects.get(id=channel_folder_id, realm=realm)
+        return channel_folder
+    except ChannelFolder.DoesNotExist:
+        raise JsonableError(_("Invalid channel folder ID"))
+
+
+def get_channel_folders_for_spectators(realm: Realm) -> list[ChannelFolderData]:
+    folder_ids_for_web_public_streams = set(
+        get_web_public_streams_queryset(realm).values_list("folder_id", flat=True)
+    )
+    folders = ChannelFolder.objects.filter(id__in=folder_ids_for_web_public_streams)
+    channel_folders = [get_channel_folder_data(channel_folder) for channel_folder in folders]
+    return sorted(channel_folders, key=lambda folder: folder.id)
+
+
+def check_channel_folder_in_use(channel_folder: ChannelFolder) -> bool:
+    if Stream.objects.filter(folder=channel_folder).exists():
+        return True
+    return False

zerver/lib/compatibility.py

@@ -0,0 +1,159 @@
+import os
+import re
+from datetime import datetime, timedelta, timezone
+
+from django.conf import settings
+from django.utils.timezone import now as timezone_now
+
+from version import DESKTOP_MINIMUM_VERSION, DESKTOP_WARNING_VERSION
+from zerver.lib.user_agent import parse_user_agent
+from zerver.models import UserProfile
+from zerver.signals import get_device_browser
+
+# LAST_SERVER_UPGRADE_TIME is the last time the server had a version deployed.
+if settings.PRODUCTION:  # nocoverage
+    timestamp = os.path.basename(os.path.abspath(settings.DEPLOY_ROOT))
+    LAST_SERVER_UPGRADE_TIME = datetime.strptime(timestamp, "%Y-%m-%d-%H-%M-%S").replace(
+        tzinfo=timezone.utc
+    )
+else:
+    LAST_SERVER_UPGRADE_TIME = timezone_now()
+
+
+def is_outdated_server(user_profile: UserProfile | None) -> bool:
+    # Release tarballs are unpacked via `tar -xf`, which means the
+    # `mtime` on files in them is preserved from when the release
+    # tarball was built.  Checking this allows us to catch cases where
+    # someone has upgraded in the last year but to a release more than
+    # a year old.
+    git_version_path = os.path.join(settings.DEPLOY_ROOT, "version.py")
+    release_build_time = datetime.fromtimestamp(os.path.getmtime(git_version_path), timezone.utc)
+
+    version_no_newer_than = min(LAST_SERVER_UPGRADE_TIME, release_build_time)
+    deadline = version_no_newer_than + timedelta(days=settings.SERVER_UPGRADE_NAG_DEADLINE_DAYS)
+
+    if user_profile is None or not user_profile.is_realm_admin:
+        # Administrators get warned at the deadline; all users 30 days later.
+        deadline += timedelta(days=30)
+
+    if timezone_now() > deadline:
+        return True
+    return False
+
+
+def pop_numerals(ver: str) -> tuple[list[int], str]:
+    match = re.search(r"^( \d+ (?: \. \d+ )* ) (.*)", ver, re.VERBOSE)
+    if match is None:
+        return [], ver
+    numerals, rest = match.groups()
+    numbers = [int(n) for n in numerals.split(".")]
+    return numbers, rest
+
+
+def version_lt(ver1: str, ver2: str) -> bool | None:
+    """
+    Compare two Zulip-style version strings.
+
+    Versions are dot-separated sequences of decimal integers,
+    followed by arbitrary trailing decoration.  Comparison is
+    lexicographic on the integer sequences, and refuses to
+    guess how any trailing decoration compares to any other,
+    to further numerals, or to nothing.
+
+    Returns:
+      True if ver1 < ver2
+      False if ver1 >= ver2
+      None if can't tell.
+    """
+    num1, rest1 = pop_numerals(ver1)
+    num2, rest2 = pop_numerals(ver2)
+    if not num1 or not num2:
+        return None
+    common_len = min(len(num1), len(num2))
+    common_num1, rest_num1 = num1[:common_len], num1[common_len:]
+    common_num2, rest_num2 = num2[:common_len], num2[common_len:]
+
+    # Leading numbers win.
+    if common_num1 != common_num2:
+        return common_num1 < common_num2
+
+    # More numbers beats end-of-string, but ??? vs trailing text.
+    # (NB at most one of rest_num1, rest_num2 is nonempty.)
+    if not rest1 and rest_num2:
+        return True
+    if rest_num1 and not rest2:
+        return False
+    if rest_num1 or rest_num2:
+        return None
+
+    # Trailing text we can only compare for equality.
+    if rest1 == rest2:
+        return False
+    return None
+
+
+def find_mobile_os(user_agent: str) -> str | None:
+    if re.search(r"\b Android \b", user_agent, re.IGNORECASE | re.VERBOSE):
+        return "android"
+    if re.search(r"\b(?: iOS | iPhone\ OS )\b", user_agent, re.IGNORECASE | re.VERBOSE):
+        return "ios"
+    return None
+
+
+def is_outdated_desktop_app(user_agent_str: str) -> tuple[bool, bool, bool]:
+    # Returns (insecure, banned, auto_update_broken)
+    user_agent = parse_user_agent(user_agent_str)
+    if user_agent["name"] == "ZulipDesktop":
+        # The deprecated QT/webkit based desktop app, last updated in ~2016.
+        return (True, True, True)
+
+    if user_agent["name"] != "ZulipElectron":
+        return (False, False, False)
+
+    if version_lt(user_agent["version"], "4.0.0"):
+        # Version 2.3.82 and older (aka <4.0.0) of the modern
+        # Electron-based Zulip desktop app with known security issues.
+        # won't auto-update; we may want a special notice to
+        # distinguish those from modern releases.
+        return (True, True, True)
+
+    if version_lt(user_agent["version"], DESKTOP_MINIMUM_VERSION):
+        # Below DESKTOP_MINIMUM_VERSION, we reject access as well.
+        return (True, True, False)
+
+    if version_lt(user_agent["version"], DESKTOP_WARNING_VERSION):
+        # Other insecure versions should just warn.
+        return (True, False, False)
+
+    return (False, False, False)
+
+
+def is_banned_browser(user_agent: str) -> tuple[bool, str | None]:
+    browser_name = get_device_browser(user_agent)
+    if browser_name == "Internet Explorer":
+        return (True, browser_name)
+    return (False, browser_name)
+
+
+def is_pronouns_field_type_supported(user_agent_str: str | None) -> bool:
+    # In order to avoid users having a bad experience with these
+    # custom profile fields disappearing after applying migration
+    # 0421_migrate_pronouns_custom_profile_fields, we provide this
+    # compatibility shim to show such custom profile fields as
+    # SHORT_TEXT to older mobile app clients.
+    #
+    # TODO/compatibility(7.0): Because this is a relatively minor
+    # detail, we can remove this compatibility hack once most users
+    # have upgraded to a sufficiently new mobile client.
+    if user_agent_str is None:
+        return True
+
+    user_agent = parse_user_agent(user_agent_str)
+    if user_agent["name"] != "ZulipMobile":
+        return True
+
+    FIRST_VERSION_TO_SUPPORT_PRONOUNS_FIELD = "27.192"
+    if version_lt(user_agent["version"], FIRST_VERSION_TO_SUPPORT_PRONOUNS_FIELD):
+        return False
+
+    return True

zerver/lib/context_managers.py

@@ -0,0 +1,50 @@
+"""
+Context managers, i.e. things you can use with the 'with' statement.
+"""
+
+import fcntl
+from collections.abc import Iterator
+from contextlib import contextmanager
+from typing import IO, Any
+
+
+@contextmanager
+def flock(lockfile: int | IO[Any], shared: bool = False) -> Iterator[None]:
+    """Lock a file object using flock(2) for the duration of a 'with' statement.
+
+    If shared is True, use a LOCK_SH lock, otherwise LOCK_EX."""
+
+    fcntl.flock(lockfile, fcntl.LOCK_SH if shared else fcntl.LOCK_EX)
+    try:
+        yield
+    finally:
+        fcntl.flock(lockfile, fcntl.LOCK_UN)
+
+
+@contextmanager
+def lockfile(filename: str, shared: bool = False) -> Iterator[None]:
+    """Lock a file using flock(2) for the duration of a 'with' statement.
+
+    If shared is True, use a LOCK_SH lock, otherwise LOCK_EX.
+
+    The file is given by name and will be created if it does not exist."""
+    with open(filename, "w") as lock, flock(lock, shared=shared):
+        yield
+
+
+@contextmanager
+def lockfile_nonblocking(filename: str) -> Iterator[bool]:  # nocoverage
+    """Lock a file using flock(2) for the duration of a 'with' statement.
+
+    Doesn't block, yields False immediately if the lock can't be acquired."""
+    with open(filename, "w") as f:
+        lock_acquired = False
+        try:
+            fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            lock_acquired = True
+            yield lock_acquired
+        except BlockingIOError:
+            yield False
+        finally:
+            if lock_acquired:
+                fcntl.flock(f, fcntl.LOCK_UN)

zerver/lib/create_user.py

@@ -0,0 +1,244 @@
+import re
+from datetime import datetime
+from email.headerregistry import Address
+
+from django.contrib.auth.models import UserManager
+from django.utils.timezone import now as timezone_now
+
+from zerver.lib.i18n import get_default_language_for_new_user
+from zerver.lib.onboarding_steps import copy_onboarding_steps
+from zerver.lib.timezone import canonicalize_timezone
+from zerver.lib.upload import copy_avatar
+from zerver.models import Realm, RealmUserDefault, Stream, UserBaseSettings, UserProfile
+from zerver.models.realms import get_fake_email_domain
+
+
+def copy_default_settings(
+    settings_source: UserProfile | RealmUserDefault, target_profile: UserProfile
+) -> None:
+    # Important note: Code run from here to configure the user's
+    # settings should not send events, as that would cause clients
+    # to throw an exception (we haven't sent the realm_user/add event
+    # yet, so that event will include the updated details of target_profile).
+    #
+    # Note that this function will do at least one save() on target_profile.
+    for settings_name in UserBaseSettings.property_types:
+        if settings_name in ["default_language", "enable_login_emails"] and isinstance(
+            settings_source, RealmUserDefault
+        ):
+            continue
+
+        if settings_name == "email_address_visibility":
+            # For email_address_visibility, the value selected in registration form
+            # is preferred over the realm-level default value and value of source
+            # profile.
+            continue
+        value = getattr(settings_source, settings_name)
+        setattr(target_profile, settings_name, value)
+
+    if isinstance(settings_source, RealmUserDefault):
+        target_profile.save()
+        return
+
+    target_profile.full_name = settings_source.full_name
+    target_profile.timezone = canonicalize_timezone(settings_source.timezone)
+    target_profile.save()
+
+    if settings_source.avatar_source == UserProfile.AVATAR_FROM_USER:
+        from zerver.actions.user_settings import do_change_avatar_fields
+
+        copy_avatar(settings_source, target_profile)
+        do_change_avatar_fields(
+            target_profile,
+            UserProfile.AVATAR_FROM_USER,
+            skip_notify=True,
+            acting_user=target_profile,
+        )
+
+    copy_onboarding_steps(settings_source, target_profile)
+
+
+def get_dummy_email_address_for_display_regex(realm: Realm) -> str:
+    """
+    Returns a regex that matches the format of dummy email addresses we
+    generate for the .email of users with limit email_address_visibility.
+
+    The reason we need a regex is that we want something that we can use both
+    for generating the dummy email addresses and recognizing them together with extraction
+    of the user ID.
+    """
+
+    # We can't directly have (\d+) in the username passed to Address, because it gets
+    # mutated by the underlying logic for escaping special characters.
+    # So we use a trick by using $ as a placeholder which will be preserved, and then
+    # replace it with (\d+) to obtain our intended regex.
+    address_template = Address(username="user$", domain=get_fake_email_domain(realm.host)).addr_spec
+    regex = re.escape(address_template).replace(r"\$", r"(\d+)", 1)
+    return regex
+
+
+def get_display_email_address(user_profile: UserProfile) -> str:
+    if not user_profile.email_address_is_realm_public():
+        # The format of the dummy email address created here needs to stay consistent
+        # with get_dummy_email_address_for_display_regex.
+        return Address(
+            username=f"user{user_profile.id}", domain=get_fake_email_domain(user_profile.realm.host)
+        ).addr_spec
+    return user_profile.delivery_email
+
+
+# create_user_profile is based on Django's User.objects.create_user,
+# except that we don't save to the database so it can used in
+# bulk_creates
+#
+# Only use this for bulk_create -- for normal usage one should use
+# create_user (below) which will also make the Subscription and
+# Recipient objects
+def create_user_profile(
+    realm: Realm,
+    email: str,
+    password: str | None,
+    active: bool,
+    bot_type: int | None,
+    full_name: str,
+    bot_owner: UserProfile | None,
+    is_mirror_dummy: bool,
+    tos_version: str | None,
+    timezone: str,
+    default_language: str,
+    is_deleted: bool = False,
+    force_date_joined: datetime | None = None,
+    is_imported_stub: bool = False,
+    *,
+    email_address_visibility: int,
+) -> UserProfile:
+    if force_date_joined is None:
+        date_joined = timezone_now()
+    else:  # nocoverage
+        date_joined = force_date_joined
+
+    email = UserManager.normalize_email(email)
+
+    user_profile = UserProfile(
+        is_staff=False,
+        is_active=active,
+        full_name=full_name,
+        last_login=date_joined,
+        date_joined=date_joined,
+        realm=realm,
+        is_bot=bool(bot_type),
+        bot_type=bot_type,
+        bot_owner=bot_owner,
+        is_mirror_dummy=is_mirror_dummy,
+        is_deleted=is_deleted,
+        tos_version=tos_version,
+        timezone=timezone,
+        default_language=default_language,
+        delivery_email=email,
+        email_address_visibility=email_address_visibility,
+        is_imported_stub=is_imported_stub,
+    )
+    if bot_type or not active:
+        password = None
+    if user_profile.email_address_is_realm_public():
+        # If emails are visible to everyone, we can set this here and save a DB query
+        user_profile.email = get_display_email_address(user_profile)
+    user_profile.set_password(password)
+    return user_profile
+
+
+def create_user(
+    email: str,
+    password: str | None,
+    realm: Realm,
+    full_name: str,
+    active: bool = True,
+    role: int | None = None,
+    bot_type: int | None = None,
+    bot_owner: UserProfile | None = None,
+    tos_version: str | None = None,
+    timezone: str = "",
+    avatar_source: str | None = None,
+    is_mirror_dummy: bool = False,
+    is_deleted: bool = False,
+    default_language: str | None = None,
+    default_sending_stream: Stream | None = None,
+    default_events_register_stream: Stream | None = None,
+    default_all_public_streams: bool | None = None,
+    source_profile: UserProfile | None = None,
+    force_date_joined: datetime | None = None,
+    enable_marketing_emails: bool | None = None,
+    email_address_visibility: int | None = None,
+) -> UserProfile:
+    realm_user_default = RealmUserDefault.objects.get(realm=realm)
+    if bot_type is None:
+        if email_address_visibility is not None:
+            user_email_address_visibility = email_address_visibility
+        else:
+            user_email_address_visibility = realm_user_default.email_address_visibility
+    else:
+        # There is no privacy motivation for limiting access to bot email addresses,
+        # so we hardcode them to EMAIL_ADDRESS_VISIBILITY_EVERYONE.
+        user_email_address_visibility = UserProfile.EMAIL_ADDRESS_VISIBILITY_EVERYONE
+
+    # Users created via the API or LDAP/SAML syncing code paths will
+    # usually not have a default_language value, and should fall back
+    # to the realm default.
+    if default_language is None:
+        default_language = get_default_language_for_new_user(realm, request=None)
+
+    user_profile = create_user_profile(
+        realm,
+        email,
+        password,
+        active,
+        bot_type,
+        full_name,
+        bot_owner,
+        is_mirror_dummy,
+        tos_version,
+        timezone,
+        default_language,
+        is_deleted,
+        force_date_joined=force_date_joined,
+        email_address_visibility=user_email_address_visibility,
+    )
+    if avatar_source is None:
+        avatar_source = realm.default_avatar_source
+    user_profile.avatar_source = avatar_source
+    user_profile.timezone = timezone
+    user_profile.default_sending_stream = default_sending_stream
+    user_profile.default_events_register_stream = default_events_register_stream
+    if role is not None:
+        user_profile.role = role
+    # Allow the ORM default to be used if not provided
+    if default_all_public_streams is not None:
+        user_profile.default_all_public_streams = default_all_public_streams
+    # If a source profile was specified, we copy settings from that
+    # user.  Note that this is positioned in a way that overrides
+    # other arguments passed in, which is correct for most defaults
+    # like time zone where the source profile likely has a better value
+    # than the guess. As we decide on details like avatars and full
+    # names for this feature, we may want to move it.
+    if source_profile is not None:
+        # copy_default_settings saves the attribute values so a secondary
+        # save is not required.
+        copy_default_settings(source_profile, user_profile)
+    elif bot_type is None:
+        copy_default_settings(realm_user_default, user_profile)
+    else:
+        # This will be executed only for bots.
+        user_profile.save()
+
+    if bot_type is None and enable_marketing_emails is not None:
+        user_profile.enable_marketing_emails = enable_marketing_emails
+        user_profile.save(update_fields=["enable_marketing_emails"])
+
+    if not user_profile.email_address_is_realm_public():
+        # With restricted access to email addresses, we can't generate
+        # the fake email addresses we use for display purposes without
+        # a User ID, which isn't generated until the .save() above.
+        user_profile.email = get_display_email_address(user_profile)
+        user_profile.save(update_fields=["email"])
+
+    return user_profile

zerver/lib/db.py

@@ -0,0 +1,54 @@
+import time
+from collections.abc import Callable, Iterable, Mapping, Sequence
+from typing import Any, TypeAlias, TypeVar
+
+from psycopg2.extensions import connection, cursor
+from psycopg2.sql import Composable
+from typing_extensions import override
+
+CursorObj = TypeVar("CursorObj", bound=cursor)
+Query: TypeAlias = str | bytes | Composable
+Params: TypeAlias = Sequence[object] | Mapping[str, object] | None
+ParamsT = TypeVar("ParamsT")
+
+
+# Similar to the tracking done in Django's CursorDebugWrapper, but done at the
+# psycopg2 cursor level so it works with SQLAlchemy.
+def wrapper_execute(
+    self: CursorObj, action: Callable[[Query, ParamsT], None], sql: Query, params: ParamsT
+) -> None:
+    start = time.time()
+    try:
+        action(sql, params)
+    finally:
+        stop = time.time()
+        duration = stop - start
+        assert isinstance(self.connection, TimeTrackingConnection)
+        self.connection.queries.append(
+            {
+                "time": f"{duration:.3f}",
+            }
+        )
+
+
+class TimeTrackingCursor(cursor):
+    """A psycopg2 cursor class that tracks the time spent executing queries."""
+
+    @override
+    def execute(self, query: Query, vars: Params = None) -> None:
+        wrapper_execute(self, super().execute, query, vars)
+
+    @override
+    def executemany(self, query: Query, vars_list: Iterable[Params]) -> None:  # nocoverage
+        wrapper_execute(self, super().executemany, query, vars_list)
+
+
+CursorT = TypeVar("CursorT", bound=cursor)
+
+
+class TimeTrackingConnection(connection):
+    """A psycopg2 connection class that uses TimeTrackingCursors."""
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        self.queries: list[dict[str, str]] = []
+        super().__init__(*args, **kwargs)

zerver/lib/db_connections.py

@@ -0,0 +1,7 @@
+from django.db import connections
+
+
+def reset_queries() -> None:
+    for conn in connections.all():
+        if conn.connection is not None:
+            conn.connection.queries = []

zerver/lib/debug.py

@@ -0,0 +1,117 @@
+import code
+import gc
+import logging
+import os
+import signal
+import socket
+import threading
+import traceback
+import tracemalloc
+from types import FrameType
+
+from django.conf import settings
+from django.utils.timezone import now as timezone_now
+
+logger = logging.getLogger("zulip.debug")
+
+
+# Interactive debugging code from
+# https://stackoverflow.com/questions/132058/showing-the-stack-trace-from-a-running-python-application
+# (that link also points to code for an interactive remote debugger
+# setup, which we might want if we move Tornado to run in a daemon
+# rather than via screen).
+def interactive_debug(sig: int, frame: FrameType | None) -> None:
+    """Interrupt running process, and provide a python prompt for
+    interactive debugging."""
+    d = {"_frame": frame}  # Allow access to frame object.
+    if frame is not None:
+        d.update(frame.f_globals)  # Unless shadowed by global
+        d.update(frame.f_locals)
+
+    message = "Signal received : entering python shell.\nTraceback:\n"
+    message += "".join(traceback.format_stack(frame))
+    i = code.InteractiveConsole(d)
+    i.interact(message)
+
+
+# SIGUSR1 => Just print the stack
+# SIGUSR2 => Print stack + open interactive debugging shell
+def interactive_debug_listen() -> None:
+    signal.signal(signal.SIGUSR1, lambda sig, stack: traceback.print_stack(stack))
+    signal.signal(signal.SIGUSR2, interactive_debug)
+
+
+def tracemalloc_dump() -> None:
+    if not tracemalloc.is_tracing():
+        logger.warning("pid %s: tracemalloc off, nothing to dump", os.getpid())
+        return
+    # Despite our name for it, `timezone_now` always deals in UTC.
+    basename = "snap.{}.{}".format(
+        os.getpid(), timezone_now().replace(tzinfo=None).isoformat("-", "seconds")
+    )
+    path = os.path.join(settings.TRACEMALLOC_DUMP_DIR, basename)
+    os.makedirs(settings.TRACEMALLOC_DUMP_DIR, exist_ok=True)
+
+    gc.collect()
+    tracemalloc.take_snapshot().dump(path)
+
+    with open(f"/proc/{os.getpid()}/stat", "rb") as f:
+        procstat = f.read().split()
+    rss_pages = int(procstat[23])
+    logger.info(
+        "tracemalloc dump: tracing %s MiB (%s MiB peak), using %s MiB; rss %s MiB; dumped %s",
+        tracemalloc.get_traced_memory()[0] // 1048576,
+        tracemalloc.get_traced_memory()[1] // 1048576,
+        tracemalloc.get_tracemalloc_memory() // 1048576,
+        rss_pages // 256,
+        basename,
+    )
+
+
+def tracemalloc_listen_sock(sock: socket.socket) -> None:
+    logger.debug("pid %s: tracemalloc_listen_sock started!", os.getpid())
+    while True:
+        sock.recv(1)
+        tracemalloc_dump()
+
+
+listener_pid: int | None = None
+
+
+def tracemalloc_listen() -> None:
+    global listener_pid
+    if listener_pid == os.getpid():
+        # Already set up -- and in this process, not just its parent.
+        return
+    logger.debug("pid %s: tracemalloc_listen working...", os.getpid())
+    listener_pid = os.getpid()
+
+    sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
+    os.makedirs(settings.TRACEMALLOC_DUMP_DIR, exist_ok=True)
+    path = os.path.join(settings.TRACEMALLOC_DUMP_DIR, f"tracemalloc.{os.getpid()}")
+    sock.bind(path)
+    thread = threading.Thread(target=lambda: tracemalloc_listen_sock(sock), daemon=True)
+    thread.start()
+    logger.debug("pid %s: tracemalloc_listen done: %s", os.getpid(), path)
+
+
+def maybe_tracemalloc_listen() -> None:
+    """If tracemalloc tracing enabled, listen for requests to dump a snapshot.
+
+    To trigger once this is listening:
+      echo | socat -u stdin unix-sendto:/var/log/zulip/tracemalloc/tracemalloc.$pid
+
+    To enable in the Zulip web server: edit /etc/zulip/uwsgi.ini ,
+    and add e.g. ` PYTHONTRACEMALLOC=5` to the `env=` line.
+    This function is called in middleware, so the process will
+    automatically start listening.
+
+    To enable in other contexts: see upstream docs
+    https://docs.python.org/3/library/tracemalloc .
+    You may also have to add a call to this function somewhere.
+
+    """
+    if os.environ.get("PYTHONTRACEMALLOC"):
+        # If the server was started with `tracemalloc` tracing on, then
+        # listen for a signal to dump `tracemalloc` snapshots.
+        tracemalloc_listen()

zerver/lib/default_streams.py

@@ -0,0 +1,19 @@
+from zerver.models import DefaultStream, Stream
+
+
+def get_slim_realm_default_streams(realm_id: int) -> set[Stream]:
+    # We really want this query to be simple and just get "thin" Stream objects
+    # in one round trip.
+    #
+    # The above is enforced by at least three tests that verify query counts,
+    # and test_query_count in test_subs.py makes sure that the query itself is
+    # not like 11000 bytes, which is what we had in a prior version that used
+    # select_related() with no arguments (and thus joined to too many tables).
+    #
+    # Please be careful about modifying this code, as it has had a history
+    # of performance problems.
+    return set(Stream.objects.filter(defaultstream__realm_id=realm_id))
+
+
+def get_default_stream_ids_for_realm(realm_id: int) -> set[int]:
+    return set(DefaultStream.objects.filter(realm_id=realm_id).values_list("stream_id", flat=True))

zerver/lib/demo_organizations.py

@@ -0,0 +1,78 @@
+import os
+from datetime import timedelta
+from functools import lru_cache
+
+import orjson
+from django.conf import settings
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+
+from zerver.actions.scheduled_messages import check_schedule_message
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.timestamp import ceiling_to_day, datetime_to_global_time
+from zerver.models.clients import get_client
+from zerver.models.realms import Realm
+from zerver.models.users import UserProfile, get_system_bot
+
+
+@lru_cache(None)
+def get_demo_organization_wordlists() -> dict[str, list[str]]:
+    path = os.path.join(settings.DEPLOY_ROOT, "zerver/lib", "demo_organization_words.json")
+    with open(path, "rb") as reader:
+        return orjson.loads(reader.read())
+
+
+def demo_organization_owner_email_exists(realm: Realm) -> bool:
+    human_owner_emails = set(realm.get_human_owner_users().values_list("delivery_email", flat=True))
+    return human_owner_emails != {""}
+
+
+def check_demo_organization_has_set_email(realm: Realm) -> None:
+    # This should be called after checking that the realm has
+    # a demo_organization_scheduled_deletion_date set.
+    assert realm.demo_organization_scheduled_deletion_date is not None
+    if not demo_organization_owner_email_exists(realm):
+        raise JsonableError(_("Configure owner account email address."))
+
+
+def get_demo_organization_deadline_days_remaining(realm: Realm) -> int:
+    assert realm.demo_organization_scheduled_deletion_date is not None
+    days_remaining = (realm.demo_organization_scheduled_deletion_date - timezone_now()).days
+    return days_remaining
+
+
+def get_scheduled_deletion_global_time(realm: Realm) -> str:
+    assert realm.demo_organization_scheduled_deletion_date is not None
+    scheduled_deletion_cronjob_timestamp = ceiling_to_day(
+        realm.demo_organization_scheduled_deletion_date
+    ) + timedelta(hours=6)
+    return datetime_to_global_time(scheduled_deletion_cronjob_timestamp)
+
+
+def schedule_demo_organization_deletion_reminder(user: UserProfile) -> None:
+    assert user.is_realm_owner
+    realm = user.realm
+    assert realm.demo_organization_scheduled_deletion_date is not None
+    deliver_at = realm.demo_organization_scheduled_deletion_date - timedelta(days=7)
+    sender = get_system_bot(settings.NOTIFICATION_BOT, realm.id)
+    client = get_client("Internal")
+
+    message_content = _("""
+As a reminder, this [demo organization]({demo_help}) will be automatically deleted on {deletion_time}, unless it's [converted into a permanent organization]({convert_demo}).
+""").format(
+        demo_help="/help/demo-organizations",
+        deletion_time=get_scheduled_deletion_global_time(realm),
+        convert_demo="/help/demo-organizations#convert-a-demo-organization-to-a-permanent-organization",
+    )
+
+    check_schedule_message(
+        sender,
+        client,
+        "private",
+        [user.id],
+        None,
+        message_content,
+        deliver_at,
+        realm,
+        skip_events=True,
+    )

zerver/lib/dev_ldap_directory.py

@@ -0,0 +1,90 @@
+import glob
+import logging
+import os
+from email.headerregistry import Address
+from typing import Any
+
+from django.conf import settings
+
+from zerver.lib.storage import static_path
+
+# See https://jackstromberg.com/2013/01/useraccountcontrol-attributeflag-values/
+# for docs on what these values mean.
+LDAP_USER_ACCOUNT_CONTROL_NORMAL = "512"
+LDAP_USER_ACCOUNT_CONTROL_DISABLED = "514"
+
+
+def generate_dev_ldap_dir(mode: str, num_users: int = 8) -> dict[str, dict[str, Any]]:
+    mode = mode.lower()
+    ldap_data = []
+    for i in range(1, num_users + 1):
+        name = f"LDAP User {i}"
+        email = f"ldapuser{i}@zulip.com"
+        phone_number = f"999999999{i}"
+        birthdate = f"19{i:02}-{i:02}-{i:02}"
+        ldap_data.append((name, email, phone_number, birthdate))
+
+    profile_images = []
+    for path in glob.glob(os.path.join(static_path("images/test-images/avatars"), "*")):
+        with open(path, "rb") as f:
+            profile_images.append(f.read())
+    ldap_dir = {}
+    for i, user_data in enumerate(ldap_data):
+        email = user_data[1].lower()
+        email_username = Address(addr_spec=email).username
+        common_data = {
+            "cn": [user_data[0]],
+            "userPassword": [email_username],
+            "phoneNumber": [user_data[2]],
+            "birthDate": [user_data[3]],
+        }
+        if mode == "a":
+            ldap_dir[f"uid={email},ou=users,dc=zulip,dc=com"] = dict(
+                uid=[email],
+                thumbnailPhoto=[profile_images[i % len(profile_images)]],
+                userAccountControl=[LDAP_USER_ACCOUNT_CONTROL_NORMAL],
+                **common_data,
+            )
+        elif mode == "b":
+            ldap_dir[f"uid={email_username},ou=users,dc=zulip,dc=com"] = dict(
+                uid=[email_username],
+                jpegPhoto=[profile_images[i % len(profile_images)]],
+                **common_data,
+            )
+        elif mode == "c":
+            ldap_dir[f"uid={email_username},ou=users,dc=zulip,dc=com"] = dict(
+                uid=[email_username], email=[email], **common_data
+            )
+
+    return ldap_dir
+
+
+def init_fakeldap(
+    directory: dict[str, dict[str, list[str]]] | None = None,
+) -> None:  # nocoverage
+    # We only use this in development.  Importing mock inside
+    # this function is an import time optimization, which
+    # avoids the expensive import of the mock module (slow
+    # because its dependency pbr uses pkgresources, which is
+    # really slow to import.)
+    from unittest import mock
+
+    from fakeldap import MockLDAP
+
+    # Silent `django_auth_ldap` logger in dev mode to avoid
+    # spammy user not found log messages.
+    ldap_auth_logger = logging.getLogger("django_auth_ldap")
+    ldap_auth_logger.setLevel(logging.CRITICAL)
+
+    fakeldap_logger = logging.getLogger("fakeldap")
+    fakeldap_logger.setLevel(logging.CRITICAL)
+
+    ldap_patcher = mock.patch("django_auth_ldap.config.ldap.initialize")
+    mock_initialize = ldap_patcher.start()
+    mock_ldap = MockLDAP()
+    mock_initialize.return_value = mock_ldap
+
+    assert settings.FAKE_LDAP_MODE is not None
+    mock_ldap.directory = directory or generate_dev_ldap_dir(
+        settings.FAKE_LDAP_MODE, settings.FAKE_LDAP_NUM_USERS
+    )

zerver/lib/devices.py

@@ -0,0 +1,70 @@
+import base64
+import binascii
+
+from django.utils.translation import gettext as _
+from typing_extensions import TypedDict
+
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.timestamp import datetime_to_timestamp
+from zerver.models.devices import Device
+from zerver.models.users import UserProfile
+
+
+class DeviceInfoDict(TypedDict):
+    push_key_id: int | None
+    push_token_id: str | None
+    pending_push_token_id: str | None
+    push_token_last_updated_timestamp: int | None
+    push_registration_error_code: str | None
+
+
+def b64encode_token_id_int(token_id_int: int) -> str:
+    token_id_bytes = token_id_int.to_bytes(8, byteorder="big", signed=True)
+    return base64.b64encode(token_id_bytes).decode()
+
+
+def b64decode_token_id_base64(token_id_base64: str) -> int:
+    try:
+        token_id_bytes = base64.b64decode(token_id_base64, validate=True)
+    except binascii.Error:
+        raise JsonableError(_("{var_name} is not Base64 encoded").format(var_name="`token_id`"))
+
+    return int.from_bytes(token_id_bytes, byteorder="big", signed=True)
+
+
+def get_devices(user_profile: UserProfile) -> dict[str, DeviceInfoDict]:
+    devices = Device.objects.filter(user=user_profile)
+
+    devices_dict: dict[str, DeviceInfoDict] = dict()
+    for device in devices:
+        push_token_id_base64 = None
+        pending_push_token_id_base64 = None
+        push_token_last_updated_timestamp = None
+
+        if device.push_token_id is not None:
+            push_token_id_base64 = b64encode_token_id_int(device.push_token_id)
+        if device.pending_push_token_id is not None:
+            pending_push_token_id_base64 = b64encode_token_id_int(device.pending_push_token_id)
+        if device.push_token_last_updated_timestamp is not None:
+            push_token_last_updated_timestamp = datetime_to_timestamp(
+                device.push_token_last_updated_timestamp
+            )
+
+        devices_dict[str(device.id)] = DeviceInfoDict(
+            push_key_id=device.push_key_id,
+            push_token_id=push_token_id_base64,
+            pending_push_token_id=pending_push_token_id_base64,
+            push_token_last_updated_timestamp=push_token_last_updated_timestamp,
+            push_registration_error_code=device.push_registration_error_code,
+        )
+
+    return devices_dict
+
+
+def check_device_id(device_id: int, user_id: int) -> Device:
+    try:
+        device = Device.objects.get(id=device_id, user_id=user_id)
+    except Device.DoesNotExist:
+        raise JsonableError(_("Invalid `device_id`"))
+
+    return device

zerver/lib/digest.py

@@ -0,0 +1,560 @@
+import functools
+import heapq
+import logging
+from collections import defaultdict
+from collections.abc import Collection, Iterator
+from dataclasses import dataclass
+from datetime import datetime, timedelta, timezone
+from typing import Any, TypeAlias
+
+from django.conf import settings
+from django.db import transaction
+from django.db.models import Exists, OuterRef, QuerySet
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+from markupsafe import Markup
+
+from confirmation.models import one_click_unsubscribe_link
+from zerver.context_processors import common_context
+from zerver.lib.email_notifications import (
+    MessageListPayload,
+    build_message_list,
+    get_channel_privacy_icon,
+    message_content_allowed_in_missedmessage_emails,
+)
+from zerver.lib.logging_util import log_to_file
+from zerver.lib.message import get_last_message_id
+from zerver.lib.queue import queue_json_publish_rollback_unsafe
+from zerver.lib.send_email import FromAddress, send_future_email
+from zerver.lib.url_encoding import stream_narrow_url
+from zerver.models import (
+    Message,
+    Realm,
+    RealmAuditLog,
+    Recipient,
+    Stream,
+    Subscription,
+    UserActivityInterval,
+    UserMessage,
+    UserProfile,
+    UserTopic,
+)
+from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.models.streams import get_active_streams
+
+logger = logging.getLogger(__name__)
+log_to_file(logger, settings.DIGEST_LOG_PATH)
+
+DIGEST_CUTOFF = 5
+MAX_HOT_TOPICS_TO_BE_INCLUDED_IN_DIGEST = 4
+
+TopicKey: TypeAlias = tuple[int, str]
+
+
+@dataclass
+class DigestTeaserData:
+    participants: list[str]
+    count: int
+    first_few_messages: MessageListPayload
+
+
+class DigestTopic:
+    def __init__(self, topic_key: TopicKey) -> None:
+        self.topic_key = topic_key
+        self.human_senders: set[str] = set()
+        self.sample_messages: list[Message] = []
+        self.num_human_messages = 0
+
+    def stream_id(self) -> int:
+        # topic_key is (stream_id, topic_name)
+        return self.topic_key[0]
+
+    def topic_name(self) -> str:
+        return self.topic_key[1]
+
+    def add_message(self, message: Message) -> None:
+        if len(self.sample_messages) < 2:
+            self.sample_messages.append(message)
+
+        if not message.sender.is_bot:
+            self.human_senders.add(message.sender.full_name)
+            self.num_human_messages += 1
+
+    def length(self) -> int:
+        return self.num_human_messages
+
+    def diversity(self) -> int:
+        return len(self.human_senders)
+
+    def teaser_data(self, user: UserProfile, stream_id_map: dict[int, Stream]) -> DigestTeaserData:
+        teaser_count = self.num_human_messages - len(self.sample_messages)
+        first_few_messages = build_message_list(
+            user=user,
+            messages=self.sample_messages,
+            stream_id_map=stream_id_map,
+        )
+        return DigestTeaserData(
+            participants=sorted(self.human_senders),
+            count=teaser_count,
+            first_few_messages=first_few_messages,
+        )
+
+
+# Digests accumulate 2 types of interesting traffic for a user:
+# 1. New streams
+# 2. Interesting stream traffic, as determined by the longest and most
+#    diversely comment upon topics.
+
+
+# Changes to this should also be reflected in
+# zerver/worker/digest_emails.py:DigestWorker.consume()
+def queue_digest_user_ids(user_ids: list[int], cutoff: datetime) -> None:
+    # Convert cutoff to epoch seconds for transit.
+    event = {"user_ids": user_ids, "cutoff": cutoff.strftime("%s")}
+    queue_json_publish_rollback_unsafe("digest_emails", event)
+
+
+def enqueue_emails(cutoff: datetime) -> None:
+    if not settings.SEND_DIGEST_EMAILS:
+        return
+
+    weekday = timezone_now().weekday()
+    for realm in Realm.objects.filter(
+        deactivated=False, digest_emails_enabled=True, digest_weekday=weekday
+    ).exclude(string_id__in=settings.SYSTEM_ONLY_REALMS):
+        _enqueue_emails_for_realm(realm, cutoff)
+
+
+def _enqueue_emails_for_realm(realm: Realm, cutoff: datetime) -> None:
+    # This should only be called directly by tests.  Use enqueue_emails
+    # to process all realms that are set up for processing on any given day.
+    twelve_hours_ago = timezone_now() - timedelta(hours=12)
+
+    target_users = (
+        UserProfile.objects.filter(
+            realm=realm,
+            is_active=True,
+            is_bot=False,
+            enable_digest_emails=True,
+        )
+        .alias(
+            recent_activity=Exists(
+                UserActivityInterval.objects.filter(user_profile_id=OuterRef("id"), end__gt=cutoff)
+            )
+        )
+        .filter(recent_activity=False)
+        .alias(
+            sent_recent_digest=Exists(
+                RealmAuditLog.objects.filter(
+                    realm_id=realm.id,
+                    event_type=AuditLogEventType.USER_DIGEST_EMAIL_CREATED,
+                    event_time__gt=twelve_hours_ago,
+                    modified_user_id=OuterRef("id"),
+                )
+            )
+        )
+        .filter(sent_recent_digest=False)
+    )
+
+    user_ids = list(target_users.order_by("id").values_list("id", flat=True))
+
+    # We process batches of 30.  We want a big enough batch
+    # to amortize work, but not so big that a single item
+    # from the queue takes too long to process.
+    chunk_size = 30
+    for i in range(0, len(user_ids), chunk_size):
+        chunk_user_ids = user_ids[i : i + chunk_size]
+        queue_digest_user_ids(chunk_user_ids, cutoff)
+        logger.info(
+            "Queuing user_ids for potential digest: %s",
+            chunk_user_ids,
+        )
+
+
+last_realm_id: int | None = None
+last_cutoff: float | None = None
+
+
+def maybe_clear_recent_topics_cache(realm_id: int, cutoff: float) -> None:
+    # As an optimization, we clear the digest caches when we switch to
+    # a new realm or cutoff value.  Since these values are part of the
+    # cache key, this is not necessary for correctness -- it merely
+    # helps reduce the memory footprint of the cache.
+    global last_realm_id, last_cutoff
+    if last_realm_id != realm_id or last_cutoff != cutoff:
+        logger.info("Flushing stream cache: %s", get_recent_topics.cache_info())
+        get_recent_topics.cache_clear()
+    last_realm_id = realm_id
+    last_cutoff = cutoff
+
+
+# We cache both by stream-id and cutoff, which ensures the per-stream
+# cache also does not contain data from old digests
+@functools.lru_cache(maxsize=5000)
+def get_recent_topics(
+    realm_id: int,
+    stream_id: int,
+    cutoff_date: datetime,
+) -> list[DigestTopic]:
+    # Gather information about topic conversations, then
+    # classify by:
+    #   * topic length
+    #   * number of senders
+
+    messages = (
+        # Uses index: zerver_message_realm_recipient_date_sent
+        Message.objects.filter(
+            realm_id=realm_id,
+            recipient__type=Recipient.STREAM,
+            recipient__type_id=stream_id,
+            date_sent__gt=cutoff_date,
+        )
+        .order_by(
+            "id",  # we will sample the first few messages
+        )
+        .select_related(
+            "recipient",  # build_message_list looks up recipient.type
+            "sender",  # we need the sender's full name
+        )
+        .defer(
+            # This construction, to only fetch the sender's full_name and is_bot,
+            # is because `.only()` doesn't work with select_related tables.
+            *{
+                f"sender__{f.name}"
+                for f in UserProfile._meta.fields
+                if f.name not in {"full_name", "is_bot"}
+            }
+        )
+    )
+
+    digest_topic_map: dict[TopicKey, DigestTopic] = {}
+    for message in messages:
+        topic_key = (stream_id, message.topic_name())
+
+        if topic_key not in digest_topic_map:
+            digest_topic_map[topic_key] = DigestTopic(topic_key)
+
+        digest_topic_map[topic_key].add_message(message)
+
+    topics = list(digest_topic_map.values())
+
+    return topics
+
+
+def get_hot_topics(
+    all_topics: list[DigestTopic],
+    stream_ids: set[int],
+) -> list[DigestTopic]:
+    topics = [topic for topic in all_topics if topic.stream_id() in stream_ids]
+
+    hot_topics = heapq.nlargest(2, topics, key=DigestTopic.diversity)
+
+    for topic in heapq.nlargest(
+        MAX_HOT_TOPICS_TO_BE_INCLUDED_IN_DIGEST, topics, key=DigestTopic.length
+    ):
+        if topic not in hot_topics:
+            hot_topics.append(topic)
+        if len(hot_topics) == MAX_HOT_TOPICS_TO_BE_INCLUDED_IN_DIGEST:
+            break
+
+    return hot_topics
+
+
+def get_recently_created_streams(realm: Realm, threshold: datetime) -> list[Stream]:
+    fields = ["id", "name", "is_web_public", "invite_only"]
+    return list(get_active_streams(realm).filter(date_created__gt=threshold).only(*fields))
+
+
+def gather_new_streams(
+    realm: Realm,
+    recently_created_streams: list[Stream],  # streams only need id and name
+    can_access_public: bool,
+) -> tuple[int, dict[str, list[Markup] | list[str]]]:
+    if can_access_public:
+        new_streams = [stream for stream in recently_created_streams if not stream.invite_only]
+    else:
+        new_streams = [stream for stream in recently_created_streams if stream.is_web_public]
+
+    channels_html = []
+    channels_plain = []
+
+    for stream in new_streams:
+        narrow_url = stream_narrow_url(realm, stream)
+        channel_link = Markup(
+            "<a href='{narrow_url}'>{channel_privacy_icon}{stream_name}</a>"
+        ).format(
+            narrow_url=narrow_url,
+            channel_privacy_icon=get_channel_privacy_icon(stream),
+            stream_name=stream.name,
+        )
+        channels_html.append(channel_link)
+        channels_plain.append(stream.name)
+
+    return len(new_streams), {"html": channels_html, "plain": channels_plain}
+
+
+def get_min_recent_message_id(realm_id: int, threshold: datetime) -> int | None:
+    # ID of the realm's earliest message at or after the threshold,
+    # or None if there are none.
+    return (
+        # Uses index: zerver_message_realm_date_sent
+        Message.objects.filter(realm_id=realm_id, date_sent__gte=threshold)
+        .order_by("date_sent")
+        .values_list("id", flat=True)
+        .first()
+    )
+
+
+def get_new_messages_count(user: UserProfile, min_recent_message_id: int | None) -> int:
+    if min_recent_message_id is None:
+        # No message was sent since the threshold.
+        return 0
+
+    # We filter on `message_id` for performance, so the planner drives
+    # from the unique (user_profile_id, message_id) index on `UserMessage`.
+    #
+    # Since we want "messages the user received since threshold",
+    # the intuition is to filter on `user_profile` and `message__date_sent`.
+    # When filtering without `message_id`, the planner drives from Message's
+    # `date_sent` index, which is not scoped to a user: it reads every message
+    # sent since threshold and joins against all UserMessage rows for the user
+    # to determine whether they received each message - which is too slow.
+    count = UserMessage.objects.filter(
+        user_profile=user,
+        message_id__gte=min_recent_message_id,
+        message__sender__is_bot=False,
+    ).count()
+    return count
+
+
+def enough_traffic(
+    hot_conversations: int, new_streams: int, new_messages: int, show_message_content: bool
+) -> bool:
+    if new_streams > 0:
+        return True
+    if not show_message_content:
+        return new_messages > 0
+    return hot_conversations > 0
+
+
+def get_user_stream_map(user_ids: list[int], cutoff_date: datetime) -> dict[int, set[int]]:
+    """Skipping streams where the user's subscription status has changed
+    when constructing digests is critical to ensure correctness for
+    streams without shared history, guest users, and long-term idle
+    users, because it means that every user has the same view of the
+    history of a given stream whose message history is being included
+    (and thus we can share a lot of work).
+
+    The downside is that newly created streams are never included in
+    the first digest email after their creation.  Should we wish to
+    change that, we will need to be very careful to avoid creating
+    bugs for any of those classes of users.
+    """
+    events = [
+        AuditLogEventType.SUBSCRIPTION_CREATED,
+        AuditLogEventType.SUBSCRIPTION_ACTIVATED,
+        AuditLogEventType.SUBSCRIPTION_DEACTIVATED,
+    ]
+    # This uses the zerver_realmauditlog_user_subscriptions_idx
+    # partial index on RealmAuditLog which is specifically for those
+    # three event types.
+    rows = (
+        Subscription.objects.filter(
+            user_profile_id__in=user_ids,
+            recipient__type=Recipient.STREAM,
+            active=True,
+            is_muted=False,
+        )
+        .alias(
+            was_modified=Exists(
+                RealmAuditLog.objects.filter(
+                    modified_stream_id=OuterRef("recipient__type_id"),
+                    modified_user_id=OuterRef("user_profile_id"),
+                    event_time__gt=cutoff_date,
+                    event_type__in=events,
+                )
+            )
+        )
+        .filter(was_modified=False)
+        .values("user_profile_id", "recipient__type_id")
+    )
+
+    # maps user_id -> {stream_id, stream_id, ...}
+    dct: dict[int, set[int]] = defaultdict(set)
+    for row in rows:
+        dct[row["user_profile_id"]].add(row["recipient__type_id"])
+
+    return dct
+
+
+def get_user_muted_topics_map(user_ids: list[int]) -> dict[int, set[tuple[int, str]]]:
+    # maps user_id -> {(stream_id, lowercased topic_name), ...};
+    # topic names are matched case-insensitively.
+    rows = UserTopic.objects.filter(
+        user_profile_id__in=user_ids,
+        visibility_policy=UserTopic.VisibilityPolicy.MUTED,
+    ).values("user_profile_id", "stream_id", "topic_name")
+
+    dct: dict[int, set[tuple[int, str]]] = defaultdict(set)
+    for row in rows:
+        dct[row["user_profile_id"]].add((row["stream_id"], row["topic_name"].lower()))
+
+    return dct
+
+
+def get_slim_stream_id_map(stream_ids: set[int]) -> dict[int, Stream]:
+    # "slim" because it only fetches the names of the stream objects,
+    # suitable for passing into build_message_list.
+    streams = Stream.objects.filter(id__in=stream_ids).only("id", "name")
+    return {stream.id: stream for stream in streams}
+
+
+def bulk_get_digest_context(
+    users: Collection[UserProfile] | QuerySet[UserProfile], cutoff: float
+) -> Iterator[tuple[UserProfile, dict[str, Any]]]:
+    # We expect a non-empty list of users all from the same realm.
+    assert users
+    realm = next(iter(users)).realm
+    for user in users:
+        assert user.realm_id == realm.id
+
+    # Convert from epoch seconds to a datetime object.
+    cutoff_date = datetime.fromtimestamp(int(cutoff), tz=timezone.utc)
+
+    maybe_clear_recent_topics_cache(realm.id, cutoff)
+
+    recently_created_streams = get_recently_created_streams(realm, cutoff_date)
+
+    user_ids = [user.id for user in users]
+    user_stream_map = get_user_stream_map(user_ids, cutoff_date)
+    stream_ids = set().union(*user_stream_map.values())
+    stream_id_map = get_slim_stream_id_map(stream_ids)
+    user_muted_topics_map = get_user_muted_topics_map(user_ids)
+
+    # ID of the realm's earliest message at or after the `cutoff`.
+    # Shared across all users, and computed lazily since it is only
+    # needed for users who have message content hidden in emails.
+    min_recent_message_id: int | None = None
+    have_min_recent_message_id = False
+
+    for user in users:
+        context = common_context(user)
+
+        # Start building email template data.
+        unsubscribe_link = one_click_unsubscribe_link(user, "digest")
+        context.update(unsubscribe_link=unsubscribe_link)
+
+        # Gather new streams.
+        new_streams_count, new_streams = gather_new_streams(
+            realm=realm,
+            recently_created_streams=recently_created_streams,
+            can_access_public=user.can_access_public_streams(),
+        )
+
+        context["new_streams_count"] = new_streams_count
+        context[
+            "message_content_disabled_by_realm"
+        ] = not realm.message_content_allowed_in_email_notifications
+        context[
+            "message_content_disabled_by_user"
+        ] = not user.message_content_in_email_notifications
+
+        if not message_content_allowed_in_missedmessage_emails(user):
+            # Count new messages when message content is hidden in email notifications.
+            if not have_min_recent_message_id:
+                min_recent_message_id = get_min_recent_message_id(realm.id, cutoff_date)
+                have_min_recent_message_id = True
+            context["new_messages_count"] = get_new_messages_count(user, min_recent_message_id)
+            context["hot_conversations"] = []
+            context["show_message_content"] = False
+        else:
+            # Otherwise, get context data for hot conversations.
+            stream_ids = user_stream_map[user.id]
+            muted_topics = user_muted_topics_map[user.id]
+            recent_topics = []
+            for stream_id in stream_ids:
+                recent_topics += [
+                    digest_topic
+                    for digest_topic in get_recent_topics(realm.id, stream_id, cutoff_date)
+                    # Topics the user has muted should not be featured
+                    # in their digest.
+                    if (digest_topic.stream_id(), digest_topic.topic_name().lower())
+                    not in muted_topics
+                ]
+            hot_topics = get_hot_topics(recent_topics, stream_ids)
+
+            context["hot_conversations"] = [
+                hot_topic.teaser_data(user, stream_id_map) for hot_topic in hot_topics
+            ]
+            context["new_channels"] = new_streams
+            context["new_messages_count"] = 0
+            context["show_message_content"] = True
+
+        yield user, context
+
+
+def get_digest_context(user: UserProfile, cutoff: float) -> dict[str, Any]:
+    for ignored, context in bulk_get_digest_context([user], cutoff):
+        return context
+    raise AssertionError("Unreachable")
+
+
+@transaction.atomic(durable=True)
+def bulk_handle_digest_email(user_ids: list[int], cutoff: float) -> None:
+    # We go directly to the database to get user objects,
+    # since inactive users are likely to not be in the cache.
+    users = (
+        UserProfile.objects.filter(id__in=user_ids, is_active=True, realm__deactivated=False)
+        .order_by("id")
+        .select_related("realm")
+    )
+    digest_users = []
+
+    for user, context in bulk_get_digest_context(users, cutoff):
+        # We don't want to send emails containing almost no information.
+        if not enough_traffic(
+            len(context["hot_conversations"]),
+            context["new_streams_count"],
+            context["new_messages_count"],
+            context["show_message_content"],
+        ):
+            continue
+
+        digest_users.append(user)
+        logger.info("Enqueuing digest email for user %s", user.id)
+
+        # Send now, as a ScheduledEmail
+        send_future_email(
+            "zerver/emails/digest",
+            user.realm,
+            to_user_ids=[user.id],
+            from_name=_("{service_name} digest").format(service_name=settings.INSTALLATION_NAME),
+            from_address=FromAddress.no_reply_placeholder,
+            context=context,
+        )
+
+    bulk_write_realm_audit_logs(digest_users)
+
+
+def bulk_write_realm_audit_logs(users: list[UserProfile]) -> None:
+    if not users:
+        return
+
+    # We write RealmAuditLog rows for auditing, and we will also
+    # use these rows during the next run to possibly exclude the
+    # users (if insufficient time has passed).
+    last_message_id = get_last_message_id()
+    now = timezone_now()
+
+    log_rows = [
+        RealmAuditLog(
+            realm_id=user.realm_id,
+            modified_user_id=user.id,
+            event_last_message_id=last_message_id,
+            event_time=now,
+            event_type=AuditLogEventType.USER_DIGEST_EMAIL_CREATED,
+        )
+        for user in users
+    ]
+
+    RealmAuditLog.objects.bulk_create(log_rows)

zerver/lib/display_recipient.py

@@ -0,0 +1,253 @@
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Optional, TypedDict
+
+from django.db.models import QuerySet
+
+from zerver.lib.cache import (
+    bulk_cached_fetch,
+    cache_with_key,
+    display_recipient_cache_key,
+    generic_bulk_cached_fetch,
+    single_user_display_recipient_cache_key,
+)
+from zerver.lib.per_request_cache import cache_for_current_request
+from zerver.lib.types import DisplayRecipientT, UserDisplayRecipient
+
+if TYPE_CHECKING:
+    from zerver.models import Recipient
+
+display_recipient_fields = [
+    "id",
+    "email",
+    "full_name",
+    "is_mirror_dummy",
+]
+
+
+class TinyStreamResult(TypedDict):
+    recipient_id: int
+    name: str
+
+
+def get_display_recipient_cache_key(
+    recipient_id: int, recipient_type: int, recipient_type_id: int | None
+) -> str:
+    return display_recipient_cache_key(recipient_id)
+
+
+# Note that the _same_ cache key is used for streams, which contain a
+# string, not a list[UserDisplayRecipient]!  This works because the
+# recipient space is distinct between the two.
+@cache_with_key(get_display_recipient_cache_key, timeout=3600 * 24 * 7)
+def get_display_recipient_remote_cache(
+    recipient_id: int, recipient_type: int, recipient_type_id: int | None
+) -> list[UserDisplayRecipient]:
+    """
+    This returns an appropriate object describing the recipient of a
+    direct message (whether individual or group).
+
+    It will be an array of dicts for each recipient.
+
+    Do not use this for streams.
+    """
+
+    from zerver.models import Recipient, UserProfile
+
+    assert recipient_type != Recipient.STREAM
+
+    # The main priority for ordering here is being deterministic.
+    # Right now, we order by ID, which matches the ordering of user
+    # names in the left sidebar.
+    user_profile_list = (
+        UserProfile.objects.filter(
+            subscription__recipient_id=recipient_id,
+        )
+        .order_by("id")
+        .values(*display_recipient_fields)
+    )
+    return list(user_profile_list)
+
+
+def user_dict_id_fetcher(user_dict: UserDisplayRecipient) -> int:
+    return user_dict["id"]
+
+
+def bulk_fetch_single_user_display_recipients(uids: list[int]) -> dict[int, UserDisplayRecipient]:
+    from zerver.models import UserProfile
+
+    return bulk_cached_fetch(
+        # Use a separate cache key to protect us from conflicts with
+        # the get_user_profile_by_id cache.
+        # (Since we fetch only several fields here)
+        cache_key_function=single_user_display_recipient_cache_key,
+        query_function=lambda ids: list(
+            UserProfile.objects.filter(id__in=ids).values(*display_recipient_fields)
+        ),
+        object_ids=uids,
+        id_fetcher=user_dict_id_fetcher,
+    )
+
+
+def bulk_fetch_stream_names(
+    recipient_tuples: set[tuple[int, int, int]],
+) -> dict[int, str]:
+    """
+    Takes set of tuples of the form (recipient_id, recipient_type, recipient_type_id)
+    Returns dict mapping recipient_id to corresponding display_recipient
+    """
+
+    from zerver.models import Stream
+
+    if len(recipient_tuples) == 0:
+        return {}
+
+    recipient_id_to_stream_id = {tup[0]: tup[2] for tup in recipient_tuples}
+    recipient_ids = [tup[0] for tup in recipient_tuples]
+
+    def get_tiny_stream_rows(
+        recipient_ids: list[int],
+    ) -> QuerySet[Stream, TinyStreamResult]:
+        stream_ids = [recipient_id_to_stream_id[recipient_id] for recipient_id in recipient_ids]
+        return Stream.objects.filter(id__in=stream_ids).values("recipient_id", "name")
+
+    def get_recipient_id(row: TinyStreamResult) -> int:
+        return row["recipient_id"]
+
+    def get_name(row: TinyStreamResult) -> str:
+        return row["name"]
+
+    # ItemT = TinyStreamResult, CacheItemT = str (name), ObjKT = int (recipient_id)
+    stream_display_recipients: dict[int, str] = generic_bulk_cached_fetch(
+        cache_key_function=display_recipient_cache_key,
+        query_function=get_tiny_stream_rows,
+        object_ids=recipient_ids,
+        id_fetcher=get_recipient_id,
+        cache_transformer=get_name,
+        setter=lambda obj: obj,
+        extractor=lambda obj: obj,
+        pickled_tupled=False,
+    )
+
+    return stream_display_recipients
+
+
+def bulk_fetch_user_display_recipients(
+    recipient_tuples: set[tuple[int, int, int]],
+    recipient_dm_user_ids: Mapping[int, set[int]] = {},
+) -> dict[int, list[UserDisplayRecipient]]:
+    """
+    Takes set of tuples of the form (recipient_id, recipient_type, recipient_type_id)
+    Returns dict mapping recipient_id to corresponding display_recipient
+
+    The participants of a DirectMessageGroup are fixed when it is created, so
+    a caller that already knows them for some recipients can pass them in via
+    dm_involved_user_ids to avoid looking them up again.
+    """
+
+    from zerver.models.recipients import bulk_get_direct_message_group_user_ids
+
+    if len(recipient_tuples) == 0:
+        return {}
+
+    get_recipient_id = lambda tup: tup[0]
+
+    direct_message_group_recipient_ids = [get_recipient_id(tup) for tup in recipient_tuples]
+
+    # Query only DirectMessageGroup participants who are not already loaded.
+    user_ids_in_direct_message_groups = bulk_get_direct_message_group_user_ids(
+        [
+            recipient_id
+            for recipient_id in direct_message_group_recipient_ids
+            if recipient_id not in recipient_dm_user_ids
+        ]
+    )
+    user_ids_in_direct_message_groups.update(recipient_dm_user_ids)
+
+    # Find all user ids whose UserProfiles we will need to fetch:
+    user_ids_to_fetch: set[int] = set()
+
+    for recipient_id in direct_message_group_recipient_ids:
+        direct_message_group_user_ids = user_ids_in_direct_message_groups[recipient_id]
+        user_ids_to_fetch |= direct_message_group_user_ids
+
+    # Fetch the needed user dictionaries.
+    user_display_recipients = bulk_fetch_single_user_display_recipients(list(user_ids_to_fetch))
+
+    result = {}
+
+    for recipient_id in direct_message_group_recipient_ids:
+        user_ids = sorted(user_ids_in_direct_message_groups[recipient_id])
+        display_recipients = [user_display_recipients[user_id] for user_id in user_ids]
+        result[recipient_id] = display_recipients
+
+    return result
+
+
+def bulk_fetch_display_recipients(
+    recipient_tuples: set[tuple[int, int, int]],
+    recipient_dm_user_ids: Mapping[int, set[int]] = {},
+) -> dict[int, DisplayRecipientT]:
+    """
+    Takes set of tuples of the form (recipient_id, recipient_type, recipient_type_id)
+    Returns dict mapping recipient_id to corresponding display_recipient
+    """
+
+    from zerver.models import Recipient
+
+    stream_recipients = {
+        recipient for recipient in recipient_tuples if recipient[1] == Recipient.STREAM
+    }
+    direct_message_recipients = recipient_tuples - stream_recipients
+
+    stream_display_recipients = bulk_fetch_stream_names(stream_recipients)
+    direct_message_display_recipients = bulk_fetch_user_display_recipients(
+        direct_message_recipients, recipient_dm_user_ids
+    )
+
+    # Glue the dicts together and return:
+    return {**stream_display_recipients, **direct_message_display_recipients}
+
+
+@cache_for_current_request
+def get_display_recipient_by_id(
+    recipient_id: int, recipient_type: int, recipient_type_id: int | None
+) -> list[UserDisplayRecipient]:
+    """
+    returns: an object describing the recipient (using a cache).
+    If the type is a stream, the type_id must be an int; a string is returned.
+    Otherwise, type_id may be None; an array of recipient dicts is returned.
+    """
+    return get_display_recipient_remote_cache(recipient_id, recipient_type, recipient_type_id)
+
+
+def get_display_recipient(recipient: "Recipient") -> list[UserDisplayRecipient]:
+    return get_display_recipient_by_id(
+        recipient.id,
+        recipient.type,
+        recipient.type_id,
+    )
+
+
+def get_recipient_ids(
+    recipient: Optional["Recipient"], user_profile_id: int
+) -> tuple[list[int], str]:
+    from zerver.models import Recipient
+
+    if recipient is None:
+        recipient_type_str = ""
+        to = []
+    elif recipient.type == Recipient.STREAM:
+        recipient_type_str = "stream"
+        to = [recipient.type_id]
+    else:
+        # We're somewhat weirdly using the display recipient cache,
+        # for the fact that it is a probably-cached lookup which knows
+        # the user-ids which are members of a group DM.
+        recipient_type_str = "private"
+        to = []
+        recipients = get_display_recipient(recipient)
+        for r in recipients:
+            assert not isinstance(r, str)  # It will only be a string for streams
+            if r["id"] != user_profile_id or len(recipients) == 1:
+                to.append(r["id"])
+    return to, recipient_type_str

zerver/lib/domains.py

@@ -0,0 +1,22 @@
+import re
+
+from django.core.exceptions import ValidationError
+from django.utils.translation import gettext as _
+
+
+def validate_domain(domain: str | None) -> None:
+    if domain is None or len(domain) == 0:
+        raise ValidationError(_("Domain can't be empty."))
+    if "." not in domain:
+        raise ValidationError(_("Domain must have at least one dot (.)"))
+    if len(domain) > 255:
+        raise ValidationError(_("Domain is too long"))
+    if domain[0] == "." or domain[-1] == ".":
+        raise ValidationError(_("Domain cannot start or end with a dot (.)"))
+    for subdomain in domain.split("."):
+        if not subdomain:
+            raise ValidationError(_("Consecutive '.' are not allowed."))
+        if subdomain[0] == "-" or subdomain[-1] == "-":
+            raise ValidationError(_("Subdomains cannot start or end with a '-'."))
+        if not re.match(r"^[a-z0-9-]*$", subdomain):
+            raise ValidationError(_("Domain can only have letters, numbers, '.' and '-'s."))

zerver/lib/drafts.py

@@ -0,0 +1,183 @@
+import time
+from collections.abc import Callable
+from dataclasses import dataclass
+from datetime import datetime
+from functools import wraps
+from typing import Annotated, Concatenate, Literal
+
+from django.core.exceptions import ValidationError
+from django.db import transaction
+from django.http import HttpRequest, HttpResponse
+from django.utils.translation import gettext as _
+from pydantic import BaseModel, ConfigDict
+from typing_extensions import ParamSpec
+
+from zerver.lib.addressee import get_user_profiles_by_ids
+from zerver.lib.exceptions import JsonableError, ResourceNotFoundError
+from zerver.lib.message import normalize_body, truncate_topic
+from zerver.lib.recipient_users import (
+    check_sender_can_access_recipients,
+    recipient_for_user_profiles,
+)
+from zerver.lib.streams import access_stream_by_id
+from zerver.lib.timestamp import timestamp_to_datetime
+from zerver.lib.typed_endpoint import RequiredStringConstraint
+from zerver.models import Draft, UserProfile
+from zerver.tornado.django_api import send_event_on_commit
+
+ParamT = ParamSpec("ParamT")
+
+
+@dataclass
+class ValidatedDraftData:
+    recipient_id: int | None
+    topic: str
+    content: str
+    last_edit_time: datetime
+
+
+class DraftData(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    type: Literal["private", "stream", ""]
+    to: list[int]
+    topic: str
+    content: Annotated[str, RequiredStringConstraint()]
+    timestamp: int | float | None = None
+
+
+def further_validated_draft_dict(
+    draft_dict: DraftData, user_profile: UserProfile
+) -> ValidatedDraftData:
+    """Take a DraftData object that was already validated by the @typed_endpoint
+    decorator then further sanitize, validate, and transform it.
+    Ultimately return this "further validated" draft dict.
+    It will have a slightly different set of keys the values
+    for which can be used to directly create a Draft object."""
+
+    content = normalize_body(draft_dict.content)
+
+    timestamp = draft_dict.timestamp
+    if timestamp is None:
+        timestamp = time.time()
+    timestamp = round(timestamp, 6)
+    if timestamp < 0:
+        # While it's not exactly an invalid timestamp, it's not something
+        # we want to allow either.
+        raise JsonableError(_("Timestamp must not be negative."))
+    last_edit_time = timestamp_to_datetime(timestamp)
+
+    topic_name = ""
+    recipient_id = None
+    to = draft_dict.to
+    if draft_dict.type == "stream":
+        topic_name = truncate_topic(draft_dict.topic)
+        if "\0" in topic_name:
+            raise JsonableError(_("Topic must not contain null bytes"))
+        if len(to) != 1:
+            raise JsonableError(_("Must specify exactly 1 channel ID for channel messages"))
+        stream, _sub = access_stream_by_id(user_profile, to[0])
+        recipient_id = stream.recipient_id
+    elif draft_dict.type == "private" and len(to) != 0:
+        to_users = get_user_profiles_by_ids(set(to), user_profile.realm)
+        # Since recipient_for_user_profiles creates the
+        # DirectMessageGroup object, and thus Subscription objects
+        # granting the sender access to the recipients, we need to
+        # check access before calling it.
+        check_sender_can_access_recipients(user_profile, to_users)
+        try:
+            recipient_id = recipient_for_user_profiles(to_users, False, None, user_profile).id
+        except ValidationError as e:  # nocoverage
+            raise JsonableError(e.messages[0])
+
+    return ValidatedDraftData(
+        recipient_id=recipient_id,
+        topic=topic_name,
+        content=content,
+        last_edit_time=last_edit_time,
+    )
+
+
+def draft_endpoint(
+    view_func: Callable[Concatenate[HttpRequest, UserProfile, ParamT], HttpResponse],
+) -> Callable[Concatenate[HttpRequest, UserProfile, ParamT], HttpResponse]:
+    @wraps(view_func)
+    def draft_view_func(
+        request: HttpRequest,
+        user_profile: UserProfile,
+        /,
+        *args: ParamT.args,
+        **kwargs: ParamT.kwargs,
+    ) -> HttpResponse:
+        if not user_profile.enable_drafts_synchronization:
+            raise JsonableError(_("User has disabled synchronizing drafts."))
+        return view_func(request, user_profile, *args, **kwargs)
+
+    return draft_view_func
+
+
+def do_create_drafts(drafts: list[DraftData], user_profile: UserProfile) -> list[Draft]:
+    """Create drafts in bulk for a given user based on the DraftData objects. Since
+    currently, the only place this method is being used (apart from tests) is from
+    the create_draft view, we assume that these are syntactically valid
+    (i.e. they satisfy the @typed_endpoint validation for DraftData)."""
+    draft_objects = []
+    for draft in drafts:
+        valid_draft_dict = further_validated_draft_dict(draft, user_profile)
+        draft_objects.append(
+            Draft(
+                user_profile=user_profile,
+                recipient_id=valid_draft_dict.recipient_id,
+                topic=valid_draft_dict.topic,
+                content=valid_draft_dict.content,
+                last_edit_time=valid_draft_dict.last_edit_time,
+            )
+        )
+
+    with transaction.atomic(durable=True):
+        created_draft_objects = Draft.objects.bulk_create(draft_objects)
+
+        event = {
+            "type": "drafts",
+            "op": "add",
+            "drafts": [draft.to_dict() for draft in created_draft_objects],
+        }
+        send_event_on_commit(user_profile.realm, event, [user_profile.id])
+
+    return created_draft_objects
+
+
+def do_edit_draft(draft_id: int, draft: DraftData, user_profile: UserProfile) -> None:
+    """Edit/update a single draft for a given user. Since the only place this method is being
+    used from (apart from tests) is the edit_draft view, we assume that the DraftData object
+    is syntactically valid (i.e. it satisfies the @typed_endpoint validation for DraftData)."""
+    try:
+        draft_object = Draft.objects.get(id=draft_id, user_profile=user_profile)
+    except Draft.DoesNotExist:
+        raise ResourceNotFoundError(_("Draft does not exist"))
+    valid_draft_dict = further_validated_draft_dict(draft, user_profile)
+    draft_object.content = valid_draft_dict.content
+    draft_object.topic = valid_draft_dict.topic
+    draft_object.recipient_id = valid_draft_dict.recipient_id
+    draft_object.last_edit_time = valid_draft_dict.last_edit_time
+
+    with transaction.atomic(durable=True):
+        draft_object.save()
+
+        event = {"type": "drafts", "op": "update", "draft": draft_object.to_dict()}
+        send_event_on_commit(user_profile.realm, event, [user_profile.id])
+
+
+@transaction.atomic(durable=True)
+def do_delete_draft(draft_id: int, user_profile: UserProfile) -> None:
+    """Delete a draft belonging to a particular user."""
+    try:
+        draft_object = Draft.objects.get(id=draft_id, user_profile=user_profile)
+    except Draft.DoesNotExist:
+        raise ResourceNotFoundError(_("Draft does not exist"))
+
+    draft_id = draft_object.id
+    draft_object.delete()
+
+    event = {"type": "drafts", "op": "remove", "draft_id": draft_id}
+    send_event_on_commit(user_profile.realm, event, [user_profile.id])

zerver/lib/email_mirror.py

@@ -0,0 +1,607 @@
+import encodings.aliases
+import logging
+import re
+import secrets
+import sys
+from email.headerregistry import Address, AddressHeader
+from email.message import EmailMessage
+from email.utils import parseaddr
+from re import Match
+
+from django.conf import settings
+from django.db.models import Q
+from django.utils.translation import gettext as _
+from typing_extensions import override
+
+from zerver.actions.message_send import (
+    check_send_message,
+    internal_send_group_direct_message,
+    internal_send_private_message,
+    internal_send_stream_message,
+)
+from zerver.lib.display_recipient import get_display_recipient
+from zerver.lib.email_mirror_helpers import (
+    ZulipEmailForwardError,
+    ZulipEmailForwardUserError,
+    decode_email_address,
+    get_email_gateway_message_string_from_address,
+)
+from zerver.lib.exceptions import JsonableError, RateLimitedError
+from zerver.lib.markdown import get_markdown_link_for_url
+from zerver.lib.markdown.from_html import convert_html_to_markdown
+from zerver.lib.message import normalize_body, truncate_content, truncate_topic
+from zerver.lib.rate_limiter import RateLimitedObject
+from zerver.lib.send_email import FromAddress
+from zerver.lib.streams import access_stream_for_send_message
+from zerver.lib.string_validation import is_character_printable
+from zerver.lib.upload import upload_message_attachment
+from zerver.models import (
+    ChannelEmailAddress,
+    Message,
+    MissedMessageEmailAddress,
+    Realm,
+    Recipient,
+    Stream,
+    UserProfile,
+)
+from zerver.models.clients import get_client
+from zerver.models.constants import MAX_TOPIC_NAME_LENGTH
+from zerver.models.streams import StreamTopicsPolicyEnum, get_stream_by_id_in_realm
+from zerver.models.users import get_system_bot
+from zproject.backends import is_user_active
+
+if sys.version_info < (3, 14):  # nocoverage
+    # https://github.com/python/cpython/issues/62824
+    encodings.aliases.aliases.update(
+        {
+            "iso_8859_8_i": "iso8859_8",
+            "iso_8859_8_e": "iso8859_8",
+        }
+    )
+
+logger = logging.getLogger(__name__)
+
+
+def redact_email_address(error_message: str) -> str:
+    if not settings.EMAIL_GATEWAY_EXTRA_PATTERN_HACK:
+        domain = Address(addr_spec=settings.EMAIL_GATEWAY_PATTERN).domain
+    else:
+        # EMAIL_GATEWAY_EXTRA_PATTERN_HACK is of the form '@example.com'
+        domain = settings.EMAIL_GATEWAY_EXTRA_PATTERN_HACK.removeprefix("@")
+
+    def redact(address_match: Match[str]) -> str:
+        email_address = address_match[0]
+        # Annotate basic info about the address before scrubbing:
+        if is_missed_message_address(email_address):
+            annotation = " <Missed message address>"
+        else:
+            try:
+                target_stream_id = decode_stream_email_address(email_address)[0].channel_id
+                annotation = f" <Address to stream id: {target_stream_id}>"
+            except ZulipEmailForwardError:
+                annotation = " <Invalid address>"
+
+        # Scrub the address from the message, to the form XXXXX@example.com:
+        return "X" * len(address_match[1]) + address_match[2] + annotation
+
+    return re.sub(rf"\b(\S*?)(@{re.escape(domain)})", redact, error_message)
+
+
+def log_error(email_message: EmailMessage, error_message: str, to: str | None) -> None:
+    recipient = to or "No recipient found"
+    error_message = "Sender: {}\nTo: {}\n{}".format(
+        email_message.get("From"), recipient, error_message
+    )
+
+    error_message = redact_email_address(error_message)
+    logger.error(error_message)
+
+
+# Temporary missed message addresses
+
+
+def generate_missed_message_token() -> str:
+    return "mm" + secrets.token_hex(16)
+
+
+def is_missed_message_address(address: str) -> bool:
+    try:
+        msg_string = get_email_gateway_message_string_from_address(address)
+    except ZulipEmailForwardError:
+        return False
+
+    return is_mm_32_format(msg_string)
+
+
+def is_mm_32_format(msg_string: str | None) -> bool:
+    """
+    Missed message strings are formatted with a little "mm" prefix
+    followed by a randomly generated 32-character string.
+    """
+    return msg_string is not None and msg_string.startswith("mm") and len(msg_string) == 34
+
+
+def get_missed_message_token_from_address(address: str) -> str:
+    msg_string = get_email_gateway_message_string_from_address(address)
+
+    if not is_mm_32_format(msg_string):
+        raise ZulipEmailForwardError("Could not parse missed message address")
+
+    return msg_string
+
+
+def get_usable_missed_message_address(address: str) -> MissedMessageEmailAddress:
+    token = get_missed_message_token_from_address(address)
+    try:
+        mm_address = MissedMessageEmailAddress.objects.select_related(
+            "user_profile",
+            "user_profile__realm",
+            "message",
+            "message__sender",
+            "message__recipient",
+        ).get(email_token=token)
+    except MissedMessageEmailAddress.DoesNotExist:
+        raise ZulipEmailForwardError("Zulip notification reply address is invalid.")
+
+    return mm_address
+
+
+def create_missed_message_address(user_profile: UserProfile, message: Message) -> str:
+    # If the email gateway isn't configured, we specify a reply
+    # address, since there's no useful way for the user to reply into
+    # Zulip.
+    if settings.EMAIL_GATEWAY_PATTERN == "":
+        return FromAddress.NOREPLY
+
+    mm_address = MissedMessageEmailAddress.objects.create(
+        message=message,
+        user_profile=user_profile,
+        email_token=generate_missed_message_token(),
+    )
+    return str(mm_address)
+
+
+def construct_zulip_body(
+    message: EmailMessage,
+    subject: str,
+    realm: Realm,
+    *,
+    sender: UserProfile,
+    show_sender: bool = False,
+    include_quotes: bool = False,
+    include_footer: bool = False,
+    prefer_text: bool = True,
+    subject_in_body: bool = False,
+) -> str:
+    body = extract_body(message, include_quotes, prefer_text)
+    # Remove null characters, since Zulip will reject
+    body = body.replace("\x00", "")
+    if not include_footer:
+        body = filter_footer(body)
+
+    if not body.endswith("\n"):
+        body += "\n"
+    if not body.rstrip():
+        body = "(No email body)"
+
+    preamble = ""
+    if show_sender:
+        from_address = str(message.get("From", ""))
+        preamble = f"**From:** {from_address}\n"
+    if subject_in_body:
+        preamble += f"**Subject:** {subject}\n"
+    if preamble != "":
+        preamble += "\n"
+
+    postamble = extract_and_upload_attachments(message, realm, sender)
+    if postamble != "":
+        postamble = "\n" + postamble
+
+    # Truncate the content ourselves, to ensure that the attachments
+    # all make it into the body-as-posted
+    body = truncate_content(
+        body,
+        settings.MAX_MESSAGE_LENGTH - len(preamble) - len(postamble),
+        "\n[message truncated]",
+    )
+    return preamble + body + postamble
+
+
+## Sending the Zulip ##
+
+
+def send_zulip(sender: UserProfile, stream: Stream, topic_name: str, content: str) -> None:
+    internal_send_stream_message(
+        sender,
+        stream,
+        truncate_topic(topic_name),
+        normalize_body(content),
+        email_gateway=True,
+    )
+
+
+def send_mm_reply_to_stream(
+    user_profile: UserProfile, stream: Stream, topic_name: str, body: str
+) -> None:
+    try:
+        check_send_message(
+            sender=user_profile,
+            client=get_client("Internal"),
+            recipient_type_name="stream",
+            message_to=[stream.id],
+            topic_name=topic_name,
+            message_content=body,
+        )
+    except JsonableError as error:
+        error_message = _(
+            "Error sending message to channel {channel_name} via message notification email reply:\n{error_message}"
+        ).format(channel_name=stream.name, error_message=error.msg)
+        internal_send_private_message(
+            get_system_bot(settings.NOTIFICATION_BOT, user_profile.realm_id),
+            user_profile,
+            error_message,
+        )
+
+
+def get_message_part_by_type(message: EmailMessage, content_type: str) -> str | None:
+    for part in message.walk():
+        if part.get_content_type() == content_type:
+            content = part.get_payload(decode=True)
+            assert isinstance(content, bytes)
+
+            charset = part.get_content_charset()
+            if charset is None:
+                # If no charset has been specified in the header, assume us-ascii,
+                # by RFC6657: https://tools.ietf.org/html/rfc6657
+                charset = "us-ascii"
+
+            try:
+                return content.decode(charset, errors="ignore")
+            except LookupError:
+                # The RFCs do not define how to handle unknown charsets,
+                # but treating as US-ASCII seems reasonable.
+                return content.decode("us-ascii", errors="ignore")
+
+    return None
+
+
+def extract_body(
+    message: EmailMessage, include_quotes: bool = False, prefer_text: bool = True
+) -> str:
+    plaintext_content = extract_plaintext_body(message, include_quotes)
+    html_content = extract_html_body(message, include_quotes)
+
+    if plaintext_content is None and html_content is None:
+        logger.warning("Content types: %s", [part.get_content_type() for part in message.walk()])
+        raise ZulipEmailForwardUserError("Unable to find plaintext or HTML message body")
+    if not plaintext_content and not html_content:
+        raise ZulipEmailForwardUserError("Email has no nonempty body sections; ignoring.")
+
+    if prefer_text:
+        if plaintext_content:
+            return plaintext_content
+        else:
+            assert html_content  # Needed for mypy. Ensured by the validating block above.
+            return html_content
+    else:
+        if html_content:
+            return html_content
+        else:
+            assert plaintext_content  # Needed for mypy. Ensured by the validating block above.
+            return plaintext_content
+
+
+talon_initialized = False
+
+
+def extract_plaintext_body(message: EmailMessage, include_quotes: bool = False) -> str | None:
+    import talon_core
+
+    global talon_initialized
+    if not talon_initialized:
+        talon_core.init()
+        talon_initialized = True
+
+    plaintext_content = get_message_part_by_type(message, "text/plain")
+    if plaintext_content is not None:
+        if include_quotes:
+            return plaintext_content
+        else:
+            return talon_core.quotations.extract_from_plain(plaintext_content)
+    else:
+        return None
+
+
+def extract_html_body(message: EmailMessage, include_quotes: bool = False) -> str | None:
+    import talon_core
+
+    global talon_initialized
+    if not talon_initialized:  # nocoverage
+        talon_core.init()
+        talon_initialized = True
+
+    html_content = get_message_part_by_type(message, "text/html")
+    if html_content is not None:
+        if include_quotes:
+            return convert_html_to_markdown(html_content)
+        else:
+            return convert_html_to_markdown(talon_core.quotations.extract_from_html(html_content))
+    else:
+        return None
+
+
+def filter_footer(text: str) -> str:
+    # Try to filter out obvious footers.
+    possible_footers = [line for line in text.split("\n") if line.strip() == "--"]
+    if len(possible_footers) != 1:
+        # Be conservative and don't try to scrub content if there
+        # isn't a trivial footer structure.
+        return text
+
+    return re.split(r"^\s*--\s*$", text, maxsplit=1, flags=re.MULTILINE)[0].strip()
+
+
+def extract_and_upload_attachments(message: EmailMessage, realm: Realm, sender: UserProfile) -> str:
+    attachment_links = []
+    for part in message.walk():
+        content_type = part.get_content_type()
+        filename = part.get_filename()
+        if filename:
+            attachment = part.get_payload(decode=True)
+            if isinstance(attachment, bytes):
+                upload_url, filename = upload_message_attachment(
+                    filename,
+                    content_type,
+                    attachment,
+                    sender,
+                    target_realm=realm,
+                )
+                formatted_link = get_markdown_link_for_url(filename, upload_url)
+                attachment_links.append(formatted_link)
+            else:
+                logger.warning(
+                    "Payload is not bytes (invalid attachment %s in message from %s).",
+                    filename,
+                    message.get("From"),
+                )
+
+    return "\n".join(attachment_links)
+
+
+def decode_stream_email_address(email: str) -> tuple[ChannelEmailAddress, dict[str, bool]]:
+    token, options = decode_email_address(email)
+
+    try:
+        channel_email_address = ChannelEmailAddress.objects.select_related(
+            "channel", "sender", "creator", "realm"
+        ).get(email_token=token)
+    except ChannelEmailAddress.DoesNotExist:
+        raise ZulipEmailForwardError("Bad stream token from email recipient " + email)
+
+    return channel_email_address, options
+
+
+def find_emailgateway_recipient(message: EmailMessage) -> str:
+    # We can't use Delivered-To; if there is a X-Gm-Original-To
+    # it is more accurate, so try to find the most-accurate
+    # recipient list in descending priority order
+    recipient_headers = [
+        "X-Gm-Original-To",
+        "Delivered-To",
+        "Envelope-To",
+        "Resent-To",
+        "Resent-CC",
+        "To",
+        "CC",
+    ]
+
+    pattern_parts = [re.escape(part) for part in settings.EMAIL_GATEWAY_PATTERN.split("%s")]
+    match_email_re = re.compile(r".*?".join(pattern_parts))
+
+    for header_name in recipient_headers:
+        for header_value in message.get_all(header_name, []):
+            if isinstance(header_value, AddressHeader):
+                emails = [addr.addr_spec for addr in header_value.addresses]
+            else:
+                emails = [parseaddr(str(header_value))[1]]
+
+            for email in emails:
+                if match_email_re.match(email):
+                    return email
+
+    raise ZulipEmailForwardError("Missing recipient in mirror email")
+
+
+def strip_from_subject(subject: str) -> str:
+    # strips RE and FWD from the subject
+    # from: https://stackoverflow.com/questions/9153629/regex-code-for-removing-fwd-re-etc-from-email-subject
+    reg = r"([\[\(] *)?\b(RE|AW|SV|FWD?) *(\[\d+\])?([-:;)\]][ :;\])-]*|$)|\]+ *$"
+    stripped = re.sub(reg, "", subject, flags=re.IGNORECASE | re.MULTILINE)
+    return stripped.strip()
+
+
+def is_forwarded(subject: str) -> bool:
+    # regex taken from strip_from_subject, we use it to detect various forms
+    # of FWD at the beginning of the subject.
+    reg = r"([\[\(] *)?\b(FWD?) *([-:;)\]][ :;\])-]*|$)|\]+ *$"
+    return bool(re.match(reg, subject, flags=re.IGNORECASE))
+
+
+def check_access_for_channel_email_address(channel_email_address: ChannelEmailAddress) -> None:
+    channel = channel_email_address.channel
+    sender = channel_email_address.sender
+    creator = channel_email_address.creator
+    realm = channel_email_address.realm
+    email_gateway_bot = get_system_bot(settings.EMAIL_GATEWAY_BOT, realm.id)
+
+    if sender.id == email_gateway_bot.id and creator is not None:
+        user_for_access_check = creator
+    else:
+        user_for_access_check = sender
+
+    # Raises JsonableError on permission denied
+    access_stream_for_send_message(user_for_access_check, channel, forwarder_user_profile=None)
+
+
+def process_stream_message(to: str, message: EmailMessage) -> None:
+    subject_header = message.get("Subject", "")
+
+    channel_email_address, options = decode_stream_email_address(to)
+    channel = channel_email_address.channel
+    sender = channel_email_address.sender
+    realm = channel_email_address.realm
+    try:
+        check_access_for_channel_email_address(channel_email_address)
+    except JsonableError as e:
+        logger.info("Failed to process email to %s (%s): %s", channel.name, realm.string_id, e)
+        return
+
+    # Don't remove quotations if message is forwarded, unless otherwise specified:
+    if "include_quotes" not in options:
+        options["include_quotes"] = is_forwarded(subject_header)
+
+    subject = strip_from_subject(subject_header)
+    # We don't want to reject email messages with disallowed characters in the Subject,
+    # so we just remove them to make it a valid Zulip topic name.
+    subject = "".join([char for char in subject if is_character_printable(char)])
+    if channel.topics_policy == StreamTopicsPolicyEnum.empty_topic_only.value:
+        options["subject_in_body"] = True
+        topic = ""
+    elif subject == "":
+        topic = _("Email with no subject")
+    else:
+        topic = subject
+
+    if len(subject) > MAX_TOPIC_NAME_LENGTH:
+        options["subject_in_body"] = True
+
+    body = construct_zulip_body(message, subject, realm, sender=sender, **options)
+    send_zulip(sender, channel, topic, body)
+    logger.info(
+        "Successfully processed email to %s (%s)",
+        channel.name,
+        realm.string_id,
+    )
+
+
+def process_missed_message(to: str, message: EmailMessage) -> None:
+    auto_submitted = message.get("Auto-Submitted", "")
+    if auto_submitted in ("auto-replied", "auto-generated"):
+        logger.info("Dropping %s message from %s", auto_submitted, message.get("From"))
+        return
+
+    mm_address = get_usable_missed_message_address(to)
+    mm_address.increment_times_used()
+
+    user_profile = mm_address.user_profile
+    topic_name = mm_address.message.topic_name()
+    recipient = mm_address.message.recipient
+
+    if not is_user_active(user_profile):
+        logger.warning("Sending user is not active. Ignoring this message notification email.")
+        return
+
+    body = construct_zulip_body(message, topic_name, user_profile.realm, sender=user_profile)
+
+    assert recipient is not None
+    if recipient.type == Recipient.STREAM:
+        stream = get_stream_by_id_in_realm(recipient.type_id, user_profile.realm)
+        send_mm_reply_to_stream(user_profile, stream, topic_name, body)
+        recipient_str = stream.name
+    elif recipient.type == Recipient.DIRECT_MESSAGE_GROUP:
+        display_recipient = get_display_recipient(recipient)
+        # Drop members who have been deactivated, matching the check in
+        # validate_recipient_user_profiles. The web app blocks composing
+        # to such a group entirely, but the email gateway can receive
+        # replies to notification emails that predate the deactivation.
+        user_ids = [user_dict["id"] for user_dict in display_recipient]
+        emails = list(
+            UserProfile.objects.filter(id__in=user_ids)
+            .filter(Q(is_active=True) | Q(is_mirror_dummy=True))
+            .values_list("email", flat=True)
+        )
+        if len(emails) <= 1:
+            # Everyone but the sender (whom we verified active above)
+            # has been deactivated; there is nobody left to receive
+            # the reply.
+            logger.info(
+                "Dropping message notification email reply from user %s to a group direct message with no active recipients",
+                user_profile.id,
+            )
+            return
+        recipient_str = ", ".join(emails)
+        internal_send_group_direct_message(user_profile.realm, user_profile, body, emails=emails)
+    else:
+        raise AssertionError("Invalid recipient type!")
+
+    logger.info(
+        "Successfully processed email from user %s to %s",
+        user_profile.id,
+        recipient_str,
+    )
+
+
+def process_message(message: EmailMessage, rcpt_to: str | None = None) -> None:
+    to: str | None = None
+
+    try:
+        if rcpt_to is not None:
+            to = rcpt_to
+        else:
+            to = find_emailgateway_recipient(message)
+
+        if is_missed_message_address(to):
+            process_missed_message(to, message)
+        else:
+            process_stream_message(to, message)
+    except ZulipEmailForwardUserError as e:
+        # TODO: notify sender of error, retry if appropriate.
+        logger.info(e.args[0])
+    except ZulipEmailForwardError as e:
+        log_error(message, e.args[0], to)
+
+
+def validate_to_address(address: str, rate_limit: bool = True) -> None:
+    if is_missed_message_address(address):
+        mm_address = get_usable_missed_message_address(address)
+        if mm_address.message.recipient.type == Recipient.STREAM:
+            # ACL's on DMs are harder to apply simply, so we
+            # just check channel messages.
+            access_stream_for_send_message(
+                mm_address.user_profile,
+                get_stream_by_id_in_realm(
+                    mm_address.message.recipient.type_id,
+                    mm_address.user_profile.realm,
+                ),
+                forwarder_user_profile=None,
+            )
+    else:
+        channel_email = decode_stream_email_address(address)[0]
+        if rate_limit:
+            # Only channel email addresses are rate-limited, since
+            # they are likely to be used as the destination for
+            # mails from automated systems.
+            rate_limit_mirror_by_realm(channel_email.realm)
+        check_access_for_channel_email_address(channel_email)
+
+
+# Email mirror rate limiter code:
+class RateLimitedRealmMirror(RateLimitedObject):
+    def __init__(self, realm: Realm) -> None:
+        self.realm = realm
+        super().__init__()
+
+    @override
+    def key(self) -> str:
+        return f"{type(self).__name__}:{self.realm.string_id}"
+
+    @override
+    def rules(self) -> list[tuple[int, int]]:
+        return settings.RATE_LIMITING_MIRROR_REALM_RULES
+
+
+def rate_limit_mirror_by_realm(recipient_realm: Realm) -> None:
+    ratelimited, secs_to_freedom = RateLimitedRealmMirror(recipient_realm).rate_limit()
+
+    if ratelimited:
+        raise RateLimitedError(secs_to_freedom)

zerver/lib/email_mirror_helpers.py

@@ -0,0 +1,122 @@
+import re
+from collections.abc import Callable
+from typing import Any
+
+from django.conf import settings
+from django.utils.text import slugify
+
+from zerver.models import ChannelEmailAddress, Stream, UserProfile
+
+
+def default_option_handler_factory(address_option: str) -> Callable[[dict[str, Any]], None]:
+    def option_setter(options_dict: dict[str, Any]) -> None:
+        options_dict[address_option.replace("-", "_")] = True
+
+    return option_setter
+
+
+optional_address_tokens = {
+    "show-sender": default_option_handler_factory("show-sender"),
+    "include-footer": default_option_handler_factory("include-footer"),
+    "include-quotes": default_option_handler_factory("include-quotes"),
+    "prefer-text": lambda options: options.update(prefer_text=True),
+    "prefer-html": lambda options: options.update(prefer_text=False),
+}
+
+
+class ZulipEmailForwardError(Exception):
+    pass
+
+
+class ZulipEmailForwardUserError(ZulipEmailForwardError):
+    pass
+
+
+def get_email_gateway_message_string_from_address(address: str) -> str:
+    if settings.EMAIL_GATEWAY_PATTERN == "":
+        raise ZulipEmailForwardError("This server is not configured for incoming email.")
+    pattern_parts = [re.escape(part) for part in settings.EMAIL_GATEWAY_PATTERN.split("%s")]
+    if settings.EMAIL_GATEWAY_EXTRA_PATTERN_HACK:
+        # Accept mails delivered to any Zulip server
+        pattern_parts[-1] = settings.EMAIL_GATEWAY_EXTRA_PATTERN_HACK
+    match_email_re = re.compile(r"(.*?)".join(pattern_parts))
+    match = match_email_re.match(address)
+
+    if not match:
+        raise ZulipEmailForwardError("Address not recognized by gateway.")
+    msg_string = match.group(1)
+
+    return msg_string
+
+
+def get_channel_email_token(stream: Stream, *, creator: UserProfile, sender: UserProfile) -> str:
+    channel_email_address, _created = ChannelEmailAddress.objects.get_or_create(
+        realm=stream.realm,
+        channel=stream,
+        creator=creator,
+        sender=sender,
+    )
+    return channel_email_address.email_token
+
+
+def encode_email_address(name: str, email_token: str, show_sender: bool = False) -> str:
+    # Some deployments may not use the email gateway
+    if settings.EMAIL_GATEWAY_PATTERN == "":
+        return ""
+
+    # Given the fact that we have almost no restrictions on stream names and
+    # that what characters are allowed in e-mail addresses is complicated and
+    # dependent on context in the address, we opt for a simple scheme:
+    # 1. Replace all substrings of non-alphanumeric characters with a single hyphen.
+    # 2. Use Django's slugify to convert the resulting name to ascii.
+    # 3. If the resulting name is shorter than the name we got in step 1,
+    # it means some letters can't be reasonably turned to ascii and have to be dropped,
+    # which would mangle the name, so we just skip the name part of the address.
+    name = re.sub(r"\W+", "-", name)
+    slug_name = slugify(name)
+    encoded_name = slug_name if len(slug_name) == len(name) else ""
+
+    # If encoded_name ends up empty, we just skip this part of the address:
+    if encoded_name:
+        encoded_token = f"{encoded_name}.{email_token}"
+    else:
+        encoded_token = email_token
+
+    if show_sender:
+        encoded_token += ".show-sender"
+
+    return settings.EMAIL_GATEWAY_PATTERN % (encoded_token,)
+
+
+def decode_email_address(email: str) -> tuple[str, dict[str, bool]]:
+    # Perform the reverse of encode_email_address. Returns a tuple of
+    # (email_token, options)
+    msg_string = get_email_gateway_message_string_from_address(email)
+
+    # Support both + and . as separators.  For background, the `+` is
+    # more aesthetically pleasing, but because Google groups silently
+    # drops the use of `+` in email addresses, which would completely
+    # break the integration, we now favor `.` as the separator between
+    # tokens in the email addresses we generate.
+    #
+    # We need to keep supporting `+` indefinitely for backwards
+    # compatibility with older versions of Zulip that offered users
+    # email addresses prioritizing using `+` for better aesthetics.
+    msg_string = msg_string.replace(".", "+")
+
+    parts = msg_string.split("+")
+    options: dict[str, bool] = {}
+    for part in parts:
+        if part in optional_address_tokens:
+            optional_address_tokens[part](options)
+
+    remaining_parts = [part for part in parts if part not in optional_address_tokens]
+
+    # There should be one or two parts left:
+    # [stream_name, email_token] or just [email_token]
+    if len(remaining_parts) == 1:
+        token = remaining_parts[0]
+    else:
+        token = remaining_parts[1]
+
+    return token, options

zerver/lib/email_mirror_server.py

@@ -0,0 +1,275 @@
+import asyncio
+import base64
+import email
+import grp
+import logging
+import os
+import pwd
+import signal
+import smtplib
+import socket
+from collections.abc import Awaitable
+from contextlib import suppress
+from ssl import SSLContext, SSLError
+from typing import Any
+from urllib.parse import SplitResult
+
+from aiosmtpd.controller import UnthreadedController
+from aiosmtpd.handlers import Message as MessageHandler
+from aiosmtpd.smtp import SMTP, Envelope, Session, TLSSetupException
+from asgiref.sync import sync_to_async
+from django.conf import settings
+from django.core.mail import EmailMultiAlternatives as DjangoEmailMultiAlternatives
+from typing_extensions import override
+
+from version import ZULIP_VERSION
+from zerver.lib.email_mirror import decode_stream_email_address, validate_to_address
+from zerver.lib.email_mirror_helpers import (
+    ZulipEmailForwardError,
+    get_email_gateway_message_string_from_address,
+)
+from zerver.lib.exceptions import JsonableError, RateLimitedError
+from zerver.lib.logging_util import log_to_file
+from zerver.lib.queue import queue_json_publish_rollback_unsafe
+
+# We add a file handler to this later, once we've dropped privileges
+logger = logging.getLogger("zerver.lib.email_mirror")
+
+
+def send_to_postmaster(msg: email.message.Message) -> None:
+    # RFC5321 says:
+    #   Any system that includes an SMTP server supporting mail relaying or
+    #   delivery MUST support the reserved mailbox "postmaster" as a case-
+    #   insensitive local name.  This postmaster address is not strictly
+    #   necessary if the server always returns 554 on connection opening (as
+    #   described in Section 3.1).  The requirement to accept mail for
+    #   postmaster implies that RCPT commands that specify a mailbox for
+    #   postmaster at any of the domains for which the SMTP server provides
+    #   mail service, as well as the special case of "RCPT TO:<Postmaster>"
+    #   (with no domain specification), MUST be supported.
+    #
+    # We forward such mail to the ZULIP_ADMINISTRATOR.
+    mail = DjangoEmailMultiAlternatives(
+        subject=f"Mail to postmaster: {msg['Subject']}",
+        from_email=settings.NOREPLY_EMAIL_ADDRESS,
+        to=[settings.ZULIP_ADMINISTRATOR],
+    )
+    mail.attach(None, msg, "message/rfc822")
+    try:
+        mail.send()
+    except smtplib.SMTPResponseException as e:
+        logger.exception(
+            "Error sending bounce email to %s with error code %s: %s",
+            mail.to,
+            e.smtp_code,
+            e.smtp_error,
+            stack_info=True,
+        )
+    except smtplib.SMTPException:
+        logger.exception("Error sending bounce email to %s", mail.to, stack_info=True)
+
+
+class ZulipMessageHandler(MessageHandler):
+    def __init__(self) -> None:
+        super().__init__(email.message.Message)
+
+    async def handle_RCPT(
+        self,
+        server: SMTP,
+        session: Session,
+        envelope: Envelope,
+        address: str,
+        rcpt_options: list[str],
+    ) -> str:
+        # Rewrite all postmaster email addresses to just "postmaster"
+        if address.lower() == "postmaster":
+            envelope.rcpt_tos.append("postmaster")
+            return "250 Continue"
+
+        with suppress(ZulipEmailForwardError):
+            if get_email_gateway_message_string_from_address(address).lower() == "postmaster":
+                envelope.rcpt_tos.append("postmaster")
+                return "250 Continue"
+
+        try:
+            # Attempt to do the ACL-check now, just to reject early.
+            # The authoritative check is done in the worker, but we
+            # wish to reject now if we can do so simply.
+            await sync_to_async(validate_to_address)(address)
+
+        except RateLimitedError:
+            recipient_realm = await sync_to_async(
+                lambda a: decode_stream_email_address(a)[0].realm
+            )(address)
+            logger.info(
+                "Rejecting a MAIL FROM: %s to realm: %s via %s - rate limited.",
+                envelope.mail_from,
+                recipient_realm.string_id,
+                str(session.peer),
+            )
+            return "550 4.7.0 Rate-limited due to too many emails on this realm."
+
+        except ZulipEmailForwardError as e:
+            return f"550 5.1.1 Bad destination mailbox address: {e}"
+
+        except JsonableError as e:
+            return f"550 5.7.1 Permission denied: {e}"
+
+        envelope.rcpt_tos.append(address)
+        return "250 Continue"
+
+    @override
+    def handle_message(self, message: email.message.Message) -> None:
+        msg_base64 = base64.b64encode(bytes(message))
+
+        logger.info(
+            "Received email to %s, from %s via %s",
+            message["X-RcptTo"],
+            message["X-MailFrom"],
+            message["X-Peer"],
+        )
+        for address in message["X-RcptTo"].split(", "):
+            if address == "postmaster":
+                send_to_postmaster(message)
+            else:
+                queue_json_publish_rollback_unsafe(
+                    "email_mirror",
+                    {
+                        "rcpt_to": address,
+                        "msg_base64": msg_base64.decode(),
+                    },
+                )
+
+    async def handle_exception(self, error: Exception) -> str:
+        if isinstance(error, TLSSetupException):  # nocoverage
+            if isinstance(error.__cause__, SSLError):
+                reason = error.__cause__.reason
+            else:
+                reason = str(error.__cause__)
+            logger.debug("Dropping invalid TLS connection: %s", reason)
+            return f"421 4.7.6 TLS error: {reason}"
+        else:
+            logger.error("SMTP session exception", exc_info=error)
+            return "500 Server error"
+
+
+def parse_listen_address(listen: str) -> tuple[str, int]:
+    host: str | None
+    port: int | None
+    if listen.isdigit():
+        host, port = None, int(listen)
+    else:
+        r = SplitResult("", listen, "", "", "")
+        try:
+            port = r.port
+        except ValueError as e:
+            # An unbracketed IPv6 address is ambiguous with
+            # address:port, and parses as a host and a non-numeric
+            # port.
+            raise RuntimeError(
+                f"{listen!r} is not a valid port, or address:port; IPv6 addresses "
+                "must be enclosed in square brackets."
+            ) from e
+        if port is None:
+            raise RuntimeError(f"{listen!r} does not have a valid port number.")
+        host = r.hostname
+    if host is None:
+        # With no address specified, bind the IPv6 wildcard address if
+        # IPv6 is available, since (with IPV6_V6ONLY disabled, as
+        # _create_server does) that accepts IPv4 connections as well.
+        host = "::" if socket.has_dualstack_ipv6() else "0.0.0.0"  # noqa: S104
+    return host, port
+
+
+class PermissionDroppingUnthreadedController(UnthreadedController):  # nocoverage
+    @override
+    def __init__(
+        self,
+        user: str | None = None,
+        group: str | None = None,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__(**kwargs)
+
+        # These may remain None in development, when elevated
+        # privileges are not needed because a non-low port is chosen.
+        self.user_id: int | None = None
+        self.group_id: int | None = None
+        if user is not None:
+            self.user_id = pwd.getpwnam(user).pw_uid
+        if group is not None:
+            self.group_id = grp.getgrnam(group).gr_gid
+
+    @override
+    def _create_server(self) -> Awaitable[asyncio.AbstractServer]:
+        # Make the listen socket, then drop privileges before starting
+        # the server
+        family, type_, proto, _, sockaddr = socket.getaddrinfo(
+            self.hostname,
+            self.port,
+            type=socket.SOCK_STREAM,
+            flags=socket.AI_PASSIVE,
+        )[0]
+        server_socket = socket.socket(family, type_, proto)
+        server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        if family == socket.AF_INET6:
+            # Also accept IPv4 connections when binding the IPv6
+            # wildcard address; this has no effect when binding a
+            # specific address.
+            server_socket.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0)
+        server_socket.bind(sockaddr)
+        # Log the address which was actually bound; if the requested
+        # address was a hostname with more than one address, only the
+        # first one that the resolver returned is used.
+        bound_host, bound_port = server_socket.getsockname()[:2]
+        if family == socket.AF_INET6:
+            bound_host = f"[{bound_host}]"
+        logger.info("Listening on %s:%d", bound_host, bound_port)
+        if os.geteuid() == 0:
+            assert self.user_id is not None
+            assert self.group_id is not None
+            # We may have a logfile owned by root, from before we
+            # fixed it to be owned by zulip; chown it if it exists, so
+            # we don't fail below.
+            if os.path.exists(settings.EMAIL_MIRROR_LOG_PATH):
+                os.chown(settings.EMAIL_MIRROR_LOG_PATH, self.user_id, self.group_id)
+
+            logger.info("Dropping privileges to uid %d / gid %d", self.user_id, self.group_id)
+            os.setgid(self.group_id)
+            os.setuid(self.user_id)
+
+        log_to_file(logger, settings.EMAIL_MIRROR_LOG_PATH)
+
+        server = self.loop.create_server(
+            self._factory_invoker,
+            sock=server_socket,
+            ssl=self.ssl_context,
+        )
+
+        return server
+
+
+def run_smtp_server(
+    user: str | None, group: str | None, host: str, port: int, tls_context: SSLContext | None
+) -> None:  # nocoverage
+    logger.info("Starting SMTP server on %s:%d", f"[{host}]" if ":" in host else host, port)
+    server = PermissionDroppingUnthreadedController(
+        user=user,
+        group=group,
+        hostname=host,
+        port=port,
+        handler=ZulipMessageHandler(),
+        tls_context=tls_context,
+        ident=f"Zulip Server {ZULIP_VERSION}",
+    )
+
+    server.loop.add_signal_handler(signal.SIGINT, server.loop.stop)
+
+    server.begin()
+    with suppress(KeyboardInterrupt):
+        # The KeyboardInterrupt will exit the loop, but there's no
+        # reason to throw a stacktrace rather than conduct an ordered
+        # exit.
+        server.loop.run_forever()
+
+    server.end()

zerver/lib/email_validation.py

@@ -0,0 +1,232 @@
+from collections.abc import Callable
+from email.errors import HeaderParseError
+from email.headerregistry import Address
+
+import idna
+from django.core import validators
+from django.core.exceptions import ValidationError
+from django.utils.translation import gettext as _
+
+from zerver.lib.name_restrictions import is_disposable_domain
+
+# TODO: Move DisposableEmailError, etc. into here.
+from zerver.models import Realm, RealmDomain
+from zerver.models.realms import (
+    DisposableEmailError,
+    DomainNotAllowedForRealmError,
+    EmailContainsPlusError,
+)
+from zerver.models.users import get_users_by_delivery_email, is_cross_realm_bot_email
+
+
+def validate_is_not_disposable(email: str) -> None:
+    try:
+        domain = Address(addr_spec=email).domain
+    except (HeaderParseError, ValueError):
+        raise DisposableEmailError
+
+    if is_disposable_domain(domain):
+        raise DisposableEmailError
+
+
+def get_realm_email_validator(realm: Realm) -> Callable[[str], None]:
+    if not realm.emails_restricted_to_domains:
+        # Should we also do '+' check for non-restricted realms?
+        if realm.disallow_disposable_email_addresses:
+            return validate_is_not_disposable
+
+        # allow any email through
+        return lambda email: None
+
+    """
+    RESTRICTIVE REALMS:
+
+    Some realms only allow emails within a set
+    of domains that are configured in RealmDomain.
+
+    We get the set of domains up front so that
+    folks can validate multiple emails without
+    multiple round trips to the database.
+    """
+
+    query = RealmDomain.objects.filter(realm=realm)
+    rows = list(query.values("allow_subdomains", "domain"))
+
+    allowed_domains = {r["domain"] for r in rows}
+
+    allowed_subdomains = {r["domain"] for r in rows if r["allow_subdomains"]}
+
+    def validate(email: str) -> None:
+        """
+        We don't have to do a "disposable" check for restricted
+        domains, since the realm is already giving us
+        a small whitelist.
+        """
+
+        address = Address(addr_spec=email)
+        if "+" in address.username:
+            raise EmailContainsPlusError
+
+        domain = address.domain.lower()
+
+        if domain in allowed_domains:
+            return
+
+        while len(domain) > 0:
+            _subdomain, _sep, domain = domain.partition(".")
+            if domain in allowed_subdomains:
+                return
+
+        raise DomainNotAllowedForRealmError
+
+    return validate
+
+
+# Is a user with the given email address allowed to be in the given realm?
+# (This function does not check whether the user has been invited to the realm.
+# So for invite-only realms, this is the test for whether a user can be invited,
+# not whether the user can sign up currently.)
+def email_allowed_for_realm(email: str, realm: Realm) -> None:
+    """
+    Avoid calling this in a loop!
+    Instead, call get_realm_email_validator()
+    outside of the loop.
+    """
+    get_realm_email_validator(realm)(email)
+
+
+def validate_email_is_valid(
+    email: str,
+    validate_email_allowed_in_realm: Callable[[str], None],
+) -> str | None:
+    try:
+        validators.validate_email(email)
+    except ValidationError:
+        return _("Invalid address.")
+
+    # Django's EmailValidator is looser than IDNA 2008 about what it
+    # accepts in a domain, and also accepts RFC 5321 address-literal
+    # domains (user@[192.168.0.1]) which we don't want to deliver to.
+    # Require the domain to IDNA-encode cleanly, which is what SMTP
+    # delivery will need to do.
+    try:
+        domain = Address(addr_spec=email).domain
+        idna.encode(domain, uts46=True)
+    except (HeaderParseError, ValueError, idna.IDNAError):
+        return _("Invalid address.")
+
+    try:
+        validate_email_allowed_in_realm(email)
+    except DomainNotAllowedForRealmError:
+        return _("Outside your domain.")
+    except DisposableEmailError:
+        return _("Please use your real email address.")
+    except EmailContainsPlusError:
+        return _("Email addresses containing + are not allowed.")
+
+    return None
+
+
+def email_reserved_for_system_bots_error(email: str) -> str:
+    return f"{email} is reserved for system bots"
+
+
+def get_existing_user_errors(
+    target_realm: Realm,
+    emails: set[str],
+    *,
+    allow_inactive_mirror_dummies: bool,
+    verbose: bool = False,
+) -> dict[str, tuple[str, bool]]:
+    """
+    We use this function even for a list of one emails.
+
+    It checks "new" emails to make sure that they don't
+    already exist.  There's a bit of fiddly logic related
+    to cross-realm bots and mirror dummies too.
+    """
+
+    errors: dict[str, tuple[str, bool]] = {}
+
+    users = get_users_by_delivery_email(emails, target_realm).only(
+        "delivery_email",
+        "is_active",
+        "is_mirror_dummy",
+    )
+
+    """
+    A note on casing: We will preserve the casing used by
+    the user for email in most of this code.  The only
+    exception is when we do existence checks against
+    the `user_dict` dictionary.  (We don't allow two
+    users in the same realm to have the same effective
+    delivery email.)
+    """
+    user_dict = {user.delivery_email.lower(): user for user in users}
+
+    def process_email(email: str) -> None:
+        if is_cross_realm_bot_email(email):
+            if verbose:
+                msg = email_reserved_for_system_bots_error(email)
+            else:
+                msg = _("Reserved for system bots.")
+            deactivated = False
+            errors[email] = (msg, deactivated)
+            return
+
+        existing_user_profile = user_dict.get(email.lower())
+
+        if existing_user_profile is None:
+            # HAPPY PATH!  Most people invite users that don't exist yet.
+            return
+
+        if existing_user_profile.is_mirror_dummy and allow_inactive_mirror_dummies:
+            if existing_user_profile.is_active:
+                raise AssertionError("Mirror dummy user is already active!")
+            return
+
+        """
+        Email has already been taken by a "normal" user.
+        """
+        deactivated = not existing_user_profile.is_active
+
+        if existing_user_profile.is_active:
+            if verbose:
+                msg = _("{email} already has an account").format(email=email)
+            else:
+                msg = _("Already has an account.")
+        else:
+            msg = _("Account has been deactivated.")
+
+        errors[email] = (msg, deactivated)
+
+    for email in emails:
+        process_email(email)
+
+    return errors
+
+
+def validate_email_not_already_in_realm(
+    target_realm: Realm, email: str, *, allow_inactive_mirror_dummies: bool, verbose: bool = True
+) -> None:
+    """
+    NOTE:
+        Only use this to validate that a single email
+        is not already used in the realm.
+
+        We should start using bulk_check_new_emails()
+        for any endpoint that takes multiple emails,
+        such as the "invite" interface.
+    """
+    error_dict = get_existing_user_errors(
+        target_realm,
+        {email},
+        allow_inactive_mirror_dummies=allow_inactive_mirror_dummies,
+        verbose=verbose,
+    )
+
+    # Loop through errors, the only key should be our email.
+    for key, error_info in error_dict.items():
+        assert key == email
+        msg, _deactivated = error_info
+        raise ValidationError(msg)

zerver/lib/emoji.py

@@ -0,0 +1,163 @@
+import hashlib
+import os
+import re
+from dataclasses import dataclass
+
+import orjson
+from django.conf import settings
+from django.contrib.staticfiles.storage import staticfiles_storage
+from django.utils.translation import gettext as _
+
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.mime_types import guess_extension
+from zerver.lib.storage import static_path
+from zerver.models import Reaction, Realm, RealmEmoji, UserProfile
+from zerver.models.realm_emoji import (
+    get_all_custom_emoji_for_realm,
+    get_name_keyed_dict_for_active_realm_emoji,
+)
+
+emoji_codes_path = static_path("generated/emoji/emoji_codes.json")
+if not os.path.exists(emoji_codes_path):  # nocoverage
+    # During the collectstatic step of build-release-tarball,
+    # prod-static/serve/generated/emoji won't exist yet.
+    emoji_codes_path = os.path.join(
+        os.path.dirname(__file__),
+        "../../static/generated/emoji/emoji_codes.json",
+    )
+
+with open(emoji_codes_path, "rb") as fp:
+    emoji_codes = orjson.loads(fp.read())
+
+name_to_codepoint = emoji_codes["name_to_codepoint"]
+codepoint_to_name = emoji_codes["codepoint_to_name"]
+EMOTICON_CONVERSIONS = emoji_codes["emoticon_conversions"]
+
+possible_emoticons = EMOTICON_CONVERSIONS.keys()
+possible_emoticon_regexes = (re.escape(emoticon) for emoticon in possible_emoticons)
+terminal_symbols = r",.;?!()\[\] \"'\n\t"  # from composebox_typeahead.ts
+EMOTICON_RE = (
+    rf"(?<![^{terminal_symbols}])(?P<emoticon>("
+    + r")|(".join(possible_emoticon_regexes)
+    + rf"))(?![^{terminal_symbols}])"
+)
+
+
+def data_url() -> str:
+    # This bakes a hash into the URL, which looks something like
+    # static/webpack-bundles/files/64.0cdafdf0b6596657a9be.png
+    # This is how Django deals with serving static files in a cacheable way.
+    # See PR #22275 for details.
+    return staticfiles_storage.url("generated/emoji/emoji_api.json")
+
+
+# Translates emoticons to their colon syntax, e.g. `:smiley:`.
+def translate_emoticons(text: str) -> str:
+    translated = text
+
+    for emoticon in EMOTICON_CONVERSIONS:
+        translated = re.sub(re.escape(emoticon), EMOTICON_CONVERSIONS[emoticon], translated)
+
+    return translated
+
+
+@dataclass
+class EmojiData:
+    emoji_code: str
+    reaction_type: str
+
+
+def get_emoji_data(realm_id: int, emoji_name: str) -> EmojiData:
+    # Even if emoji_name is either in name_to_codepoint or named "zulip",
+    # we still need to call get_realm_active_emoji.
+    realm_emoji_dict = get_name_keyed_dict_for_active_realm_emoji(realm_id)
+    realm_emoji = realm_emoji_dict.get(emoji_name)
+
+    if realm_emoji is not None:
+        emoji_code = str(realm_emoji["id"])
+        return EmojiData(emoji_code=emoji_code, reaction_type=Reaction.REALM_EMOJI)
+
+    if emoji_name == "zulip":
+        return EmojiData(emoji_code=emoji_name, reaction_type=Reaction.ZULIP_EXTRA_EMOJI)
+
+    if emoji_name in name_to_codepoint:
+        emoji_code = name_to_codepoint[emoji_name]
+        return EmojiData(emoji_code=emoji_code, reaction_type=Reaction.UNICODE_EMOJI)
+
+    raise JsonableError(_("Emoji '{emoji_name}' does not exist").format(emoji_name=emoji_name))
+
+
+def check_emoji_request(realm: Realm, emoji_name: str, emoji_code: str, emoji_type: str) -> None:
+    # For a given realm and emoji type, checks whether an emoji
+    # code is valid for new reactions, or not.
+    if emoji_type == "realm_emoji":
+        # We cache emoji, so this generally avoids a round trip,
+        # but it does require deserializing a bigger data structure
+        # than we need.
+        realm_emojis = get_all_custom_emoji_for_realm(realm.id)
+        realm_emoji = realm_emojis.get(emoji_code)
+        if realm_emoji is None:
+            raise JsonableError(_("Invalid custom emoji."))
+        if realm_emoji["name"] != emoji_name:
+            raise JsonableError(_("Invalid custom emoji name."))
+        if realm_emoji["deactivated"]:
+            raise JsonableError(_("This custom emoji has been deactivated."))
+    elif emoji_type == "zulip_extra_emoji":
+        if emoji_code != "zulip":
+            raise JsonableError(_("Invalid emoji code."))
+        if emoji_name != emoji_code:
+            raise JsonableError(_("Invalid emoji name."))
+    elif emoji_type == "unicode_emoji":
+        if emoji_code not in codepoint_to_name:
+            raise JsonableError(_("Invalid emoji code."))
+        if name_to_codepoint.get(emoji_name) != emoji_code:
+            raise JsonableError(_("Invalid emoji name."))
+    else:
+        # The above are the only valid emoji types
+        raise JsonableError(_("Invalid emoji type."))
+
+
+def check_remove_custom_emoji(user_profile: UserProfile, emoji_name: str) -> None:
+    # normal users can remove emoji they themselves added
+    if user_profile.is_realm_admin:
+        return
+
+    emoji = RealmEmoji.objects.filter(
+        realm=user_profile.realm, name=emoji_name, deactivated=False
+    ).first()
+    current_user_is_author = emoji is not None and emoji.author.id == user_profile.id
+    if not current_user_is_author:
+        raise JsonableError(_("Must be an organization administrator or emoji author"))
+
+
+def check_valid_emoji_name(emoji_name: str) -> None:
+    if emoji_name:
+        if re.match(r"^[0-9a-z\-_]+(?<![\-_])$", emoji_name):
+            return
+        if re.match(r"^[0-9a-z\-_]+$", emoji_name):
+            raise JsonableError(_("Emoji names must end with either a letter or digit."))
+        raise JsonableError(
+            _(
+                "Emoji names must contain only lowercase English letters, digits, spaces, dashes, and underscores.",
+            )
+        )
+    raise JsonableError(_("Emoji name is missing"))
+
+
+def get_emoji_file_name(content_type: str, emoji_id: int) -> str:
+    image_ext = guess_extension(content_type, strict=False)
+    # The only callsite of this pre-limits the content_type to a
+    # reasonable set that we know have extensions.
+    assert image_ext is not None
+
+    # We salt this with a server-side secret so that it is not
+    # enumerable by clients, and will not collide on the server.  New
+    # realm imports may pass a synthetic emoji_id, which is fine as
+    # long as it starts at 1, and as such later emoji cannot collide
+    # unless there is a legit hash collision.
+    #
+    # We truncate the hash at 8 characters, as this is enough entropy
+    # to make collisions vanishingly unlikely.  In the event of a
+    # collusion, the id will advance and a manual retry will succeed.
+    hash_key = settings.AVATAR_SALT.encode() + b":" + str(emoji_id).encode()
+    return "".join((hashlib.sha256(hash_key).hexdigest()[0:8], image_ext))

zerver/lib/emoji_utils.py

@@ -0,0 +1,25 @@
+# This file doesn't import from django so that we can use it in `build_emoji`
+
+
+def unqualify_emoji(emoji: str) -> str:
+    # Starting from version 4.0.0, `emoji_datasource` package has started to
+    # add an emoji presentation variation selector for certain emojis which
+    # have defined variation sequences. The emoji presentation selector
+    # "qualifies" an emoji, and an "unqualified" version of an emoji does
+    # not have an emoji presentation selector.
+    #
+    # Since in informal environments(like texting and chat), it is more
+    # appropriate for an emoji to have a colorful display so until emoji
+    # characters have a text presentation selector, it should have a
+    # colorful display. Hence we can continue using emoji characters
+    # without appending emoji presentation selector.
+    # (http://unicode.org/reports/tr51/index.html#Presentation_Style)
+    return emoji.replace("\ufe0f", "")
+
+
+def emoji_to_hex_codepoint(emoji: str) -> str:
+    return "-".join(f"{ord(c):04x}" for c in emoji)
+
+
+def hex_codepoint_to_emoji(hex: str) -> str:
+    return "".join(chr(int(h, 16)) for h in hex.split("-"))

zerver/lib/event_schema.py

@@ -0,0 +1,766 @@
+# This module is a collection of testing helpers for validating the
+# schema of "events" sent by Zulip's server-to-client push system.
+#
+# By policy, every event generated by Zulip's API should be validated
+# by a test in test_events.py with a schema checker here.
+#
+# See https://zulip.readthedocs.io/en/latest/subsystems/events-system.html
+import inspect
+from collections.abc import Callable
+from enum import Enum
+from pprint import PrettyPrinter
+from typing import cast
+
+from pydantic import BaseModel
+
+from zerver.lib.event_types import (
+    AlertWordsEvent,
+    AllowMessageEditingData,
+    AttachmentAddEvent,
+    AttachmentRemoveEvent,
+    AttachmentUpdateEvent,
+    AuthenticationData,
+    BaseEvent,
+    BotServicesEmbedded,
+    BotServicesOutgoing,
+    ChannelFolderAddEvent,
+    ChannelFolderReorderEvent,
+    ChannelFolderUpdateEvent,
+    CustomProfileFieldsEvent,
+    DefaultStreamGroupsEvent,
+    DefaultStreamsEvent,
+    DeleteMessageEvent,
+    DeviceAddEvent,
+    DeviceRemoveEvent,
+    DeviceUpdateEvent,
+    DirectMessageEvent,
+    DraftsAddEvent,
+    DraftsRemoveEvent,
+    DraftsUpdateEvent,
+    GroupSettingUpdateData,
+    HasWebexTokenEvent,
+    HasZoomTokenEvent,
+    HeartbeatEvent,
+    IconData,
+    InvitesChangedEvent,
+    LegacyPresenceEvent,
+    LogoData,
+    MessageContentEditLimitSecondsData,
+    MessageEvent,
+    ModernPresenceEvent,
+    MutedTopicsEvent,
+    MutedUsersEvent,
+    NavigationViewAddEvent,
+    NavigationViewRemoveEvent,
+    NavigationViewUpdateEvent,
+    NightLogoData,
+    OnboardingStepsEvent,
+    PersonAvatarFields,
+    PersonBotOwnerId,
+    PersonCustomProfileField,
+    PersonDateJoined,
+    PersonDeliveryEmail,
+    PersonEmail,
+    PersonFullName,
+    PersonIsActive,
+    PersonIsImportedStub,
+    PersonRole,
+    PersonTimezone,
+    PlanTypeData,
+    ReactionAddEvent,
+    ReactionRemoveEvent,
+    RealmBotAddEvent,
+    RealmBotDeleteEvent,
+    RealmBotUpdateEvent,
+    RealmDeactivatedEvent,
+    RealmDescriptionData,
+    RealmDomainsAddEvent,
+    RealmDomainsChangeEvent,
+    RealmDomainsRemoveEvent,
+    RealmEmojiAddEvent,
+    RealmEmojiUpdateEvent,
+    RealmEmojiUpdateOneEvent,
+    RealmExportConsentEvent,
+    RealmExportEvent,
+    RealmLinkifiersEvent,
+    RealmPlaygroundsEvent,
+    RealmTopicsPolicyData,
+    RealmUpdateDictEvent,
+    RealmUpdateEvent,
+    RealmUserAddEvent,
+    RealmUserRemoveEvent,
+    RealmUserSettingsDefaultsUpdateEvent,
+    RealmUserUpdateEvent,
+    RemindersAddEvent,
+    RemindersRemoveEvent,
+    RestartEvent,
+    SavedSnippetsAddEvent,
+    SavedSnippetsRemoveEvent,
+    SavedSnippetsUpdateEvent,
+    ScheduledMessagesAddEvent,
+    ScheduledMessagesRemoveEvent,
+    ScheduledMessagesUpdateEvent,
+    StreamCreateEvent,
+    StreamDeleteEvent,
+    StreamUpdateEvent,
+    SubmessageEvent,
+    SubscriptionAddEvent,
+    SubscriptionPeerAddEvent,
+    SubscriptionPeerRemoveEvent,
+    SubscriptionRemoveEvent,
+    SubscriptionUpdateEvent,
+    TypingEditMessageStartEvent,
+    TypingEditMessageStopEvent,
+    TypingStartEvent,
+    TypingStopEvent,
+    UpdateMessageEvent,
+    UpdateMessageFlagsAddEvent,
+    UpdateMessageFlagsRemoveEvent,
+    UserGroupAddEvent,
+    UserGroupAddMembersEvent,
+    UserGroupAddSubgroupsEvent,
+    UserGroupRemoveEvent,
+    UserGroupRemoveMembersEvent,
+    UserGroupRemoveSubgroupsEvent,
+    UserGroupUpdateEvent,
+    UserSettingsUpdateEvent,
+    UserStatusEvent,
+    UserTopicEvent,
+    WebReloadClientEvent,
+)
+from zerver.lib.topic import ORIG_TOPIC, TOPIC_NAME
+from zerver.lib.types import UserGroupMembersDict
+from zerver.models import Realm, RealmUserDefault, Stream, UserProfile
+from zerver.models.streams import StreamTopicsPolicyEnum
+
+
+def validate_with_model(data: dict[str, object], model: type[BaseModel]) -> None:
+    allowed_fields = set(model.model_fields.keys())
+    if not set(data.keys()).issubset(allowed_fields):  # nocoverage
+        raise ValueError(f"Extra fields not allowed: {set(data.keys()) - allowed_fields}")
+
+    model.model_validate(data, strict=True)
+
+
+def make_checker(base_model: type[BaseEvent]) -> Callable[[str, dict[str, object]], None]:
+    def f(label: str, event: dict[str, object]) -> None:
+        assert isinstance(event["id"], int)
+        event = dict(event)
+        del event["id"]
+        try:
+            validate_with_model(event, base_model)
+        except Exception:  # nocoverage
+            print(f"""
+FAILURE:
+
+The event below fails the check to make sure it has the
+correct "shape" of data:
+
+    {label}
+
+Often this is a symptom that the following type definition
+is either broken or needs to be updated due to other
+changes that you have made:
+
+    {base_model}
+
+A traceback should follow to help you debug this problem.
+
+Here is the event:
+""")
+
+            PrettyPrinter(indent=4).pprint(event)
+            raise
+
+    return f
+
+
+check_alert_words = make_checker(AlertWordsEvent)
+check_attachment_add = make_checker(AttachmentAddEvent)
+check_attachment_remove = make_checker(AttachmentRemoveEvent)
+check_attachment_update = make_checker(AttachmentUpdateEvent)
+check_channel_folder_add = make_checker(ChannelFolderAddEvent)
+check_channel_folder_reorder = make_checker(ChannelFolderReorderEvent)
+check_custom_profile_fields = make_checker(CustomProfileFieldsEvent)
+check_default_stream_groups = make_checker(DefaultStreamGroupsEvent)
+check_default_streams = make_checker(DefaultStreamsEvent)
+check_device_add = make_checker(DeviceAddEvent)
+check_device_remove = make_checker(DeviceRemoveEvent)
+check_device_update = make_checker(DeviceUpdateEvent)
+check_direct_message = make_checker(DirectMessageEvent)
+check_draft_add = make_checker(DraftsAddEvent)
+check_draft_remove = make_checker(DraftsRemoveEvent)
+check_draft_update = make_checker(DraftsUpdateEvent)
+check_heartbeat = make_checker(HeartbeatEvent)
+check_invites_changed = make_checker(InvitesChangedEvent)
+check_message = make_checker(MessageEvent)
+check_muted_users = make_checker(MutedUsersEvent)
+check_navigation_view_add = make_checker(NavigationViewAddEvent)
+check_navigation_view_remove = make_checker(NavigationViewRemoveEvent)
+check_navigation_view_update = make_checker(NavigationViewUpdateEvent)
+check_onboarding_steps = make_checker(OnboardingStepsEvent)
+check_reaction_add = make_checker(ReactionAddEvent)
+check_reaction_remove = make_checker(ReactionRemoveEvent)
+check_realm_bot_delete = make_checker(RealmBotDeleteEvent)
+check_realm_deactivated = make_checker(RealmDeactivatedEvent)
+check_realm_domains_add = make_checker(RealmDomainsAddEvent)
+check_realm_domains_change = make_checker(RealmDomainsChangeEvent)
+check_realm_domains_remove = make_checker(RealmDomainsRemoveEvent)
+check_realm_emoji_add = make_checker(RealmEmojiAddEvent)
+check_realm_emoji_update_one = make_checker(RealmEmojiUpdateOneEvent)
+check_realm_export_consent = make_checker(RealmExportConsentEvent)
+check_realm_linkifiers = make_checker(RealmLinkifiersEvent)
+check_realm_playgrounds = make_checker(RealmPlaygroundsEvent)
+check_realm_user_add = make_checker(RealmUserAddEvent)
+check_realm_user_remove = make_checker(RealmUserRemoveEvent)
+check_reminder_add = make_checker(RemindersAddEvent)
+check_reminder_remove = make_checker(RemindersRemoveEvent)
+check_restart = make_checker(RestartEvent)
+check_saved_snippets_add = make_checker(SavedSnippetsAddEvent)
+check_saved_snippets_remove = make_checker(SavedSnippetsRemoveEvent)
+check_saved_snippets_update = make_checker(SavedSnippetsUpdateEvent)
+check_scheduled_message_add = make_checker(ScheduledMessagesAddEvent)
+check_scheduled_message_remove = make_checker(ScheduledMessagesRemoveEvent)
+check_scheduled_message_update = make_checker(ScheduledMessagesUpdateEvent)
+check_stream_create = make_checker(StreamCreateEvent)
+check_stream_delete = make_checker(StreamDeleteEvent)
+check_submessage = make_checker(SubmessageEvent)
+check_subscription_add = make_checker(SubscriptionAddEvent)
+check_subscription_peer_add = make_checker(SubscriptionPeerAddEvent)
+check_subscription_peer_remove = make_checker(SubscriptionPeerRemoveEvent)
+check_subscription_remove = make_checker(SubscriptionRemoveEvent)
+check_typing_start = make_checker(TypingStartEvent)
+check_typing_stop = make_checker(TypingStopEvent)
+check_typing_edit_message_start = make_checker(TypingEditMessageStartEvent)
+check_typing_edit_message_stop = make_checker(TypingEditMessageStopEvent)
+check_update_message_flags_add = make_checker(UpdateMessageFlagsAddEvent)
+check_update_message_flags_remove = make_checker(UpdateMessageFlagsRemoveEvent)
+check_user_group_add = make_checker(UserGroupAddEvent)
+check_user_group_add_members = make_checker(UserGroupAddMembersEvent)
+check_user_group_add_subgroups = make_checker(UserGroupAddSubgroupsEvent)
+check_user_group_remove = make_checker(UserGroupRemoveEvent)
+check_user_group_remove_members = make_checker(UserGroupRemoveMembersEvent)
+check_user_group_remove_subgroups = make_checker(UserGroupRemoveSubgroupsEvent)
+check_user_topic = make_checker(UserTopicEvent)
+check_web_reload_client_event = make_checker(WebReloadClientEvent)
+
+
+# Now for the slightly more tricky bits.  All the following functions
+# get wrapped with more stringent checkers.  Some of the wrappers are
+# reasonably sane functions that just check the data, not the shape of
+# the data.
+#
+# But there are some ugly wrappers that work around the fact that
+# a) our events are not very consistent and/or b) our types aren't
+# robust.
+#
+# TODO: work through the bottom of this file to try to find ways to
+#       simplify our types or make them more robust
+
+_check_channel_folder_update = make_checker(ChannelFolderUpdateEvent)
+_check_delete_message = make_checker(DeleteMessageEvent)
+_check_has_zoom_token = make_checker(HasZoomTokenEvent)
+_check_has_webex_token = make_checker(HasWebexTokenEvent)
+_check_legacy_presence = make_checker(LegacyPresenceEvent)
+_check_modern_presence = make_checker(ModernPresenceEvent)
+_check_muted_topics = make_checker(MutedTopicsEvent)
+_check_realm_bot_add = make_checker(RealmBotAddEvent)
+_check_realm_bot_update = make_checker(RealmBotUpdateEvent)
+_check_realm_default_update = make_checker(RealmUserSettingsDefaultsUpdateEvent)
+_check_realm_emoji_update = make_checker(RealmEmojiUpdateEvent)
+_check_realm_export = make_checker(RealmExportEvent)
+_check_realm_update = make_checker(RealmUpdateEvent)
+_check_realm_update_dict = make_checker(RealmUpdateDictEvent)
+_check_realm_user_update = make_checker(RealmUserUpdateEvent)
+_check_stream_update = make_checker(StreamUpdateEvent)
+_check_subscription_update = make_checker(SubscriptionUpdateEvent)
+_check_update_message = make_checker(UpdateMessageEvent)
+_check_user_group_update = make_checker(UserGroupUpdateEvent)
+_check_user_settings_update = make_checker(UserSettingsUpdateEvent)
+_check_user_status = make_checker(UserStatusEvent)
+
+
+PERSON_TYPES: dict[str, type[BaseModel]] = dict(
+    avatar_fields=PersonAvatarFields,
+    bot_owner_id=PersonBotOwnerId,
+    custom_profile_field=PersonCustomProfileField,
+    date_joined=PersonDateJoined,
+    delivery_email=PersonDeliveryEmail,
+    email=PersonEmail,
+    full_name=PersonFullName,
+    role=PersonRole,
+    timezone=PersonTimezone,
+    is_active=PersonIsActive,
+    is_imported_stub=PersonIsImportedStub,
+)
+
+
+def check_channel_folder_update(var_name: str, event: dict[str, object], fields: set[str]) -> None:
+    _check_channel_folder_update(var_name, event)
+
+    assert isinstance(event["data"], dict)
+    assert set(event["data"].keys()) == fields
+
+
+def check_delete_message(
+    var_name: str,
+    event: dict[str, object],
+    message_type: str,
+    num_message_ids: int,
+    is_legacy: bool,
+) -> None:
+    _check_delete_message(var_name, event)
+
+    keys = {"id", "type", "message_type"}
+
+    assert event["message_type"] == message_type
+
+    if message_type == "stream":
+        keys |= {"stream_id", "topic"}
+    elif message_type == "private":
+        pass
+    else:
+        raise AssertionError("unexpected message_type")
+
+    if is_legacy:
+        assert num_message_ids == 1
+        keys.add("message_id")
+    else:
+        assert isinstance(event["message_ids"], list)
+        assert num_message_ids == len(event["message_ids"])
+        keys.add("message_ids")
+
+    assert set(event.keys()) == keys
+
+
+def check_has_zoom_token(
+    var_name: str,
+    event: dict[str, object],
+    value: bool,
+) -> None:
+    _check_has_zoom_token(var_name, event)
+    assert event["value"] == value
+
+
+def check_has_webex_token(
+    var_name: str,
+    event: dict[str, object],
+    value: bool,
+) -> None:
+    _check_has_webex_token(var_name, event)
+    assert event["value"] == value
+
+
+def check_muted_topics(
+    var_name: str,
+    event: dict[str, object],
+) -> None:
+    _check_muted_topics(var_name, event)
+    muted_topics = event["muted_topics"]
+    assert isinstance(muted_topics, list)
+    for muted_topic in muted_topics:
+        muted_topic_tuple = tuple(muted_topic)
+        assert list(map(type, muted_topic_tuple)) == [str, str, int]
+
+
+def check_legacy_presence(
+    var_name: str,
+    event: dict[str, object],
+    has_email: bool,
+    presence_key: str,
+    status: str,
+) -> None:
+    _check_legacy_presence(var_name, event)
+
+    assert ("email" in event) == has_email
+
+    assert isinstance(event["presence"], dict)
+
+    # Our tests only have one presence value.
+    [(event_presence_key, event_presence_value)] = event["presence"].items()
+    assert event_presence_key == presence_key
+    assert event_presence_value["status"] == status
+
+
+def check_modern_presence(var_name: str, event: dict[str, object], user_id: int) -> None:
+    _check_modern_presence(var_name, event)
+
+    assert isinstance(event["presences"], dict)
+
+    [(event_presences_key, _event_presences_value)] = event["presences"].items()
+    assert event_presences_key == str(user_id)
+
+
+def check_realm_bot_add(
+    var_name: str,
+    event: dict[str, object],
+    bot_type: int,
+) -> None:
+    _check_realm_bot_add(var_name, event)
+
+    assert isinstance(event["bot"], dict)
+
+    services = event["bot"]["services"]
+
+    if bot_type == UserProfile.DEFAULT_BOT:
+        assert services == []
+    elif bot_type == UserProfile.OUTGOING_WEBHOOK_BOT:
+        assert len(services) == 1
+        validate_with_model(services[0], BotServicesOutgoing)
+    elif bot_type == UserProfile.EMBEDDED_BOT:
+        assert len(services) == 1
+        validate_with_model(services[0], BotServicesEmbedded)
+    else:
+        raise AssertionError(f"Unknown bot_type: {bot_type}")
+
+
+def check_realm_bot_update(
+    # Check schema plus the field.
+    var_name: str,
+    event: dict[str, object],
+    field: str,
+) -> None:
+    # Check the overall schema first.
+    _check_realm_bot_update(var_name, event)
+
+    assert isinstance(event["bot"], dict)
+    assert {"user_id", field} == set(event["bot"].keys())
+
+
+def check_realm_emoji_update(var_name: str, event: dict[str, object]) -> None:
+    """
+    The way we send realm emojis is kinda clumsy--we
+    send a dict mapping the emoji id to a sub_dict with
+    the fields (including the id).  Ideally we can streamline
+    this and just send a list of dicts.  The clients can make
+    a Map as needed.
+    """
+    _check_realm_emoji_update(var_name, event)
+
+    assert isinstance(event["realm_emoji"], dict)
+    for k, v in event["realm_emoji"].items():
+        assert v["id"] == k
+
+
+def check_realm_export(
+    var_name: str,
+    event: dict[str, object],
+    has_export_url: bool,
+    has_deleted_timestamp: bool,
+    has_failed_timestamp: bool,
+) -> None:
+    # Check the overall event first, knowing it has some
+    # optional types.
+    _check_realm_export(var_name, event)
+
+    # It's possible to have multiple data exports, but the events tests do not
+    # exercise that case, so we do strict validation for a single export here.
+    assert isinstance(event["exports"], list)
+    assert len(event["exports"]) == 1
+    export = event["exports"][0]
+
+    # Now verify which fields have non-None values.
+    assert has_export_url == (export["export_url"] is not None)
+    assert has_deleted_timestamp == (export["deleted_timestamp"] is not None)
+    assert has_failed_timestamp == (export["failed_timestamp"] is not None)
+
+
+def check_realm_update(
+    var_name: str,
+    event: dict[str, object],
+    prop: str,
+) -> None:
+    """
+    Realm updates have these two fields:
+
+        property
+        value
+
+    We check not only the basic schema, but also that
+    the value people actually matches the type from
+    Realm.property_types that we have configured
+    for the property.
+
+    For certain properties, there are extra fields.
+    For example, when property is "description", there's also
+    "rendered_description".
+    """
+    _check_realm_update(var_name, event)
+
+    assert prop == event["property"]
+    value = event["value"]
+
+    if prop in [
+        "moderation_request_channel_id",
+        "new_stream_announcements_stream_id",
+        "signup_announcements_stream_id",
+        "zulip_update_announcements_stream_id",
+        "org_type",
+    ]:
+        assert isinstance(value, int)
+        return
+
+    property_type = Realm.property_types[prop]
+    if inspect.isclass(property_type) and issubclass(property_type, Enum):
+        assert isinstance(value, str)
+        property_type[value]
+    else:
+        assert isinstance(value, property_type)
+
+
+def check_realm_default_update(
+    var_name: str,
+    event: dict[str, object],
+    prop: str,
+) -> None:
+    _check_realm_default_update(var_name, event)
+
+    assert prop == event["property"]
+    assert prop != "default_language"
+    assert prop in RealmUserDefault.property_types
+
+    prop_type = RealmUserDefault.property_types[prop]
+    value = event["value"]
+    if inspect.isclass(prop_type) and issubclass(prop_type, Enum):
+        assert isinstance(value, str)
+        prop_type[value]
+    else:
+        assert isinstance(value, prop_type)
+
+
+def check_realm_update_dict(
+    # handle union types
+    var_name: str,
+    event: dict[str, object],
+) -> None:
+    _check_realm_update_dict(var_name, event)
+
+    if event["property"] == "default":
+        assert isinstance(event["data"], dict)
+
+        if "allow_message_editing" in event["data"]:
+            sub_type: type[BaseModel] = AllowMessageEditingData
+        elif "message_content_edit_limit_seconds" in event["data"]:
+            sub_type = MessageContentEditLimitSecondsData
+        elif "authentication_methods" in event["data"]:
+            sub_type = AuthenticationData
+        elif any(
+            setting_name in event["data"] for setting_name in Realm.REALM_PERMISSION_GROUP_SETTINGS
+        ):
+            sub_type = GroupSettingUpdateData
+        elif "plan_type" in event["data"]:
+            sub_type = PlanTypeData
+        elif "topics_policy" in event["data"]:
+            sub_type = RealmTopicsPolicyData
+        elif "description" in event["data"]:
+            sub_type = RealmDescriptionData
+        else:
+            raise AssertionError("unhandled fields in data")
+
+    elif event["property"] == "icon":
+        sub_type = IconData
+    elif event["property"] == "logo":
+        sub_type = LogoData
+    elif event["property"] == "night_logo":
+        sub_type = NightLogoData
+    else:
+        raise AssertionError("unhandled property: {event['property']}")
+
+    validate_with_model(cast(dict[str, object], event["data"]), sub_type)
+
+
+def check_realm_user_update(
+    # person_flavor tells us which extra fields we need
+    var_name: str,
+    event: dict[str, object],
+    person_flavor: str,
+) -> None:
+    _check_realm_user_update(var_name, event)
+
+    sub_type = PERSON_TYPES[person_flavor]
+    validate_with_model(cast(dict[str, object], event["person"]), sub_type)
+
+
+def check_stream_update(
+    var_name: str,
+    event: dict[str, object],
+) -> None:
+    _check_stream_update(var_name, event)
+    prop = event["property"]
+    value = event["value"]
+
+    extra_keys = set(event.keys()) - {
+        "id",
+        "type",
+        "op",
+        "property",
+        "value",
+        "name",
+        "stream_id",
+        "first_message_id",
+        "is_archived",
+        "folder_id",
+    }
+
+    if prop == "description":
+        assert extra_keys == {"rendered_description"}
+        assert isinstance(value, str)
+    elif prop == "invite_only":
+        assert extra_keys == {"history_public_to_subscribers", "is_web_public"}
+        assert isinstance(value, bool)
+    elif prop == "message_retention_days":
+        assert extra_keys == set()
+        if value is not None:
+            assert isinstance(value, int)
+    elif prop == "name":
+        assert extra_keys == set()
+        assert isinstance(value, str)
+    elif prop == "stream_post_policy":
+        assert extra_keys == set()
+        assert value in Stream.STREAM_POST_POLICY_TYPES
+    elif prop in Stream.stream_permission_group_settings:
+        assert extra_keys == set()
+        assert isinstance(value, int | dict)
+        # We cannot validate a TypedDict using isinstance, thus
+        # requiring this check.
+        if isinstance(value, dict):
+            expected_keys = set(inspect.get_annotations(UserGroupMembersDict).keys())
+            keys = set(value.keys())
+            assert expected_keys == keys
+    elif prop == "first_message_id":
+        assert extra_keys == set()
+        assert isinstance(value, int)
+    elif prop == "topics_policy":
+        assert extra_keys == set()
+        assert value in [e.name for e in StreamTopicsPolicyEnum]
+    elif prop == "is_recently_active":
+        assert extra_keys == set()
+        assert isinstance(value, bool)
+    elif prop == "is_announcement_only":
+        assert extra_keys == set()
+        assert isinstance(value, bool)
+    elif prop == "is_archived":
+        assert extra_keys == set()
+        assert isinstance(value, bool)
+    elif prop == "folder_id":
+        assert extra_keys == set()
+        assert value is None or isinstance(value, int)
+    elif prop == "default_push_notifications":
+        assert extra_keys == set()
+        assert isinstance(value, bool)
+    else:
+        raise AssertionError(f"Unknown property: {prop}")
+
+
+def check_subscription_update(
+    var_name: str, event: dict[str, object], property: str, value: bool
+) -> None:
+    _check_subscription_update(var_name, event)
+    assert event["property"] == property
+    assert event["value"] == value
+
+
+def check_user_settings_update(
+    var_name: str,
+    event: dict[str, object],
+) -> None:
+    _check_user_settings_update(var_name, event)
+    setting_name = event["property"]
+    value = event["value"]
+
+    assert isinstance(setting_name, str)
+    if setting_name == "timezone":
+        assert isinstance(value, str)
+    else:
+        setting_type = UserProfile.property_types[setting_name]
+        if inspect.isclass(setting_type) and issubclass(setting_type, Enum):
+            assert isinstance(value, str)
+            setting_type[value]
+        else:
+            assert isinstance(value, setting_type)
+
+    if setting_name == "default_language":
+        assert "language_name" in event
+    else:
+        assert "language_name" not in event
+
+
+def check_update_message(
+    var_name: str,
+    event: dict[str, object],
+    is_stream_message: bool,
+    has_content: bool,
+    has_topic: bool,
+    has_new_stream_id: bool,
+    is_embedded_update_only: bool,
+) -> None:
+    # Always check the basic schema first.
+    _check_update_message(var_name, event)
+
+    actual_keys = set(event.keys())
+    expected_keys = {
+        "id",
+        "type",
+        "user_id",
+        "edit_timestamp",
+        "message_id",
+        "flags",
+        "message_ids",
+        "rendering_only",
+    }
+
+    if is_stream_message:
+        expected_keys |= {
+            "stream_id",
+            "stream_name",
+        }
+
+    if has_content:
+        expected_keys |= {
+            "is_me_message",
+            "orig_content",
+            "orig_rendered_content",
+            "content",
+            "rendered_content",
+        }
+
+    if has_topic:
+        expected_keys |= {
+            "topic_links",
+            ORIG_TOPIC,
+            TOPIC_NAME,
+            "propagate_mode",
+        }
+
+    if has_new_stream_id:
+        expected_keys |= {
+            "new_stream_id",
+            ORIG_TOPIC,
+            "propagate_mode",
+        }
+
+    if is_embedded_update_only:
+        expected_keys |= {
+            "content",
+            "rendered_content",
+        }
+        assert event["user_id"] is None
+    else:
+        assert isinstance(event["user_id"], int)
+
+    assert event["rendering_only"] == is_embedded_update_only
+    assert expected_keys == actual_keys
+
+
+def check_user_group_update(var_name: str, event: dict[str, object], fields: set[str]) -> None:
+    _check_user_group_update(var_name, event)
+
+    assert isinstance(event["data"], dict)
+
+    assert set(event["data"].keys()) == fields
+
+
+def check_user_status(var_name: str, event: dict[str, object], fields: set[str]) -> None:
+    _check_user_status(var_name, event)
+
+    assert set(event.keys()) == {"id", "type", "user_id"} | fields

zerver/lib/exceptions.py

@@ -0,0 +1,908 @@
+from enum import Enum, auto
+from typing import Any
+
+from django.core.exceptions import ValidationError
+from django.utils.translation import gettext as _
+from django_stubs_ext import StrPromise
+from typing_extensions import override
+
+
+class ErrorCode(Enum):
+    BAD_REQUEST = auto()  # Generic name, from the name of HTTP 400.
+    REQUEST_VARIABLE_MISSING = auto()
+    REQUEST_VARIABLE_INVALID = auto()
+    INVALID_JSON = auto()
+    BAD_IMAGE = auto()
+    REALM_UPLOAD_QUOTA = auto()
+    BAD_NARROW = auto()
+    CANNOT_DEACTIVATE_LAST_USER = auto()
+    MISSING_HTTP_EVENT_HEADER = auto()
+    CHANNEL_ALREADY_EXISTS = auto()
+    STREAM_DOES_NOT_EXIST = auto()
+    UNAUTHORIZED_PRINCIPAL = auto()
+    UNSUPPORTED_WEBHOOK_EVENT_TYPE = auto()
+    ANOMALOUS_WEBHOOK_PAYLOAD = auto()
+    BAD_EVENT_QUEUE_ID = auto()
+    CSRF_FAILED = auto()
+    INVITATION_FAILED = auto()
+    INVALID_ZULIP_SERVER = auto()
+    INVALID_PUSH_DEVICE_TOKEN = auto()
+    INVALID_REMOTE_PUSH_DEVICE_TOKEN = auto()
+    INVALID_MARKDOWN_INCLUDE_STATEMENT = auto()
+    REQUEST_CONFUSING_VAR = auto()
+    INVALID_API_KEY = auto()
+    INVALID_VIDEO_CALL_PROVIDER_TOKEN = auto()
+    UNKNOWN_ZOOM_USER = auto()
+    UNAUTHENTICATED_USER = auto()
+    NONEXISTENT_SUBDOMAIN = auto()
+    RATE_LIMIT_HIT = auto()
+    USER_DEACTIVATED = auto()
+    REALM_DEACTIVATED = auto()
+    REMOTE_SERVER_DEACTIVATED = auto()
+    PASSWORD_AUTH_DISABLED = auto()
+    PASSWORD_RESET_REQUIRED = auto()
+    AUTHENTICATION_FAILED = auto()
+    UNAUTHORIZED = auto()
+    REQUEST_TIMEOUT = auto()
+    MOVE_MESSAGES_TIME_LIMIT_EXCEEDED = auto()
+    REACTION_ALREADY_EXISTS = auto()
+    REACTION_DOES_NOT_EXIST = auto()
+    SERVER_NOT_READY = auto()
+    MISSING_REMOTE_REALM = auto()
+    TOPIC_WILDCARD_MENTION_NOT_ALLOWED = auto()
+    STREAM_WILDCARD_MENTION_NOT_ALLOWED = auto()
+    REMOTE_BILLING_UNAUTHENTICATED_USER = auto()
+    REMOTE_REALM_SERVER_MISMATCH_ERROR = auto()
+    PUSH_NOTIFICATIONS_DISALLOWED = auto()
+    EXPECTATION_MISMATCH = auto()
+    SYSTEM_GROUP_REQUIRED = auto()
+    CANNOT_DEACTIVATE_GROUP_IN_USE = auto()
+    CANNOT_ADMINISTER_CHANNEL = auto()
+    REMOTE_SERVER_VERIFICATION_SECRET_NOT_PREPARED = auto()
+    HOSTNAME_ALREADY_IN_USE_BOUNCER_ERROR = auto()
+    INVALID_BOUNCER_PUBLIC_KEY = auto()
+    REQUEST_EXPIRED = auto()
+    PUSH_SERVICE_NOT_CONFIGURED = auto()
+    NO_ACTIVE_PUSH_DEVICE = auto()
+    FAILED_TO_CONNECT_BOUNCER = auto()
+    INTERNAL_SERVER_ERROR_ON_BOUNCER = auto()
+    ADMIN_ACTION_REQUIRED = auto()
+    PERMISSION_DENIED = auto()
+
+
+class JsonableError(Exception):
+    """A standardized error format we can turn into a nice JSON HTTP response.
+
+    This class can be invoked in a couple ways.
+
+     * Easiest, but completely machine-unreadable:
+
+         raise JsonableError(_("No such widget: {}").format(widget_name))
+
+       The message may be passed through to clients and shown to a user,
+       so translation is required.  Because the text will vary depending
+       on the user's language, it's not possible for code to distinguish
+       this error from others in a non-buggy way.
+
+     * Fully machine-readable, with an error code and structured data:
+
+         class NoSuchWidgetError(JsonableError):
+             code = ErrorCode.NO_SUCH_WIDGET
+             data_fields = ['widget_name']
+
+             def __init__(self, widget_name: str) -> None:
+                 self.widget_name: str = widget_name
+
+             @staticmethod
+             def msg_format() -> str:
+                 return _("No such widget: {widget_name}")
+
+         raise NoSuchWidgetError(widget_name)
+
+       Now both server and client code see a `widget_name` attribute
+       and an error code.
+
+    Subclasses may also override `http_status_code`.
+    """
+
+    # Override this in subclasses, as needed.
+    code: ErrorCode = ErrorCode.BAD_REQUEST
+
+    # Override this in subclasses if providing structured data.
+    data_fields: list[str] = []
+
+    # Optionally override this in subclasses to return a different HTTP status,
+    # like 403 or 404.
+    http_status_code: int = 400
+
+    def __init__(self, msg: str | StrPromise) -> None:
+        # `_msg` is an implementation detail of `JsonableError` itself.
+        self._msg = msg
+
+    @staticmethod
+    def msg_format() -> str:
+        """Override in subclasses.  Gets the items in `data_fields` as format args.
+
+        This should return (a translation of) a string literal.
+        The reason it's not simply a class attribute is to allow
+        translation to work.
+        """
+        # Secretly this gets one more format arg not in `data_fields`: `_msg`.
+        # That's for the sake of the `JsonableError` base logic itself, for
+        # the simplest form of use where we just get a plain message string
+        # at construction time.
+        return "{_msg}"
+
+    @property
+    def extra_headers(self) -> dict[str, Any]:
+        return {}
+
+    #
+    # Infrastructure -- not intended to be overridden in subclasses.
+    #
+
+    @property
+    def msg(self) -> str:
+        format_data = dict(
+            ((f, getattr(self, f)) for f in self.data_fields), _msg=getattr(self, "_msg", None)
+        )
+        return self.msg_format().format(**format_data)
+
+    @property
+    def data(self) -> dict[str, Any]:
+        return dict(((f, getattr(self, f)) for f in self.data_fields), code=self.code.name)
+
+    @override
+    def __str__(self) -> str:
+        return self.msg
+
+
+class UnauthorizedError(JsonableError):
+    code: ErrorCode = ErrorCode.UNAUTHORIZED
+    http_status_code: int = 401
+
+    def __init__(self, msg: str | None = None, www_authenticate: str | None = None) -> None:
+        if msg is None:
+            msg = _("Not logged in: API authentication or user session required")
+        super().__init__(msg)
+        if www_authenticate is None:
+            self.www_authenticate = 'Basic realm="zulip"'
+        elif www_authenticate == "session":
+            self.www_authenticate = 'Session realm="zulip"'
+        else:
+            raise AssertionError("Invalid www_authenticate value!")
+
+    @property
+    @override
+    def extra_headers(self) -> dict[str, Any]:
+        extra_headers_dict = super().extra_headers
+        extra_headers_dict["WWW-Authenticate"] = self.www_authenticate
+        return extra_headers_dict
+
+
+class ChannelExistsError(JsonableError):
+    code = ErrorCode.CHANNEL_ALREADY_EXISTS
+    http_status_code = 409
+    data_fields = ["channel_name"]
+
+    def __init__(self, channel_name: str) -> None:
+        self.channel_name = channel_name
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("Channel '{channel_name}' already exists")
+
+
+class StreamDoesNotExistError(JsonableError):
+    code = ErrorCode.STREAM_DOES_NOT_EXIST
+    data_fields = ["stream"]
+
+    def __init__(self, stream: str) -> None:
+        self.stream = stream
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("Channel '{stream}' does not exist")
+
+
+class StreamWithIDDoesNotExistError(JsonableError):
+    code = ErrorCode.STREAM_DOES_NOT_EXIST
+    data_fields = ["stream_id"]
+
+    def __init__(self, stream_id: int) -> None:
+        self.stream_id = stream_id
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("Channel with ID '{stream_id}' does not exist")
+
+
+class IncompatibleParametersError(JsonableError):
+    data_fields = ["parameters"]
+
+    def __init__(self, parameters: list[str]) -> None:
+        self.parameters = ", ".join(parameters)
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("Unsupported parameter combination: {parameters}")
+
+
+class MissingDependentParameterError(JsonableError):
+    data_fields = ["required_parameter", "set_parameter"]
+
+    def __init__(self, required_parameter: str, set_parameter: str) -> None:
+        self.required_parameter = required_parameter
+        self.set_parameter = set_parameter
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("{required_parameter} is required when {set_parameter} is set.")
+
+
+class CannotDeactivateLastUserError(JsonableError):
+    code = ErrorCode.CANNOT_DEACTIVATE_LAST_USER
+    data_fields = ["is_last_owner", "entity"]
+
+    def __init__(self, is_last_owner: bool) -> None:
+        self.is_last_owner = is_last_owner
+        self.entity = _("organization owner") if is_last_owner else _("user")
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("Cannot deactivate the only {entity}.")
+
+
+class InvalidMarkdownIncludeStatementError(JsonableError):
+    code = ErrorCode.INVALID_MARKDOWN_INCLUDE_STATEMENT
+    data_fields = ["include_statement"]
+
+    def __init__(self, include_statement: str) -> None:
+        self.include_statement = include_statement
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("Invalid Markdown include statement: {include_statement}")
+
+
+class RateLimitedError(JsonableError):
+    code = ErrorCode.RATE_LIMIT_HIT
+    http_status_code = 429
+
+    def __init__(self, secs_to_freedom: float | None = None) -> None:
+        self.secs_to_freedom = secs_to_freedom
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _(
+            "API usage exceeded rate limit; see https://zulip.com/api/http-headers#rate-limiting-response-headers"
+        )
+
+    @property
+    @override
+    def extra_headers(self) -> dict[str, Any]:
+        extra_headers_dict = super().extra_headers
+        if self.secs_to_freedom is not None:
+            extra_headers_dict["Retry-After"] = self.secs_to_freedom
+
+        return extra_headers_dict
+
+    @property
+    @override
+    def data(self) -> dict[str, Any]:
+        data_dict = super().data
+        data_dict["retry-after"] = self.secs_to_freedom
+
+        return data_dict
+
+
+class InvalidJSONError(JsonableError):
+    code = ErrorCode.INVALID_JSON
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("Malformed JSON")
+
+
+class OrganizationAdministratorRequiredError(JsonableError):
+    code: ErrorCode = ErrorCode.UNAUTHORIZED_PRINCIPAL
+
+    def __init__(self) -> None:
+        pass
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("Must be an organization administrator")
+
+
+class OrganizationOwnerRequiredError(JsonableError):
+    code: ErrorCode = ErrorCode.UNAUTHORIZED_PRINCIPAL
+
+    def __init__(self) -> None:
+        pass
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("Must be an organization owner")
+
+
+class BotRequiredError(JsonableError):
+    code: ErrorCode = ErrorCode.PERMISSION_DENIED
+
+    def __init__(self) -> None:
+        pass
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("Must be a bot user")
+
+
+class AuthenticationFailedError(JsonableError):
+    # Generic class for authentication failures
+    code: ErrorCode = ErrorCode.AUTHENTICATION_FAILED
+    http_status_code = 401
+
+    def __init__(self) -> None:
+        pass
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("Your username or password is incorrect")
+
+
+class UserDeactivatedError(AuthenticationFailedError):
+    code: ErrorCode = ErrorCode.USER_DEACTIVATED
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("Account is deactivated")
+
+
+class RealmDeactivatedError(AuthenticationFailedError):
+    code: ErrorCode = ErrorCode.REALM_DEACTIVATED
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("This organization has been deactivated")
+
+
+class RemoteServerDeactivatedError(AuthenticationFailedError):
+    code: ErrorCode = ErrorCode.REALM_DEACTIVATED
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _(
+            "The mobile push notification service registration for your server has been deactivated"
+        )
+
+
+class PasswordAuthDisabledError(AuthenticationFailedError):
+    code: ErrorCode = ErrorCode.PASSWORD_AUTH_DISABLED
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("Password authentication is disabled in this organization")
+
+
+class PasswordResetRequiredError(AuthenticationFailedError):
+    code: ErrorCode = ErrorCode.PASSWORD_RESET_REQUIRED
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("Your password has been disabled and needs to be reset")
+
+
+class MarkdownRenderingError(Exception):
+    pass
+
+
+class InvalidAPIKeyError(JsonableError):
+    code = ErrorCode.INVALID_API_KEY
+    http_status_code = 401
+
+    def __init__(self) -> None:
+        pass
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("Invalid API key")
+
+
+class InvalidAPIKeyFormatError(InvalidAPIKeyError):
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("Malformed API key")
+
+
+class WebhookError(JsonableError):
+    """
+    Intended as a generic exception raised by specific webhook
+    integrations. This class is subclassed by more specific exceptions
+    such as UnsupportedWebhookEventTypeError and AnomalousWebhookPayloadError.
+    """
+
+    data_fields = ["webhook_name"]
+
+    def __init__(self) -> None:
+        # webhook_name is often set by decorators such as webhook_view
+        # in zerver/decorator.py
+        self.webhook_name = "(unknown)"
+
+
+class UnsupportedWebhookEventTypeError(WebhookError):
+    """Intended as an exception for event formats that we know the
+    third-party service generates but which Zulip doesn't support /
+    generate a message for.
+
+    Exceptions where we cannot parse the event type, possibly because
+    the event isn't actually from the service in question, should
+    raise AnomalousWebhookPayloadError.
+    """
+
+    code = ErrorCode.UNSUPPORTED_WEBHOOK_EVENT_TYPE
+    http_status_code = 200
+    data_fields = ["webhook_name", "event_type"]
+
+    def __init__(self, event_type: str | None) -> None:
+        super().__init__()
+        self.event_type = event_type
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _(
+            "The '{event_type}' event isn't currently supported by the {webhook_name} webhook; ignoring"
+        )
+
+
+class AnomalousWebhookPayloadError(WebhookError):
+    """Intended as an exception for incoming webhook requests that we
+    cannot recognize as having been generated by the service in
+    question. (E.g. because someone pointed a Jira server at the
+    GitHub integration URL).
+
+    If we can parse the event but don't support it, use
+    UnsupportedWebhookEventTypeError.
+
+    """
+
+    code = ErrorCode.ANOMALOUS_WEBHOOK_PAYLOAD
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("Unable to parse request: Did {webhook_name} generate this event?")
+
+
+class MissingAuthenticationError(JsonableError):
+    code = ErrorCode.UNAUTHENTICATED_USER
+    http_status_code = 401
+
+    def __init__(self) -> None:
+        pass
+
+    # No msg_format is defined since this exception is caught and
+    # converted into json_unauthorized in Zulip's middleware.
+
+
+class RemoteBillingAuthenticationError(JsonableError):
+    # We want this as a distinct class from MissingAuthenticationError,
+    # as we don't want the json_unauthorized conversion mechanism to apply
+    # to this.
+    code = ErrorCode.REMOTE_BILLING_UNAUTHENTICATED_USER
+    http_status_code = 401
+
+    def __init__(self) -> None:
+        pass
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("User not authenticated")
+
+
+class InvalidSubdomainError(JsonableError):
+    code = ErrorCode.NONEXISTENT_SUBDOMAIN
+    http_status_code = 404
+
+    def __init__(self) -> None:
+        pass
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("Invalid subdomain")
+
+
+class InvitationError(JsonableError):
+    code = ErrorCode.INVITATION_FAILED
+    data_fields = [
+        "errors",
+        "sent_invitations",
+        "license_limit_reached",
+        "daily_limit_reached",
+    ]
+
+    def __init__(
+        self,
+        msg: str,
+        errors: list[tuple[str, str, bool]],
+        sent_invitations: bool,
+        license_limit_reached: bool = False,
+        daily_limit_reached: bool = False,
+    ) -> None:
+        self._msg: str = msg
+        self.errors: list[tuple[str, str, bool]] = errors
+        self.sent_invitations: bool = sent_invitations
+        self.license_limit_reached: bool = license_limit_reached
+        self.daily_limit_reached: bool = daily_limit_reached
+
+
+class DirectMessageInitiationError(JsonableError):
+    def __init__(self) -> None:
+        pass
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("You do not have permission to initiate direct message conversations.")
+
+
+class MessagesNotAllowedInEmptyTopicError(JsonableError):
+    data_fields = ["empty_topic_display_name"]
+
+    def __init__(self, empty_topic_display_name: str) -> None:
+        self.empty_topic_display_name = empty_topic_display_name
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _(
+            "Sending messages to the {empty_topic_display_name} is not allowed in this channel."
+        )
+
+
+class TopicsNotAllowedError(JsonableError):
+    data_fields = ["empty_topic_display_name"]
+
+    def __init__(self, empty_topic_display_name: str) -> None:
+        self.empty_topic_display_name = empty_topic_display_name
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("Only the {empty_topic_display_name} topic is allowed in this channel.")
+
+
+class CannotSetTopicsPolicyError(JsonableError):
+    data_fields = ["empty_topic_display_name"]
+
+    def __init__(self, empty_topic_display_name: str) -> None:
+        self.empty_topic_display_name = empty_topic_display_name
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _(
+            "To enable this configuration, all messages in this channel must be in the {empty_topic_display_name} topic. Consider renaming or deleting other topics."
+        )
+
+
+class DirectMessagePermissionError(JsonableError):
+    def __init__(self, is_nobody_group: bool) -> None:
+        if is_nobody_group:
+            msg = _("Direct messages are disabled in this organization.")
+        else:
+            msg = _("This conversation does not include any users who can authorize it.")
+        super().__init__(msg)
+
+
+class AccessDeniedError(JsonableError):
+    http_status_code = 403
+
+    def __init__(self) -> None:
+        pass
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("Access denied")
+
+
+class ResourceNotFoundError(JsonableError):
+    http_status_code = 404
+
+
+class ValidationFailureError(JsonableError):
+    # This class translations a Django ValidationError into a
+    # Zulip-style JsonableError, sending back just the first error for
+    # consistency of API.
+    data_fields = ["errors"]
+
+    def __init__(self, error: ValidationError) -> None:
+        super().__init__(error.messages[0])
+        self.errors = error.message_dict
+
+
+class MessageMoveError(JsonableError):
+    code = ErrorCode.MOVE_MESSAGES_TIME_LIMIT_EXCEEDED
+    data_fields = [
+        "first_message_id_allowed_to_move",
+        "total_messages_in_topic",
+        "total_messages_allowed_to_move",
+    ]
+
+    def __init__(
+        self,
+        first_message_id_allowed_to_move: int,
+        total_messages_in_topic: int,
+        total_messages_allowed_to_move: int,
+    ) -> None:
+        self.first_message_id_allowed_to_move = first_message_id_allowed_to_move
+        self.total_messages_in_topic = total_messages_in_topic
+        self.total_messages_allowed_to_move = total_messages_allowed_to_move
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _(
+            "You only have permission to move the {total_messages_allowed_to_move}/{total_messages_in_topic} most recent messages in this topic."
+        )
+
+
+class ReactionExistsError(JsonableError):
+    code = ErrorCode.REACTION_ALREADY_EXISTS
+
+    def __init__(self) -> None:
+        pass
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("Reaction already exists.")
+
+
+class ReactionDoesNotExistError(JsonableError):
+    code = ErrorCode.REACTION_DOES_NOT_EXIST
+
+    def __init__(self) -> None:
+        pass
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("Reaction doesn't exist.")
+
+
+class ApiParamValidationError(JsonableError):
+    def __init__(self, msg: str, error_type: str) -> None:
+        super().__init__(msg)
+        self.error_type = error_type
+
+
+class ServerNotReadyError(JsonableError):
+    code = ErrorCode.SERVER_NOT_READY
+    http_status_code = 500
+
+
+class RemoteRealmServerMismatchError(JsonableError):  # nocoverage
+    code = ErrorCode.REMOTE_REALM_SERVER_MISMATCH_ERROR
+    http_status_code = 403
+
+    def __init__(self) -> None:
+        pass
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _(
+            "Your organization is registered to a different Zulip server. Please contact Zulip support for assistance in resolving this issue."
+        )
+
+
+class MissingRemoteRealmError(JsonableError):  # nocoverage
+    code: ErrorCode = ErrorCode.MISSING_REMOTE_REALM
+    http_status_code = 403
+
+    def __init__(self) -> None:
+        pass
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("Organization not registered")
+
+
+class StreamWildcardMentionNotAllowedError(JsonableError):
+    code: ErrorCode = ErrorCode.STREAM_WILDCARD_MENTION_NOT_ALLOWED
+
+    def __init__(self) -> None:
+        pass
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("You do not have permission to use channel wildcard mentions in this channel.")
+
+
+class TopicWildcardMentionNotAllowedError(JsonableError):
+    code: ErrorCode = ErrorCode.TOPIC_WILDCARD_MENTION_NOT_ALLOWED
+
+    def __init__(self) -> None:
+        pass
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("You do not have permission to use topic wildcard mentions in this topic.")
+
+
+class ExpectationMismatchError(JsonableError):
+    code: ErrorCode = ErrorCode.EXPECTATION_MISMATCH
+    data_fields = ["field_name"]
+
+    def __init__(self) -> None:
+        self.field_name = "field"
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("'{field_name}' value does not match the expected value.")
+
+
+class PreviousSettingValueMismatchedError(ExpectationMismatchError):
+    def __init__(self) -> None:
+        super().__init__()
+        self.field_name = "old"
+
+
+class PreviousMessageContentMismatchedError(ExpectationMismatchError):
+    def __init__(self) -> None:
+        super().__init__()
+        self.field_name = "prev_content_sha256"
+
+
+class SystemGroupRequiredError(JsonableError):
+    code = ErrorCode.SYSTEM_GROUP_REQUIRED
+    data_fields = ["setting_name"]
+
+    def __init__(self, setting_name: str) -> None:
+        self.setting_name = setting_name
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("'{setting_name}' must be a system user group.")
+
+
+class CannotDeactivateGroupInUseError(JsonableError):
+    code = ErrorCode.CANNOT_DEACTIVATE_GROUP_IN_USE
+    data_fields = ["objections"]
+
+    def __init__(
+        self,
+        objections: list[dict[str, Any]],
+    ) -> None:
+        self.objections = objections
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("Cannot deactivate user group in use.")
+
+
+class CannotAdministerChannelError(JsonableError):
+    def __init__(self) -> None:
+        pass
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("You do not have permission to administer this channel.")
+
+
+class CannotManageDefaultChannelError(JsonableError):
+    def __init__(self) -> None:
+        pass
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("You do not have permission to change default channels.")
+
+
+class EmailAlreadyInUseError(JsonableError):
+    def __init__(self) -> None:
+        pass
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("Email is already in use.")
+
+
+class DeliveryTimeNotInFutureError(JsonableError):
+    def __init__(self) -> None:
+        pass
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("Scheduled delivery time must be in the future.")
+
+
+class SlackImportInvalidFileError(Exception):
+    """
+    An error that is raised during the Slack import process
+    and is intended to be shown to the user.
+    """
+
+    def __init__(self, message: str) -> None:
+        super().__init__(message)
+        self.message = message
+
+
+class InvalidBouncerPublicKeyError(JsonableError):
+    code = ErrorCode.INVALID_BOUNCER_PUBLIC_KEY
+
+    def __init__(self) -> None:
+        pass
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("Invalid bouncer_public_key")
+
+
+class RequestExpiredError(JsonableError):
+    code = ErrorCode.REQUEST_EXPIRED
+
+    def __init__(self) -> None:
+        pass
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("Request expired")
+
+
+class InvalidEncryptedPushRegistrationError(JsonableError):
+    def __init__(self) -> None:
+        pass
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("Invalid encrypted_push_registration")
+
+
+class PushServiceNotConfiguredError(JsonableError):
+    code = ErrorCode.PUSH_SERVICE_NOT_CONFIGURED
+
+    def __init__(self) -> None:
+        pass
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("Server is not configured to use push notification service.")

zerver/lib/external_accounts.py

@@ -0,0 +1,194 @@
+"""
+This module stores data for "external account" custom profile field.
+"""
+
+from dataclasses import dataclass
+
+from django.core.exceptions import ValidationError
+from django.utils.translation import gettext as _
+from django.utils.translation import gettext_lazy
+from django_stubs_ext import StrPromise
+
+from zerver.lib.types import ProfileFieldData
+from zerver.lib.validator import (
+    check_dict_only,
+    check_external_account_url_pattern,
+    check_required_string,
+)
+
+
+# Default external account fields are by default available
+# to realm admins, where realm admin only need to select
+# the default field and other values(i.e. name, url) will be
+# fetch from this dictionary.
+@dataclass
+class ExternalAccount:
+    text: str  # Field text for admins - custom profile field in org settings view
+    name: StrPromise  # Field label or name - user profile in user settings view
+    hint: str  # Field hint for realm users
+    url_pattern: str  # Field URL linkifier
+
+
+# Set url_pattern as "%(username)s" if there's no common URL pattern,
+# and the user should enter the full URL of their link.
+# Set url_pattern as "" to make the field a text field.
+DEFAULT_EXTERNAL_ACCOUNTS = {
+    "atlassian": ExternalAccount(
+        text="Atlassian",
+        url_pattern="",
+        name=gettext_lazy("Atlassian account ID"),
+        hint="",
+    ),
+    "behance": ExternalAccount(
+        text="Behance",
+        url_pattern="https://www.behance.net/%(username)s",
+        name=gettext_lazy("Behance username"),
+        hint="",
+    ),
+    "bitbucket": ExternalAccount(
+        text="Bitbucket",
+        url_pattern="https://bitbucket.org/%(username)s",
+        name=gettext_lazy("Bitbucket username"),
+        hint="",
+    ),
+    "bluesky": ExternalAccount(
+        text="Bluesky",
+        url_pattern="https://bsky.app/profile/%(username)s",
+        name=gettext_lazy("Bluesky handle"),
+        hint="",
+    ),
+    "codeberg": ExternalAccount(
+        text="Codeberg",
+        url_pattern="https://codeberg.org/%(username)s",
+        name=gettext_lazy("Codeberg username"),
+        hint="",
+    ),
+    "discord": ExternalAccount(
+        text="Discord",
+        url_pattern="",
+        name=gettext_lazy("Discord tag"),
+        hint="",
+    ),
+    "dribbble": ExternalAccount(
+        text="Dribbble",
+        url_pattern="https://dribbble.com/%(username)s",
+        name=gettext_lazy("Dribbble username"),
+        hint="",
+    ),
+    "facebook": ExternalAccount(
+        text="Facebook",
+        url_pattern="https://www.facebook.com/%(username)s",
+        name=gettext_lazy("Facebook username"),
+        hint="",
+    ),
+    "github": ExternalAccount(
+        text="GitHub",
+        url_pattern="https://github.com/%(username)s",
+        name=gettext_lazy("GitHub username"),
+        hint="",
+    ),
+    "gitlab": ExternalAccount(
+        text="GitLab",
+        url_pattern="https://gitlab.com/%(username)s",
+        name=gettext_lazy("GitLab username"),
+        hint="",
+    ),
+    "instagram": ExternalAccount(
+        text="Instagram",
+        url_pattern="https://www.instagram.com/%(username)s",
+        name=gettext_lazy("Instagram username"),
+        hint="",
+    ),
+    "linkedin": ExternalAccount(
+        text="LinkedIn",
+        url_pattern="https://www.linkedin.com/in/%(username)s",
+        name=gettext_lazy("LinkedIn profile name"),
+        hint="",
+    ),
+    "mastodon": ExternalAccount(
+        text="Mastodon",
+        url_pattern="%(username)s",
+        name=gettext_lazy("Mastodon profile"),
+        hint="The full URL to your profile",
+    ),
+    "medium": ExternalAccount(
+        text="Medium",
+        url_pattern="https://%(username)s.medium.com",
+        name=gettext_lazy("Medium username"),
+        hint="",
+    ),
+    "pinterest": ExternalAccount(
+        text="Pinterest",
+        url_pattern="https://www.pinterest.com/%(username)s",
+        name=gettext_lazy("Pinterest username"),
+        hint="",
+    ),
+    "reddit": ExternalAccount(
+        text="Reddit",
+        url_pattern="https://www.reddit.com/user/%(username)s",
+        name=gettext_lazy("Reddit username"),
+        hint="",
+    ),
+    "snapchat": ExternalAccount(
+        text="Snapchat",
+        url_pattern="https://www.snapchat.com/@%(username)s",
+        name=gettext_lazy("Snapchat username"),
+        hint="",
+    ),
+    "threads": ExternalAccount(
+        text="Threads",
+        url_pattern="https://www.threads.com/@%(username)s",
+        name=gettext_lazy("Threads username"),
+        hint="",
+    ),
+    "tiktok": ExternalAccount(
+        text="TikTok",
+        url_pattern="https://www.tiktok.com/@%(username)s",
+        name=gettext_lazy("TikTok username"),
+        hint="",
+    ),
+    "twitch": ExternalAccount(
+        text="Twitch",
+        url_pattern="https://www.twitch.tv/%(username)s",
+        name=gettext_lazy("Twitch username"),
+        hint="",
+    ),
+    "x": ExternalAccount(
+        text="X",
+        url_pattern="https://x.com/%(username)s",
+        name=gettext_lazy("X username"),
+        hint="",
+    ),
+    "youtube": ExternalAccount(
+        text="YouTube",
+        url_pattern="https://www.youtube.com/%(username)s",
+        name=gettext_lazy("YouTube handle"),
+        hint="",
+    ),
+}
+
+
+def get_default_external_accounts() -> dict[str, dict[str, str]]:
+    return {
+        subtype: {
+            "text": external_account.text,
+            "url_pattern": external_account.url_pattern,
+            "name": str(external_account.name),
+            "hint": external_account.hint,
+        }
+        for subtype, external_account in DEFAULT_EXTERNAL_ACCOUNTS.items()
+    }
+
+
+def validate_external_account_field_data(field_data: ProfileFieldData) -> ProfileFieldData:
+    field_validator = check_dict_only(
+        [("subtype", check_required_string)],
+        [("url_pattern", check_external_account_url_pattern)],
+    )
+    field_validator("field_data", field_data)
+
+    field_subtype = field_data.get("subtype")
+    if field_subtype not in DEFAULT_EXTERNAL_ACCOUNTS and field_subtype != "custom":
+        raise ValidationError(_("Invalid external account type"))
+
+    return field_data

zerver/lib/fix_unreads.py

@@ -0,0 +1,128 @@
+import logging
+import time
+from collections.abc import Callable
+from typing import TypeVar
+
+from django.db import connection
+from django.db.backends.utils import CursorWrapper
+from psycopg2.sql import SQL
+
+from zerver.models import UserProfile
+
+T = TypeVar("T")
+
+"""
+NOTE!  Be careful modifying this library, as it is used
+in a migration, and it needs to be valid for the state
+of the database that is in place when the 0104_fix_unreads
+migration runs.
+"""
+
+logger = logging.getLogger("zulip.fix_unreads")
+logger.setLevel(logging.WARNING)
+
+
+def update_unread_flags(cursor: CursorWrapper, user_message_ids: list[int]) -> None:
+    query = SQL(
+        """
+        UPDATE zerver_usermessage
+        SET flags = flags | 1
+        WHERE id IN %(user_message_ids)s
+    """
+    )
+
+    cursor.execute(query, {"user_message_ids": tuple(user_message_ids)})
+
+
+def get_timing(message: str, f: Callable[[], T]) -> T:
+    start = time.time()
+    logger.info(message)
+    ret = f()
+    elapsed = time.time() - start
+    logger.info("elapsed time: %.03f\n", elapsed)
+    return ret
+
+
+def fix_unsubscribed(cursor: CursorWrapper, user_profile: UserProfile) -> None:
+    def find_recipients() -> list[int]:
+        query = SQL(
+            """
+            SELECT
+                zerver_subscription.recipient_id
+            FROM
+                zerver_subscription
+            INNER JOIN zerver_recipient ON (
+                zerver_recipient.id = zerver_subscription.recipient_id
+            )
+            WHERE (
+                zerver_subscription.user_profile_id = %(user_profile_id)s AND
+                zerver_recipient.type = 2 AND
+                (NOT zerver_subscription.active)
+            )
+        """
+        )
+        cursor.execute(query, {"user_profile_id": user_profile.id})
+        rows = cursor.fetchall()
+        recipient_ids = [row[0] for row in rows]
+        logger.info("%s", recipient_ids)
+        return recipient_ids
+
+    recipient_ids = get_timing(
+        "get recipients",
+        find_recipients,
+    )
+
+    if not recipient_ids:
+        return
+
+    def find() -> list[int]:
+        query = SQL(
+            """
+            SELECT
+                zerver_usermessage.id
+            FROM
+                zerver_usermessage
+            INNER JOIN zerver_message ON (
+                zerver_message.id = zerver_usermessage.message_id
+            )
+            WHERE (
+                zerver_usermessage.user_profile_id = %(user_profile_id)s AND
+                (zerver_usermessage.flags & 1) = 0 AND
+                zerver_message.recipient_id in %(recipient_ids)s
+            )
+        """
+        )
+
+        cursor.execute(
+            query,
+            {
+                "user_profile_id": user_profile.id,
+                "recipient_ids": tuple(recipient_ids),
+            },
+        )
+        rows = cursor.fetchall()
+        user_message_ids = [row[0] for row in rows]
+        logger.info("rows found: %d", len(user_message_ids))
+        return user_message_ids
+
+    user_message_ids = get_timing(
+        "finding unread messages for non-active streams",
+        find,
+    )
+
+    if not user_message_ids:
+        return
+
+    def fix() -> None:
+        update_unread_flags(cursor, user_message_ids)
+
+    get_timing(
+        "fixing unread messages for non-active streams",
+        fix,
+    )
+
+
+def fix(user_profile: UserProfile) -> None:
+    logger.info("\n---\nFixing %s:", user_profile.id)
+    with connection.cursor() as cursor:
+        fix_unsubscribed(cursor, user_profile)

zerver/lib/generate_test_data.py

@@ -0,0 +1,232 @@
+import itertools
+import os
+import random
+from typing import Any
+
+import orjson
+
+from scripts.lib.zulip_tools import get_or_create_dev_uuid_var_path
+from zerver.lib.message import truncate_topic
+from zerver.lib.topic import RESOLVED_TOPIC_PREFIX
+
+
+def load_config() -> dict[str, Any]:
+    with open("zerver/tests/fixtures/config.generate_data.json", "rb") as infile:
+        config = orjson.loads(infile.read())
+
+    return config
+
+
+def generate_topics(num_topics: int, percent_topic_links: float = 0) -> list[str]:
+    config = load_config()["gen_fodder"]
+
+    # Make single word topics account for 30% of total topics.
+    # Single word topics are most common, thus
+    # it is important we test on it.
+    num_single_word_topics = num_topics // 3
+    topic_names = random.choices(config["nouns"], k=num_single_word_topics)
+
+    sentence = ["adjectives", "nouns", "connectors", "verbs", "adverbs"]
+    for pos in sentence:
+        # Add an empty string so that we can generate variable length topics.
+        config[pos].append("")
+
+    topic_names.extend(
+        " ".join(word for pos in sentence if (word := random.choice(config[pos])) != "")
+        for _ in range(num_topics - num_single_word_topics)
+    )
+
+    # Mark a small subset of topics as resolved in some streams, and
+    # many topics in a few streams. Note that these don't have the
+    # "Marked as resolved" messages, so don't match the normal user
+    # experience perfectly.
+    if random.random() < 0.15:
+        resolved_topic_probability = 0.5
+    else:
+        resolved_topic_probability = 0.05
+
+    results = []
+    for topic_name in topic_names:
+        if percent_topic_links > 0 and random.random() < percent_topic_links / 100.0:
+            topic_name = add_link_to_topic(topic_name)
+        if random.random() < resolved_topic_probability:
+            topic_name = truncate_topic(RESOLVED_TOPIC_PREFIX + topic_name)
+        results.append(topic_name)
+    return results
+
+
+def load_generators(config: dict[str, Any]) -> dict[str, Any]:
+    results = {}
+    cfg = config["gen_fodder"]
+
+    results["nouns"] = itertools.cycle(cfg["nouns"])
+    results["adjectives"] = itertools.cycle(cfg["adjectives"])
+    results["connectors"] = itertools.cycle(cfg["connectors"])
+    results["verbs"] = itertools.cycle(cfg["verbs"])
+    results["adverbs"] = itertools.cycle(cfg["adverbs"])
+    results["emojis"] = itertools.cycle(cfg["emoji"])
+    results["links"] = itertools.cycle(cfg["links"])
+
+    results["maths"] = itertools.cycle(cfg["maths"])
+    results["inline-code"] = itertools.cycle(cfg["inline-code"])
+    results["code-blocks"] = itertools.cycle(cfg["code-blocks"])
+    results["quote-blocks"] = itertools.cycle(cfg["quote-blocks"])
+    results["images"] = itertools.cycle(cfg["images"])
+
+    results["lists"] = itertools.cycle(cfg["lists"])
+
+    return results
+
+
+def parse_file(config: dict[str, Any], gens: dict[str, Any], corpus_file: str) -> list[str]:
+    # First, load the entire file into a dictionary,
+    # then apply our custom filters to it as needed.
+
+    paragraphs: list[str] = []
+
+    with open(corpus_file) as infile:
+        # OUR DATA: we need to separate the person talking and what they say
+        paragraphs = remove_line_breaks(infile)
+        paragraphs = add_flair(paragraphs, gens)
+
+    return paragraphs
+
+
+def get_flair_gen(length: int) -> list[str]:
+    # Grab the percentages from the config file
+    # create a list that we can consume that will guarantee the distribution
+    result = []
+
+    for k, v in config["dist_percentages"].items():
+        result.extend([k] * int(v * length / 100))
+
+    result.extend(["None"] * (length - len(result)))
+
+    random.shuffle(result)
+    return result
+
+
+def add_flair(paragraphs: list[str], gens: dict[str, Any]) -> list[str]:
+    # roll the dice and see what kind of flair we should add, if any
+    results = []
+
+    flair = get_flair_gen(len(paragraphs))
+
+    for i in range(len(paragraphs)):
+        key = flair[i]
+        if key == "None":
+            txt = paragraphs[i]
+        elif key == "italic":
+            txt = add_md("*", paragraphs[i])
+        elif key == "bold":
+            txt = add_md("**", paragraphs[i])
+        elif key == "strike-thru":
+            txt = add_md("~~", paragraphs[i])
+        elif key == "quoted":
+            txt = ">" + paragraphs[i]
+        elif key == "quote-block":
+            txt = paragraphs[i] + "\n" + next(gens["quote-blocks"])
+        elif key == "inline-code":
+            txt = paragraphs[i] + "\n" + next(gens["inline-code"])
+        elif key == "code-block":
+            txt = paragraphs[i] + "\n" + next(gens["code-blocks"])
+        elif key == "math":
+            txt = paragraphs[i] + "\n" + next(gens["maths"])
+        elif key == "list":
+            txt = paragraphs[i] + "\n" + next(gens["lists"])
+        elif key == "emoji":
+            txt = add_emoji(paragraphs[i], next(gens["emojis"]))
+        elif key == "link":
+            txt = add_link(paragraphs[i], next(gens["links"]))
+        elif key == "images":
+            # Ideally, this would actually be a 2-step process that
+            # first hits the `upload` endpoint and then adds that URL;
+            # this is the hacky version where we just use inline image
+            # previews of files already in the project (which are the
+            # only files we can link to as being definitely available
+            # even when developing offline).
+            txt = paragraphs[i] + "\n" + next(gens["images"])
+
+        results.append(txt)
+
+    return results
+
+
+def add_md(mode: str, text: str) -> str:
+    # mode means: bold, italic, etc.
+    # to add a list at the end of a paragraph, * item one\n * item two
+
+    # find out how long the line is, then insert the mode before the end
+
+    vals = text.split()
+    start = random.randrange(len(vals))
+    end = random.randrange(len(vals) - start) + start
+    vals[start] = mode + vals[start]
+    vals[end] += mode
+
+    return " ".join(vals).strip()
+
+
+def add_emoji(text: str, emoji: str) -> str:
+    vals = text.split()
+    start = random.randrange(len(vals))
+
+    vals[start] = vals[start] + " " + emoji + " "
+    return " ".join(vals)
+
+
+def add_link(text: str, link: str) -> str:
+    vals = text.split()
+    start = random.randrange(len(vals))
+
+    vals[start] = vals[start] + " " + link + " "
+
+    return " ".join(vals)
+
+
+def add_link_to_topic(topic: str) -> str:
+    url = random.choice(config["gen_fodder"]["topic-links"])
+    return truncate_topic(f"{url} {topic}")
+
+
+def remove_line_breaks(fh: Any) -> list[str]:
+    # We're going to remove line breaks from paragraphs
+    results = []  # save the dialogs as tuples with (author, dialog)
+
+    para = []  # we'll store the lines here to form a paragraph
+
+    for line in fh:
+        text = line.strip()
+        if text != "":
+            para.append(text)
+        else:
+            if para:
+                results.append(" ".join(para))
+            # reset the paragraph
+            para = []
+    if para:
+        results.append(" ".join(para))
+
+    return results
+
+
+def write_file(paragraphs: list[str], filename: str) -> None:
+    with open(filename, "wb") as outfile:
+        outfile.write(orjson.dumps(paragraphs))
+
+
+def create_test_data() -> None:
+    gens = load_generators(config)  # returns a dictionary of generators
+
+    paragraphs = parse_file(config, gens, config["corpus"]["filename"])
+
+    write_file(
+        paragraphs,
+        os.path.join(get_or_create_dev_uuid_var_path("test-backend"), "test_messages.json"),
+    )
+
+
+config = load_config()
+
+if __name__ == "__main__":
+    create_test_data()

zerver/lib/github.py

@@ -0,0 +1,63 @@
+import json
+import logging
+from typing import Any
+
+import requests
+
+from zerver.lib.cache import cache_with_key
+from zerver.lib.outgoing_http import OutgoingSession
+
+logger = logging.getLogger(__name__)
+
+
+class GithubSession(OutgoingSession):
+    def __init__(self, **kwargs: Any) -> None:
+        super().__init__(role="github", timeout=5, **kwargs)
+
+
+def get_latest_github_release_version_for_repo(repo: str) -> str:
+    api_url = f"https://api.github.com/repos/zulip/{repo}/releases/latest"
+    try:
+        return GithubSession().get(api_url).json()["tag_name"]
+    except (requests.RequestException, json.JSONDecodeError, KeyError):
+        logger.exception(
+            "Unable to fetch the latest release version from GitHub %s", api_url, stack_info=True
+        )
+        return ""
+
+
+def verify_release_download_link(link: str) -> bool:
+    try:
+        GithubSession().head(link).raise_for_status()
+        return True
+    except requests.RequestException:
+        logger.error("App download link is broken %s", link)
+        return False
+
+
+PLATFORM_TO_SETUP_FILE = {
+    "linux": "Zulip-{version}-x86_64.AppImage",
+    "mac": "Zulip-{version}-arm64.dmg",
+    "mac-intel": "Zulip-{version}-x64.dmg",
+    "mac-arm64": "Zulip-{version}-arm64.dmg",
+    "windows": "Zulip-Web-Setup-{version}.exe",
+}
+
+
+class InvalidPlatformError(Exception):
+    pass
+
+
+@cache_with_key(lambda platform: f"download_link:{platform}", timeout=60 * 30)
+def get_latest_github_release_download_link_for_platform(platform: str) -> str:
+    if platform not in PLATFORM_TO_SETUP_FILE:
+        raise InvalidPlatformError
+
+    latest_version = get_latest_github_release_version_for_repo("zulip-desktop")
+    if latest_version:
+        latest_version = latest_version.removeprefix("v")
+        setup_file = PLATFORM_TO_SETUP_FILE[platform].format(version=latest_version)
+        link = f"https://desktop-download.zulip.com/v{latest_version}/{setup_file}"
+        if verify_release_download_link(link):
+            return link
+    return "https://github.com/zulip/zulip-desktop/releases/latest"

zerver/lib/home.py

@@ -0,0 +1,221 @@
+import calendar
+import time
+from dataclasses import dataclass
+from urllib.parse import urlsplit
+
+from django.conf import settings
+from django.http import HttpRequest
+from django.utils import translation
+from two_factor.utils import default_device
+
+from zerver.context_processors import get_apps_page_url
+from zerver.lib.events import ClientCapabilities, do_events_register
+from zerver.lib.i18n import (
+    get_and_set_request_language,
+    get_language_list,
+    get_language_translation_data,
+)
+from zerver.lib.narrow_helpers import NeverNegatedNarrowTerm
+from zerver.lib.realm_description import get_realm_rendered_description
+from zerver.lib.request import RequestNotes
+from zerver.lib.workplace_users import (
+    realm_eligible_for_non_workplace_pricing,
+    realm_on_discounted_cloud_plan,
+)
+from zerver.models import Realm, Stream, UserProfile
+from zerver.views.message_flags import get_latest_update_message_flag_activity
+
+
+@dataclass
+class BillingInfo:
+    show_billing: bool
+    show_plans: bool
+    sponsorship_pending: bool
+    show_remote_billing: bool
+
+
+@dataclass
+class UserPermissionInfo:
+    color_scheme: int
+    is_guest: bool
+    is_realm_admin: bool
+    is_realm_owner: bool
+
+
+def get_furthest_read_time(user_profile: UserProfile | None) -> float | None:
+    if user_profile is None:
+        return time.time()
+
+    user_activity = get_latest_update_message_flag_activity(user_profile)
+    if user_activity is None:
+        return None
+
+    return calendar.timegm(user_activity.last_visit.utctimetuple())
+
+
+def promote_sponsoring_zulip_in_realm(realm: Realm) -> bool:
+    if not settings.PROMOTE_SPONSORING_ZULIP:
+        return False
+
+    # If PROMOTE_SPONSORING_ZULIP is enabled, advertise sponsoring
+    # Zulip in the gear menu of non-paying organizations.
+    return realm.plan_type in [Realm.PLAN_TYPE_STANDARD_FREE, Realm.PLAN_TYPE_SELF_HOSTED]
+
+
+def get_user_permission_info(user_profile: UserProfile | None) -> UserPermissionInfo:
+    if user_profile is not None:
+        return UserPermissionInfo(
+            color_scheme=user_profile.color_scheme,
+            is_guest=user_profile.is_guest,
+            is_realm_owner=user_profile.is_realm_owner,
+            is_realm_admin=user_profile.is_realm_admin,
+        )
+    else:
+        return UserPermissionInfo(
+            color_scheme=UserProfile.COLOR_SCHEME_AUTOMATIC,
+            is_guest=False,
+            is_realm_admin=False,
+            is_realm_owner=False,
+        )
+
+
+def build_page_params_for_home_page_load(
+    request: HttpRequest,
+    user_profile: UserProfile | None,
+    realm: Realm,
+    insecure_desktop_app: bool,
+    narrow: list[NeverNegatedNarrowTerm],
+    narrow_stream: Stream | None,
+    narrow_topic_name: str | None,
+) -> tuple[int, dict[str, object]]:
+    """
+    This function computes page_params for when we load the home page.
+
+    The page_params data structure gets sent to the client.
+    """
+
+    client_capabilities = ClientCapabilities(
+        notification_settings_null=True,
+        bulk_message_deletion=True,
+        user_avatar_url_field_optional=True,
+        stream_typing_notifications=True,
+        linkifier_url_template=True,
+        user_list_incomplete=True,
+        include_deactivated_groups=True,
+        archived_channels=True,
+        empty_topic_name=True,
+        simplified_presence_events=True,
+        individual_emoji_changes=True,
+    )
+
+    # When the client triggers a reload (e.g., after an expired event
+    # queue), it navigates to /?state_data=deferred. In that case, we
+    # skip the expensive do_events_register() call and let the client
+    # fetch state via the /json/register API instead. This makes the
+    # HTML response much smaller and avoids partial-transfer failures
+    # on flaky networks (common in Firefox background tabs resuming
+    # from laptop suspend). See #36094.
+    is_client_reload = request.GET.get("state_data") == "deferred" and user_profile is not None
+
+    if user_profile is not None and not is_client_reload:
+        client = RequestNotes.get_notes(request).client
+        assert client is not None
+        state_data = do_events_register(
+            user_profile,
+            realm,
+            client,
+            apply_markdown=True,
+            client_gravatar=True,
+            slim_presence=True,
+            presence_last_update_id_fetched_by_client=-1,
+            presence_history_limit_days=settings.PRESENCE_HISTORY_LIMIT_DAYS_FOR_WEB_APP,
+            client_capabilities=client_capabilities,
+            narrow=narrow,
+            include_streams=False,
+            include_subscribers="partial",
+        )
+        queue_id = state_data["queue_id"]
+        default_language = state_data["user_settings"]["default_language"]
+    elif user_profile is not None:
+        # Client-triggered reload: the client will fetch state_data
+        # via /json/register after the page loads.
+        state_data = None
+        queue_id = None
+        default_language = user_profile.default_language
+    else:
+        # The spectator client will be fetching the /register response
+        # for spectators via the API.
+        state_data = None
+        queue_id = None
+        default_language = realm.default_language
+
+    if user_profile is None:
+        request_language = request.COOKIES.get(settings.LANGUAGE_COOKIE_NAME, default_language)
+        split_url = urlsplit(request.build_absolute_uri())
+        show_try_zulip_modal = (
+            settings.DEVELOPMENT or split_url.hostname == "chat.zulip.org"
+        ) and split_url.query == "show_try_zulip_modal"
+    else:
+        request_language = get_and_set_request_language(
+            request,
+            default_language,
+            translation.get_language_from_path(request.path_info),
+        )
+        show_try_zulip_modal = False
+
+    furthest_read_time = get_furthest_read_time(user_profile)
+    two_fa_enabled = settings.TWO_FACTOR_AUTHENTICATION_ENABLED and user_profile is not None
+
+    # Pass parameters to the client-side JavaScript code.
+    # These end up in a JavaScript Object named 'page_params'.
+    #
+    # Sync this with home_params_schema in base_page_params.ts.
+    page_params: dict[str, object] = dict(
+        page_type="home",
+        ## Server settings.
+        test_suite=settings.TEST_SUITE,
+        insecure_desktop_app=insecure_desktop_app,
+        login_page=settings.HOME_NOT_LOGGED_IN,
+        warn_no_email=settings.WARN_NO_EMAIL,
+        # Only show marketing email settings if on Zulip Cloud
+        corporate_enabled=settings.CORPORATE_ENABLED,
+        ## Misc. extra data.
+        language_list=get_language_list(),
+        furthest_read_time=furthest_read_time,
+        embedded_bots_enabled=settings.EMBEDDED_BOTS_ENABLED,
+        two_fa_enabled=two_fa_enabled,
+        apps_page_url=get_apps_page_url(),
+        promote_sponsoring_zulip=promote_sponsoring_zulip_in_realm(realm),
+        # Adding two_fa_enabled as condition saves us 3 queries when
+        # 2FA is not enabled.
+        two_fa_enabled_user=two_fa_enabled and bool(default_device(user_profile)),
+        is_spectator=user_profile is None,
+        presence_history_limit_days_for_web_app=settings.PRESENCE_HISTORY_LIMIT_DAYS_FOR_WEB_APP,
+        # The client will fetch state_data (and create an event
+        # queue) via /json/register for spectators and reloads.
+        no_event_queue=user_profile is None or is_client_reload,
+        show_try_zulip_modal=show_try_zulip_modal,
+        non_workplace_pricing_eligible=realm_eligible_for_non_workplace_pricing(realm),
+        is_cloud_realm_with_discounted_plan=realm_on_discounted_cloud_plan(realm),
+    )
+
+    page_params["state_data"] = state_data
+
+    if narrow_stream is not None:
+        page_params["narrow_stream"] = narrow_stream.name
+        if narrow_topic_name is not None:
+            page_params["narrow_topic"] = narrow_topic_name
+        page_params["narrow"] = [
+            dict(operator=term.operator, operand=term.operand) for term in narrow
+        ]
+
+    page_params["translation_data"] = get_language_translation_data(request_language)
+
+    # This is used by `admin.ts` to display realm description for non-administrator
+    # logged-in users.
+    page_params["realm_rendered_description"] = get_realm_rendered_description(realm)
+
+    if user_profile is None:
+        page_params["language_cookie_name"] = settings.LANGUAGE_COOKIE_NAME
+
+    return queue_id, page_params

zerver/lib/html_diff.py

@@ -0,0 +1,19 @@
+import lxml.html
+from lxml.html.diff import htmldiff
+
+
+def highlight_html_differences(s1: str, s2: str, msg_id: int | None = None) -> str:
+    retval = htmldiff(s1, s2)
+    fragment = lxml.html.fragment_fromstring(retval, create_parent=True)
+
+    for elem in fragment.cssselect("del"):
+        elem.tag = "span"
+        elem.set("class", "highlight_text_deleted")
+
+    for elem in fragment.cssselect("ins"):
+        elem.tag = "span"
+        elem.set("class", "highlight_text_inserted")
+
+    retval = lxml.html.tostring(fragment, encoding="unicode")
+
+    return retval

zerver/lib/html_to_text.py

@@ -0,0 +1,38 @@
+from collections.abc import Mapping
+
+from bs4 import BeautifulSoup
+from django.utils.html import escape
+
+from zerver.lib.cache import cache_with_key, open_graph_description_cache_key
+
+
+def html_to_text(content: str | bytes, tags: Mapping[str, str] = {"p": " | "}) -> str:
+    bs = BeautifulSoup(content, features="lxml")
+    # Skip any admonition (warning) blocks, since they're
+    # usually something about users needing to be an
+    # organization administrator, and not useful for
+    # describing the page.
+    for tag in bs.find_all("div", class_="admonition"):
+        tag.clear()
+
+    # Skip tabbed-sections, which just contain navigation instructions.
+    for tag in bs.find_all("div", class_="tabbed-section"):
+        tag.clear()
+
+    text = ""
+    for element in bs.find_all(tags.keys()):
+        # Ignore empty elements
+        if not element.text:
+            continue
+        # .text converts it from HTML to text
+        if text:
+            text += tags[element.name]
+        text += element.text
+        if len(text) > 500:
+            break
+    return escape(" ".join(text.split()))
+
+
+@cache_with_key(open_graph_description_cache_key, timeout=3600 * 24)
+def get_content_description(content: bytes, request_url: str) -> str:
+    return html_to_text(content)

zerver/lib/i18n.py

@@ -0,0 +1,109 @@
+# See https://zulip.readthedocs.io/en/latest/translating/internationalization.html
+
+import logging
+import os
+from functools import lru_cache
+from typing import Any
+
+import orjson
+from django.conf import settings
+from django.http import HttpRequest
+from django.utils import translation
+from django.utils.translation.trans_real import parse_accept_lang_header
+
+from zerver.lib.request import RequestNotes
+from zerver.models import Realm
+
+
+@lru_cache(None)
+def get_language_list() -> list[dict[str, Any]]:
+    path = os.path.join(settings.DEPLOY_ROOT, "locale", "language_name_map.json")
+    with open(path, "rb") as reader:
+        languages = orjson.loads(reader.read())
+        return languages["name_map"]
+
+
+def get_language_name(code: str) -> str:
+    for lang in get_language_list():
+        if code in (lang["code"], lang["locale"]):
+            return lang["name"]
+    # Log problem, but still return a name
+    logging.error("Unknown language code '%s'", code)
+    return "Unknown"
+
+
+def get_available_language_codes() -> list[str]:
+    language_list = get_language_list()
+    codes = [
+        language["code"]
+        for language in language_list
+        if language["code"] == "en" or language["percent_translated"] >= 5
+    ]
+    return codes
+
+
+def get_language_translation_data(language: str) -> dict[str, str]:
+    if language == "en":
+        return {}
+    locale = translation.to_locale(language)
+    path = os.path.join(settings.DEPLOY_ROOT, "locale", locale, "translations.json")
+    try:
+        with open(path, "rb") as reader:
+            return orjson.loads(reader.read())
+    except FileNotFoundError:
+        print(f"Translation for {language} not found at {path}")
+        return {}
+
+
+def get_and_set_request_language(
+    request: HttpRequest, user_configured_language: str, testing_url_language: str | None = None
+) -> str:
+    # We pick a language for the user as follows:
+    # * First priority is the language in the URL, for debugging.
+    # * If not in the URL, we use the language from the user's settings.
+    request_language = testing_url_language
+    if request_language is None:
+        request_language = user_configured_language
+    translation.activate(request_language)
+
+    # We also want to save the language to the user's cookies, so that
+    # something reasonable will happen in logged-in portico pages.
+    # We accomplish that by setting a flag on the request which signals
+    # to LocaleMiddleware to set the cookie on the response.
+    RequestNotes.get_notes(request).set_language = translation.get_language()
+
+    return request_language
+
+
+def get_browser_language_code(request: HttpRequest) -> str | None:
+    accept_lang_header = request.headers.get("Accept-Language")
+    if accept_lang_header is None:
+        return None
+
+    available_language_codes = get_available_language_codes()
+    for accept_lang, priority in parse_accept_lang_header(accept_lang_header):
+        if accept_lang == "*":
+            return None
+        if accept_lang in available_language_codes:
+            return accept_lang
+    return None
+
+
+def get_default_language_for_new_user(realm: Realm, *, request: HttpRequest | None) -> str:
+    if request is None:
+        # Users created via the API or LDAP will not have a
+        # browser/request associated with them, and should just use
+        # the realm's default language.
+        return realm.default_language
+
+    browser_language_code = get_browser_language_code(request)
+    if browser_language_code is not None:
+        return browser_language_code
+    return realm.default_language
+
+
+def get_default_language_for_anonymous_user(request: HttpRequest) -> str:
+    browser_language_code = get_browser_language_code(request)
+    if browser_language_code is not None:
+        return browser_language_code
+    return settings.LANGUAGE_CODE

zerver/lib/initial_password.py

@@ -0,0 +1,21 @@
+import base64
+import hashlib
+
+from django.conf import settings
+
+
+def initial_password(email: str) -> str | None:
+    """Given an email address, returns the initial password for that account, as
+    created by populate_db."""
+
+    if settings.INITIAL_PASSWORD_SALT is not None:
+        # We check settings.DEVELOPMENT, not settings.PRODUCTION,
+        # because some tests mock settings.PRODUCTION and then use
+        # self.login, which will call this function.
+        assert settings.DEVELOPMENT, "initial_password_salt should not be set in production."
+        encoded_key = (settings.INITIAL_PASSWORD_SALT + email).encode()
+        digest = hashlib.sha256(encoded_key).digest()
+        return base64.b64encode(digest)[:16].decode()
+    else:
+        # None as a password for a user tells Django to set an unusable password
+        return None

zerver/lib/invites.py

@@ -0,0 +1,50 @@
+from django.db.models import Q
+from django.utils.timezone import now as timezone_now
+
+from confirmation.models import Confirmation
+from zerver.models import MultiuseInvite, PreregistrationUser, Realm, UserProfile
+from zerver.models.prereg_users import filter_to_valid_prereg_users
+from zerver.tornado.django_api import send_event_on_commit
+
+
+def notify_invites_changed(
+    realm: Realm, *, changed_invite_referrer: UserProfile | None = None
+) -> None:
+    event = dict(type="invites_changed")
+    admin_ids = [user.id for user in realm.get_admin_users_and_bots()]
+    recipient_ids = admin_ids
+    if changed_invite_referrer and changed_invite_referrer.id not in recipient_ids:
+        recipient_ids.append(changed_invite_referrer.id)
+    send_event_on_commit(realm, event, recipient_ids)
+
+
+def get_valid_invite_confirmations_generated_by_user(
+    user_profile: UserProfile,
+) -> list[Confirmation]:
+    prereg_user_ids = filter_to_valid_prereg_users(
+        PreregistrationUser.objects.filter(referred_by=user_profile)
+    ).values_list("id", flat=True)
+    confirmations = list(
+        Confirmation.objects.filter(type=Confirmation.INVITATION, object_id__in=prereg_user_ids)
+    )
+
+    multiuse_invite_ids = MultiuseInvite.objects.filter(referred_by=user_profile).values_list(
+        "id", flat=True
+    )
+    confirmations += Confirmation.objects.filter(
+        type=Confirmation.MULTIUSE_INVITE,
+        object_id__in=multiuse_invite_ids,
+    ).filter(Q(expiry_date__gte=timezone_now()) | Q(expiry_date=None))
+
+    return confirmations
+
+
+def revoke_invites_generated_by_user(user_profile: UserProfile) -> None:
+    confirmations_to_revoke = get_valid_invite_confirmations_generated_by_user(user_profile)
+    now = timezone_now()
+    for confirmation in confirmations_to_revoke:
+        confirmation.expiry_date = now
+
+    Confirmation.objects.bulk_update(confirmations_to_revoke, ["expiry_date"])
+    if len(confirmations_to_revoke):
+        notify_invites_changed(realm=user_profile.realm)

zerver/lib/logging_util.py

@@ -0,0 +1,275 @@
+# System documented in https://zulip.readthedocs.io/en/latest/subsystems/logging.html
+import hashlib
+import logging
+import threading
+import traceback
+from collections.abc import Iterable
+from contextlib import suppress
+from datetime import datetime, timedelta, timezone
+from logging import Logger
+
+import orjson
+from django.conf import settings
+from django.core.cache import cache
+from django.http import HttpRequest
+from django.utils.timezone import now as timezone_now
+from typing_extensions import override
+
+
+class _RateLimitFilter:
+    """This class is designed to rate-limit Django error reporting
+    notifications so that it won't send thousands of emails if the
+    database or cache is completely down.  It uses a remote shared
+    cache (shared by all Django processes) for its default behavior
+    (so that the deduplication is global, not per-process), and a
+    local in-process cache for when it can't access the remote cache.
+
+    This is critical code because it is called every time
+    `logging.error` or `logging.exception` (or an exception) happens
+    in the codebase.
+
+    Adapted from https://djangosnippets.org/snippets/2242/.
+
+    """
+
+    last_error = datetime.min.replace(tzinfo=timezone.utc)
+    # This thread-local variable is used to detect recursive
+    # exceptions during exception handling (primarily intended for
+    # when accessing the shared cache throws an exception).
+    handling_exception = threading.local()
+    should_reset_handling_exception = False
+
+    def can_use_remote_cache(self) -> tuple[bool, bool]:
+        if getattr(self.handling_exception, "value", False):
+            # If we're processing an exception that occurred
+            # while handling an exception, this almost
+            # certainly was because interacting with the
+            # remote cache is failing (e.g. because the cache
+            # is down).  Fall back to tracking duplicate
+            # exceptions in memory without the remote shared cache.
+            return False, False
+
+        # Now we test if the remote cache is accessible.
+        #
+        # This code path can only be reached if we are not potentially
+        # handling a recursive exception, so here we set
+        # self.handling_exception (in case the cache access we're
+        # about to do triggers a `logging.error` or exception that
+        # might recurse into this filter class), and actually record
+        # that this is the main exception handler thread.
+        try:
+            self.handling_exception.value = True
+            cache.set("RLF_TEST_KEY", 1, 1)
+            return cache.get("RLF_TEST_KEY") == 1, True
+        except Exception:
+            return False, True
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        # When the original filter() call finishes executing, it's
+        # going to change handling_exception.value to False. The
+        # local variable below tracks whether the *current*,
+        # potentially recursive, filter() call is allowed to touch
+        # that value (only the original will find this to be True
+        # at the end of its execution)
+        should_reset_handling_exception = False
+        try:
+            # Track duplicate errors
+            duplicate = False
+            rate = getattr(settings, f"{type(self).__name__.upper()}_LIMIT", 600)  # seconds
+
+            if rate > 0:
+                (use_cache, should_reset_handling_exception) = self.can_use_remote_cache()
+                if use_cache:
+                    if record.exc_info is not None and isinstance(record.exc_info, Iterable):
+                        tb = "\n".join(traceback.format_exception(*record.exc_info))
+                    else:
+                        tb = str(record)
+                    key = type(self).__name__.upper() + hashlib.sha1(tb.encode()).hexdigest()
+                    duplicate = cache.get(key) == 1
+                    if not duplicate:
+                        cache.set(key, 1, rate)
+                else:
+                    min_date = timezone_now() - timedelta(seconds=rate)
+                    duplicate = self.last_error >= min_date
+                    if not duplicate:
+                        self.last_error = timezone_now()
+
+            return not duplicate
+        finally:
+            if should_reset_handling_exception:
+                self.handling_exception.value = False
+
+
+class ZulipLimiter(_RateLimitFilter):
+    pass
+
+
+class EmailLimiter(_RateLimitFilter):
+    pass
+
+
+class ReturnTrue(logging.Filter):
+    @override
+    def filter(self, record: logging.LogRecord) -> bool:
+        return True
+
+
+class RequireReallyDeployed(logging.Filter):
+    @override
+    def filter(self, record: logging.LogRecord) -> bool:
+        return settings.PRODUCTION
+
+
+def find_log_caller_module(record: logging.LogRecord) -> str | None:
+    """Find the module name corresponding to where this record was logged.
+
+    Sadly `record.module` is just the innermost component of the full
+    module name, so we have to go reconstruct this ourselves.
+    """
+    # Repeat a search similar to that in logging.Logger.findCaller.
+    # The logging call should still be on the stack somewhere; search until
+    # we find something in the same source file, and that should give the
+    # right module name.
+    f = logging.currentframe()
+    while True:
+        if f.f_code.co_filename == record.pathname:
+            return f.f_globals.get("__name__")
+        if f.f_back is None:
+            return None
+        f = f.f_back
+
+
+logger_nicknames = {
+    "root": "",  # This one is more like undoing a nickname.
+    "zulip.requests": "zr",  # Super common.
+}
+
+
+def find_log_origin(record: logging.LogRecord) -> str:
+    logger_name = logger_nicknames.get(record.name, record.name)
+
+    if settings.LOGGING_SHOW_MODULE:
+        module_name = find_log_caller_module(record)
+        if module_name in (logger_name, record.name):
+            # Abbreviate a bit.
+            pass
+        else:
+            logger_name = "{}/{}".format(logger_name, module_name or "?")
+
+    if settings.RUNNING_INSIDE_TORNADO:
+        # In multi-sharded Tornado, it's often valuable to have which shard is
+        # responsible for the request in the logs.
+        from zerver.tornado.ioloop_logging import logging_data
+
+        shard = logging_data.get("port", "unknown")
+        logger_name = f"{logger_name}:{shard}"
+
+    return logger_name
+
+
+log_level_abbrevs = {
+    "DEBUG": "DEBG",
+    "INFO": "INFO",
+    "WARNING": "WARN",
+    "ERROR": "ERR",
+    "CRITICAL": "CRIT",
+}
+
+
+def abbrev_log_levelname(levelname: str) -> str:
+    # It's unlikely someone will set a custom log level with a custom name,
+    # but it's an option, so we shouldn't crash if someone does.
+    return log_level_abbrevs.get(levelname, levelname[:4])
+
+
+class ZulipFormatter(logging.Formatter):
+    # Used in the base implementation.  Default uses `,`.
+    default_msec_format = "%s.%03d"
+
+    def __init__(self) -> None:
+        super().__init__(fmt=self._compute_fmt())
+
+    def _compute_fmt(self) -> str:
+        pieces = ["%(asctime)s", "%(zulip_level_abbrev)-4s"]
+        if settings.LOGGING_SHOW_PID:
+            pieces.append("pid:%(process)d")
+        pieces.extend(["[%(zulip_origin)s]", "%(message)s"])
+        return " ".join(pieces)
+
+    @override
+    def format(self, record: logging.LogRecord) -> str:
+        if not hasattr(record, "zulip_decorated"):
+            record.zulip_level_abbrev = abbrev_log_levelname(record.levelname)
+            record.zulip_origin = find_log_origin(record)
+            record.zulip_decorated = True
+        return super().format(record)
+
+
+class ZulipWebhookFormatter(ZulipFormatter):
+    @override
+    def _compute_fmt(self) -> str:
+        basic = super()._compute_fmt()
+        multiline = [
+            basic,
+            "user: %(user)s",
+            "client: %(client)s",
+            "url: %(url)s",
+            "content_type: %(content_type)s",
+            "custom_headers:",
+            "%(custom_headers)s",
+            "payload:",
+            "%(payload)s",
+        ]
+        return "\n".join(multiline)
+
+    @override
+    def format(self, record: logging.LogRecord) -> str:
+        request: HttpRequest | None = getattr(record, "request", None)
+        if request is None:
+            record.user = None
+            record.client = None
+            record.url = None
+            record.content_type = None
+            record.custom_headers = None
+            record.payload = None
+            return super().format(record)
+
+        if request.content_type == "application/json":
+            payload: str | bytes = request.body
+        else:
+            payload = request.POST["payload"]
+
+        with suppress(orjson.JSONDecodeError):
+            payload = orjson.dumps(orjson.loads(payload), option=orjson.OPT_INDENT_2).decode()
+
+        header_text = "".join(
+            f"{header}: {value}\n"
+            for header, value in request.headers.items()
+            if header.lower().startswith("x-")
+        )
+
+        from zerver.lib.request import RequestNotes
+
+        client = RequestNotes.get_notes(request).client
+        assert client is not None
+
+        assert request.user.is_authenticated
+        record.user = f"{request.user.delivery_email} ({request.user.realm.string_id})"
+        record.client = client.name
+        record.url = request.META.get("PATH_INFO", None)
+        record.content_type = request.content_type
+        record.custom_headers = header_text or None
+        record.payload = payload
+        return super().format(record)
+
+
+def log_to_file(
+    logger: Logger,
+    filename: str,
+    log_format: str = "%(asctime)s %(levelname)-8s %(message)s",
+) -> None:
+    """Note: `filename` should be declared in zproject/computed_settings.py with zulip_path."""
+    formatter = logging.Formatter(log_format)
+    handler = logging.FileHandler(filename)
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)

zerver/lib/management.py

@@ -0,0 +1,320 @@
+# Library code for use in management commands
+import logging
+import os
+import sys
+import time
+from argparse import ArgumentParser, BooleanOptionalAction, RawTextHelpFormatter, _ActionsContainer
+from dataclasses import dataclass
+from functools import reduce, wraps
+from typing import Any, Protocol
+
+from django.conf import settings
+from django.core import validators
+from django.core.exceptions import MultipleObjectsReturned, ValidationError
+from django.core.management.base import BaseCommand, CommandError, CommandParser
+from django.db.models import Q, QuerySet
+from typing_extensions import override
+
+from scripts.lib.zulip_tools import LOCK_DIR as DEPLOYMENT_LOCK_DIR
+from zerver.lib.context_managers import lockfile_nonblocking
+from zerver.lib.initial_password import initial_password
+from zerver.models import Client, Realm, UserProfile
+from zerver.models.clients import get_client
+
+
+def is_integer_string(val: str) -> bool:
+    try:
+        int(val)
+        return True
+    except ValueError:
+        return False
+
+
+class HandleMethod(Protocol):
+    def __call__(self, *args: Any, **kwargs: Any) -> None: ...
+
+
+def lockfile_path(handle_func: HandleMethod) -> str:
+    os.makedirs(settings.LOCKFILE_DIRECTORY, exist_ok=True)
+    # Trim out just the last part of the module name, which is the
+    # command name, to use as the lockfile name;
+    # `zerver.management.commands.send_zulip_update_announcements`
+    # becomes `/srv/zulip-locks/send_zulip_update_announcements.lock`
+    lockfile_name = handle_func.__module__.split(".")[-1]
+    return settings.LOCKFILE_DIRECTORY + "/" + lockfile_name + ".lock"
+
+
+def abort_unless_locked(handle_func: HandleMethod) -> HandleMethod:
+    """Take a per-command lock, exiting with an error if it is already
+    held.  For cron jobs wrapped in Sentry's `monitors run`, that error
+    surfaces as a failed check-in; use this only for commands whose
+    runtime is reliably shorter than their cron interval, where an
+    overlapping run signals something is wrong and is worth alerting on."""
+
+    @wraps(handle_func)
+    def our_handle(self: BaseCommand, *args: Any, **kwargs: Any) -> None:
+        path = lockfile_path(handle_func)
+        with lockfile_nonblocking(path) as lock_acquired:
+            if not lock_acquired:  # nocoverage
+                self.stdout.write(self.style.ERROR(f"Lock {path} is unavailable; exiting."))
+                sys.exit(1)
+            handle_func(self, *args, **kwargs)
+
+    return our_handle
+
+
+def skip_unless_locked(handle_func: HandleMethod) -> HandleMethod:
+    """Like abort_unless_locked, but for commands where an overlapping
+    run is an expected, benign condition -- for instance a network-bound
+    job polled every minute, where a still-running previous invocation
+    just means this one should do nothing.  If the lock is already held,
+    exit successfully and silently -- in particular emitting no output,
+    since under cron any output is mailed to the administrator."""
+
+    @wraps(handle_func)
+    def our_handle(self: BaseCommand, *args: Any, **kwargs: Any) -> None:
+        path = lockfile_path(handle_func)
+        with lockfile_nonblocking(path) as lock_acquired:
+            if not lock_acquired:  # nocoverage
+                return
+            handle_func(self, *args, **kwargs)
+
+    return our_handle
+
+
+def abort_cron_during_deploy(handle_func: HandleMethod) -> HandleMethod:
+    @wraps(handle_func)
+    def our_handle(self: BaseCommand, *args: Any, **kwargs: Any) -> None:
+        # For safety, we only trust the lock directory if it was
+        # created within the last hour -- otherwise, a spurious
+        # deploy lock could linger and block all hourly crons.
+        if (
+            os.environ.get("RUNNING_UNDER_CRON")
+            and os.path.exists(DEPLOYMENT_LOCK_DIR)
+            and time.time() - os.path.getctime(DEPLOYMENT_LOCK_DIR) < 3600
+        ):  # nocoverage
+            self.stdout.write(
+                self.style.ERROR("Deployment in process; aborting cron management command.")
+            )
+            sys.exit(1)
+        handle_func(self, *args, **kwargs)
+
+    return our_handle
+
+
+@dataclass
+class CreateUserParameters:
+    email: str
+    full_name: str
+    password: str | None
+
+
+class ZulipBaseCommand(BaseCommand):
+    # Fix support for multi-line usage
+    @override
+    def create_parser(self, prog_name: str, subcommand: str, **kwargs: Any) -> CommandParser:
+        parser = super().create_parser(prog_name, subcommand, **kwargs)
+        parser.add_argument(
+            "--automated",
+            help="This command is run non-interactively (enables Sentry, etc)",
+            action=BooleanOptionalAction,
+            default=not sys.stdin.isatty(),
+        )
+        parser.formatter_class = RawTextHelpFormatter
+        return parser
+
+    @override
+    def execute(self, *args: Any, **options: Any) -> None:
+        if settings.SENTRY_DSN and not options["automated"]:  # nocoverage
+            import sentry_sdk
+
+            # This deactivates Sentry
+            sentry_sdk.init()
+        super().execute(*args, **options)
+
+    def add_realm_args(
+        self, parser: _ActionsContainer, *, required: bool = False, help: str | None = None
+    ) -> None:
+        if help is None:
+            help = """The numeric or string ID (subdomain) of the Zulip organization to modify.
+You can use the command list_realms to find ID of the realms in this server."""
+
+        parser.add_argument("-r", "--realm", dest="realm_id", required=required, help=help)
+
+    def add_create_user_args(self, parser: ArgumentParser) -> None:
+        parser.add_argument(
+            "email",
+            metavar="<email>",
+            nargs="?",
+            help="Email address for the new user",
+        )
+        parser.add_argument(
+            "full_name",
+            metavar="<full name>",
+            nargs="?",
+            help="Full name for the new user",
+        )
+        parser.add_argument(
+            "--password",
+            help="""\
+Password for the new user. Recommended only in a development environment.
+
+Sending passwords via command-line arguments is insecure,
+since it can be snooped by any process running on the
+server via `ps -ef` or reading bash history. Prefer
+--password-file.""",
+        )
+        parser.add_argument("--password-file", help="File containing a password for the new user.")
+
+    def add_user_list_args(
+        self,
+        parser: _ActionsContainer,
+        help: str = "A comma-separated list of email addresses.",
+        all_users_help: str = "All users in realm.",
+    ) -> None:
+        parser.add_argument("-u", "--users", help=help)
+
+        parser.add_argument("-a", "--all-users", action="store_true", help=all_users_help)
+
+    def get_realm(self, options: dict[str, Any]) -> Realm | None:
+        val = options["realm_id"]
+        if val is None:
+            return None
+
+        # If they specified a realm argument, we need to ensure the
+        # realm exists.  We allow two formats: the numeric ID for the
+        # realm and the string ID of the realm.
+        try:
+            if is_integer_string(val):
+                return Realm.objects.get(id=val)
+            return Realm.objects.get(string_id=val)
+        except Realm.DoesNotExist:
+            raise CommandError(
+                "There is no realm with id '{}'. Aborting.".format(options["realm_id"])
+            )
+
+    def get_users(
+        self,
+        options: dict[str, Any],
+        realm: Realm | None,
+        is_bot: bool | None = None,
+        include_deactivated: bool = False,
+    ) -> QuerySet[UserProfile]:
+        if "all_users" in options:
+            all_users = options["all_users"]
+
+            if not options["users"] and not all_users:
+                raise CommandError("You have to pass either -u/--users or -a/--all-users.")
+
+            if options["users"] and all_users:
+                raise CommandError("You can't use both -u/--users and -a/--all-users.")
+
+            if all_users and realm is None:
+                raise CommandError("The --all-users option requires a realm; please pass --realm.")
+
+            if all_users:
+                user_profiles = UserProfile.objects.filter(realm=realm)
+                if not include_deactivated:
+                    user_profiles = user_profiles.filter(is_active=True)
+                if is_bot is not None:
+                    return user_profiles.filter(is_bot=is_bot)
+                return user_profiles
+
+        if options["users"] is None:
+            return UserProfile.objects.none()
+        emails = {email.strip() for email in options["users"].split(",")}
+        # This is inefficient, but we fetch (and throw away) the
+        # get_user of each email, so that we verify that the email
+        # address/realm returned only one result; it may return more
+        # if realm is not specified but email address was.
+        for email in emails:
+            self.get_user(email, realm)
+
+        user_profiles = UserProfile.objects.all().select_related("realm")
+        if realm is not None:
+            user_profiles = user_profiles.filter(realm=realm)
+        email_matches = [Q(delivery_email__iexact=e) for e in emails]
+        user_profiles = user_profiles.filter(reduce(lambda a, b: a | b, email_matches)).order_by(
+            "id"
+        )
+
+        # Return the single query, for ease of composing.
+        return user_profiles
+
+    def get_user(self, email: str, realm: Realm | None) -> UserProfile:
+        # If a realm is specified, try to find the user there, and
+        # throw an error if they don't exist.
+        if realm is not None:
+            try:
+                return UserProfile.objects.select_related("realm").get(
+                    delivery_email__iexact=email.strip(), realm=realm
+                )
+            except UserProfile.DoesNotExist:
+                raise CommandError(
+                    f"The realm '{realm}' does not contain a user with email '{email}'"
+                )
+
+        # Realm is None in the remaining code path.  Here, we
+        # optimistically try to see if there is exactly one user with
+        # that email; if so, we'll return it.
+        try:
+            return UserProfile.objects.select_related("realm").get(
+                delivery_email__iexact=email.strip()
+            )
+        except MultipleObjectsReturned:
+            raise CommandError(
+                "This Zulip server contains multiple users with that email (in different realms);"
+                " please pass `--realm` to specify which one to modify."
+            )
+        except UserProfile.DoesNotExist:
+            raise CommandError(f"This Zulip server does not contain a user with email '{email}'")
+
+    def get_client(self) -> Client:
+        """Returns a Zulip Client object to be used for things done in management commands"""
+        return get_client("ZulipServer")
+
+    def get_create_user_params(self, options: dict[str, Any]) -> CreateUserParameters:  # nocoverage
+        """
+        Parses parameters for user creation defined in add_create_user_args.
+        """
+        if options["email"] is None:
+            email = input("Email: ")
+        else:
+            email = options["email"]
+
+        try:
+            validators.validate_email(email)
+        except ValidationError:
+            raise CommandError("Invalid email address.")
+
+        if options["full_name"] is None:
+            full_name = input("Full name: ")
+        else:
+            full_name = options["full_name"]
+
+        if options["password_file"] is not None:
+            with open(options["password_file"]) as f:
+                password: str | None = f.read().strip()
+        elif options["password"] is not None:
+            logging.warning(
+                "Passing password on the command line is insecure; prefer --password-file."
+            )
+            password = options["password"]
+        else:
+            # initial_password will return a random password that
+            # is a salted hash of the email address in a
+            # development environment, and None in a production
+            # environment.
+            user_initial_password = initial_password(email)
+            if user_initial_password is None:
+                logging.info("User will be created with a disabled password.")
+            else:
+                assert settings.DEVELOPMENT
+                logging.info("Password will be available via `./manage.py print_initial_password`.")
+            password = user_initial_password
+
+        return CreateUserParameters(
+            email=email,
+            full_name=full_name,
+            password=password,
+        )

zerver/lib/markdown/api_arguments_table_generator.py

@@ -0,0 +1,316 @@
+import json
+import re
+from collections.abc import Mapping, Sequence
+from typing import Any
+
+import markdown
+from django.utils.html import escape as escape_html
+from markdown.extensions import Extension
+from markdown.preprocessors import Preprocessor
+from typing_extensions import override
+
+from zerver.lib.markdown.priorities import PREPROCESSOR_PRIORITIES
+from zerver.openapi.openapi import (
+    Parameter,
+    check_deprecated_consistency,
+    get_openapi_parameters,
+    get_parameters_description,
+)
+
+REGEXP = re.compile(r"\{generate_api_arguments_table\|\s*(.+?)\s*\|\s*(.+)\s*\}")
+
+API_PARAMETER_TEMPLATE = """
+<div class="api-argument" id="parameter-{argument}">
+    <p class="api-argument-name"><strong>{argument}</strong> <span class="api-field-type">{type}</span> {required} {deprecated} <a href="#parameter-{argument}" class="api-argument-hover-link"><i class="fa fa-chain"></i></a></p>
+    <div class="api-example">
+        <span class="api-argument-example-label">Example</span>: <code>{example}</code>
+    </div>
+    <div class="api-description">{description}{object_details}</div>
+    <hr>
+</div>
+""".strip()
+
+OBJECT_DETAILS_TEMPLATE = """
+<p><strong>{argument}</strong> object details:</p>
+<ul>
+{values}
+</ul>
+""".strip()
+
+ONEOF_OBJECT_DETAILS_TEMPLATE = """
+<p>An object with the following fields:</p>
+<ul>
+{values}
+</ul>
+""".strip()
+
+OBJECT_LIST_ITEM_TEMPLATE = """
+<li>
+<code>{value}</code>: <span class=api-field-type>{data_type}</span> {required} {description}{object_details}
+</li>
+""".strip()
+
+OBJECT_DESCRIPTION_TEMPLATE = """
+{description}
+<p>{additional_information}</p>
+""".strip()
+
+OBJECT_CODE_TEMPLATE = "<code>{value}</code>".strip()
+
+ONEOF_DETAILS_TEMPLATE = """
+<p>This parameter must be one of the following:</p>
+<ol>
+{values}
+</ol>
+""".strip()
+
+ONEOF_LIST_ITEM_TEMPLATE = """
+<li>{item}</li>
+""".strip()
+
+
+class MarkdownArgumentsTableGenerator(Extension):
+    @override
+    def extendMarkdown(self, md: markdown.Markdown) -> None:
+        md.preprocessors.register(
+            APIArgumentsTablePreprocessor(md, self.getConfigs()),
+            "generate_api_arguments",
+            PREPROCESSOR_PRIORITIES["generate_api_arguments"],
+        )
+
+
+class APIArgumentsTablePreprocessor(Preprocessor):
+    def __init__(self, md: markdown.Markdown, config: Mapping[str, Any]) -> None:
+        super().__init__(md)
+
+    @override
+    def run(self, lines: list[str]) -> list[str]:
+        done = False
+        while not done:
+            for line in lines:
+                loc = lines.index(line)
+                match = REGEXP.search(line)
+
+                if not match:
+                    continue
+
+                doc_name = match.group(2)
+                endpoint, method = doc_name.rsplit(":", 1)
+                parameters = get_openapi_parameters(endpoint, method)
+                if parameters:
+                    text = self.render_parameters(parameters)
+                # We want to show this message only if the parameters
+                # description doesn't say anything else.
+                elif get_parameters_description(endpoint, method) == "":
+                    text = ["This endpoint does not accept any parameters."]
+                else:
+                    text = []
+                # The line that contains the directive to include the macro
+                # may be preceded or followed by text or tags, in that case
+                # we need to make sure that any preceding or following text
+                # stays the same.
+                line_split = REGEXP.split(line, maxsplit=0)
+                preceding = line_split[0]
+                following = line_split[-1]
+                text = [preceding, *text, following]
+                lines = lines[:loc] + text + lines[loc + 1 :]
+                break
+            else:
+                done = True
+        return lines
+
+    def render_oneof_block(self, object_schema: dict[str, Any], name: str) -> str:
+        md_engine = markdown.Markdown(extensions=[])
+        content = ""
+        for element in object_schema["oneOf"]:
+            if "items" in element and "properties" in element["items"]:
+                content += ONEOF_LIST_ITEM_TEMPLATE.format(
+                    item=self.render_object_details(element["items"], str(name), True)
+                )
+            elif "properties" in element:
+                content += ONEOF_LIST_ITEM_TEMPLATE.format(
+                    item=self.render_object_details(element, str(name), True)
+                )
+            elif "description" in element:
+                content += ONEOF_LIST_ITEM_TEMPLATE.format(
+                    item=md_engine.convert(element["description"])
+                )
+        return ONEOF_DETAILS_TEMPLATE.format(values=content)
+
+    def render_parameters(self, parameters: Sequence[Parameter]) -> list[str]:
+        lines = []
+
+        md_engine = markdown.Markdown(extensions=[])
+        parameters = sorted(parameters, key=lambda parameter: parameter.deprecated)
+        for parameter in parameters:
+            name = parameter.name
+            description = parameter.description
+            enums = parameter.value_schema.get("enum")
+            if enums is not None:
+                formatted_enums = [
+                    OBJECT_CODE_TEMPLATE.format(value=json.dumps(enum)) for enum in enums
+                ]
+                description += "\nMust be one of: {}. ".format(", ".join(formatted_enums))
+
+            default = parameter.value_schema.get("default")
+            if default is not None:
+                description += f"\nDefaults to `{json.dumps(default)}`."
+            data_type = generate_data_type(parameter.value_schema)
+
+            # TODO: OpenAPI allows indicating where the argument goes
+            # (path, querystring, form data...).  We should document this detail.
+
+            # We use this style without explicit JSON encoding for
+            # integers, strings, and booleans.
+            # * For booleans, JSON encoding correctly corrects for Python's
+            #   str(True)="True" not matching the encoding of "true".
+            # * For strings, doing so nicely results in strings being quoted
+            #   in the documentation, improving readability.
+            # * For integers, it is a noop, since json.dumps(3) == str(3) == "3".
+            example = json.dumps(parameter.example)
+
+            required_string: str = "required"
+            if parameter.kind == "path":
+                # Any path variable is required
+                assert parameter.required
+                required_string = "required in path"
+
+            if parameter.required:
+                required_block = f'<span class="api-argument-required">{required_string}</span>'
+            else:
+                required_block = '<span class="api-argument-optional">optional</span>'
+
+            check_deprecated_consistency(parameter.deprecated, description)
+            if parameter.deprecated:
+                deprecated_block = '<span class="api-argument-deprecated">Deprecated</span>'
+            else:
+                deprecated_block = ""
+
+            object_block = ""
+            # TODO: There are some endpoint parameters with object properties
+            # that are not defined in `zerver/openapi/zulip.yaml`
+            if "object" in data_type:
+                object_schema = parameter.value_schema
+
+                if "items" in object_schema and "properties" in object_schema["items"]:
+                    object_block = self.render_object_details(object_schema["items"], str(name))
+                elif "properties" in object_schema:
+                    object_block = self.render_object_details(object_schema, str(name))
+                elif "oneOf" in object_schema:
+                    object_block = self.render_oneof_block(object_schema, str(name))
+
+            lines.append(
+                API_PARAMETER_TEMPLATE.format(
+                    argument=name,
+                    example=escape_html(example),
+                    required=required_block,
+                    deprecated=deprecated_block,
+                    description=md_engine.convert(description),
+                    type=data_type,
+                    object_details=object_block,
+                )
+            )
+
+        return lines
+
+    def render_object_details(
+        self, schema: Mapping[str, Any], name: str, oneof: bool = False
+    ) -> str:
+        md_engine = markdown.Markdown(extensions=[])
+        li_elements = []
+
+        object_values = schema.get("properties", {})
+        for value in object_values:
+            if object_values[value].get("readOnly", False):
+                # readOnly object properties are included in responses
+                # but not in requests.
+                continue
+
+            description = ""
+            if "description" in object_values[value]:
+                description = object_values[value]["description"]
+
+            # check for default, enum, required or example in documentation
+            additions: list[str] = []
+
+            default = object_values.get(value, {}).get("default")
+            if default is not None:
+                formatted_default = OBJECT_CODE_TEMPLATE.format(value=json.dumps(default))
+                additions += f"Defaults to {formatted_default}. "
+
+            enums = object_values.get(value, {}).get("enum")
+            if enums is not None:
+                formatted_enums = [
+                    OBJECT_CODE_TEMPLATE.format(value=json.dumps(enum)) for enum in enums
+                ]
+                additions += "Must be one of: {}. ".format(", ".join(formatted_enums))
+
+            if "example" in object_values[value]:
+                example = json.dumps(object_values[value]["example"])
+                formatted_example = OBJECT_CODE_TEMPLATE.format(value=escape_html(example))
+                additions += (
+                    f'<span class="api-argument-example-label">Example</span>: {formatted_example}'
+                )
+
+            if len(additions) > 0:
+                additional_information = "".join(additions).strip()
+                description_final = OBJECT_DESCRIPTION_TEMPLATE.format(
+                    description=md_engine.convert(description),
+                    additional_information=additional_information,
+                )
+            else:
+                description_final = md_engine.convert(description)
+
+            required_block = ""
+            if "required" in schema:
+                if value in schema["required"]:
+                    required_block = '<span class="api-argument-required">required</span>'
+                else:
+                    required_block = '<span class="api-argument-optional">optional</span>'
+
+            data_type = generate_data_type(object_values[value])
+
+            details = ""
+            if "object" in data_type and "properties" in object_values[value]:
+                details += self.render_object_details(object_values[value], str(value))
+            elif "items" in object_values[value] and "properties" in object_values[value]["items"]:
+                details += self.render_object_details(object_values[value]["items"], str(value))
+            elif "oneOf" in object_values[value]:
+                details += self.render_oneof_block(object_values[value], str(value))
+
+            li = OBJECT_LIST_ITEM_TEMPLATE.format(
+                value=value,
+                data_type=data_type,
+                required=required_block,
+                description=description_final,
+                object_details=details,
+            )
+
+            li_elements.append(li)
+        if oneof:
+            object_details = ONEOF_OBJECT_DETAILS_TEMPLATE.format(
+                values="\n".join(li_elements),
+            )
+        else:
+            object_details = OBJECT_DETAILS_TEMPLATE.format(
+                argument=name,
+                values="\n".join(li_elements),
+            )
+        return object_details
+
+
+def makeExtension(*args: Any, **kwargs: str) -> MarkdownArgumentsTableGenerator:
+    return MarkdownArgumentsTableGenerator(*args, **kwargs)
+
+
+def generate_data_type(schema: Mapping[str, Any]) -> str:
+    data_type = ""
+    if "oneOf" in schema:
+        data_type = " | ".join(generate_data_type(item) for item in schema["oneOf"])
+    elif "items" in schema:
+        data_type = "(" + generate_data_type(schema["items"]) + ")[]"
+    else:
+        data_type = schema["type"]
+        if schema.get("nullable", False):
+            data_type += " | null"
+    return data_type

zerver/lib/markdown/api_return_values_table_generator.py

@@ -0,0 +1,323 @@
+import copy
+import json
+import re
+from collections import OrderedDict
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Any
+
+import markdown
+from markdown.extensions import Extension
+from markdown.preprocessors import Preprocessor
+from typing_extensions import override
+
+from zerver.lib.markdown.priorities import PREPROCESSOR_PRIORITIES
+from zerver.openapi.openapi import check_deprecated_consistency, get_openapi_return_values
+
+from .api_arguments_table_generator import generate_data_type
+
+REGEXP = re.compile(r"\{generate_return_values_table\|\s*(.+?)\s*\|\s*(.+)\s*\}")
+
+EVENT_HEADER_TEMPLATE = """
+<div class="api-event-header">
+    <h3 id="{id}"><span class="api-event-name">{event}</span></h3>
+</div>
+""".strip()
+
+OP_TEMPLATE = '<span class="api-event-op">op: {op_type}</span>'
+
+EVENTS_TABLE_TEMPLATE = """
+<div class="api-events-table">
+{events_list}
+</div>
+<hr>
+""".strip()
+
+TABLE_OPS_TEMPLATE = """
+<div class="api-event-type">{event_name}:</div>
+<div class="api-event-ops">
+{ops}
+</div>
+""".strip()
+
+TABLE_LINK_TEMPLATE = """
+<div class="api-event-link">
+    <a href="#{url}">{link_name}</a>
+</div>
+""".strip()
+
+
+@dataclass
+class EventData:
+    type: str
+    description: str
+    properties: dict[str, Any]
+    example: str
+    op_type: str | None = None
+
+
+class MarkdownReturnValuesTableGenerator(Extension):
+    @override
+    def extendMarkdown(self, md: markdown.Markdown) -> None:
+        md.preprocessors.register(
+            APIReturnValuesTablePreprocessor(md, self.getConfigs()),
+            "generate_return_values",
+            PREPROCESSOR_PRIORITIES["generate_return_values"],
+        )
+
+
+class APIReturnValuesTablePreprocessor(Preprocessor):
+    def __init__(self, md: markdown.Markdown, config: Mapping[str, Any]) -> None:
+        super().__init__(md)
+
+    @override
+    def run(self, lines: list[str]) -> list[str]:
+        done = False
+        while not done:
+            for line in lines:
+                loc = lines.index(line)
+                match = REGEXP.search(line)
+
+                if not match:
+                    continue
+
+                doc_name = match.group(2)
+                endpoint, method = doc_name.rsplit(":", 1)
+                return_values = get_openapi_return_values(endpoint, method)
+                if doc_name == "/events:get":
+                    return_values = copy.deepcopy(return_values)
+                    events = return_values["events"].pop("items", None)
+                    text = self.render_table(return_values, 0)
+                    # Another heading for the events documentation
+                    text.append("\n\n## Events by `type`\n\n")
+                    text += self.render_events(events)
+                else:
+                    text = self.render_table(return_values, 0)
+                if len(text) > 0:
+                    text = ["#### Return values", *text]
+                line_split = REGEXP.split(line, maxsplit=0)
+                preceding = line_split[0]
+                following = line_split[-1]
+                text = [preceding, *text, following]
+                lines = lines[:loc] + text + lines[loc + 1 :]
+                break
+            else:
+                done = True
+        return lines
+
+    def render_desc(
+        self, description: str, spacing: int, data_type: str, return_value: str | None = None
+    ) -> str:
+        description = description.replace("\n", "\n" + ((spacing + 4) * " "))
+        if return_value is None:
+            # HACK: It's not clear how to use OpenAPI data to identify
+            # the `key` fields for objects where e.g. the keys are
+            # user/stream IDs being mapped to data associated with
+            # those IDs.  We hackily describe those fields by
+            # requiring that the descriptions be written as `key_name:
+            # key_description` and parsing for that pattern; we need
+            # to be careful to skip cases where we'd have `Note: ...`
+            # on a later line.
+            #
+            # More correctly, we should be doing something that looks at the types;
+            # print statements and test_api_doc_endpoint is useful for testing.
+            arr = description.split(": ", 1)
+            if len(arr) == 1 or "\n" in arr[0]:
+                return (spacing * " ") + "* " + description
+            (key_name, key_description) = arr
+            return (
+                (spacing * " ")
+                + "* "
+                + key_name
+                + ": "
+                + '<span class="api-field-type">'
+                + data_type
+                + "</span>\n\n"
+                + (spacing + 4) * " "
+                + key_description
+            )
+        return (
+            (spacing * " ")
+            + "* `"
+            + return_value
+            + "`: "
+            + '<span class="api-field-type">'
+            + data_type
+            + "</span>\n\n"
+            + (spacing + 4) * " "
+            + description
+        )
+
+    def render_oneof_block(self, object_schema: dict[str, Any], spacing: int) -> list[str]:
+        ans = []
+        block_spacing = spacing
+        for element in object_schema["oneOf"]:
+            spacing = block_spacing
+            if "description" not in element:
+                # If the description is not present, we still need to render the rest
+                # of the documentation of the element shifted towards left of the page.
+                spacing -= 4
+            else:
+                # Add the specialized description of the oneOf element.
+                data_type = generate_data_type(element)
+                ans.append(self.render_desc(element["description"], spacing, data_type))
+            # If the oneOf element is an object schema then render the documentation
+            # of its keys.
+            if "properties" in element:
+                ans += self.render_table(element["properties"], spacing + 4)
+
+            if "items" in element:
+                if "properties" in element["items"]:
+                    ans += self.render_table(element["items"]["properties"], spacing + 4)
+                elif "oneOf" in element["items"]:  #  nocoverage
+                    # This block is for completeness.
+                    ans += self.render_oneof_block(element["items"], spacing + 4)
+
+            if element.get("additionalProperties", False):
+                additional_properties = element["additionalProperties"]
+                if "description" in additional_properties:
+                    data_type = generate_data_type(additional_properties)
+                    ans.append(
+                        self.render_desc(
+                            additional_properties["description"], spacing + 4, data_type
+                        )
+                    )
+                if "properties" in additional_properties:
+                    ans += self.render_table(
+                        additional_properties["properties"],
+                        spacing + 8,
+                    )
+        return ans
+
+    def render_table(self, return_values: dict[str, Any], spacing: int) -> list[str]:
+        IGNORE = ["result", "msg", "ignored_parameters_unsupported"]
+        ans = []
+        for return_value, schema in return_values.items():
+            if return_value in IGNORE:
+                continue
+            if "oneOf" in schema:
+                # For elements using oneOf there are two descriptions. The first description
+                # should be at level with the oneOf and should contain the basic non-specific
+                # description of the endpoint. Then for each element of oneOf there is a
+                # specialized description for that particular case. The description used
+                # right below is the main description.
+                data_type = generate_data_type(schema)
+                ans.append(
+                    self.render_desc(schema["description"], spacing, data_type, return_value)
+                )
+                ans += self.render_oneof_block(schema, spacing + 4)
+                continue
+            description = schema["description"]
+            data_type = generate_data_type(schema)
+            check_deprecated_consistency(schema.get("deprecated", False), description)
+            ans.append(self.render_desc(description, spacing, data_type, return_value))
+            if "properties" in schema:
+                ans += self.render_table(schema["properties"], spacing + 4)
+            if schema.get("additionalProperties", False):
+                data_type = generate_data_type(schema["additionalProperties"])
+                ans.append(
+                    self.render_desc(
+                        schema["additionalProperties"]["description"],
+                        spacing + 4,
+                        data_type,
+                    )
+                )
+                if "properties" in schema["additionalProperties"]:
+                    ans += self.render_table(
+                        schema["additionalProperties"]["properties"],
+                        spacing + 8,
+                    )
+                elif "oneOf" in schema["additionalProperties"]:
+                    ans += self.render_oneof_block(schema["additionalProperties"], spacing + 8)
+                elif schema["additionalProperties"].get("additionalProperties", False):
+                    data_type = generate_data_type(
+                        schema["additionalProperties"]["additionalProperties"]
+                    )
+                    ans.append(
+                        self.render_desc(
+                            schema["additionalProperties"]["additionalProperties"]["description"],
+                            spacing + 8,
+                            data_type,
+                        )
+                    )
+
+                    ans += self.render_table(
+                        schema["additionalProperties"]["additionalProperties"]["properties"],
+                        spacing + 12,
+                    )
+            if "items" in schema:
+                if "properties" in schema["items"]:
+                    ans += self.render_table(schema["items"]["properties"], spacing + 4)
+                elif "oneOf" in schema["items"]:
+                    ans += self.render_oneof_block(schema["items"], spacing + 4)
+        return ans
+
+    def generate_event_strings(self, event_data: EventData) -> list[str]:
+        event_strings: list[str] = []
+        if event_data.op_type is None:
+            event_strings.append(
+                EVENT_HEADER_TEMPLATE.format(id=event_data.type, event=event_data.type)
+            )
+        else:
+            op_detail = OP_TEMPLATE.format(op_type=event_data.op_type)
+            event_strings.append(
+                EVENT_HEADER_TEMPLATE.format(
+                    id=f"{event_data.type}-{event_data.op_type}",
+                    event=f"{event_data.type} {op_detail}",
+                )
+            )
+        event_strings.append(f"\n{event_data.description}\n\n\n")
+        event_strings += self.render_table(event_data.properties, 0)
+        event_strings.append("**Example**")
+        event_strings.append("\n```json\n")
+        event_strings.append(event_data.example)
+        event_strings.append("```\n\n")
+        event_strings.append("<hr>")
+        return event_strings
+
+    def generate_events_table(self, events_by_type: OrderedDict[str, list[str]]) -> list[str]:
+        event_links: list[str] = []
+        for event_type, event_ops in events_by_type.items():
+            if not event_ops:
+                event_links.append(TABLE_LINK_TEMPLATE.format(link_name=event_type, url=event_type))
+            else:
+                event_ops.sort()
+                ops_list = [
+                    TABLE_LINK_TEMPLATE.format(link_name=f"op: {op}", url=f"{event_type}-{op}")
+                    for op in event_ops
+                ]
+                event_links.append(
+                    TABLE_OPS_TEMPLATE.format(event_name=event_type, ops="\n".join(ops_list))
+                )
+        return [EVENTS_TABLE_TEMPLATE.format(events_list="\n".join(event_links))]
+
+    def render_events(self, events_dict: dict[str, Any]) -> list[str]:
+        events: list[str] = []
+        events_for_table: OrderedDict[str, list[str]] = OrderedDict()
+        for event in events_dict["oneOf"]:
+            # The op property doesn't have a description, so it must be removed
+            # before any calls to self.render_table, which expects a description.
+            op: dict[str, Any] | None = event["properties"].pop("op", None)
+            op_type: str | None = None
+            if op is not None:
+                op_type = op["enum"][0]
+            event_data = EventData(
+                type=event["properties"]["type"]["enum"][0],
+                description=event["description"],
+                properties=event["properties"],
+                example=json.dumps(event["example"], indent=4, sort_keys=True),
+                op_type=op_type,
+            )
+            events += self.generate_event_strings(event_data)
+            if event_data.op_type is None:
+                events_for_table[event_data.type] = []
+            elif event_data.type in events_for_table:
+                events_for_table[event_data.type] += [event_data.op_type]
+            else:
+                events_for_table[event_data.type] = [event_data.op_type]
+        events_table = self.generate_events_table(OrderedDict(sorted(events_for_table.items())))
+        return events_table + events
+
+
+def makeExtension(*args: Any, **kwargs: str) -> MarkdownReturnValuesTableGenerator:
+    return MarkdownReturnValuesTableGenerator(*args, **kwargs)

zerver/lib/markdown/fenced_code.py

@@ -0,0 +1,599 @@
+"""
+Fenced Code Extension for Python Markdown
+=========================================
+
+This extension adds Fenced Code Blocks to Python-Markdown.
+
+    >>> import markdown
+    >>> text = '''
+    ... A paragraph before a fenced code block:
+    ...
+    ... ~~~
+    ... Fenced code block
+    ... ~~~
+    ... '''
+    >>> html = markdown.markdown(text, extensions=['fenced_code'])
+    >>> print html
+    <p>A paragraph before a fenced code block:</p>
+    <pre><code>Fenced code block
+    </code></pre>
+
+Works with safe_mode also (we check this because we are using the HtmlStash):
+
+    >>> print markdown.markdown(text, extensions=['fenced_code'], safe_mode='replace')
+    <p>A paragraph before a fenced code block:</p>
+    <pre><code>Fenced code block
+    </code></pre>
+
+Include tilde's in a code block and wrap with blank lines:
+
+    >>> text = '''
+    ... ~~~~~~~~
+    ...
+    ... ~~~~
+    ... ~~~~~~~~'''
+    >>> print markdown.markdown(text, extensions=['fenced_code'])
+    <pre><code>
+    ~~~~
+    </code></pre>
+
+Removes trailing whitespace from code blocks that cause horizontal scrolling
+    >>> import markdown
+    >>> text = '''
+    ... A paragraph before a fenced code block:
+    ...
+    ... ~~~
+    ... Fenced code block    \t\t\t\t\t\t\t
+    ... ~~~
+    ... '''
+    >>> html = markdown.markdown(text, extensions=['fenced_code'])
+    >>> print html
+    <p>A paragraph before a fenced code block:</p>
+    <pre><code>Fenced code block
+    </code></pre>
+
+Language tags:
+
+    >>> text = '''
+    ... ~~~~{.python}
+    ... # Some python code
+    ... ~~~~'''
+    >>> print markdown.markdown(text, extensions=['fenced_code'])
+    <pre><code class="python"># Some python code
+    </code></pre>
+
+Copyright 2007-2008 [Waylan Limberg](http://achinghead.com/).
+
+Project website: <http://packages.python.org/Markdown/extensions/fenced_code_blocks.html>
+Contact: markdown@freewisdom.org
+
+License: BSD (see ../docs/LICENSE for details)
+
+Dependencies:
+* [Python 2.4+](http://python.org)
+* [Markdown 2.0+](http://packages.python.org/Markdown/)
+* [Pygments (optional)](http://pygments.org)
+
+"""
+
+import logging
+import re
+from collections.abc import Callable, Iterable, Mapping, MutableSequence, Sequence
+from typing import Any
+
+import lxml.html
+from django.utils.html import escape
+from markdown import Markdown
+from markdown.extensions import Extension, codehilite
+from markdown.extensions.codehilite import CodeHiliteExtension, parse_hl_lines
+from markdown.preprocessors import Preprocessor
+from pygments.lexers import find_lexer_class_by_name
+from pygments.util import ClassNotFound
+from typing_extensions import override
+
+from zerver.lib.exceptions import MarkdownRenderingError
+from zerver.lib.markdown.priorities import PREPROCESSOR_PRIORITIES
+from zerver.lib.tex import render_tex
+from zerver.models.realm_playgrounds import PLAYGROUND_LANGUAGE_REGEX
+
+# Global vars
+FENCE_RE = re.compile(
+    r"""
+    # ~~~ or ```
+    (?P<fence>
+        ^(?:~{3,}|`{3,})
+    )
+
+    [ ]* # spaces
+
+    (?:
+        # language, like ".py" or "{javascript}"
+        \{?\.?
+        (?P<lang>
+            """
+    + PLAYGROUND_LANGUAGE_REGEX
+    + r"""
+        ) # "py" or "javascript"
+
+        [ ]* # spaces
+
+        # header for features that use fenced block header syntax (like spoilers)
+        (?P<header>
+            [^ ~`][^~`]*
+        )?
+        \}?
+    )?
+    $
+    """,
+    re.VERBOSE,
+)
+
+
+CODE_WRAP = "<pre><code{}>{}\n</code></pre>"
+LANG_TAG = ' class="{}"'
+
+
+def validate_curl_content(lines: list[str]) -> None:
+    error_msg = """
+Missing required -X argument in curl command:
+
+{command}
+""".strip()
+
+    for line in lines:
+        regex = r'curl [-](sS)?X "?(GET|DELETE|PATCH|POST)"?'
+        if line.startswith("curl") and re.search(regex, line) is None:
+            raise MarkdownRenderingError(error_msg.format(command=line.strip()))
+
+
+CODE_VALIDATORS: dict[str | None, Callable[[list[str]], None]] = {
+    "curl": validate_curl_content,
+}
+
+
+# This function is similar to one used in fenced_code.ts
+def get_unused_fence(content: str) -> str:
+    # Define the regular expression pattern to match ``` fences
+    fence_length_re = re.compile(r"^ {0,3}(`{3,})", re.MULTILINE)
+
+    # Initialize the length variable to 3, corresponding to default fence length
+    length = 3
+    matches = fence_length_re.findall(content)
+    for match in matches:
+        length = max(length, len(match) + 1)
+
+    return "`" * length
+
+
+class FencedCodeExtension(Extension):
+    def __init__(self, config: Mapping[str, Any] = {}) -> None:
+        self.config = {
+            "run_content_validators": [
+                config.get("run_content_validators", False),
+                "Boolean specifying whether to run content validation code in CodeHandler",
+            ],
+        }
+
+        for key, value in config.items():
+            self.setConfig(key, value)
+
+    @override
+    def extendMarkdown(self, md: Markdown) -> None:
+        """Add FencedBlockPreprocessor to the Markdown instance."""
+        md.registerExtension(self)
+        processor = FencedBlockPreprocessor(
+            md, run_content_validators=self.config["run_content_validators"][0]
+        )
+        md.preprocessors.register(
+            processor, "fenced_code_block", PREPROCESSOR_PRIORITIES["fenced_code_block"]
+        )
+
+
+class ZulipBaseHandler:
+    def __init__(
+        self,
+        processor: "FencedBlockPreprocessor",
+        output: MutableSequence[str],
+        fence: str | None = None,
+        process_contents: bool = False,
+    ) -> None:
+        self.processor = processor
+        self.output = output
+        self.fence = fence
+        self.process_contents = process_contents
+        self.lines: list[str] = []
+
+    def handle_line(self, line: str) -> None:
+        if line.rstrip() == self.fence:
+            self.done()
+        else:
+            self.lines.append(line.rstrip())
+
+    def done(self) -> None:
+        if self.lines:
+            text = "\n".join(self.lines)
+            text = self.format_text(text)
+
+            # For code blocks, the contents should not receive further
+            # processing.  Whereas with quote and spoiler blocks, we
+            # explicitly want Markdown formatting of the content
+            # inside. This behavior is controlled by the
+            # process_contents configuration flag.
+            if not self.process_contents:
+                text = self.processor.placeholder(text)
+
+            processed_lines = text.split("\n")
+            self.output.append("")
+            self.output.extend(processed_lines)
+            self.output.append("")
+        self.processor.pop()
+
+    def format_text(self, text: str) -> str:
+        """Returns a formatted text.
+        Subclasses should override this method.
+        """
+        raise NotImplementedError
+
+
+def generic_handler(
+    processor: "FencedBlockPreprocessor",
+    output: MutableSequence[str],
+    fence: str,
+    lang: str | None,
+    header: str | None,
+    run_content_validators: bool = False,
+    default_language: str | None = None,
+) -> ZulipBaseHandler:
+    if lang is not None:
+        lang = lang.lower()
+    if lang in ("quote", "quoted"):
+        return QuoteHandler(processor, output, fence, default_language)
+    elif lang == "math":
+        return TexHandler(processor, output, fence)
+    elif lang == "spoiler":
+        return SpoilerHandler(processor, output, fence, header)
+    else:
+        return CodeHandler(processor, output, fence, lang, run_content_validators)
+
+
+def check_for_new_fence(
+    processor: "FencedBlockPreprocessor",
+    output: MutableSequence[str],
+    line: str,
+    run_content_validators: bool = False,
+    default_language: str | None = None,
+) -> None:
+    m = FENCE_RE.match(line)
+    if m:
+        fence = m.group("fence")
+        lang: str | None = m.group("lang")
+        header: str | None = m.group("header")
+        if not lang and default_language:
+            lang = default_language
+        handler = generic_handler(
+            processor, output, fence, lang, header, run_content_validators, default_language
+        )
+        processor.push(handler)
+    else:
+        output.append(line)
+
+
+class OuterHandler(ZulipBaseHandler):
+    def __init__(
+        self,
+        processor: "FencedBlockPreprocessor",
+        output: MutableSequence[str],
+        run_content_validators: bool = False,
+        default_language: str | None = None,
+    ) -> None:
+        self.run_content_validators = run_content_validators
+        self.default_language = default_language
+        super().__init__(processor, output)
+
+    @override
+    def handle_line(self, line: str) -> None:
+        check_for_new_fence(
+            self.processor, self.output, line, self.run_content_validators, self.default_language
+        )
+
+
+class CodeHandler(ZulipBaseHandler):
+    def __init__(
+        self,
+        processor: "FencedBlockPreprocessor",
+        output: MutableSequence[str],
+        fence: str,
+        lang: str | None,
+        run_content_validators: bool = False,
+    ) -> None:
+        self.lang = lang
+        self.run_content_validators = run_content_validators
+        super().__init__(processor, output, fence)
+
+    @override
+    def done(self) -> None:
+        # run content validators (if any)
+        if self.run_content_validators:
+            validator = CODE_VALIDATORS.get(self.lang, lambda text: None)
+            validator(self.lines)
+        super().done()
+
+    @override
+    def format_text(self, text: str) -> str:
+        return self.processor.format_code(self.lang, text)
+
+
+class QuoteHandler(ZulipBaseHandler):
+    def __init__(
+        self,
+        processor: "FencedBlockPreprocessor",
+        output: MutableSequence[str],
+        fence: str,
+        default_language: str | None = None,
+    ) -> None:
+        self.default_language = default_language
+        super().__init__(processor, output, fence, process_contents=True)
+
+    @override
+    def handle_line(self, line: str) -> None:
+        if line.rstrip() == self.fence:
+            self.done()
+        else:
+            check_for_new_fence(
+                self.processor, self.lines, line, default_language=self.default_language
+            )
+
+    @override
+    def format_text(self, text: str) -> str:
+        return self.processor.format_quote(text)
+
+
+class SpoilerHandler(ZulipBaseHandler):
+    def __init__(
+        self,
+        processor: "FencedBlockPreprocessor",
+        output: MutableSequence[str],
+        fence: str,
+        spoiler_header: str | None,
+    ) -> None:
+        self.spoiler_header = spoiler_header
+        super().__init__(processor, output, fence, process_contents=True)
+
+    @override
+    def handle_line(self, line: str) -> None:
+        if line.rstrip() == self.fence:
+            self.done()
+        else:
+            check_for_new_fence(self.processor, self.lines, line)
+
+    @override
+    def format_text(self, text: str) -> str:
+        return self.processor.format_spoiler(self.spoiler_header, text)
+
+
+class TexHandler(ZulipBaseHandler):
+    @override
+    def format_text(self, text: str) -> str:
+        return self.processor.format_tex(text)
+
+
+class CodeHilite(codehilite.CodeHilite):
+    def _parseHeader(self) -> None:
+        # Python-Markdown has a feature to parse-and-hide shebang
+        # lines present in code blocks:
+        #
+        # https://python-markdown.github.io/extensions/code_hilite/#shebang-no-path
+        #
+        # While using shebang lines for language detection is
+        # reasonable, we don't want this feature because it can be
+        # really confusing when doing anything else in a one-line code
+        # block that starts with `!` (which would then render as an
+        # empty code block!).  So we disable the feature, by
+        # overriding this function, which implements it in CodeHilite
+        # upstream.
+
+        # split text into lines
+        lines = self.src.split("\n")
+        # Python-Markdown pops out the first line which we are avoiding here.
+        # Examine first line
+        fl = lines[0]
+
+        c = re.compile(
+            r"""
+            (?:(?:^::+)|(?P<shebang>^[#]!)) # Shebang or 2 or more colons
+            (?P<path>(?:/\w+)*[/ ])?        # Zero or 1 path
+            (?P<lang>[\w#.+-]*)             # The language
+            \s*                             # Arbitrary whitespace
+            # Optional highlight lines, single- or double-quote-delimited
+            (hl_lines=(?P<quot>"|')(?P<hl_lines>.*?)(?P=quot))?
+            """,
+            re.VERBOSE,
+        )
+        # Search first line for shebang
+        m = c.search(fl)
+        if m:
+            # We have a match
+            try:
+                self.lang = m.group("lang").lower()
+            except IndexError:  # nocoverage
+                self.lang = None
+
+            if self.options["linenos"] is None and m.group("shebang"):
+                # Overridable and Shebang exists - use line numbers
+                self.options["linenos"] = True
+
+            self.options["hl_lines"] = parse_hl_lines(m.group("hl_lines"))
+
+        self.src = "\n".join(lines).strip("\n")
+
+
+class FencedBlockPreprocessor(Preprocessor):
+    def __init__(self, md: Markdown, run_content_validators: bool = False) -> None:
+        super().__init__(md)
+
+        self.checked_for_codehilite = False
+        self.run_content_validators = run_content_validators
+        self.codehilite_conf: Mapping[str, Sequence[Any]] = {}
+
+    def push(self, handler: ZulipBaseHandler) -> None:
+        self.handlers.append(handler)
+
+    def pop(self) -> None:
+        self.handlers.pop()
+
+    @override
+    def run(self, lines: Iterable[str]) -> list[str]:
+        """Match and store Fenced Code Blocks in the HtmlStash."""
+
+        from zerver.lib.markdown import ZulipMarkdown
+
+        output: list[str] = []
+
+        processor = self
+        self.handlers: list[ZulipBaseHandler] = []
+
+        default_language = None
+        if isinstance(self.md, ZulipMarkdown) and self.md.zulip_realm is not None:
+            default_language = self.md.zulip_realm.default_code_block_language
+        handler = OuterHandler(processor, output, self.run_content_validators, default_language)
+        self.push(handler)
+
+        for line in lines:
+            self.handlers[-1].handle_line(line)
+
+        while self.handlers:
+            self.handlers[-1].done()
+
+        # This fiddly handling of new lines at the end of our output was done to make
+        # existing tests pass. Markdown is just kind of funny when it comes to new lines,
+        # but we could probably remove this hack.
+        if len(output) > 2 and output[-2] != "":
+            output.append("")
+        return output
+
+    def format_code(self, lang: str | None, text: str) -> str:
+        if lang:
+            langclass = LANG_TAG.format(lang)
+        else:
+            langclass = ""
+
+        # Check for code hilite extension
+        if not self.checked_for_codehilite:
+            for ext in self.md.registeredExtensions:
+                if isinstance(ext, CodeHiliteExtension):
+                    self.codehilite_conf = ext.config
+                    break
+
+            self.checked_for_codehilite = True
+
+        # If config is not empty, then the codehighlite extension
+        # is enabled, so we call it to highlight the code
+        if self.codehilite_conf:
+            highliter = CodeHilite(
+                text,
+                linenums=self.codehilite_conf["linenums"][0],
+                guess_lang=self.codehilite_conf["guess_lang"][0],
+                css_class=self.codehilite_conf["css_class"][0],
+                style=self.codehilite_conf["pygments_style"][0],
+                use_pygments=self.codehilite_conf["use_pygments"][0],
+                lang=lang or None,
+                noclasses=self.codehilite_conf["noclasses"][0],
+                # By default, the Pygments PHP lexers won't highlight
+                # code without a `<?php` marker at the start of the
+                # code block, which is undesired in the common case of
+                # pasting a snippet of PHP code rather than whole
+                # file. The `startinline` option overrides this
+                # behavior for PHP-descended languages and has no
+                # effect on other lexers.
+                #
+                # See https://pygments.org/docs/lexers/#lexers-for-php-and-related-languages
+                startinline=True,
+            )
+
+            try:
+                code = highliter.hilite().rstrip("\n")
+            except Exception:
+                logging.exception("Failed to highlight fenced code block")
+                code = CODE_WRAP.format(langclass, self._escape(text))
+        else:
+            code = CODE_WRAP.format(langclass, self._escape(text))
+
+        # To support our "view in playground" feature, the frontend
+        # needs to know what Pygments language was used for
+        # highlighting this code block.  We record this in a data
+        # attribute attached to the outer `pre` element.
+        # Unfortunately, the pygments API doesn't offer a way to add
+        # this, so we need to do it in a post-processing step.
+        if lang:
+            div_tag = lxml.html.fromstring(code)
+
+            # For the value of our data element, we get the lexer
+            # subclass name instead of directly using the language,
+            # since that canonicalizes aliases (Eg: `js` and
+            # `javascript` will be mapped to `JavaScript`).
+            try:
+                code_language = find_lexer_class_by_name(lang).name
+            except ClassNotFound:
+                # If there isn't a Pygments lexer by this name, we
+                # still tag it with the user's data-code-language
+                # value, since this allows hooking up a "playground"
+                # for custom "languages" that aren't known to Pygments.
+                code_language = lang
+
+            div_tag.attrib["data-code-language"] = code_language
+            code = lxml.html.tostring(div_tag, encoding="unicode")
+        return code
+
+    def format_quote(self, text: str) -> str:
+        paragraphs = text.split("\n")
+        quoted_paragraphs = []
+        for paragraph in paragraphs:
+            lines = paragraph.split("\n")
+            quoted_paragraphs.append("\n".join("> " + line for line in lines))
+        return "\n".join(quoted_paragraphs)
+
+    def format_spoiler(self, header: str | None, text: str) -> str:
+        output = []
+        header_div_open_html = '<div class="spoiler-block"><div class="spoiler-header">'
+        end_header_start_content_html = '</div><div class="spoiler-content" aria-hidden="true">'
+        footer_html = "</div></div>"
+
+        output.append(self.placeholder(header_div_open_html))
+        if header is not None:
+            output.append(header)
+        output.append(self.placeholder(end_header_start_content_html))
+        output.append(text)
+        output.append(self.placeholder(footer_html))
+        return "\n\n".join(output)
+
+    def format_tex(self, text: str) -> str:
+        paragraphs = text.split("\n\n")
+        tex_paragraphs = []
+        for paragraph in paragraphs:
+            html = render_tex(paragraph, is_inline=False)
+            if html is not None:
+                tex_paragraphs.append(html)
+            else:
+                tex_paragraphs.append('<span class="tex-error">' + escape(paragraph) + "</span>")
+        return "\n\n".join(tex_paragraphs)
+
+    def placeholder(self, code: str) -> str:
+        return self.md.htmlStash.store(code)
+
+    def _escape(self, txt: str) -> str:
+        """basic html escaping"""
+        txt = txt.replace("&", "&amp;")
+        txt = txt.replace("<", "&lt;")
+        txt = txt.replace(">", "&gt;")
+        txt = txt.replace('"', "&quot;")
+        return txt
+
+
+def makeExtension(*args: Any, **kwargs: Any) -> FencedCodeExtension:
+    return FencedCodeExtension(kwargs)
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod()

zerver/lib/markdown/from_html.py

@@ -0,0 +1,74 @@
+import posixpath
+import re
+import sys
+import warnings
+from urllib.parse import urlsplit
+
+import markdownify
+from bs4 import MarkupResemblesLocatorWarning, Tag
+
+from zerver.lib.markdown import get_markdown_link_for_url, sanitize_url
+
+
+class ZulipMarkdownConverter(markdownify.MarkdownConverter):
+    """HTML-to-Markdown converter adapted to Zulip's Markdown.
+
+    Zulip doesn't inline-render external images, so images become
+    `[label](src)` links instead of broken `![alt](src)`.
+    """
+
+    def convert_img(self, el: Tag, text: str, parent_tags: set[str]) -> str:
+        src = el.get("src", "")
+        alt = el.get("alt", "")
+        # BeautifulSoup returns a list only for multi-valued attributes
+        # (e.g. class), never for src or alt.
+        assert isinstance(src, str)
+        assert isinstance(alt, str)
+        if not src:
+            return alt
+        url = sanitize_url(src)
+        if not url:
+            # Unlinkable src (e.g. data: URIs, unsafe schemes): emit just the
+            # alt text rather than a bogus link.
+            return alt
+        label = alt or posixpath.basename(urlsplit(url).path)
+        if not label:
+            # No alt text, and no filename in the path to fall back on, as in
+            # `<img src="https://example.com/?token=abc">`.
+            return url
+        if "a" in parent_tags:
+            # Markdown has no nested links, so hand the label up for the
+            # enclosing <a> to use as its text, with brackets stripped.
+            return re.sub(r"\[|\]", "", label)
+        return get_markdown_link_for_url(label, url)
+
+    def convert_title(self, el: Tag, text: str, parent_tags: set[str]) -> str:
+        return ""
+
+
+def convert_html_to_markdown(
+    html: str, *, converter_class: type[ZulipMarkdownConverter] = ZulipMarkdownConverter
+) -> str:
+    # A bare URL is expected input here (the text/html part of an incoming
+    # email), but BeautifulSoup warns about it ("looks like a URL, not markup").
+    # Suppress the warning so that expected input doesn't spam our logs.
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", MarkupResemblesLocatorWarning)
+        converter = converter_class(
+            # Zulip's Markdown supports only ATX headings, not Setext headings.
+            heading_style="ATX",
+            # Zulip's Markdown has no backslash escaping; it renders "\" as a
+            # literal character.
+            escape_asterisks=False,
+            escape_underscores=False,
+            # HTML treats a newline as insignificant whitespace, but Zulip
+            # renders it as a line break. Collapse source newlines to spaces.
+            # <br>, <pre>, and block-level elements still produce line breaks.
+            wrap=True,
+            # An effectively infinite width disables column wrapping. Don't
+            # use None, which also disables wrapping but skips the pass that
+            # strips indentation from lines following a <br>.
+            wrap_width=sys.maxsize,
+        )
+        markdown: str = converter.convert(html).strip()
+    return markdown

zerver/lib/markdown/help_settings_links.py

@@ -0,0 +1,187 @@
+import re
+from re import Match
+from typing import Any
+
+from markdown import Markdown
+from markdown.extensions import Extension
+from markdown.preprocessors import Preprocessor
+from typing_extensions import override
+
+from zerver.lib.markdown.priorities import PREPROCESSOR_PRIORITIES
+
+REGEXP = re.compile(r"\{settings_tab\|(?P<setting_identifier>.*?)\}")
+
+
+# If any changes to this link mapping are made,
+# `starlight_help/src/components/NavigationSteps.astro` should be updated accordingly.
+# This manual update mechanism will cease to exist once we have switched to the
+# starlight_help system.
+link_mapping = {
+    # a mapping from the setting identifier that is the same as the final URL
+    # breadcrumb to that setting to the name of its setting type, the setting
+    # name as it appears in the user interface, and a relative link that can
+    # be used to get to that setting
+    "profile": ["Personal settings", "Profile", "/#settings/profile"],
+    "account-and-privacy": [
+        "Personal settings",
+        "Account & privacy",
+        "/#settings/account-and-privacy",
+    ],
+    "preferences": ["Personal settings", "Preferences", "/#settings/preferences"],
+    "notifications": ["Personal settings", "Notifications", "/#settings/notifications"],
+    "your-bots": ["Personal settings", "Bots", "/#settings/your-bots"],
+    "alert-words": ["Personal settings", "Alert words", "/#settings/alert-words"],
+    "uploaded-files": ["Personal settings", "Uploaded files", "/#settings/uploaded-files"],
+    "topics": ["Personal settings", "Topics", "/#settings/topics"],
+    "muted-users": ["Personal settings", "Muted users", "/#settings/muted-users"],
+    "organization-profile": [
+        "Organization settings",
+        "Organization profile",
+        "/#organization/organization-profile",
+    ],
+    "organization-settings": [
+        "Organization settings",
+        "Organization settings",
+        "/#organization/organization-settings",
+    ],
+    "organization-permissions": [
+        "Organization settings",
+        "Organization permissions",
+        "/#organization/organization-permissions",
+    ],
+    "default-user-settings": [
+        "Organization settings",
+        "Default user settings",
+        "/#organization/organization-level-user-defaults",
+    ],
+    "emoji-settings": ["Organization settings", "Custom emoji", "/#organization/emoji-settings"],
+    "auth-methods": [
+        "Organization settings",
+        "Authentication methods",
+        "/#organization/auth-methods",
+    ],
+    "users": [
+        "Organization settings",
+        "Users",
+        "/#organization/users/active",
+    ],
+    "deactivated": [
+        "Organization settings",
+        "Users",
+        "/#organization/users/deactivated",
+    ],
+    "invitations": [
+        "Organization settings",
+        "Users",
+        "/#organization/users/invitations",
+    ],
+    "bots": [
+        "Organization settings",
+        "Bots",
+        "/#organization/bots",
+    ],
+    "default-channels-list": [
+        "Organization settings",
+        "Default channels",
+        "/#organization/default-channels-list",
+    ],
+    "linkifier-settings": [
+        "Organization settings",
+        "Linkifiers",
+        "/#organization/linkifier-settings",
+    ],
+    "playground-settings": [
+        "Organization settings",
+        "Code playgrounds",
+        "/#organization/playground-settings",
+    ],
+    "profile-field-settings": [
+        "Organization settings",
+        "Custom profile fields",
+        "/#organization/profile-field-settings",
+    ],
+    "channel-folder-settings": [
+        "Organization settings",
+        "Channel folders",
+        "/#organization/channel-folders",
+    ],
+    "data-exports-admin": [
+        "Organization settings",
+        "Data exports",
+        "/#organization/data-exports-admin",
+    ],
+}
+
+settings_markdown = """
+1. Click on the **gear** (<i class="zulip-icon zulip-icon-gear"></i>) icon in the upper
+   right corner of the web or desktop app.
+
+1. Select **{setting_type_name}**.
+
+1. On the left, click {setting_reference}.
+"""
+
+
+def getMarkdown(setting_type_name: str, setting_name: str, setting_link: str) -> str:
+    if relative_settings_links:
+        relative_link = f"[{setting_name}]({setting_link})"
+        # The "Bots" label appears in both Personal and Organization settings
+        # in the user interface so we need special text for this setting.
+        if setting_name in ["Bots", "Users"]:
+            return f"1. Navigate to the {relative_link} \
+                    tab of the **{setting_type_name}** menu."
+        return f"1. Go to {relative_link}."
+    return settings_markdown.format(  # nocoverage
+        setting_type_name=setting_type_name,
+        setting_reference=f"**{setting_name}**",
+    )
+
+
+class SettingHelpExtension(Extension):
+    @override
+    def extendMarkdown(self, md: Markdown) -> None:
+        """Add SettingHelpExtension to the Markdown instance."""
+        md.registerExtension(self)
+        md.preprocessors.register(Setting(), "setting", PREPROCESSOR_PRIORITIES["setting"])
+
+
+relative_settings_links: bool = False
+
+
+def set_relative_settings_links(value: bool) -> None:
+    global relative_settings_links
+    relative_settings_links = value
+
+
+class Setting(Preprocessor):
+    @override
+    def run(self, lines: list[str]) -> list[str]:
+        done = False
+        while not done:
+            for line in lines:
+                loc = lines.index(line)
+                match = REGEXP.search(line)
+
+                if match:
+                    text = [self.handleMatch(match)]
+                    # The line that contains the directive to include the macro
+                    # may be preceded or followed by text or tags, in that case
+                    # we need to make sure that any preceding or following text
+                    # stays the same.
+                    line_split = REGEXP.split(line, maxsplit=0)
+                    preceding = line_split[0]
+                    following = line_split[-1]
+                    text = [preceding, *text, following]
+                    lines = lines[:loc] + text + lines[loc + 1 :]
+                    break
+            else:
+                done = True
+        return lines
+
+    def handleMatch(self, match: Match[str]) -> str:
+        setting_identifier = match.group("setting_identifier")
+        return getMarkdown(*link_mapping[setting_identifier])
+
+
+def makeExtension(*args: Any, **kwargs: Any) -> SettingHelpExtension:
+    return SettingHelpExtension(*args, **kwargs)

zerver/lib/markdown/include.py

@@ -0,0 +1,60 @@
+import os
+import re
+from re import Match
+from xml.etree.ElementTree import Element
+
+from markdown import Extension, Markdown
+from markdown.blockparser import BlockParser
+from markdown.blockprocessors import BlockProcessor
+from typing_extensions import override
+
+from zerver.lib.exceptions import InvalidMarkdownIncludeStatementError
+from zerver.lib.markdown.priorities import BLOCK_PROCESSOR_PRIORITIES
+
+
+class IncludeExtension(Extension):
+    def __init__(self, base_path: str) -> None:
+        super().__init__()
+        self.base_path = base_path
+
+    @override
+    def extendMarkdown(self, md: Markdown) -> None:
+        md.parser.blockprocessors.register(
+            IncludeBlockProcessor(md.parser, self.base_path),
+            "include",
+            BLOCK_PROCESSOR_PRIORITIES["include"],
+        )
+
+
+class IncludeBlockProcessor(BlockProcessor):
+    RE = re.compile(r"^ {,3}\{!([^!]+)!\} *$", re.MULTILINE)
+
+    def __init__(self, parser: BlockParser, base_path: str) -> None:
+        super().__init__(parser)
+        self.base_path = base_path
+
+    @override
+    def test(self, parent: Element, block: str) -> bool:
+        return bool(self.RE.search(block))
+
+    def expand_include(self, m: Match[str]) -> str:
+        try:
+            with open(os.path.normpath(os.path.join(self.base_path, m[1]))) as f:
+                lines = f.read().splitlines()
+        except OSError as e:
+            raise InvalidMarkdownIncludeStatementError(m[0].strip()) from e
+
+        for prep in self.parser.md.preprocessors:
+            lines = prep.run(lines)
+
+        return "\n".join(lines)
+
+    @override
+    def run(self, parent: Element, blocks: list[str]) -> None:
+        self.parser.state.set("include")
+        self.parser.parseChunk(parent, self.RE.sub(self.expand_include, blocks.pop(0)))
+        self.parser.state.reset()
+
+
+def makeExtension(base_path: str) -> IncludeExtension:
+    return IncludeExtension(base_path=base_path)

zerver/lib/markdown/nested_code_blocks.py

@@ -0,0 +1,87 @@
+from collections.abc import Mapping
+from typing import Any
+from xml.etree.ElementTree import Element, SubElement
+
+import markdown
+from markdown.extensions import Extension
+from typing_extensions import override
+
+from zerver.lib.markdown import ResultWithFamily, walk_tree_with_family
+from zerver.lib.markdown.priorities import PREPROCESSOR_PRIORITIES
+
+
+class NestedCodeBlocksRenderer(Extension):
+    @override
+    def extendMarkdown(self, md: markdown.Markdown) -> None:
+        md.treeprocessors.register(
+            NestedCodeBlocksRendererTreeProcessor(md, self.getConfigs()),
+            "nested_code_blocks",
+            PREPROCESSOR_PRIORITIES["nested_code_blocks"],
+        )
+
+
+class NestedCodeBlocksRendererTreeProcessor(markdown.treeprocessors.Treeprocessor):
+    def __init__(self, md: markdown.Markdown, config: Mapping[str, Any]) -> None:
+        super().__init__(md)
+
+    @override
+    def run(self, root: Element) -> None:
+        code_tags = walk_tree_with_family(root, self.get_code_tags)
+        nested_code_blocks = self.get_nested_code_blocks(code_tags)
+        for block in nested_code_blocks:
+            _tag, text = block.result
+            codehilite_block = self.get_codehilite_block(text)
+            self.replace_element(block.family.grandparent, codehilite_block, block.family.parent)
+
+    def get_code_tags(self, e: Element) -> tuple[str, str | None] | None:
+        if e.tag == "code":
+            return (e.tag, e.text)
+        return None
+
+    def get_nested_code_blocks(
+        self,
+        code_tags: list[ResultWithFamily[tuple[str, str | None]]],
+    ) -> list[ResultWithFamily[tuple[str, str | None]]]:
+        nested_code_blocks = []
+        for code_tag in code_tags:
+            parent: Any = code_tag.family.parent
+            grandparent: Any = code_tag.family.grandparent
+            if (
+                parent.tag == "p"
+                and grandparent.tag == "li"
+                and parent.text is None
+                and len(parent) == 1
+                and sum(1 for text in parent.itertext()) == 1
+            ):
+                # if the parent (<p>) has no text, and no children,
+                # that means that the <code> element inside is its
+                # only thing inside the bullet, we can confidently say
+                # that this is a nested code block
+                nested_code_blocks.append(code_tag)
+
+        return nested_code_blocks
+
+    def get_codehilite_block(self, code_block_text: str | None) -> Element:
+        div = Element("div")
+        div.set("class", "codehilite")
+        pre = SubElement(div, "pre")
+        pre.text = code_block_text
+        return div
+
+    def replace_element(
+        self,
+        parent: Element | None,
+        replacement: Element,
+        element_to_replace: Element,
+    ) -> None:
+        if parent is None:
+            return
+
+        for index, child in enumerate(parent):
+            if child is element_to_replace:
+                parent.insert(index, replacement)
+                parent.remove(element_to_replace)
+
+
+def makeExtension(*args: Any, **kwargs: str) -> NestedCodeBlocksRenderer:
+    return NestedCodeBlocksRenderer(**kwargs)

zerver/lib/markdown/priorities.py

@@ -0,0 +1,24 @@
+# Note that in the Markdown preprocessor registry, the highest
+# numeric value is considered the highest priority, so the dict
+# below is ordered from highest-to-lowest priority.
+# Priorities for the built-in preprocessors are commented out.
+PREPROCESSOR_PRIORITIES = {
+    "generate_parameter_description": 535,
+    "generate_response_description": 531,
+    "generate_api_header": 530,
+    "generate_code_example": 525,
+    "generate_return_values": 510,
+    "generate_api_arguments": 505,
+    "setting": 450,
+    # "normalize_whitespace": 30,
+    "fenced_code_block": 25,
+    # "html_block": 20,
+    "tabbed_sections": -500,
+    "nested_code_blocks": -500,
+    "emoticon_translations": -505,
+    "static_images": -510,
+}
+
+BLOCK_PROCESSOR_PRIORITIES = {
+    "include": 51,
+}

zerver/lib/markdown/static.py

@@ -0,0 +1,36 @@
+from typing import Any
+from xml.etree.ElementTree import Element
+
+import markdown
+from django.contrib.staticfiles.storage import staticfiles_storage
+from markdown.extensions import Extension
+from typing_extensions import override
+
+from zerver.lib.markdown.priorities import PREPROCESSOR_PRIORITIES
+
+
+class MarkdownStaticImagesGenerator(Extension):
+    @override
+    def extendMarkdown(self, md: markdown.Markdown) -> None:
+        md.treeprocessors.register(
+            StaticImageProcessor(md),
+            "static_images",
+            PREPROCESSOR_PRIORITIES["static_images"],
+        )
+
+
+class StaticImageProcessor(markdown.treeprocessors.Treeprocessor):
+    """
+    Rewrite img tags which refer to /static/ to use staticfiles
+    """
+
+    @override
+    def run(self, root: Element) -> None:
+        for img in root.iter("img"):
+            url = img.get("src")
+            if url is not None and url.startswith("/static/"):
+                img.set("src", staticfiles_storage.url(url.removeprefix("/static/")))
+
+
+def makeExtension(*args: Any, **kwargs: str) -> MarkdownStaticImagesGenerator:
+    return MarkdownStaticImagesGenerator(*args, **kwargs)

zerver/lib/markdown/tabbed_sections.py

@@ -0,0 +1,170 @@
+import re
+from collections.abc import Mapping
+from typing import Any
+
+import markdown
+from markdown.extensions import Extension
+from markdown.preprocessors import Preprocessor
+from typing_extensions import override
+
+from zerver.lib.markdown.priorities import PREPROCESSOR_PRIORITIES
+
+START_TABBED_SECTION_REGEX = re.compile(r"^\{start_tabs\}$")
+END_TABBED_SECTION_REGEX = re.compile(r"^\{end_tabs\}$")
+TAB_CONTENT_REGEX = re.compile(r"^\{tab\|([^}]+)\}$")
+
+TABBED_SECTION_TEMPLATE = """
+<div class="tabbed-section {tab_class}" markdown="1">
+{nav_bar}
+<div class="blocks">
+{blocks}
+</div>
+</div>
+""".strip()
+
+NAV_BAR_TEMPLATE = """
+<ul class="nav">
+{tabs}
+</ul>
+""".strip()
+
+NAV_LIST_ITEM_TEMPLATE = """
+<li class="{class_}" data-tab-key="{data_tab_key}" tabindex="0">{label}</li>
+""".strip()
+
+DIV_TAB_CONTENT_TEMPLATE = """
+<div class="tab-content {class_}" data-tab-key="{data_tab_key}" markdown="1">
+{content}
+</div>
+""".strip()
+
+# If adding new entries here, also check if you need to update
+# tabbed-instructions.js
+TAB_SECTION_LABELS = {
+    "desktop-web": "Desktop/Web",
+    "ios": "iOS",
+    "android": "Android",
+    "python": "Python",
+    "js": "JavaScript",
+    "curl": "curl",
+    "zulip-send": "zulip-send",
+    "instructions-for-all-platforms": "Instructions for all platforms",
+    "for-a-bot": "For a bot",
+    "for-yourself": "For yourself",
+    "grafana-latest": "Grafana 8.3+",
+    "grafana-older-version": "Grafana 8.2 and below",
+    "send-channel-message": "Send a channel message",
+    "send-dm": "Send a DM",
+}
+
+
+class TabbedSectionsGenerator(Extension):
+    @override
+    def extendMarkdown(self, md: markdown.Markdown) -> None:
+        md.preprocessors.register(
+            TabbedSectionsPreprocessor(md, self.getConfigs()),
+            "tabbed_sections",
+            PREPROCESSOR_PRIORITIES["tabbed_sections"],
+        )
+
+
+def parse_tabs(lines: list[str]) -> dict[str, Any] | None:
+    block: dict[str, Any] = {}
+    for index, line in enumerate(lines):
+        start_match = START_TABBED_SECTION_REGEX.search(line)
+        if start_match:
+            block["start_tabs_index"] = index
+
+        tab_content_match = TAB_CONTENT_REGEX.search(line)
+        if tab_content_match:
+            block.setdefault("tabs", [])
+            tab = {"start": index, "tab_key": tab_content_match.group(1)}
+            block["tabs"].append(tab)
+
+        end_match = END_TABBED_SECTION_REGEX.search(line)
+        if end_match:
+            block["end_tabs_index"] = index
+            break
+    return block
+
+
+def generate_content_blocks(
+    tab_section: dict[str, Any], lines: list[str], tab_content_template: str
+) -> str:
+    tab_content_blocks = []
+    for index, tab in enumerate(tab_section["tabs"]):
+        start_index = tab["start"] + 1
+        try:
+            # If there are more tabs, we can use the starting index
+            # of the next tab as the ending index of the previous one
+            end_index = tab_section["tabs"][index + 1]["start"]
+        except IndexError:
+            # Otherwise, just use the end of the entire section
+            end_index = tab_section["end_tabs_index"]
+
+        content = "\n".join(lines[start_index:end_index]).strip()
+        tab_content_block = tab_content_template.format(
+            class_="active" if index == 0 else "",
+            data_tab_key=tab["tab_key"],
+            # This attribute is not used directly in this file here,
+            # we need this for the current conversion script in for
+            # starlight_help where this function is being imported.
+            tab_label=TAB_SECTION_LABELS[tab["tab_key"]],
+            # Wrapping the content in two newlines is necessary here.
+            # If we don't do this, the inner Markdown does not get
+            # rendered properly.
+            content=f"\n{content}\n",
+        )
+        tab_content_blocks.append(tab_content_block)
+    return "\n".join(tab_content_blocks)
+
+
+class TabbedSectionsPreprocessor(Preprocessor):
+    def __init__(self, md: markdown.Markdown, config: Mapping[str, Any]) -> None:
+        super().__init__(md)
+
+    @override
+    def run(self, lines: list[str]) -> list[str]:
+        tab_section = parse_tabs(lines)
+        while tab_section:
+            if "tabs" in tab_section:
+                tab_class = "has-tabs"
+            else:
+                tab_class = "no-tabs"
+                tab_section["tabs"] = [
+                    {
+                        "tab_key": "instructions-for-all-platforms",
+                        "start": tab_section["start_tabs_index"],
+                    }
+                ]
+            nav_bar = self.generate_nav_bar(tab_section)
+            content_blocks = generate_content_blocks(tab_section, lines, DIV_TAB_CONTENT_TEMPLATE)
+            rendered_tabs = TABBED_SECTION_TEMPLATE.format(
+                tab_class=tab_class, nav_bar=nav_bar, blocks=content_blocks
+            )
+
+            start = tab_section["start_tabs_index"]
+            end = tab_section["end_tabs_index"] + 1
+            lines = [*lines[:start], rendered_tabs, *lines[end:]]
+            tab_section = parse_tabs(lines)
+        return lines
+
+    def generate_nav_bar(self, tab_section: dict[str, Any]) -> str:
+        li_elements = []
+        for index, tab in enumerate(tab_section["tabs"]):
+            tab_key = tab.get("tab_key")
+            tab_label = TAB_SECTION_LABELS.get(tab_key)
+            if tab_label is None:
+                raise ValueError(
+                    f"Tab '{tab_key}' is not present in TAB_SECTION_LABELS in zerver/lib/markdown/tabbed_sections.py"
+                )
+            class_ = "active" if index == 0 else ""
+
+            li = NAV_LIST_ITEM_TEMPLATE.format(class_=class_, data_tab_key=tab_key, label=tab_label)
+            li_elements.append(li)
+
+        return NAV_BAR_TEMPLATE.format(tabs="\n".join(li_elements))
+
+
+def makeExtension(*args: Any, **kwargs: str) -> TabbedSectionsGenerator:
+    return TabbedSectionsGenerator(**kwargs)

zerver/lib/mdiff.py

@@ -0,0 +1,19 @@
+import logging
+import os
+import subprocess
+
+
+class DiffError(Exception):
+    pass
+
+
+def diff_strings(output: str, expected_output: str) -> str:
+    mdiff_path = "web/tests/lib/mdiff.cjs"
+    if not os.path.isfile(mdiff_path):  # nocoverage
+        msg = "Cannot find mdiff for Markdown diff rendering"
+        logging.error(msg)
+        raise DiffError(msg)
+
+    command = ["node", mdiff_path, output, expected_output]
+    diff = subprocess.check_output(command, text=True)
+    return diff

zerver/lib/mention.py

@@ -0,0 +1,508 @@
+import functools
+import re
+from collections import defaultdict
+from dataclasses import dataclass
+from re import Match
+from typing import Literal
+
+from django.conf import settings
+from django.db.models import Q
+from django_stubs_ext import StrPromise
+
+from zerver.lib.streams import get_content_access_streams
+from zerver.lib.topic import get_latest_message_for_user_in_topic
+from zerver.lib.types import UserDisplayRecipient
+from zerver.lib.user_groups import (
+    UserGroupMembershipDetails,
+    get_root_id_annotated_recursive_subgroups_for_groups,
+    user_has_permission_for_group_setting,
+)
+from zerver.lib.users import get_inaccessible_user_ids
+from zerver.models import NamedUserGroup, UserProfile
+from zerver.models.groups import SystemGroups
+from zerver.models.streams import Stream
+from zerver.models.users import is_cross_realm_bot_email
+
+BEFORE_MENTION_ALLOWED_REGEX = r"(?<![^\s\'\"\(\{\[\/<])"
+
+# For mentions that generate a link of their own, don't match syntax
+# that is formatted with another link. e.g., "[#**channel**]({link})"
+BEFORE_LINK_PRODUCING_MENTION_ALLOWED_REGEX = r"(?<![^\s\'\"\(\{\/<])"
+
+# Match multi-word string between @** ** or match any one-word
+# sequences after @
+MENTIONS_RE = re.compile(
+    rf"{BEFORE_MENTION_ALLOWED_REGEX}@(?P<silent>_?)(\*\*(?P<match>[^\*]+)\*\*)"
+)
+USER_GROUP_MENTIONS_RE = re.compile(
+    rf"{BEFORE_MENTION_ALLOWED_REGEX}@(?P<silent>_?)(\*(?P<match>[^\*]+)\*)"
+)
+
+topic_wildcards = frozenset(["topic"])
+stream_wildcards = frozenset(["all", "everyone", "stream", "channel"])
+
+
+@dataclass
+class FullNameInfo:
+    id: int
+    full_name: str
+    is_active: bool
+
+
+@dataclass
+class UserFilter:
+    id: int | None
+    full_name: str | None
+
+    def Q(self) -> Q:
+        if self.full_name is not None and self.id is not None:
+            return Q(full_name__iexact=self.full_name, id=self.id)
+        elif self.id is not None:
+            return Q(id=self.id)
+        elif self.full_name is not None:
+            return Q(full_name__iexact=self.full_name)
+        else:
+            raise AssertionError("totally empty filter makes no sense")
+
+
+@dataclass
+class MentionText:
+    text: str | None
+    is_topic_wildcard: bool
+    is_stream_wildcard: bool
+
+
+@dataclass
+class PossibleMentions:
+    mention_texts: set[str]
+    message_has_topic_wildcards: bool
+    message_has_stream_wildcards: bool
+
+
+@dataclass(frozen=True)
+class ChannelTopicInfo:
+    channel_name: str
+    topic_name: str
+
+
+@dataclass
+class ChannelInfo:
+    channel_id: int
+    recipient_id: int
+    history_public_to_subscribers: bool
+    # TODO: Track whether the current user has only metadata access or
+    # content access, so that we can allow mentioning channels with
+    # only metadata access, while still enforcing content access to
+    # mention topics or messages within channels.
+
+
+class MentionBackend:
+    # Be careful about reuse: MentionBackend contains caches which are
+    # designed to only have the lifespan of a sender user (typically a
+    # single request).
+    #
+    # In particular, user_cache is not robust to message_sender
+    # within the lifetime of a single MentionBackend lifetime.
+
+    def __init__(self, realm_id: int) -> None:
+        self.realm_id = realm_id
+        self.user_cache: dict[tuple[int, str], FullNameInfo] = {}
+        self.stream_cache: dict[str, ChannelInfo] = {}
+        self.topic_cache: dict[ChannelTopicInfo, int | None] = {}
+
+    def get_full_name_info_list(
+        self, user_filters: list[UserFilter], message_sender: UserProfile | None
+    ) -> list[FullNameInfo]:
+        result: list[FullNameInfo] = []
+        unseen_user_filters: list[UserFilter] = []
+
+        # Try to get messages from the user_cache first.
+        # This loop populates two lists:
+        #  - results are the objects we pull from cache
+        #  - unseen_user_filters are filters where need to hit the DB
+        for user_filter in user_filters:
+            # We expect callers who take advantage of our user_cache to supply both
+            # id and full_name in the user mentions in their messages.
+            if user_filter.id is not None and user_filter.full_name is not None:
+                user = self.user_cache.get((user_filter.id, user_filter.full_name), None)
+                if user is not None:
+                    result.append(user)
+                    continue
+
+            # BOO! We have to go the database.
+            unseen_user_filters.append(user_filter)
+
+        # Most of the time, we have to go to the database to get user info,
+        # unless our last loop found everything in the cache.
+        if unseen_user_filters:
+            q_list = [user_filter.Q() for user_filter in unseen_user_filters]
+
+            rows = (
+                UserProfile.objects.filter(
+                    Q(realm_id=self.realm_id) | Q(email__in=settings.CROSS_REALM_BOT_EMAILS),
+                )
+                .filter(
+                    functools.reduce(lambda a, b: a | b, q_list),
+                )
+                .only(
+                    "id",
+                    "full_name",
+                    "is_active",
+                )
+            )
+
+            possible_mention_user_ids = [row.id for row in rows]
+            inaccessible_user_ids = get_inaccessible_user_ids(
+                possible_mention_user_ids, message_sender
+            )
+
+            user_list = [
+                FullNameInfo(id=row.id, full_name=row.full_name, is_active=row.is_active)
+                for row in rows
+                if row.id not in inaccessible_user_ids
+            ]
+
+            # We expect callers who take advantage of our cache to supply both
+            # id and full_name in the user mentions in their messages.
+            for user in user_list:
+                self.user_cache[(user.id, user.full_name)] = user
+
+            result += user_list
+
+        return result
+
+    def get_stream_name_map(
+        self, stream_names: set[str], acting_user: UserProfile | None
+    ) -> dict[str, int]:
+        if not stream_names:
+            return {}
+
+        result: dict[str, int] = {}
+        unseen_stream_names: list[str] = []
+
+        for stream_name in stream_names:
+            if stream_name in self.stream_cache:
+                result[stream_name] = self.stream_cache[stream_name].channel_id
+            else:
+                unseen_stream_names.append(stream_name)
+
+        if not unseen_stream_names:
+            return result
+
+        q_list = {Q(name=name) for name in unseen_stream_names}
+        if acting_user is None:
+            rows = (
+                Stream.objects.filter(
+                    realm_id=self.realm_id,
+                )
+                .filter(
+                    functools.reduce(lambda a, b: a | b, q_list),
+                )
+                .values(
+                    "id",
+                    "name",
+                    "recipient_id",
+                    "history_public_to_subscribers",
+                )
+            )
+            for row in rows:
+                self.stream_cache[row["name"]] = ChannelInfo(
+                    row["id"], row["recipient_id"], row["history_public_to_subscribers"]
+                )
+                result[row["name"]] = row["id"]
+        else:
+            content_access_streams = get_content_access_streams(
+                acting_user,
+                list(
+                    Stream.objects.filter(
+                        realm_id=self.realm_id,
+                    ).filter(
+                        functools.reduce(lambda a, b: a | b, q_list),
+                    )
+                ),
+                user_group_membership_details=UserGroupMembershipDetails(
+                    user_recursive_group_ids=None
+                ),
+            )
+            for stream in content_access_streams:
+                assert stream.recipient_id is not None
+                self.stream_cache[stream.name] = ChannelInfo(
+                    stream.id, stream.recipient_id, stream.history_public_to_subscribers
+                )
+                result[stream.name] = stream.id
+
+        return result
+
+    def get_topic_info_map(
+        self, channel_topics: set[ChannelTopicInfo], acting_user: UserProfile | None
+    ) -> dict[ChannelTopicInfo, int | None]:
+        if not channel_topics:
+            return {}
+
+        result: dict[ChannelTopicInfo, int | None] = {}
+        unseen_channel_topic: list[ChannelTopicInfo] = []
+
+        for channel_topic in channel_topics:
+            if channel_topic in self.topic_cache:
+                result[channel_topic] = self.topic_cache[channel_topic]
+            else:
+                unseen_channel_topic.append(channel_topic)
+
+        for channel_topic in unseen_channel_topic:
+            channel_info = self.stream_cache.get(channel_topic.channel_name)
+
+            if channel_info is None:
+                # The acting user does not have access to content in this channel.
+                continue
+
+            recipient_id = channel_info.recipient_id
+            topic_name = channel_topic.topic_name
+            history_public_to_subscribers = channel_info.history_public_to_subscribers
+
+            # Any message in the topic is a valid choice for the
+            # /with/ anchor. There are two risks to manage here:
+            # - The target message could be deleted or be a mispost that
+            #   is off-topic and moved shortly.
+            # - The topic could be split into two topics.
+            #
+            # Originally, we picked the oldest message because that
+            # message is least likely to be deleted/moved for being a
+            # mispost/error -- i.e., trying to do a bit better in rare
+            # corner cases. We switched to preferring the latest
+            # message in API feature level 400. The "latest" algorithm
+            # is better when linking to an active conversation in a
+            # long topic that's a follow-up/tangent and ends up being
+            # moved/split, which users do constantly.
+            topic_latest_message = get_latest_message_for_user_in_topic(
+                self.realm_id,
+                acting_user,
+                recipient_id,
+                topic_name,
+                history_public_to_subscribers,
+                acting_user_has_channel_content_access=True,
+            )
+
+            self.topic_cache[channel_topic] = topic_latest_message
+            result[channel_topic] = topic_latest_message
+
+        return result
+
+
+def user_mention_matches_topic_wildcard(mention: str) -> bool:
+    return mention in topic_wildcards
+
+
+def user_mention_matches_stream_wildcard(mention: str) -> bool:
+    return mention in stream_wildcards
+
+
+def extract_mention_text(m: Match[str]) -> MentionText:
+    text = m.group("match")
+    if text in topic_wildcards:
+        return MentionText(text=None, is_topic_wildcard=True, is_stream_wildcard=False)
+    if text in stream_wildcards:
+        return MentionText(text=None, is_topic_wildcard=False, is_stream_wildcard=True)
+    return MentionText(text=text, is_topic_wildcard=False, is_stream_wildcard=False)
+
+
+def possible_mentions(content: str) -> PossibleMentions:
+    # mention texts can either be names, or an extended name|id syntax.
+    texts = set()
+    message_has_topic_wildcards = False
+    message_has_stream_wildcards = False
+    for m in MENTIONS_RE.finditer(content):
+        mention_text = extract_mention_text(m)
+        text = mention_text.text
+        if text:
+            texts.add(text)
+        if mention_text.is_topic_wildcard:
+            message_has_topic_wildcards = True
+        if mention_text.is_stream_wildcard:
+            message_has_stream_wildcards = True
+    return PossibleMentions(
+        mention_texts=texts,
+        message_has_topic_wildcards=message_has_topic_wildcards,
+        message_has_stream_wildcards=message_has_stream_wildcards,
+    )
+
+
+def possible_user_group_mentions(content: str) -> dict[str, Literal["silent", "non-silent"]]:
+    # maps each group name to its mention type, silent or non-silent.
+    mentions: dict[str, Literal["silent", "non-silent"]] = {}
+
+    for mention in USER_GROUP_MENTIONS_RE.finditer(content):
+        group_mention = mention.group("match")
+
+        # non-silent mention can override silent.
+        if not mention.group("silent"):
+            mentions[group_mention] = "non-silent"
+
+        # silent mention should NOT override non-silent.
+        if mention.group("silent") and group_mention not in mentions:
+            mentions[group_mention] = "silent"
+
+    return mentions
+
+
+def get_possible_mentions_info(
+    mention_backend: MentionBackend, mention_texts: set[str], message_sender: UserProfile | None
+) -> list[FullNameInfo]:
+    if not mention_texts:
+        return []
+
+    user_filters = list()
+
+    name_re = r"(?P<full_name>.+)?\|(?P<mention_id>\d+)$"
+    for mention_text in mention_texts:
+        name_syntax_match = re.match(name_re, mention_text)
+        if name_syntax_match:
+            full_name = name_syntax_match.group("full_name")
+            mention_id = name_syntax_match.group("mention_id")
+            if full_name:
+                # For **name|id** mentions as mention_id
+                # cannot be null inside this block.
+                user_filters.append(UserFilter(full_name=full_name, id=int(mention_id)))
+            else:
+                # For **|id** syntax.
+                user_filters.append(UserFilter(full_name=None, id=int(mention_id)))
+        else:
+            # For **name** syntax.
+            user_filters.append(UserFilter(full_name=mention_text, id=None))
+
+    return mention_backend.get_full_name_info_list(user_filters, message_sender)
+
+
+class MentionData:
+    def __init__(
+        self, mention_backend: MentionBackend, content: str, message_sender: UserProfile | None
+    ) -> None:
+        self.mention_backend = mention_backend
+        realm_id = mention_backend.realm_id
+        self.message_sender = message_sender
+        mentions = possible_mentions(content)
+        possible_mentions_info = get_possible_mentions_info(
+            mention_backend, mentions.mention_texts, message_sender
+        )
+        self.full_name_info = {row.full_name.lower(): row for row in possible_mentions_info}
+        self.user_id_info = {row.id: row for row in possible_mentions_info}
+        self.init_user_group_data(realm_id=realm_id, content=content)
+        self.has_stream_wildcards = mentions.message_has_stream_wildcards
+        self.has_topic_wildcards = mentions.message_has_topic_wildcards
+
+    def message_has_stream_wildcards(self) -> bool:
+        return self.has_stream_wildcards
+
+    def message_has_topic_wildcards(self) -> bool:
+        return self.has_topic_wildcards
+
+    def init_user_group_data(self, realm_id: int, content: str) -> None:
+        self.user_group_name_info: dict[str, NamedUserGroup] = {}
+        self.user_group_members: dict[int, set[int]] = defaultdict(set)
+        user_group_names_mentions = possible_user_group_mentions(content)
+        if user_group_names_mentions:
+            named_user_groups = NamedUserGroup.objects.filter(
+                realm_for_sharding_id=realm_id, name__in=user_group_names_mentions
+            )
+
+            # No filter here as we need user_group_name_info for all groups mentions.
+            self.user_group_name_info = {group.name.lower(): group for group in named_user_groups}
+
+            # We only fetch group membership mentions that can
+            # possibly trigger notifications.
+            filtered_group_ids = [
+                group.id
+                for group in named_user_groups
+                if not group.deactivated
+                and user_group_names_mentions.get(group.name) == "non-silent"
+            ]
+
+            # Avoid doing a database query if there's nothing to fetch.
+            #
+            # This isn't quite optimal -- we've not checked our user
+            # has permission to mention the group yet.
+            if len(filtered_group_ids) == 0:
+                return
+
+            # Fetch membership for the groups filtered above in a
+            # single, efficient bulk query, mapping each group to its
+            # direct and indirect members.
+            for group_root_id, member_id in (
+                get_root_id_annotated_recursive_subgroups_for_groups(filtered_group_ids, realm_id)
+                .filter(direct_members__is_active=True)
+                .values_list("root_id", "direct_members")  # type: ignore[misc]  # root_id is an annotated field.
+            ):
+                self.user_group_members[group_root_id].add(member_id)
+
+    def get_user_by_name(self, name: str) -> FullNameInfo | None:
+        # warning: get_user_by_name is not dependable if two
+        # users of the same full name are mentioned. Use
+        # get_user_by_id where possible.
+        return self.full_name_info.get(name.lower(), None)
+
+    def get_user_by_id(self, id: int) -> FullNameInfo | None:
+        return self.user_id_info.get(id, None)
+
+    def get_user_ids(self) -> set[int]:
+        """
+        Returns the user IDs that might have been mentioned by this
+        content.  Note that because this data structure has not parsed
+        the message and does not know about escaping/code blocks, this
+        will overestimate the list of user ids.
+        """
+        return set(self.user_id_info.keys())
+
+    def get_user_group(self, name: str) -> NamedUserGroup | None:
+        return self.user_group_name_info.get(name.lower(), None)
+
+    def get_group_members(self, user_group_id: int) -> set[int]:
+        return self.user_group_members.get(user_group_id, set())
+
+    def get_stream_name_map(
+        self, stream_names: set[str], acting_user: UserProfile | None
+    ) -> dict[str, int]:
+        return self.mention_backend.get_stream_name_map(stream_names, acting_user=acting_user)
+
+    def get_topic_info_map(
+        self, channel_topics: set[ChannelTopicInfo], acting_user: UserProfile | None
+    ) -> dict[ChannelTopicInfo, int | None]:
+        return self.mention_backend.get_topic_info_map(channel_topics, acting_user=acting_user)
+
+
+def silent_mention_syntax_for_user(user_profile: UserProfile | UserDisplayRecipient) -> str:
+    if isinstance(user_profile, UserProfile):
+        return f"@_**{user_profile.full_name}|{user_profile.id}**"
+    else:
+        return f"@_**{user_profile['full_name']}|{user_profile['id']}**"
+
+
+def silent_mention_syntax_for_user_group(user_group: NamedUserGroup) -> str:
+    return f"@_*{user_group.name}*"
+
+
+def get_user_group_mention_display_name(user_group: NamedUserGroup) -> StrPromise | str:
+    if user_group.is_system_group:
+        return SystemGroups.GROUP_DISPLAY_NAME_MAP[user_group.name]
+
+    return user_group.name
+
+
+def sender_can_mention_group(sender: UserProfile | None, named_group: NamedUserGroup) -> bool:
+    can_mention_group = named_group.can_mention_group
+
+    if (
+        hasattr(can_mention_group, "named_user_group")
+        and can_mention_group.named_user_group.name == SystemGroups.EVERYONE
+    ):
+        return True
+
+    assert sender is not None
+
+    if is_cross_realm_bot_email(sender.delivery_email):
+        return False
+
+    return user_has_permission_for_group_setting(
+        can_mention_group.id,
+        sender,
+        NamedUserGroup.GROUP_PERMISSION_SETTINGS["can_mention_group"],
+        direct_member_only=False,
+    )

zerver/lib/message_cache.py

@@ -0,0 +1,594 @@
+import copy
+from collections.abc import Iterable, Mapping
+from datetime import datetime
+from email.headerregistry import Address
+from typing import Any, TypedDict
+
+import orjson
+
+from zerver.lib.avatar import get_avatar_field, get_avatar_for_inaccessible_user
+from zerver.lib.cache import cache_set_many, cache_with_key, to_dict_cache_key, to_dict_cache_key_id
+from zerver.lib.display_recipient import bulk_fetch_display_recipients
+from zerver.lib.markdown import render_message_markdown, topic_links
+from zerver.lib.markdown import version as markdown_version
+from zerver.lib.query_helpers import query_for_ids
+from zerver.lib.timestamp import datetime_to_timestamp
+from zerver.lib.topic import DB_TOPIC_NAME, TOPIC_LINKS, TOPIC_NAME
+from zerver.lib.types import DisplayRecipientT, EditHistoryEvent, UserDisplayRecipient
+from zerver.models import Message, Reaction, Realm, Recipient, Stream, SubMessage, UserProfile
+from zerver.models.realms import get_fake_email_domain
+
+
+class RawReactionRow(TypedDict):
+    emoji_code: str
+    emoji_name: str
+    message_id: int
+    reaction_type: str
+    user_profile_id: int
+
+
+def sew_messages_and_reactions(
+    messages: list[dict[str, Any]], reactions: list[dict[str, Any]]
+) -> list[dict[str, Any]]:
+    """Given an iterable of messages and reactions stitch reactions
+    into messages.
+    """
+    # Add all messages with empty reaction item
+    for message in messages:
+        message["reactions"] = []
+
+    # Convert list of messages into dictionary to make reaction stitching easy
+    converted_messages = {message["id"]: message for message in messages}
+
+    for reaction in reactions:
+        converted_messages[reaction["message_id"]]["reactions"].append(reaction)
+
+    return list(converted_messages.values())
+
+
+def sew_messages_and_submessages(
+    messages: list[dict[str, Any]], submessages: list[dict[str, Any]]
+) -> None:
+    # This is super similar to sew_messages_and_reactions.
+    for message in messages:
+        message["submessages"] = []
+
+    message_dict = {message["id"]: message for message in messages}
+
+    for submessage in submessages:
+        message_id = submessage["message_id"]
+        if message_id in message_dict:
+            message = message_dict[message_id]
+            message["submessages"].append(submessage)
+
+
+def extract_message_dict(message_bytes: bytes) -> dict[str, Any]:
+    return orjson.loads(message_bytes)
+
+
+def stringify_message_dict(message_dict: dict[str, Any]) -> bytes:
+    return orjson.dumps(message_dict)
+
+
+@cache_with_key(to_dict_cache_key, timeout=3600 * 24, pickled_tupled=False)
+def message_to_encoded_cache(message: Message, realm_id: int | None = None) -> bytes:
+    return MessageDict.messages_to_encoded_cache([message], realm_id)[message.id]
+
+
+def update_message_cache(
+    changed_messages: Iterable[Message], realm_id: int | None = None
+) -> list[int]:
+    """Updates the message as stored in the to_dict cache (for serving
+    messages)."""
+    items_for_remote_cache = {}
+    changed_messages_to_dict = MessageDict.messages_to_encoded_cache(changed_messages, realm_id)
+    for msg_id, msg in changed_messages_to_dict.items():
+        items_for_remote_cache[to_dict_cache_key_id(msg_id)] = msg
+
+    cache_set_many(items_for_remote_cache)
+    return list(changed_messages_to_dict.keys())
+
+
+def save_message_rendered_content(message: Message, content: str) -> str:
+    rendering_result = render_message_markdown(message, content, realm=message.get_realm())
+    rendered_content = None
+    if rendering_result is not None:
+        rendered_content = rendering_result.rendered_content
+    message.rendered_content = rendered_content
+    message.rendered_content_version = markdown_version
+    message.save_rendered_content()
+    return rendered_content
+
+
+class ReactionDict:
+    @staticmethod
+    def build_dict_from_raw_db_row(row: RawReactionRow) -> dict[str, Any]:
+        return {
+            "emoji_name": row["emoji_name"],
+            "emoji_code": row["emoji_code"],
+            "reaction_type": row["reaction_type"],
+            "user_id": row["user_profile_id"],
+        }
+
+
+class MessageDict:
+    """MessageDict is the core class responsible for marshalling Message
+    objects obtained from the database into a format that can be sent
+    to clients via the Zulip API, whether via `GET /messages`,
+    outgoing webhooks, or other code paths.  There are two core flows through
+    which this class is used:
+
+    * For just-sent messages, we construct a single `wide_dict` object
+      containing all the data for the message and the related
+      UserProfile models (sender_info and recipient_info); this object
+      can be stored in queues, caches, etc., and then later turned
+      into an API-format JSONable dictionary via finalize_payload.
+
+    * When fetching messages from the database, we fetch their data in
+      bulk using messages_for_ids, which makes use of caching, bulk
+      fetches that skip the Django ORM, etc., to provide an optimized
+      interface for fetching hundreds of thousands of messages from
+      the database and then turning them into API-format JSON
+      dictionaries.
+
+    """
+
+    @staticmethod
+    def wide_dict(
+        message: Message,
+        realm_id: int | None = None,
+        *,
+        dm_involved_user_ids: set[int] | None = None,
+    ) -> dict[str, Any]:
+        """
+        The next two lines get the cacheable field related
+        to our message object, with the side effect of
+        populating the cache.
+        """
+        encoded_object_bytes = message_to_encoded_cache(message, realm_id)
+        obj = extract_message_dict(encoded_object_bytes)
+
+        """
+        The steps below are similar to what we do in
+        post_process_dicts(), except we don't call finalize_payload(),
+        since that step happens later in the queue
+        processor.
+        """
+        MessageDict.bulk_hydrate_sender_info([obj])
+        MessageDict.bulk_hydrate_recipient_info(
+            [obj],
+            {} if dm_involved_user_ids is None else {obj["recipient_id"]: dm_involved_user_ids},
+        )
+
+        return obj
+
+    @staticmethod
+    def post_process_dicts(
+        objs: list[dict[str, Any]],
+        *,
+        apply_markdown: bool,
+        client_gravatar: bool,
+        allow_empty_topic_name: bool,
+        realm: Realm,
+    ) -> None:
+        """
+        NOTE: This function mutates the objects in
+              the `objs` list, rather than making
+              shallow copies.  It might be safer to
+              make shallow copies here, but performance
+              is somewhat important here, as we are
+              often fetching hundreds of messages.
+        """
+        MessageDict.bulk_hydrate_sender_info(objs)
+        MessageDict.bulk_hydrate_recipient_info(objs)
+
+        for obj in objs:
+            can_access_sender = obj.get("can_access_sender", True)
+            MessageDict.finalize_payload(
+                obj,
+                apply_markdown=apply_markdown,
+                client_gravatar=client_gravatar,
+                allow_empty_topic_name=allow_empty_topic_name,
+                skip_copy=True,
+                can_access_sender=can_access_sender,
+                realm_host=realm.host,
+            )
+
+    @staticmethod
+    def finalize_payload(
+        obj: dict[str, Any],
+        *,
+        apply_markdown: bool,
+        client_gravatar: bool,
+        allow_empty_topic_name: bool,
+        keep_rendered_content: bool = False,
+        skip_copy: bool = False,
+        can_access_sender: bool,
+        realm_host: str,
+    ) -> dict[str, Any]:
+        """
+        By default, we make a shallow copy of the incoming dict to avoid
+        mutation-related bugs.  Code paths that are passing a unique object
+        can pass skip_copy=True to avoid this extra work.
+        """
+        if not skip_copy:
+            obj = copy.copy(obj)
+
+        # Compatibility code to change topic="" to topic=Message.EMPTY_TOPIC_FALLBACK_NAME
+        # for older clients with no support for empty topic name.
+        if (
+            obj["recipient_type"] == Recipient.STREAM
+            and obj["subject"] == ""
+            and not allow_empty_topic_name
+        ):
+            obj["subject"] = Message.EMPTY_TOPIC_FALLBACK_NAME
+
+        if obj["sender_email_address_visibility"] != UserProfile.EMAIL_ADDRESS_VISIBILITY_EVERYONE:
+            # If email address of the sender is only available to administrators,
+            # clients cannot compute gravatars, so we force-set it to false.
+            # If we plumbed the current user's role, we could allow client_gravatar=True
+            # here if the current user's role has access to the target user's email address.
+            client_gravatar = False
+
+        if not can_access_sender:
+            # Enforce inability to access details of inaccessible
+            # users. We should be able to remove the realm_host and
+            # can_access_user plumbing to this function if/when we
+            # shift the Zulip API to not send these denormalized
+            # fields about message senders favor of just sending the
+            # sender's user ID.
+            obj["sender_full_name"] = str(UserProfile.INACCESSIBLE_USER_NAME)
+            sender_id = obj["sender_id"]
+            obj["sender_email"] = Address(
+                username=f"user{sender_id}", domain=get_fake_email_domain(realm_host)
+            ).addr_spec
+
+        MessageDict.set_sender_avatar(obj, client_gravatar, can_access_sender)
+        if apply_markdown:
+            obj["content_type"] = "text/html"
+            obj["content"] = obj["rendered_content"]
+        else:
+            obj["content_type"] = "text/x-markdown"
+
+        for item in obj.get("edit_history", []):
+            if "prev_rendered_content_version" in item:
+                del item["prev_rendered_content_version"]
+            if not allow_empty_topic_name:
+                if "prev_topic" in item and item["prev_topic"] == "":
+                    item["prev_topic"] = Message.EMPTY_TOPIC_FALLBACK_NAME
+                if "topic" in item and item["topic"] == "":
+                    item["topic"] = Message.EMPTY_TOPIC_FALLBACK_NAME
+
+        if not keep_rendered_content:
+            del obj["rendered_content"]
+        del obj["sender_realm_id"]
+        del obj["sender_avatar_source"]
+        del obj["sender_delivery_email"]
+        del obj["sender_avatar_version"]
+
+        del obj["recipient_type"]
+        del obj["recipient_type_id"]
+        del obj["sender_is_mirror_dummy"]
+        del obj["sender_email_address_visibility"]
+        if "can_access_sender" in obj:
+            del obj["can_access_sender"]
+        return obj
+
+    @staticmethod
+    def sew_submessages_and_reactions_to_msgs(
+        messages: list[dict[str, Any]],
+    ) -> list[dict[str, Any]]:
+        msg_ids = [msg["id"] for msg in messages]
+        submessages = SubMessage.get_raw_db_rows(msg_ids)
+        sew_messages_and_submessages(messages, submessages)
+
+        reactions = Reaction.get_raw_db_rows(msg_ids)
+        return sew_messages_and_reactions(messages, reactions)
+
+    @staticmethod
+    def messages_to_encoded_cache(
+        messages: Iterable[Message], realm_id: int | None = None
+    ) -> dict[int, bytes]:
+        messages_dict = MessageDict.messages_to_encoded_cache_helper(messages, realm_id)
+        encoded_messages = {msg["id"]: stringify_message_dict(msg) for msg in messages_dict}
+        return encoded_messages
+
+    @staticmethod
+    def messages_to_encoded_cache_helper(
+        messages: Iterable[Message], realm_id: int | None = None
+    ) -> list[dict[str, Any]]:
+        # Near duplicate of the build_message_dict + get_raw_db_rows
+        # code path that accepts already fetched Message objects
+        # rather than message IDs.
+
+        def get_rendering_realm_id(message: Message) -> int:
+            # realm_id can differ among users, currently only possible
+            # with cross realm bots.
+            if realm_id is not None:
+                return realm_id
+            if message.recipient.type == Recipient.STREAM:
+                return Stream.objects.get(id=message.recipient.type_id).realm_id
+            return message.realm_id
+
+        message_rows = [
+            {
+                "id": message.id,
+                DB_TOPIC_NAME: message.topic_name(),
+                "date_sent": message.date_sent,
+                "last_edit_time": message.last_edit_time,
+                "edit_history": message.edit_history,
+                "content": message.content,
+                "rendered_content": message.rendered_content,
+                "rendered_content_version": message.rendered_content_version,
+                "recipient_id": message.recipient.id,
+                "recipient__type": message.recipient.type,
+                "recipient__type_id": message.recipient.type_id,
+                "rendering_realm_id": get_rendering_realm_id(message),
+                "sender_id": message.sender.id,
+                "sending_client__name": message.sending_client.name,
+                "sender__realm_id": message.sender.realm_id,
+            }
+            for message in messages
+        ]
+
+        MessageDict.sew_submessages_and_reactions_to_msgs(message_rows)
+        return [MessageDict.build_dict_from_raw_db_row(row) for row in message_rows]
+
+    @staticmethod
+    def ids_to_dict(needed_ids: list[int]) -> list[dict[str, Any]]:
+        # This is a special purpose function optimized for
+        # callers like get_messages_backend().
+        fields = [
+            "id",
+            DB_TOPIC_NAME,
+            "date_sent",
+            "last_edit_time",
+            "edit_history",
+            "content",
+            "rendered_content",
+            "rendered_content_version",
+            "recipient_id",
+            "recipient__type",
+            "recipient__type_id",
+            "sender_id",
+            "sending_client__name",
+            "sender__realm_id",
+        ]
+        # Uses index: zerver_message_pkey
+        messages = Message.objects.filter(id__in=needed_ids).values(*fields)
+        MessageDict.sew_submessages_and_reactions_to_msgs(messages)
+        return [MessageDict.build_dict_from_raw_db_row(row) for row in messages]
+
+    @staticmethod
+    def build_dict_from_raw_db_row(row: dict[str, Any]) -> dict[str, Any]:
+        """
+        row is a row from a .values() call, and it needs to have
+        all the relevant fields populated
+        """
+
+        def get_message_topic(row: dict[str, Any]) -> str:
+            if row["recipient__type"] == Recipient.STREAM:
+                return row[DB_TOPIC_NAME]
+            return ""
+
+        return MessageDict.build_message_dict(
+            message_id=row["id"],
+            last_edit_time=row["last_edit_time"],
+            edit_history_json=row["edit_history"],
+            content=row["content"],
+            topic_name=get_message_topic(row),
+            date_sent=row["date_sent"],
+            rendered_content=row["rendered_content"],
+            rendered_content_version=row["rendered_content_version"],
+            sender_id=row["sender_id"],
+            sender_realm_id=row["sender__realm_id"],
+            sending_client_name=row["sending_client__name"],
+            rendering_realm_id=row.get("rendering_realm_id", row["sender__realm_id"]),
+            recipient_id=row["recipient_id"],
+            recipient_type=row["recipient__type"],
+            recipient_type_id=row["recipient__type_id"],
+            reactions=row["reactions"],
+            submessages=row["submessages"],
+        )
+
+    @staticmethod
+    def build_message_dict(
+        message_id: int,
+        last_edit_time: datetime | None,
+        edit_history_json: str | None,
+        content: str,
+        topic_name: str,
+        date_sent: datetime,
+        rendered_content: str | None,
+        rendered_content_version: int | None,
+        sender_id: int,
+        sender_realm_id: int,
+        sending_client_name: str,
+        rendering_realm_id: int,
+        recipient_id: int,
+        recipient_type: int,
+        recipient_type_id: int,
+        reactions: list[RawReactionRow],
+        submessages: list[dict[str, Any]],
+    ) -> dict[str, Any]:
+        obj = dict(
+            id=message_id,
+            sender_id=sender_id,
+            content=content,
+            recipient_type_id=recipient_type_id,
+            recipient_type=recipient_type,
+            recipient_id=recipient_id,
+            timestamp=datetime_to_timestamp(date_sent),
+            client=sending_client_name,
+        )
+
+        obj[TOPIC_NAME] = topic_name
+        obj["sender_realm_id"] = sender_realm_id
+
+        # Render topic_links with the stream's realm instead of the
+        # sender's realm; this is important for messages sent by
+        # cross-realm bots like NOTIFICATION_BOT.
+        obj[TOPIC_LINKS] = topic_links(rendering_realm_id, topic_name)
+
+        if last_edit_time is not None:
+            obj["last_edit_timestamp"] = datetime_to_timestamp(last_edit_time)
+            assert edit_history_json is not None
+            edit_history: list[EditHistoryEvent] = orjson.loads(edit_history_json)
+            obj["edit_history"] = edit_history
+
+        if Message.need_to_render_content(
+            rendered_content, rendered_content_version, markdown_version
+        ):
+            # We really shouldn't be rendering objects in this method, but there is
+            # a scenario where we upgrade the version of Markdown and fail to run
+            # management commands to re-render historical messages, and then we
+            # need to have side effects.  This method is optimized to not need full
+            # blown ORM objects, but the Markdown renderer is unfortunately highly
+            # coupled to Message, and we also need to persist the new rendered content.
+            # If we don't have a message object passed in, we get one here.  The cost
+            # of going to the DB here should be overshadowed by the cost of rendering
+            # and updating the row.
+            # TODO: see #1379 to eliminate Markdown dependencies
+            message = Message.objects.select_related("sender").get(id=message_id)
+
+            assert message is not None  # Hint for mypy.
+            # It's unfortunate that we need to have side effects on the message
+            # in some cases.
+            rendered_content = save_message_rendered_content(message, content)
+
+        if rendered_content is not None:
+            obj["rendered_content"] = rendered_content
+        else:
+            obj["rendered_content"] = (
+                "<p>[Zulip note: Sorry, we could not understand the formatting of your message]</p>"
+            )
+
+        if rendered_content is not None:
+            obj["is_me_message"] = Message.is_status_message(content, rendered_content)
+        else:
+            obj["is_me_message"] = False
+
+        obj["reactions"] = [
+            ReactionDict.build_dict_from_raw_db_row(reaction) for reaction in reactions
+        ]
+        obj["submessages"] = submessages
+        return obj
+
+    @staticmethod
+    def bulk_hydrate_sender_info(objs: list[dict[str, Any]]) -> None:
+        sender_ids = list({obj["sender_id"] for obj in objs})
+
+        if not sender_ids:
+            return
+
+        query = UserProfile.objects.values(
+            "id",
+            "full_name",
+            "delivery_email",
+            "email",
+            "realm__string_id",
+            "avatar_source",
+            "avatar_version",
+            "is_mirror_dummy",
+            "email_address_visibility",
+        )
+
+        rows = query_for_ids(query, sender_ids, "zerver_userprofile.id")
+
+        sender_dict = {row["id"]: row for row in rows}
+
+        for obj in objs:
+            sender_id = obj["sender_id"]
+            user_row = sender_dict[sender_id]
+            obj["sender_full_name"] = user_row["full_name"]
+            obj["sender_email"] = user_row["email"]
+            obj["sender_delivery_email"] = user_row["delivery_email"]
+            obj["sender_realm_str"] = user_row["realm__string_id"]
+            obj["sender_avatar_source"] = user_row["avatar_source"]
+            obj["sender_avatar_version"] = user_row["avatar_version"]
+            obj["sender_is_mirror_dummy"] = user_row["is_mirror_dummy"]
+            obj["sender_email_address_visibility"] = user_row["email_address_visibility"]
+
+    @staticmethod
+    def hydrate_recipient_info(obj: dict[str, Any], display_recipient: DisplayRecipientT) -> None:
+        """
+        This method hyrdrates recipient info with things
+        like full names and emails of senders.  Eventually
+        our clients should be able to hyrdrate these fields
+        themselves with info they already have on users.
+        """
+
+        recipient_type = obj["recipient_type"]
+        recipient_type_id = obj["recipient_type_id"]
+        sender_is_mirror_dummy = obj["sender_is_mirror_dummy"]
+        sender_email = obj["sender_email"]
+        sender_full_name = obj["sender_full_name"]
+        sender_id = obj["sender_id"]
+
+        if recipient_type == Recipient.STREAM:
+            display_type = "stream"
+        elif recipient_type == Recipient.DIRECT_MESSAGE_GROUP:
+            assert not isinstance(display_recipient, str)
+            display_type = "private"
+            if len(display_recipient) == 1:
+                # add the sender in if this isn't a message between
+                # someone and themself, preserving ordering
+                recip: UserDisplayRecipient = {
+                    "email": sender_email,
+                    "full_name": sender_full_name,
+                    "id": sender_id,
+                    "is_mirror_dummy": sender_is_mirror_dummy,
+                }
+                if recip["email"] < display_recipient[0]["email"]:
+                    display_recipient = [recip, display_recipient[0]]
+                elif recip["email"] > display_recipient[0]["email"]:
+                    display_recipient = [display_recipient[0], recip]
+        else:
+            raise AssertionError(f"Invalid recipient type {recipient_type}")
+
+        obj["display_recipient"] = display_recipient
+        obj["type"] = display_type
+        if obj["type"] == "stream":
+            obj["stream_id"] = recipient_type_id
+
+    @staticmethod
+    def bulk_hydrate_recipient_info(
+        objs: list[dict[str, Any]],
+        recipient_dm_user_ids: Mapping[int, set[int]] = {},
+    ) -> None:
+        recipient_tuples = {  # We use set to eliminate duplicate tuples.
+            (
+                obj["recipient_id"],
+                obj["recipient_type"],
+                obj["recipient_type_id"],
+            )
+            for obj in objs
+        }
+        display_recipients = bulk_fetch_display_recipients(recipient_tuples, recipient_dm_user_ids)
+
+        for obj in objs:
+            MessageDict.hydrate_recipient_info(obj, display_recipients[obj["recipient_id"]])
+
+    @staticmethod
+    def set_sender_avatar(
+        obj: dict[str, Any], client_gravatar: bool, can_access_sender: bool = True
+    ) -> None:
+        if not can_access_sender:
+            obj["avatar_url"] = get_avatar_for_inaccessible_user()
+            return
+
+        sender_id = obj["sender_id"]
+        sender_realm_id = obj["sender_realm_id"]
+        sender_delivery_email = obj["sender_delivery_email"]
+        sender_avatar_source = obj["sender_avatar_source"]
+        sender_avatar_version = obj["sender_avatar_version"]
+
+        obj["avatar_url"] = get_avatar_field(
+            user_id=sender_id,
+            realm_id=sender_realm_id,
+            email=sender_delivery_email,
+            avatar_source=sender_avatar_source,
+            avatar_version=sender_avatar_version,
+            medium=False,
+            client_gravatar=client_gravatar,
+        )

zerver/lib/message_report.py

@@ -0,0 +1,198 @@
+from django.conf import settings
+from django.utils.translation import gettext as _
+from django.utils.translation import override as override_language
+
+from zerver.actions.message_send import internal_send_private_message, internal_send_stream_message
+from zerver.lib.display_recipient import get_display_recipient
+from zerver.lib.markdown.fenced_code import get_unused_fence
+from zerver.lib.mention import silent_mention_syntax_for_user
+from zerver.lib.message import (
+    get_user_mentions_for_display,
+    is_1_to_1_message,
+    is_message_to_self,
+    truncate_content,
+)
+from zerver.lib.timestamp import datetime_to_global_time
+from zerver.lib.topic_link_util import get_message_link_syntax
+from zerver.lib.url_encoding import pm_message_url, stream_message_url
+from zerver.models import Message, Realm, UserProfile
+from zerver.models.recipients import Recipient
+from zerver.models.streams import StreamTopicsPolicyEnum
+from zerver.models.users import get_system_bot
+
+# We shrink the truncate length for the reported message to ensure
+# that the "notes" included by the reporting user fit within the
+# limit. The extra 500 is an arbitrary buffer for all the extra
+# template strings.
+MAX_REPORT_MESSAGE_SNIPPET_LENGTH = (
+    settings.MAX_MESSAGE_LENGTH - Realm.MAX_REPORT_MESSAGE_EXPLANATION_LENGTH - 500
+)
+
+
+def send_message_report(
+    reporting_user: UserProfile,
+    realm: Realm,
+    reported_message: Message,
+    report_type: str,
+    description: str,
+) -> int | None:
+    moderation_request_channel = realm.moderation_request_channel
+    assert moderation_request_channel is not None
+
+    reported_user = reported_message.sender
+    reported_user_mention = silent_mention_syntax_for_user(reported_user)
+    reporting_user_mention = silent_mention_syntax_for_user(reporting_user)
+    report_reason = Realm.REPORT_MESSAGE_REASONS[report_type]
+    reported_message_date_sent = datetime_to_global_time(reported_message.date_sent)
+
+    # Build reported message header
+    if is_1_to_1_message(reported_message):
+        if reported_user != reporting_user:
+            report_header = _(
+                "{reporting_user_mention} reported a message sent by {reported_user_mention} at {reported_message_date_sent}."
+            ).format(
+                reporting_user_mention=reporting_user_mention,
+                reported_user_mention=reported_user_mention,
+                reported_message_date_sent=reported_message_date_sent,
+            )
+        else:
+            # Clearly mention the direct message recipient if someone is
+            # reporting their own message in a 1-on-1 direct message.
+            if not is_message_to_self(reported_message):
+                recipient_user = next(
+                    silent_mention_syntax_for_user(user)
+                    for user in get_display_recipient(reported_message.recipient)
+                    if user["id"] != reporting_user.id
+                )
+            else:
+                recipient_user = reporting_user_mention
+            report_header = _(
+                "{reporting_user_mention} reported a message sent by {reported_user_mention} to {recipient_user} at {reported_message_date_sent}."
+            ).format(
+                reporting_user_mention=reporting_user_mention,
+                reported_user_mention=reported_user_mention,
+                recipient_user=recipient_user,
+                reported_message_date_sent=reported_message_date_sent,
+            )
+    elif reported_message.recipient.type == Recipient.DIRECT_MESSAGE_GROUP:
+        report_header = _(
+            "{reporting_user_mention} reported a message sent by {reported_user_mention} to {list_of_direct_message_recipients} at {reported_message_date_sent}."
+        ).format(
+            reporting_user_mention=reporting_user_mention,
+            reported_user_mention=reported_user_mention,
+            list_of_direct_message_recipients=get_user_mentions_for_display(
+                [
+                    user
+                    for user in get_display_recipient(reported_message.recipient)
+                    if user["id"] != reported_user.id
+                ]
+            ),
+            reported_message_date_sent=reported_message_date_sent,
+        )
+    else:
+        assert reported_message.is_channel_message is True
+        topic_name = reported_message.topic_name()
+        channel_id = reported_message.recipient.type_id
+        channel_name = reported_message.recipient.label()
+        channel_message_link = get_message_link_syntax(
+            channel_id,
+            channel_name,
+            topic_name,
+            reported_message.id,
+        )
+        report_header = _(
+            "{reporting_user_mention} reported a message sent by {reported_user_mention} at {reported_message_date_sent}."
+        ).format(
+            reporting_user_mention=reporting_user_mention,
+            reported_user_mention=reported_user_mention,
+            reported_message_date_sent=reported_message_date_sent,
+        )
+
+    content = report_header
+
+    # Build report context and message preview block
+    if reported_message.is_channel_message:
+        original_message_string = _("Original message at {channel_message_link}").format(
+            channel_message_link=channel_message_link
+        )
+    else:
+        direct_message_link = pm_message_url(
+            realm,
+            dict(
+                id=reported_message.id,
+                display_recipient=get_display_recipient(reported_message.recipient),
+            ),
+        )
+        original_message_string = _("[Original message]({direct_message_link})").format(
+            direct_message_link=direct_message_link
+        )
+
+    reported_message_content = truncate_content(
+        reported_message.content, MAX_REPORT_MESSAGE_SNIPPET_LENGTH, "\n[message truncated]"
+    )
+    reported_message_preview_block = """
+```quote
+**{report_reason}**. {description}
+```
+
+{fence} spoiler {original_message_string}
+{reported_message}
+{fence}
+""".format(
+        report_reason=report_reason,
+        description=description,
+        original_message_string=original_message_string,
+        reported_message=reported_message_content,
+        fence=get_unused_fence(reported_message_content),
+    )
+    content += reported_message_preview_block
+
+    topic_name = _("{fullname} moderation").format(fullname=reported_user.full_name)
+    if moderation_request_channel.topics_policy == StreamTopicsPolicyEnum.empty_topic_only.value:
+        topic_name = ""
+
+    return internal_send_stream_message(
+        sender=get_system_bot(settings.NOTIFICATION_BOT, moderation_request_channel.realm.id),
+        stream=moderation_request_channel,
+        topic_name=topic_name,
+        content=content,
+        acting_user=reporting_user,
+    )
+
+
+def send_report_confirmation_dm(
+    reporting_user: UserProfile,
+    realm: Realm,
+    reported_message: Message,
+    report_type: str,
+) -> None:
+    if reported_message.is_channel_message:
+        message_url = stream_message_url(
+            realm=realm,
+            message=dict(
+                id=reported_message.id,
+                stream_id=reported_message.recipient.type_id,
+                display_recipient=reported_message.recipient.label(),
+                topic=reported_message.topic_name(),
+            ),
+        )
+    else:
+        message_url = pm_message_url(
+            realm,
+            dict(
+                id=reported_message.id,
+                display_recipient=get_display_recipient(reported_message.recipient),
+            ),
+        )
+
+    with override_language(reporting_user.default_language):
+        reason = str(Realm.REPORT_MESSAGE_REASONS[report_type]).lower()
+        content = _("You reported [a message]({message_url}) for {reason}.").format(
+            message_url=message_url, reason=reason
+        )
+
+    internal_send_private_message(
+        get_system_bot(settings.NOTIFICATION_BOT, realm.id),
+        reporting_user,
+        content,
+    )

zerver/lib/migrate.py

@@ -0,0 +1,139 @@
+import time
+from collections.abc import Callable
+from typing import Any
+
+from django.db import connection
+from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+from django.db.backends.utils import CursorWrapper
+from django.db.migrations.state import StateApps
+from psycopg2.sql import SQL, Composable, Identifier
+
+
+def do_batch_update(
+    cursor: CursorWrapper,
+    table: str,
+    assignments: list[Composable],
+    batch_size: int = 10000,
+    sleep: float = 0.1,
+) -> None:
+    # The string substitution below is complicated by our need to
+    # support multiple PostgreSQL versions.
+    stmt = SQL(
+        """
+        UPDATE {}
+        SET {}
+        WHERE id >= %s AND id < %s
+    """
+    ).format(
+        Identifier(table),
+        SQL(", ").join(assignments),
+    )
+
+    cursor.execute(SQL("SELECT MIN(id), MAX(id) FROM {}").format(Identifier(table)))
+    (min_id, max_id) = cursor.fetchone()
+    if min_id is None:
+        return
+
+    print(f"\n    Range of rows to update: [{min_id}, {max_id}]")
+    while min_id <= max_id:
+        lower = min_id
+        upper = min_id + batch_size
+        print(f"    Updating range [{lower},{upper})")
+        cursor.execute(stmt, [lower, upper])
+
+        min_id = upper
+        time.sleep(sleep)
+
+        # Once we've finished, check if any new rows were inserted to the table
+        if min_id > max_id:
+            cursor.execute(SQL("SELECT MAX(id) FROM {}").format(Identifier(table)))
+            (max_id,) = cursor.fetchone()
+
+    print("    Finishing...", end="")
+
+
+def rename_indexes_constraints(
+    old_table: str, new_table: str
+) -> Callable[[StateApps, BaseDatabaseSchemaEditor], None]:
+    # NOTE: `get_constraints`, which this uses, does not include any
+    # information about if the index name was manually set from
+    # Django, nor if it is a partial index.  This has the theoretical
+    # possibility to cause false positives in the duplicate-index
+    # check below, which would incorrectly drop one of the wanted
+    # indexes.
+    #
+    # Before using this on a table, ensure that it does not use
+    # partial indexes, nor manually-named indexes.
+    def inner_migration(apps: StateApps, schema_editor: Any) -> None:
+        seen_indexes = set()
+        with connection.cursor() as cursor:
+            constraints = connection.introspection.get_constraints(cursor, old_table)
+
+            for old_name, constraint in constraints.items():
+                if constraint["check"]:
+                    suffix = "_check"
+                    is_index = False
+                elif constraint["foreign_key"] is not None:
+                    is_index = False
+                    to_table, to_column = constraint["foreign_key"]
+                    suffix = f"_fk_{to_table}_{to_column}"
+                elif constraint["primary_key"]:
+                    suffix = "_pk"
+                    is_index = True
+                elif constraint["unique"]:
+                    suffix = "_uniq"
+                    is_index = True
+                else:
+                    suffix = "_idx" if len(constraint["columns"]) > 1 else ""
+                    is_index = True
+                new_name = schema_editor._create_index_name(
+                    new_table, constraint["columns"], suffix
+                )
+                if new_name in seen_indexes:
+                    # This index duplicates one we already renamed,
+                    # and attempting to rename it would cause a
+                    # conflict.  Drop the duplicated index.
+                    if is_index:
+                        raw_query = SQL("DROP INDEX {old_name}").format(
+                            old_name=Identifier(old_name)
+                        )
+                    else:
+                        raw_query = SQL(
+                            "ALTER TABLE {table_name} DROP CONSTRAINT {old_name}"
+                        ).format(table_name=Identifier(old_table), old_name=Identifier(old_name))
+                    cursor.execute(raw_query)
+                    continue
+
+                seen_indexes.add(new_name)
+                if is_index:
+                    raw_query = SQL("ALTER INDEX {old_name} RENAME TO {new_name}").format(
+                        old_name=Identifier(old_name), new_name=Identifier(new_name)
+                    )
+                else:
+                    raw_query = SQL(
+                        "ALTER TABLE {old_table} RENAME CONSTRAINT {old_name} TO {new_name}"
+                    ).format(
+                        old_table=Identifier(old_table),
+                        old_name=Identifier(old_name),
+                        new_name=Identifier(new_name),
+                    )
+                cursor.execute(raw_query)
+
+            for sequence in connection.introspection.get_sequences(cursor, old_table):
+                old_name = sequence["name"]
+                column = sequence["column"]
+                new_name = f"{new_table}_{column}_seq"
+
+                raw_query = SQL("ALTER SEQUENCE {old_name} RENAME TO {new_name}").format(
+                    old_name=Identifier(old_name),
+                    new_name=Identifier(new_name),
+                )
+                cursor.execute(raw_query)
+
+            cursor.execute(
+                SQL("ALTER TABLE {old_table} RENAME TO {new_table}").format(
+                    old_table=Identifier(old_table), new_table=Identifier(new_table)
+                )
+            )
+
+    return inner_migration

zerver/lib/migration_status.py

@@ -0,0 +1,169 @@
+import os
+import re
+from importlib import import_module
+from io import StringIO
+from typing import Any, TypeAlias, TypedDict
+
+from django.db import connection
+from django.db.migrations.loader import MigrationLoader
+from django.db.migrations.recorder import MigrationRecorder
+
+AppMigrations: TypeAlias = dict[str, list[str]]
+
+
+class MigrationStatusJson(TypedDict):
+    migrations_by_app: AppMigrations
+    zulip_version: str
+
+
+STALE_MIGRATIONS = [
+    # Ignore django-guardian, which we installed until 1.7.0~3134
+    ("guardian", "0001_initial"),
+    # Ignore django.contrib.sites, which we installed until 2.0.0-rc1~984.
+    ("sites", "0001_initial"),
+    ("sites", "0002_alter_domain_unique"),
+    # These migrations (0002=>0028) were squashed into 0001, in 6fbddf578a6e
+    # through a21f2d771553, 1.7.0~3135.
+    ("zerver", "0002_django_1_8"),
+    ("zerver", "0003_custom_indexes"),
+    ("zerver", "0004_userprofile_left_side_userlist"),
+    ("zerver", "0005_auto_20150920_1340"),
+    ("zerver", "0006_zerver_userprofile_email_upper_idx"),
+    ("zerver", "0007_userprofile_is_bot_active_indexes"),
+    ("zerver", "0008_preregistrationuser_upper_email_idx"),
+    ("zerver", "0009_add_missing_migrations"),
+    ("zerver", "0010_delete_streamcolor"),
+    ("zerver", "0011_remove_guardian"),
+    ("zerver", "0012_remove_appledevicetoken"),
+    ("zerver", "0013_realmemoji"),
+    ("zerver", "0014_realm_emoji_url_length"),
+    ("zerver", "0015_attachment"),
+    ("zerver", "0016_realm_create_stream_by_admins_only"),
+    ("zerver", "0017_userprofile_bot_type"),
+    ("zerver", "0018_realm_emoji_message"),
+    ("zerver", "0019_preregistrationuser_realm_creation"),
+    ("zerver", "0020_add_tracking_attachment"),
+    ("zerver", "0021_migrate_attachment_data"),
+    ("zerver", "0022_subscription_pin_to_top"),
+    ("zerver", "0023_userprofile_default_language"),
+    ("zerver", "0024_realm_allow_message_editing"),
+    ("zerver", "0025_realm_message_content_edit_limit"),
+    ("zerver", "0026_delete_mituser"),
+    ("zerver", "0027_realm_default_language"),
+    ("zerver", "0028_userprofile_tos_version"),
+    # These migrations were removed when `social_auth` was renamed to
+    # `social_django` in social-auth-app-django 0.0.1 (Zulip server 1.5.0), and
+    # were removed from `replaces` in
+    # https://github.com/python-social-auth/social-app-django/pull/25
+    # https://github.com/python-social-auth/social-app-django/pull/900
+    ("default", "0001_initial"),
+    ("default", "0002_add_related_name"),
+    ("default", "0003_alter_email_max_length"),
+    ("default", "0004_auto_20160423_0400"),
+    ("default", "0005_auto_20160727_2333"),
+    ("social_auth", "0001_initial"),
+    ("social_auth", "0002_add_related_name"),
+    ("social_auth", "0003_alter_email_max_length"),
+    ("social_auth", "0004_auto_20160423_0400"),
+    ("social_auth", "0005_auto_20160727_2333"),
+    # This was a typo (twofactor for two_factor) corrected in
+    # https://github.com/jazzband/django-two-factor-auth/pull/642
+    ("twofactor", "0001_squashed_0008_delete_phonedevice"),
+]
+
+
+def get_migration_status(**options: Any) -> str:
+    from django.apps import apps
+    from django.core.management import call_command
+    from django.db import DEFAULT_DB_ALIAS
+    from django.utils.module_loading import module_has_submodule
+
+    verbosity = options.get("verbosity", 1)
+
+    for app_config in apps.get_app_configs():
+        if module_has_submodule(app_config.module, "management"):
+            import_module(".management", app_config.name)
+
+    app_label = options["app_label"] if options.get("app_label") else None
+    db = options.get("database", DEFAULT_DB_ALIAS)
+    out = StringIO()
+    command_args = ["--list"]
+    if app_label:
+        command_args.append(app_label)
+
+    call_command(
+        "showmigrations",
+        *command_args,
+        database=db,
+        no_color=options.get("no_color", False),
+        settings=options.get("settings", os.environ["DJANGO_SETTINGS_MODULE"]),
+        stdout=out,
+        skip_checks=options.get("skip_checks", True),
+        traceback=options.get("traceback", True),
+        verbosity=verbosity,
+    )
+    out.seek(0)
+    output = out.read()
+    return re.sub(r"\x1b\[[0-9;]*m", "", output)
+
+
+def parse_migration_status(
+    stale_migrations: list[tuple[str, str]] = STALE_MIGRATIONS,
+) -> AppMigrations:
+    """
+    This is a copy of Django's `showmigrations` command, keep this in sync with
+    the actual logic from Django. The key differences are, this returns a dict
+    and filters out any migration found in the `stale_migrations` parameter.
+
+    Django's `showmigrations`:
+    https://github.com/django/django/blob/main/django/core/management/commands/showmigrations.py
+    """
+    # Load migrations from disk/DB
+    loader = MigrationLoader(connection, ignore_no_migrations=True)
+    recorder = MigrationRecorder(connection)
+    recorded_migrations = recorder.applied_migrations()
+    graph = loader.graph
+    migrations_dict: AppMigrations = {}
+    app_names = sorted(loader.migrated_apps)
+    stale_migrations_dict: dict[str, list[str]] = {}
+
+    for app, migration in stale_migrations:
+        if app not in stale_migrations_dict:
+            stale_migrations_dict[app] = []
+        stale_migrations_dict[app].append(migration)
+
+    # For each app, print its migrations in order from oldest (roots) to
+    # newest (leaves).
+    for app_name in app_names:
+        migrations_dict[app_name] = []
+        shown = set()
+        apps_stale_migrations = stale_migrations_dict.get(app_name, [])
+        for node in graph.leaf_nodes(app_name):
+            for plan_node in graph.forwards_plan(node):
+                if (
+                    plan_node not in shown
+                    and plan_node[0] == app_name
+                    and plan_node[1] not in apps_stale_migrations
+                ):
+                    # Give it a nice title if it's a squashed one
+                    title = plan_node[1]
+
+                    if graph.nodes[plan_node].replaces:
+                        title += f" ({len(graph.nodes[plan_node].replaces)} squashed migrations)"
+
+                    applied_migration = loader.applied_migrations.get(plan_node)
+                    # Mark it as applied/unapplied
+                    if applied_migration:
+                        if plan_node in recorded_migrations:
+                            output = f"[X] {title}"
+                        else:
+                            output = f"[-] {title}"
+                    else:
+                        output = f"[ ] {title}"
+                    migrations_dict[app_name].append(output)
+                    shown.add(plan_node)
+        # If there are no migrations, record as such
+        if not shown:
+            output = "(no migrations)"
+            migrations_dict[app_name].append(output)
+    return migrations_dict

zerver/lib/mime_types.py

@@ -0,0 +1,60 @@
+import sys
+from email.message import EmailMessage
+from mimetypes import add_type
+from mimetypes import guess_extension as guess_extension
+from mimetypes import guess_type as guess_type
+
+EXTRA_MIME_TYPES = [
+    ("audio/flac", ".flac"),
+    ("audio/mp4", ".m4a"),
+    ("audio/wav", ".wav"),
+    ("audio/webm", ".weba"),
+    ("image/apng", ".apng"),
+    ("video/quicktime", ".mov"),
+]
+
+if sys.version_info < (3, 11):  # nocoverage
+    # https://github.com/python/cpython/issues/89802
+    EXTRA_MIME_TYPES += [
+        ("image/avif", ".avif"),
+        ("image/webp", ".webp"),
+    ]
+
+for mime_type, extension in EXTRA_MIME_TYPES:
+    add_type(mime_type, extension)
+
+
+AUDIO_INLINE_MIME_TYPES = [
+    "audio/aac",
+    "audio/flac",
+    "audio/mp4",
+    "audio/mpeg",
+    "audio/vnd.wave",
+    "audio/wav",
+    "audio/webm",
+    "audio/x-wav",
+]
+
+INLINE_MIME_TYPES = [
+    *AUDIO_INLINE_MIME_TYPES,
+    "application/pdf",
+    "image/apng",
+    "image/avif",
+    "image/gif",
+    "image/jpeg",
+    "image/png",
+    "image/webp",
+    "text/plain",
+    "video/mp4",
+    "video/quicktime",
+    "video/webm",
+    # To avoid cross-site scripting attacks, DO NOT add types such
+    # as application/xhtml+xml, application/x-shockwave-flash,
+    # image/svg+xml, text/html, or text/xml.
+]
+
+
+def bare_content_type(content_type: str) -> str:
+    fake_msg = EmailMessage()
+    fake_msg["content-type"] = content_type
+    return fake_msg.get_content_type()

zerver/lib/mobile_auth_otp.py

@@ -0,0 +1,51 @@
+# Simple one-time-pad library, to be used for encrypting Zulip API
+# keys when sending them to the mobile apps via new standard mobile
+# authentication flow.  This encryption is used to protect against
+# credential-stealing attacks where a malicious app registers the
+# zulip:// URL on a device, which might otherwise allow it to hijack a
+# user's API key.
+#
+# The decryption logic here isn't actually used by the flow; we just
+# have it here as part of testing the overall library.
+
+from zerver.models import UserProfile
+
+
+def xor_hex_strings(bytes_a: str, bytes_b: str) -> str:
+    """Given two hex strings of equal length, return a hex string with
+    the bitwise xor of the two hex strings."""
+    assert len(bytes_a) == len(bytes_b)
+    return "".join(f"{int(x, 16) ^ int(y, 16):x}" for x, y in zip(bytes_a, bytes_b, strict=False))
+
+
+def ascii_to_hex(input_string: str) -> str:
+    """Given an ascii string, encode it as a hex string"""
+    return input_string.encode().hex()
+
+
+def hex_to_ascii(input_string: str) -> str:
+    """Given a hex array, decode it back to a string"""
+    return bytes.fromhex(input_string).decode()
+
+
+def otp_encrypt_api_key(api_key: str, otp: str) -> str:
+    assert len(otp) == UserProfile.API_KEY_LENGTH * 2
+    hex_encoded_api_key = ascii_to_hex(api_key)
+    assert len(hex_encoded_api_key) == UserProfile.API_KEY_LENGTH * 2
+    return xor_hex_strings(hex_encoded_api_key, otp)
+
+
+def otp_decrypt_api_key(otp_encrypted_api_key: str, otp: str) -> str:
+    assert len(otp) == UserProfile.API_KEY_LENGTH * 2
+    assert len(otp_encrypted_api_key) == UserProfile.API_KEY_LENGTH * 2
+    hex_encoded_api_key = xor_hex_strings(otp_encrypted_api_key, otp)
+    return hex_to_ascii(hex_encoded_api_key)
+
+
+def is_valid_otp(otp: str) -> bool:
+    try:
+        assert len(otp) == UserProfile.API_KEY_LENGTH * 2
+        [int(c, 16) for c in otp]
+        return True
+    except Exception:
+        return False

zerver/lib/muted_users.py

@@ -0,0 +1,50 @@
+from datetime import datetime
+
+from zerver.lib.cache import cache_with_key, get_muting_users_cache_key
+from zerver.lib.timestamp import datetime_to_timestamp
+from zerver.lib.utils import assert_is_not_none
+from zerver.models import MutedUser, UserProfile
+
+
+def get_user_mutes(user_profile: UserProfile) -> list[dict[str, int]]:
+    rows = MutedUser.objects.filter(user_profile=user_profile).values(
+        "muted_user_id",
+        "date_muted",
+    )
+    return [
+        {
+            "id": row["muted_user_id"],
+            "timestamp": datetime_to_timestamp(assert_is_not_none(row["date_muted"])),
+        }
+        for row in rows
+    ]
+
+
+def add_user_mute(user_profile: UserProfile, muted_user: UserProfile, date_muted: datetime) -> None:
+    MutedUser.objects.create(
+        user_profile=user_profile,
+        muted_user=muted_user,
+        date_muted=date_muted,
+    )
+
+
+def get_mute_object(user_profile: UserProfile, muted_user: UserProfile) -> MutedUser | None:
+    try:
+        return MutedUser.objects.get(user_profile=user_profile, muted_user=muted_user)
+    except MutedUser.DoesNotExist:
+        return None
+
+
+@cache_with_key(get_muting_users_cache_key, timeout=3600 * 24 * 7)
+def get_muting_users(muted_user_id: int) -> set[int]:
+    """
+    This is kind of the inverse of `get_user_mutes` above.
+    While `get_user_mutes` is mainly used for event system work,
+    this is used in the message send codepath, to get a list
+    of IDs of users who have muted a particular user.
+    The result will also include deactivated users.
+    """
+    rows = MutedUser.objects.filter(
+        muted_user_id=muted_user_id,
+    ).values("user_profile_id")
+    return {row["user_profile_id"] for row in rows}

zerver/lib/name_restrictions.py

@@ -0,0 +1,382 @@
+from disposable_email_domains import blocklist
+from django.conf import settings
+
+
+def is_reserved_subdomain(subdomain: str) -> bool:
+    if subdomain == settings.SOCIAL_AUTH_SUBDOMAIN:
+        return True
+    if subdomain == settings.SELF_HOSTING_MANAGEMENT_SUBDOMAIN:
+        return True
+    if subdomain in ZULIP_RESERVED_SUBDOMAINS:
+        return True
+    if subdomain.endswith("s") and subdomain.removesuffix("s") in ZULIP_RESERVED_SUBDOMAINS:
+        return True
+    if subdomain in GENERIC_RESERVED_SUBDOMAINS:
+        return True
+    if subdomain.endswith("s") and subdomain.removesuffix("s") in GENERIC_RESERVED_SUBDOMAINS:
+        return True
+    if settings.CORPORATE_ENABLED and ("zulip" in subdomain or "kandra" in subdomain):
+        return True
+    return False
+
+
+def is_disposable_domain(domain: str) -> bool:
+    if domain.lower() in OVERRIDE_ALLOW_EMAIL_DOMAINS:
+        return False
+    return domain.lower() in DISPOSABLE_DOMAINS
+
+
+ZULIP_RESERVED_SUBDOMAINS = {
+    # Zulip terms
+    "stream",
+    "channel",
+    "topic",
+    "thread",
+    "installation",
+    "organization",
+    "your-org",
+    "realm",
+    "team",
+    "subdomain",
+    "activity",
+    "octopus",
+    "acme",
+    "push",
+    # machines
+    "zulipdev",
+    "localhost",
+    "staging",
+    "prod",
+    "production",
+    "testing",
+    "nagios",
+    "nginx",
+    "mg",
+    "front-mail",
+    # website pages
+    "server",
+    "client",
+    "features",
+    "integration",
+    "bot",
+    "blog",
+    "history",
+    "story",
+    "stories",
+    "testimonial",
+    "compare",
+    "for",
+    "vs",
+    # competitor pages
+    "slack",
+    "mattermost",
+    "rocketchat",
+    "irc",
+    "twitter",
+    "zephyr",
+    "flowdock",
+    "spark",
+    "skype",
+    "microsoft",
+    "twist",
+    "ryver",
+    "matrix",
+    "discord",
+    "email",
+    "usenet",
+    # Zulip names
+    "zulip",
+    "tulip",
+    "humbug",
+    # platforms
+    "plan9",
+    "electron",
+    "linux",
+    "mac",
+    "windows",
+    "cli",
+    "ubuntu",
+    "android",
+    "ios",
+    # floss
+    "contribute",
+    "floss",
+    "foss",
+    "free",
+    "opensource",
+    "open",
+    "code",
+    "license",
+    # internship programs
+    "intern",
+    "outreachy",
+    "gsoc",
+    "externship",
+    # Things that sound like security
+    "auth",
+    "authentication",
+    "security",
+    # tech blogs
+    "engineering",
+    "infrastructure",
+    "tooling",
+    "tools",
+    "javascript",
+    "python",
+}
+
+# Most of this list was curated from the following sources:
+# http://wiki.dwscoalition.org/notes/List_of_reserved_subdomains (license: CC-BY-SA 3.0)
+# https://stackoverflow.com/questions/11868191/which-saas-subdomains-to-block (license: CC-BY-SA 2.5)
+GENERIC_RESERVED_SUBDOMAINS = {
+    "about",
+    "abuse",
+    "account",
+    "ad",
+    "admanager",
+    "admin",
+    "admindashboard",
+    "administrator",
+    "adsense",
+    "advice",
+    "adword",
+    "affiliate",
+    "alpha",
+    "anonymous",
+    "api",
+    "assets",
+    "audio",
+    "avatar",
+    "badges",
+    "beta",
+    "billing",
+    "biz",
+    "blog",
+    "board",
+    "bookmark",
+    "bot",
+    "bugs",
+    "buy",
+    "cache",
+    "calendar",
+    "chat",
+    "clients",
+    "cname",
+    "code",
+    "comment",
+    "communities",
+    "community",
+    "contact",
+    "contributor",
+    "control",
+    "coppa",
+    "copyright",
+    "cpanel",
+    "css",
+    "cssproxy",
+    "customer",
+    "customise",
+    "customize",
+    "dashboard",
+    "data",
+    "demo",
+    "deploy",
+    "deployment",
+    "desktop",
+    "dev",
+    "devel",
+    "developer",
+    "development",
+    "discussion",
+    "diversity",
+    "dmca",
+    "docs",
+    "donate",
+    "download",
+    "e-mail",
+    "email",
+    "embed",
+    "embedded",
+    "example",
+    "explore",
+    "faq",
+    "favorite",
+    "favourites",
+    "features",
+    "feed",
+    "feedback",
+    "files",
+    "forum",
+    "friend",
+    "ftp",
+    "general",
+    "gettingstarted",
+    "gift",
+    "git",
+    "global",
+    "graphs",
+    "guide",
+    "hack",
+    "hello",
+    "help",
+    "home",
+    "hostmaster",
+    "https",
+    "icon",
+    "im",
+    "image",
+    "img",
+    "inbox",
+    "index",
+    "investors",
+    "invite",
+    "invoice",
+    "ios",
+    "ipad",
+    "iphone",
+    "irc",
+    "jabber",
+    "jars",
+    "jobs",
+    "join",
+    "js",
+    "kb",
+    "knowledgebase",
+    "launchpad",
+    "legal",
+    "livejournal",
+    "lj",
+    "login",
+    "logs",
+    "m",
+    "mail",
+    "main",
+    "manage",
+    "map",
+    "media",
+    "memories",
+    "memory",
+    "merchandise",
+    "messages",
+    "mobile",
+    "my",
+    "mystore",
+    "networks",
+    "new",
+    "newsite",
+    "onboarding",
+    "official",
+    "ogg",
+    "online",
+    "order",
+    "paid",
+    "panel",
+    "partner",
+    "partnerpage",
+    "pay",
+    "payment",
+    "picture",
+    "policy",
+    "pop",
+    "popular",
+    "portal",
+    "post",
+    "postmaster",
+    "press",
+    "pricing",
+    "principles",
+    "privacy",
+    "private",
+    "profile",
+    "public",
+    "question",
+    "random",
+    "redirect",
+    "register",
+    "registration",
+    "resolver",
+    "root",
+    "rss",
+    "s",
+    "sandbox",
+    "school",
+    "search",
+    "secure",
+    "servers",
+    "service",
+    "setting",
+    "shop",
+    "shortcuts",
+    "signin",
+    "signup",
+    "sitemap",
+    "sitenews",
+    "sites",
+    "sms",
+    "smtp",
+    "sorry",
+    "ssl",
+    "staff",
+    "stage",
+    "staging",
+    "stars",
+    "stat",
+    "static",
+    "statistics",
+    "status",
+    "store",
+    "style",
+    "support",
+    "surveys",
+    "svn",
+    "syn",
+    "syndicated",
+    "system",
+    "tag",
+    "talk",
+    "team",
+    "termsofservice",
+    "test",
+    "testers",
+    "ticket",
+    "tool",
+    "tos",
+    "trac",
+    "translate",
+    "update",
+    "upgrade",
+    "uploads",
+    "use",
+    "user",
+    "username",
+    "validation",
+    "videos",
+    "volunteer",
+    "web",
+    "webdisk",
+    "webmail",
+    "webmaster",
+    "welcome",
+    "whm",
+    "whois",
+    "wiki",
+    "www",
+    "www0",
+    "www8",
+    "www9",
+    "xml",
+    "xmpp",
+    "xoxo",
+}
+
+DISPOSABLE_DOMAINS = set(blocklist)
+
+OVERRIDE_ALLOW_EMAIL_DOMAINS = {
+    "airsi.de",
+    # Controlled by https://www.abine.com; more legitimate than most
+    # disposable domains
+    "opayq.com",
+    "abinemail.com",
+    "blurmail.net",
+    "maskmemail.com",
+}

zerver/lib/narrow_helpers.py

@@ -0,0 +1,63 @@
+"""
+This module partly exists to prevent circular dependencies.
+
+It also isolates some fairly yucky code related to the fact
+that we have to support two formats of narrow specifications
+from users:
+
+    legacy:
+        [["stream", "devel"], ["is", mentioned"]
+
+    modern:
+        [
+            {"operator": "stream", "operand": "devel", "negated": "false"},
+            {"operator": "is", "operand": "mentioned", "negated": "false"},
+        ]
+
+    And then on top of that, we want to represent narrow
+    specification internally as dataclasses.
+"""
+
+import os
+from collections.abc import Collection, Sequence
+from dataclasses import dataclass, field
+
+from django.conf import settings
+
+
+@dataclass
+class NarrowTerm:
+    operator: str
+    operand: str | int | list[int]
+    negated: bool
+
+
+@dataclass
+class NeverNegatedNarrowTerm(NarrowTerm):
+    negated: bool = field(default=False, init=False)
+    operand: str
+
+
+def narrow_dataclasses_from_tuples(
+    tups: Collection[Sequence[str]],
+) -> Collection[NeverNegatedNarrowTerm]:
+    """
+    This method assumes that the callers are in our event-handling codepath, and
+    therefore as of summer 2023, they do not yet support the "negated" flag.
+    """
+    return [NeverNegatedNarrowTerm(operator=tup[0], operand=tup[1]) for tup in tups]
+
+
+stop_words_list: list[str] | None = None
+
+
+def read_stop_words() -> list[str]:
+    global stop_words_list
+    if stop_words_list is None:
+        file_path = os.path.join(
+            settings.DEPLOY_ROOT, "puppet/zulip/files/postgresql/zulip_english.stop"
+        )
+        with open(file_path) as f:
+            stop_words_list = f.read().splitlines()
+
+    return stop_words_list

zerver/lib/narrow_predicate.py

@@ -0,0 +1,84 @@
+from collections.abc import Collection
+from typing import Any, Protocol
+
+from django.utils.translation import gettext as _
+
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.narrow_helpers import NeverNegatedNarrowTerm
+from zerver.lib.topic import RESOLVED_TOPIC_PREFIX, get_topic_from_message_info
+
+# "stream" is a legacy alias for "channel"
+channel_operators: list[str] = ["channel", "stream"]
+# "streams" is a legacy alias for "channels"
+channels_operators: list[str] = ["channels", "streams"]
+
+
+def check_narrow_for_events(narrow: Collection[NeverNegatedNarrowTerm]) -> None:
+    supported_operators = [*channel_operators, "topic", "sender", "is"]
+    unsupported_is_operands = ["followed"]
+    for narrow_term in narrow:
+        operator = narrow_term.operator
+        if operator not in supported_operators:
+            raise JsonableError(_("Operator {operator} not supported.").format(operator=operator))
+        if operator == "is" and narrow_term.operand in unsupported_is_operands:
+            raise JsonableError(
+                _("Operand {operand} not supported.").format(operand=narrow_term.operand)
+            )
+
+
+class NarrowPredicate(Protocol):
+    def __call__(self, *, message: dict[str, Any], flags: list[str]) -> bool: ...
+
+
+def build_narrow_predicate(
+    narrow: Collection[NeverNegatedNarrowTerm],
+) -> NarrowPredicate:
+    """Changes to this function should come with corresponding changes to
+    NarrowLibraryTest."""
+    check_narrow_for_events(narrow)
+
+    def narrow_predicate(*, message: dict[str, Any], flags: list[str]) -> bool:
+        def satisfies_operator(*, operator: str, operand: str) -> bool:
+            if operator in channel_operators:
+                if message["type"] != "stream":
+                    return False
+                if operand.lower() != message["display_recipient"].lower():
+                    return False
+            elif operator == "topic":
+                if message["type"] != "stream":
+                    return False
+                topic_name = get_topic_from_message_info(message)
+                if operand.lower() != topic_name.lower():
+                    return False
+            elif operator == "sender":
+                if operand.lower() != message["sender_email"].lower():
+                    return False
+            elif operator == "is" and operand in ["dm", "private"]:
+                # "is:private" is a legacy alias for "is:dm"
+                if message["type"] != "private":
+                    return False
+            elif operator == "is" and operand == "starred":
+                if operand not in flags:
+                    return False
+            elif operator == "is" and operand == "unread":
+                if "read" in flags:
+                    return False
+            elif operator == "is" and operand in ["alerted", "mentioned"]:
+                if "mentioned" not in flags:
+                    return False
+            elif operator == "is" and operand == "resolved":
+                if message["type"] != "stream":
+                    return False
+                topic_name = get_topic_from_message_info(message)
+                if not topic_name.startswith(RESOLVED_TOPIC_PREFIX):
+                    return False
+            return True
+
+        for narrow_term in narrow:
+            # TODO: Eventually handle negated narrow terms.
+            if not satisfies_operator(operator=narrow_term.operator, operand=narrow_term.operand):
+                return False
+
+        return True
+
+    return narrow_predicate

zerver/lib/navigation_views.py

@@ -0,0 +1,33 @@
+from typing import TypedDict
+
+from django.utils.translation import gettext as _
+
+from zerver.lib.exceptions import ResourceNotFoundError
+from zerver.models import NavigationView, UserProfile
+
+
+class NavigationViewDict(TypedDict):
+    fragment: str
+    is_pinned: bool
+    name: str | None
+
+
+def get_navigation_view(user: UserProfile, fragment: str) -> NavigationView:
+    try:
+        navigation_view = NavigationView.objects.get(user=user, fragment=fragment)
+        return navigation_view
+    except NavigationView.DoesNotExist:
+        raise ResourceNotFoundError(_("Navigation view does not exist."))
+
+
+def get_navigation_view_dict(navigation_view: NavigationView) -> NavigationViewDict:
+    return NavigationViewDict(
+        fragment=navigation_view.fragment,
+        is_pinned=navigation_view.is_pinned,
+        name=navigation_view.name,
+    )
+
+
+def get_navigation_views_for_user(user: UserProfile) -> list[NavigationViewDict]:
+    navigation_views = NavigationView.objects.filter(user=user)
+    return [get_navigation_view_dict(navigation_view) for navigation_view in navigation_views]

zerver/lib/notes.py

@@ -0,0 +1,54 @@
+import weakref
+from abc import ABC, abstractmethod
+from collections.abc import MutableMapping
+from typing import Any, ClassVar, Generic, TypeVar
+
+from typing_extensions import override
+
+_KeyT = TypeVar("_KeyT")
+_DataT = TypeVar("_DataT")
+
+
+class BaseNotes(ABC, Generic[_KeyT, _DataT]):
+    """This class defines a generic type-safe mechanism for associating
+    additional data with an object (without modifying the original
+    object via subclassing or monkey-patching).
+
+    It was originally designed to avoid monkey-patching the Django
+    HttpRequest object, to which we want to associate computed state
+    (e.g. parsed state computed from the User-Agent) so that it's
+    available in code paths that receive the HttpRequest object.
+
+    The implementation uses a WeakKeyDictionary, so that the notes
+    object will be garbage-collected when the original object no
+    longer has other references (avoiding memory leaks).
+
+    We still need to be careful to avoid any of the attributes of
+    _DataT having points to the original object, as that can create a
+    cyclic reference cycle that the Python garbage collect may not
+    handle correctly.
+    """
+
+    __notes_map: ClassVar[MutableMapping[Any, Any]]
+
+    @override
+    def __init_subclass__(cls, **kwargs: object) -> None:
+        super().__init_subclass__(**kwargs)
+        if not hasattr(cls, "__notes_map"):
+            cls.__notes_map = weakref.WeakKeyDictionary()
+
+    @classmethod
+    def get_notes(cls, key: _KeyT) -> _DataT:
+        try:
+            return cls.__notes_map[key]
+        except KeyError:
+            cls.__notes_map[key] = cls.init_notes()
+            return cls.__notes_map[key]
+
+    @classmethod
+    def set_notes(cls, key: _KeyT, notes: _DataT) -> None:
+        cls.__notes_map[key] = notes
+
+    @classmethod
+    @abstractmethod
+    def init_notes(cls) -> _DataT: ...

zerver/lib/notification_data.py

@@ -0,0 +1,413 @@
+import math
+from collections.abc import Collection
+from dataclasses import dataclass
+from typing import Any
+
+from zerver.lib.mention import MentionData
+from zerver.lib.user_groups import get_user_group_member_ids
+from zerver.models import NamedUserGroup, UserProfile, UserTopic
+from zerver.models.scheduled_jobs import NotificationTriggers
+
+
+@dataclass
+class UserMessageNotificationsData:
+    user_id: int
+    online_push_enabled: bool
+    dm_email_notify: bool
+    dm_push_notify: bool
+    mention_email_notify: bool
+    mention_push_notify: bool
+    topic_wildcard_mention_email_notify: bool
+    topic_wildcard_mention_push_notify: bool
+    stream_wildcard_mention_email_notify: bool
+    stream_wildcard_mention_push_notify: bool
+    stream_push_notify: bool
+    stream_email_notify: bool
+    followed_topic_push_notify: bool
+    followed_topic_email_notify: bool
+    topic_wildcard_mention_in_followed_topic_push_notify: bool
+    topic_wildcard_mention_in_followed_topic_email_notify: bool
+    stream_wildcard_mention_in_followed_topic_push_notify: bool
+    stream_wildcard_mention_in_followed_topic_email_notify: bool
+    sender_is_muted: bool
+    disable_external_notifications: bool
+
+    def __post_init__(self) -> None:
+        # Check that there's no dubious data.
+        if self.dm_email_notify or self.dm_push_notify:
+            assert not (
+                self.stream_email_notify
+                or self.stream_push_notify
+                or self.followed_topic_email_notify
+                or self.followed_topic_push_notify
+            )
+
+        if (
+            self.stream_email_notify
+            or self.stream_push_notify
+            or self.followed_topic_email_notify
+            or self.followed_topic_push_notify
+        ):
+            assert not (self.dm_email_notify or self.dm_push_notify)
+
+    @classmethod
+    def from_user_id_sets(
+        cls,
+        *,
+        user_id: int,
+        flags: Collection[str],
+        private_message: bool,
+        disable_external_notifications: bool,
+        online_push_user_ids: set[int],
+        dm_mention_push_disabled_user_ids: set[int],
+        dm_mention_email_disabled_user_ids: set[int],
+        stream_push_user_ids: set[int],
+        stream_email_user_ids: set[int],
+        topic_wildcard_mention_user_ids: set[int],
+        stream_wildcard_mention_user_ids: set[int],
+        followed_topic_push_user_ids: set[int],
+        followed_topic_email_user_ids: set[int],
+        topic_wildcard_mention_in_followed_topic_user_ids: set[int],
+        stream_wildcard_mention_in_followed_topic_user_ids: set[int],
+        muted_sender_user_ids: set[int],
+        all_bot_user_ids: set[int],
+        push_device_registered_user_ids: set[int] | None,
+    ) -> "UserMessageNotificationsData":
+        if user_id in all_bot_user_ids:
+            # Don't send any notifications to bots
+            return cls(
+                user_id=user_id,
+                dm_email_notify=False,
+                mention_email_notify=False,
+                topic_wildcard_mention_email_notify=False,
+                stream_wildcard_mention_email_notify=False,
+                dm_push_notify=False,
+                mention_push_notify=False,
+                topic_wildcard_mention_push_notify=False,
+                stream_wildcard_mention_push_notify=False,
+                online_push_enabled=False,
+                stream_push_notify=False,
+                stream_email_notify=False,
+                followed_topic_push_notify=False,
+                followed_topic_email_notify=False,
+                topic_wildcard_mention_in_followed_topic_push_notify=False,
+                topic_wildcard_mention_in_followed_topic_email_notify=False,
+                stream_wildcard_mention_in_followed_topic_push_notify=False,
+                stream_wildcard_mention_in_followed_topic_email_notify=False,
+                sender_is_muted=False,
+                disable_external_notifications=False,
+            )
+
+        # `stream_wildcard_mention_user_ids`, `topic_wildcard_mention_user_ids`,
+        # `stream_wildcard_mention_in_followed_topic_user_ids` and `topic_wildcard_mention_in_followed_topic_user_ids`
+        # are those user IDs for whom stream or topic wildcard mentions should obey notification
+        # settings for personal mentions. Hence, it isn't an independent notification setting and acts as a wrapper.
+        dm_email_notify = user_id not in dm_mention_email_disabled_user_ids and private_message
+        mention_email_notify = (
+            user_id not in dm_mention_email_disabled_user_ids and "mentioned" in flags
+        )
+        topic_wildcard_mention_email_notify = (
+            user_id in topic_wildcard_mention_user_ids
+            and user_id not in dm_mention_email_disabled_user_ids
+            and "topic_wildcard_mentioned" in flags
+        )
+        stream_wildcard_mention_email_notify = (
+            user_id in stream_wildcard_mention_user_ids
+            and user_id not in dm_mention_email_disabled_user_ids
+            and "stream_wildcard_mentioned" in flags
+        )
+        topic_wildcard_mention_in_followed_topic_email_notify = (
+            user_id in topic_wildcard_mention_in_followed_topic_user_ids
+            and user_id not in dm_mention_email_disabled_user_ids
+            and "topic_wildcard_mentioned" in flags
+        )
+        stream_wildcard_mention_in_followed_topic_email_notify = (
+            user_id in stream_wildcard_mention_in_followed_topic_user_ids
+            and user_id not in dm_mention_email_disabled_user_ids
+            and "stream_wildcard_mentioned" in flags
+        )
+
+        # TODO/compatibility: `push_device_registered_user_ids` is None when
+        # `process_message_event`/`process_message_update_event` handles events
+        # prior to the introduction of `push_device_registered_user_ids` field in the events.
+        # We hardcode `push_device_registered` to True as the check is meant for newer events.
+        #
+        # Remove the `if` block when one can no longer directly upgrade from 11.x to main.
+        if push_device_registered_user_ids is None:
+            push_device_registered = True  # nocoverage
+        else:
+            push_device_registered = user_id in push_device_registered_user_ids
+
+        dm_push_notify = (
+            push_device_registered
+            and user_id not in dm_mention_push_disabled_user_ids
+            and private_message
+        )
+        mention_push_notify = (
+            push_device_registered
+            and user_id not in dm_mention_push_disabled_user_ids
+            and "mentioned" in flags
+        )
+        topic_wildcard_mention_push_notify = (
+            push_device_registered
+            and user_id in topic_wildcard_mention_user_ids
+            and user_id not in dm_mention_push_disabled_user_ids
+            and "topic_wildcard_mentioned" in flags
+        )
+        stream_wildcard_mention_push_notify = (
+            push_device_registered
+            and user_id in stream_wildcard_mention_user_ids
+            and user_id not in dm_mention_push_disabled_user_ids
+            and "stream_wildcard_mentioned" in flags
+        )
+        topic_wildcard_mention_in_followed_topic_push_notify = (
+            push_device_registered
+            and user_id in topic_wildcard_mention_in_followed_topic_user_ids
+            and user_id not in dm_mention_push_disabled_user_ids
+            and "topic_wildcard_mentioned" in flags
+        )
+        stream_wildcard_mention_in_followed_topic_push_notify = (
+            push_device_registered
+            and user_id in stream_wildcard_mention_in_followed_topic_user_ids
+            and user_id not in dm_mention_push_disabled_user_ids
+            and "stream_wildcard_mentioned" in flags
+        )
+        online_push_enabled = push_device_registered and user_id in online_push_user_ids
+        stream_push_notify = push_device_registered and user_id in stream_push_user_ids
+        followed_topic_push_notify = (
+            push_device_registered and user_id in followed_topic_push_user_ids
+        )
+        return cls(
+            user_id=user_id,
+            dm_email_notify=dm_email_notify,
+            mention_email_notify=mention_email_notify,
+            topic_wildcard_mention_email_notify=topic_wildcard_mention_email_notify,
+            stream_wildcard_mention_email_notify=stream_wildcard_mention_email_notify,
+            dm_push_notify=dm_push_notify,
+            mention_push_notify=mention_push_notify,
+            topic_wildcard_mention_push_notify=topic_wildcard_mention_push_notify,
+            stream_wildcard_mention_push_notify=stream_wildcard_mention_push_notify,
+            online_push_enabled=online_push_enabled,
+            stream_push_notify=stream_push_notify,
+            stream_email_notify=user_id in stream_email_user_ids,
+            followed_topic_push_notify=followed_topic_push_notify,
+            followed_topic_email_notify=user_id in followed_topic_email_user_ids,
+            topic_wildcard_mention_in_followed_topic_push_notify=topic_wildcard_mention_in_followed_topic_push_notify,
+            topic_wildcard_mention_in_followed_topic_email_notify=topic_wildcard_mention_in_followed_topic_email_notify,
+            stream_wildcard_mention_in_followed_topic_push_notify=stream_wildcard_mention_in_followed_topic_push_notify,
+            stream_wildcard_mention_in_followed_topic_email_notify=stream_wildcard_mention_in_followed_topic_email_notify,
+            sender_is_muted=user_id in muted_sender_user_ids,
+            disable_external_notifications=disable_external_notifications,
+        )
+
+    # For these functions, acting_user_id is the user sent a message
+    # (or edited a message) triggering the event for which we need to
+    # determine notifiability.
+    def trivially_should_not_notify(self, acting_user_id: int) -> bool:
+        """Common check for reasons not to trigger a notification that are
+        independent of users' notification settings and thus don't
+        depend on what type of notification (email/push) it is.
+        """
+        if self.user_id == acting_user_id:
+            return True
+
+        if self.sender_is_muted:
+            return True
+
+        if self.disable_external_notifications:
+            return True
+
+        return False
+
+    def is_notifiable(self, acting_user_id: int, idle: bool) -> bool:
+        return self.is_email_notifiable(acting_user_id, idle) or self.is_push_notifiable(
+            acting_user_id, idle
+        )
+
+    def is_push_notifiable(self, acting_user_id: int, idle: bool) -> bool:
+        return self.get_push_notification_trigger(acting_user_id, idle) is not None
+
+    def get_push_notification_trigger(self, acting_user_id: int, idle: bool) -> str | None:
+        if not idle and not self.online_push_enabled:
+            return None
+
+        if self.trivially_should_not_notify(acting_user_id):
+            return None
+
+        # The order here is important. If, for example, both
+        # `mention_push_notify` and `stream_push_notify` are True, we
+        # want to classify it as a mention, since that's more salient.
+        if self.dm_push_notify:
+            return NotificationTriggers.DIRECT_MESSAGE
+        elif self.mention_push_notify:
+            return NotificationTriggers.MENTION
+        elif self.topic_wildcard_mention_in_followed_topic_push_notify:
+            return NotificationTriggers.TOPIC_WILDCARD_MENTION_IN_FOLLOWED_TOPIC
+        elif self.stream_wildcard_mention_in_followed_topic_push_notify:
+            return NotificationTriggers.STREAM_WILDCARD_MENTION_IN_FOLLOWED_TOPIC
+        elif self.topic_wildcard_mention_push_notify:
+            return NotificationTriggers.TOPIC_WILDCARD_MENTION
+        elif self.stream_wildcard_mention_push_notify:
+            return NotificationTriggers.STREAM_WILDCARD_MENTION
+        elif self.followed_topic_push_notify:
+            return NotificationTriggers.FOLLOWED_TOPIC_PUSH
+        elif self.stream_push_notify:
+            return NotificationTriggers.STREAM_PUSH
+        else:
+            return None
+
+    def is_email_notifiable(self, acting_user_id: int, idle: bool) -> bool:
+        return self.get_email_notification_trigger(acting_user_id, idle) is not None
+
+    def get_email_notification_trigger(self, acting_user_id: int, idle: bool) -> str | None:
+        if not idle:
+            return None
+
+        if self.trivially_should_not_notify(acting_user_id):
+            return None
+
+        # The order here is important. If, for example, both
+        # `mention_email_notify` and `stream_email_notify` are True, we
+        # want to classify it as a mention, since that's more salient.
+        if self.dm_email_notify:
+            return NotificationTriggers.DIRECT_MESSAGE
+        elif self.mention_email_notify:
+            return NotificationTriggers.MENTION
+        elif self.topic_wildcard_mention_in_followed_topic_email_notify:
+            return NotificationTriggers.TOPIC_WILDCARD_MENTION_IN_FOLLOWED_TOPIC
+        elif self.stream_wildcard_mention_in_followed_topic_email_notify:
+            return NotificationTriggers.STREAM_WILDCARD_MENTION_IN_FOLLOWED_TOPIC
+        elif self.topic_wildcard_mention_email_notify:
+            return NotificationTriggers.TOPIC_WILDCARD_MENTION
+        elif self.stream_wildcard_mention_email_notify:
+            return NotificationTriggers.STREAM_WILDCARD_MENTION
+        elif self.followed_topic_email_notify:
+            return NotificationTriggers.FOLLOWED_TOPIC_EMAIL
+        elif self.stream_email_notify:
+            return NotificationTriggers.STREAM_EMAIL
+        else:
+            return None
+
+
+def user_allows_notifications_in_StreamTopic(
+    stream_is_muted: bool,
+    visibility_policy: int,
+    stream_specific_setting: bool | None,
+    global_setting: bool,
+    channel_specific_setting_overrides_mute: bool,
+) -> bool:
+    """
+    Captures the hierarchy of notification settings, where visibility policy is considered first,
+    followed by stream-specific settings, and the global-setting in the UserProfile is the fallback.
+
+    When `channel_specific_setting_overrides_mute` is True (currently used for
+    `wildcard_mentions_notify` setting), `stream_specific_setting` overrides
+    channel mute, but not topic mute.
+    """
+    # Muted topics always suppress notifications, regardless of other settings.
+    if visibility_policy == UserTopic.VisibilityPolicy.MUTED:
+        return False
+
+    if stream_is_muted and visibility_policy != UserTopic.VisibilityPolicy.UNMUTED:
+        if channel_specific_setting_overrides_mute and stream_specific_setting is not None:
+            return stream_specific_setting
+        return False
+
+    if stream_specific_setting is not None:
+        return stream_specific_setting
+
+    return global_setting
+
+
+def get_user_group_mentions_data(
+    mentioned_user_ids: set[int], mentioned_user_group_ids: list[int], mention_data: MentionData
+) -> dict[int, int]:
+    # Maps user_id -> mentioned user_group_id
+    mentioned_user_groups_map: dict[int, int] = dict()
+
+    # Add members of the mentioned user groups into `mentions_user_ids`.
+    for group_id in mentioned_user_group_ids:
+        member_ids = mention_data.get_group_members(group_id)
+        for member_id in member_ids:
+            if member_id in mentioned_user_ids:
+                # If a user is also mentioned personally, we use that as a trigger
+                # for notifications.
+                continue
+
+            if member_id in mentioned_user_groups_map:
+                # If multiple user groups are mentioned, we prefer the
+                # user group with the least members for email/mobile
+                # notifications.
+                previous_group_id = mentioned_user_groups_map[member_id]
+                previous_group_member_ids = mention_data.get_group_members(previous_group_id)
+
+                if len(previous_group_member_ids) > len(member_ids):
+                    mentioned_user_groups_map[member_id] = group_id
+            else:
+                mentioned_user_groups_map[member_id] = group_id
+
+    return mentioned_user_groups_map
+
+
+@dataclass
+class MentionedUserGroup:
+    id: int
+    name: str
+    members_count: int
+
+
+def get_mentioned_user_group(
+    messages: list[dict[str, Any]], user_profile: UserProfile
+) -> MentionedUserGroup | None:
+    """Returns the user group name to display in the email notification
+    if user group(s) are mentioned.
+
+    This implements the same algorithm as get_user_group_mentions_data
+    in zerver/lib/notification_data.py, but we're passed a list of
+    messages instead.
+    """
+    for message in messages:
+        if (
+            message.get("mentioned_user_group_id") is None
+            and message["trigger"] == NotificationTriggers.MENTION
+        ):
+            # The user has also been personally mentioned, so that gets prioritized.
+            return None
+
+    # These IDs are those of the smallest user groups mentioned in each message.
+    mentioned_user_group_ids = [
+        message["mentioned_user_group_id"]
+        for message in messages
+        if message.get("mentioned_user_group_id") is not None
+    ]
+
+    if len(mentioned_user_group_ids) == 0:
+        return None
+
+    # We now want to calculate the name of the smallest user group mentioned among
+    # all these messages.
+    # Deduplicate IDs while preserving message order.
+    unique_user_group_ids = list(dict.fromkeys(mentioned_user_group_ids))
+    user_groups_by_id = {
+        user_group.id: user_group
+        for user_group in NamedUserGroup.objects.filter(
+            id__in=unique_user_group_ids, realm_for_sharding=user_profile.realm
+        )
+    }
+
+    smallest_user_group_size = math.inf
+    for user_group_id in unique_user_group_ids:
+        current_user_group = user_groups_by_id[user_group_id]
+        current_mentioned_user_group = MentionedUserGroup(
+            id=current_user_group.id,
+            name=current_user_group.name,
+            members_count=len(get_user_group_member_ids(current_user_group)),
+        )
+
+        if current_mentioned_user_group.members_count < smallest_user_group_size:
+            # If multiple user groups are mentioned, we prefer the
+            # user group with the least members.
+            smallest_user_group_size = current_mentioned_user_group.members_count
+            smallest_mentioned_user_group = current_mentioned_user_group
+
+    return smallest_mentioned_user_group

zerver/lib/onboarding.py

@@ -0,0 +1,589 @@
+from dataclasses import dataclass
+from enum import Enum
+
+from django.conf import settings
+from django.db import transaction
+from django.db.models import Count
+from django.utils.translation import gettext as _
+from django.utils.translation import override as override_language
+
+from zerver.actions.create_realm import setup_realm_internal_bots
+from zerver.actions.message_send import (
+    do_send_messages,
+    internal_prep_private_message,
+    internal_prep_stream_message_by_name,
+    internal_send_private_message,
+)
+from zerver.actions.reactions import do_add_reaction
+from zerver.lib.demo_organizations import get_scheduled_deletion_global_time
+from zerver.lib.emoji import get_emoji_data
+from zerver.lib.markdown.fenced_code import get_unused_fence
+from zerver.lib.message import SendMessageRequest, remove_single_newlines
+from zerver.models import Message, OnboardingUserMessage, Realm, UserProfile
+from zerver.models.groups import SystemGroups
+from zerver.models.recipients import Recipient
+from zerver.models.users import get_system_bot
+
+
+@dataclass
+class InitialDirectMessageIDs:
+    welcome_bot_intro_message_id: int
+    welcome_bot_custom_message_id: int | None
+
+
+def missing_any_realm_internal_bots() -> bool:
+    bot_emails = [
+        bot["email_template"] % (settings.INTERNAL_BOT_DOMAIN,)
+        for bot in settings.REALM_INTERNAL_BOTS
+    ]
+    realm_count = Realm.objects.count()
+    return UserProfile.objects.filter(email__in=bot_emails).values("email").alias(
+        count=Count("id")
+    ).filter(count=realm_count).count() != len(bot_emails)
+
+
+def create_if_missing_realm_internal_bots() -> None:
+    """This checks if there is any realm internal bot missing.
+
+    If that is the case, it creates the missing realm internal bots.
+    """
+    if missing_any_realm_internal_bots():
+        for realm in Realm.objects.all().iterator():
+            setup_realm_internal_bots(realm)
+
+
+def get_custom_welcome_message_string(realm: Realm, welcome_message_custom_text: str) -> str:
+    # TODO: This could use silent_mention_syntax_for_user_group, but
+    # it's not worth doing a database query to get the NamedUserGroup,
+    # so some refactoring is required to do that.
+    mention_syntax = f"@_*{SystemGroups.ADMINISTRATORS}*"
+    fence = get_unused_fence(welcome_message_custom_text)
+    welcome_bot_custom_message_intro_string = _("A note from {admin_group_syntax}:").format(
+        admin_group_syntax=mention_syntax
+    )
+
+    welcome_bot_custom_message_string = f"""
+{welcome_bot_custom_message_intro_string}
+{fence}quote
+{welcome_message_custom_text}
+{fence}
+"""
+
+    return welcome_bot_custom_message_string
+
+
+def send_initial_direct_messages_to_user(
+    user: UserProfile,
+    *,
+    realm_creation: bool = False,
+    welcome_message_custom_text: str = "",
+) -> InitialDirectMessageIDs:
+    # We adjust the initial Welcome Bot direct message for education organizations.
+    education_organization = user.realm.org_type in (
+        Realm.ORG_TYPES["education_nonprofit"]["id"],
+        Realm.ORG_TYPES["education"]["id"],
+    )
+    welcome_bot_messages = []
+
+    # We need to override the language in this code path, because it's
+    # called from account registration, which is a pre-account API
+    # request and thus may not have the user's language context yet.
+    with override_language(user.default_language):
+        if education_organization:
+            getting_started_string = _("""
+To learn more, check out our [using Zulip for a class guide]({getting_started_url})!
+""").format(getting_started_url="/help/using-zulip-for-a-class")
+        else:
+            getting_started_string = _("""
+To learn more, check out our [getting started guide]({getting_started_url})!
+""").format(getting_started_url="/help/getting-started-with-zulip")
+
+        organization_setup_string = ""
+        # Add extra content on setting up a new organization for administrators.
+        if user.is_realm_admin:
+            if education_organization:
+                organization_setup_string = _("""
+We also have a guide for [setting up Zulip for a class]({organization_setup_url}).
+""").format(organization_setup_url="/help/setting-up-zulip-for-a-class")
+            else:
+                organization_setup_string = _("""
+We also have a guide for [moving your organization to Zulip]({organization_setup_url}).
+""").format(organization_setup_url="/help/moving-to-zulip")
+
+        demo_organization_warning_string = ""
+        # Add extra content about automatic deletion for demo organization creators.
+        if (
+            realm_creation
+            and user.is_realm_owner
+            and user.realm.demo_organization_scheduled_deletion_date is not None
+        ):
+            demo_organization_warning_string = _("""
+This [demo organization]({demo_organization_help_url}) will be **automatically
+deleted** on {deletion_global_time}, unless it's [converted into
+a permanent organization]({convert_demo_organization_help_url}).
+""").format(
+                demo_organization_help_url="/help/demo-organizations",
+                deletion_global_time=get_scheduled_deletion_global_time(user.realm),
+                convert_demo_organization_help_url="/help/demo-organizations#convert-a-demo-organization-to-a-permanent-organization",
+            )
+
+        inform_about_tracked_onboarding_messages_text = ""
+        if OnboardingUserMessage.objects.filter(realm_id=user.realm_id).exists():
+            inform_about_tracked_onboarding_messages_text = _("""
+I've kicked off some conversations to help you get started. You can find
+them in your [Inbox](/#inbox).
+""")
+
+        navigation_tour_video_string = _("""
+You can always come back to the [Welcome to Zulip video]({navigation_tour_video_url}) for a quick app overview.
+""").format(navigation_tour_video_url=settings.NAVIGATION_TOUR_VIDEO_URL)
+
+        welcome_bot_custom_message_string = ""
+        # Add welcome bot custom message.
+        if welcome_message_custom_text:
+            welcome_bot_custom_message_string = get_custom_welcome_message_string(
+                user.realm, welcome_message_custom_text
+            )
+
+        content = _("""
+Hello, and welcome to Zulip!👋 {inform_about_tracked_onboarding_messages_text}
+
+{getting_started_text} {organization_setup_text}
+
+{navigation_tour_video_text}
+
+{demo_organization_text}
+
+""").format(
+            inform_about_tracked_onboarding_messages_text=inform_about_tracked_onboarding_messages_text,
+            getting_started_text=getting_started_string,
+            organization_setup_text=organization_setup_string,
+            navigation_tour_video_text=navigation_tour_video_string,
+            demo_organization_text=demo_organization_warning_string,
+        )
+
+    welcome_bot_messages.append(
+        internal_prep_private_message(
+            get_system_bot(settings.WELCOME_BOT, user.realm_id),
+            user,
+            remove_single_newlines(content),
+            # Note: Welcome bot doesn't trigger email/push notifications,
+            # as this is intended to be seen contextually in the application.
+            disable_external_notifications=True,
+        )
+    )
+
+    if welcome_bot_custom_message_string:
+        welcome_bot_messages.append(
+            internal_prep_private_message(
+                get_system_bot(settings.WELCOME_BOT, user.realm_id),
+                user,
+                welcome_bot_custom_message_string,
+                disable_external_notifications=True,
+            )
+        )
+
+    sent_messages = do_send_messages(welcome_bot_messages)
+
+    welcome_bot_custom_message_id = None
+    welcome_bot_intro_message_id = sent_messages[0].message_id
+    if welcome_bot_custom_message_string:
+        welcome_bot_custom_message_id = sent_messages[1].message_id
+
+    return InitialDirectMessageIDs(
+        welcome_bot_intro_message_id=welcome_bot_intro_message_id,
+        welcome_bot_custom_message_id=welcome_bot_custom_message_id,
+    )
+
+
+def bot_commands(no_help_command: bool = False) -> str:
+    commands = [
+        "apps",
+        "profile",
+        "theme",
+        "channels",
+        "topics",
+        "message formatting",
+        "keyboard shortcuts",
+    ]
+    if not no_help_command:
+        commands.append("help")
+    return ", ".join("`" + command + "`" for command in commands) + "."
+
+
+def select_welcome_bot_response(human_response_lower: str) -> str:
+    # Given the raw (pre-markdown-rendering) content for a private
+    # message from the user to Welcome Bot, select the appropriate reply.
+    if human_response_lower in ["app", "apps"]:
+        return _("""
+You can [download](/apps/) the [mobile and desktop apps](/apps/).
+Zulip also works great in a browser.
+""")
+    elif human_response_lower == "profile":
+        return _("""
+Go to [Profile settings](#settings/profile) to add a [profile picture](/help/change-your-profile-picture)
+and edit your [profile information](/help/edit-your-profile).
+""")
+    elif human_response_lower == "theme":
+        return _("""
+You can switch between [light and dark theme](/help/dark-theme), [pick your
+favorite emoji set](/help/emoji-and-emoticons#change-your-emoji-set), [change
+your language](/help/change-your-language), and otherwise customize your Zulip
+experience in your [Preferences](#settings/preferences).
+""")
+    elif human_response_lower in ["stream", "streams", "channel", "channels"]:
+        return _("""
+Channels organize conversations based on who needs to see them. For example,
+it's common to have a channel for each team in an organization.
+
+[Browse and subscribe to channels]({settings_link}).
+""").format(help_link="/help/introduction-to-channels", settings_link="#channels/all")
+    elif human_response_lower in ["topic", "topics"]:
+        return _("""
+[Topics](/help/introduction-to-topics) summarize what each conversation in Zulip
+is about. You can read Zulip one topic at a time, seeing each message in
+context, no matter how many other conversations are going on.
+
+When you start a conversation, label it with a new topic. For a good topic name,
+think about finishing the sentence: “Hey, can we chat about…?”
+
+Check out [Recent conversations](#recent) for a list of topics that are being
+discussed.
+""")
+    elif human_response_lower in ["keyboard", "shortcuts", "keyboard shortcuts"]:
+        return _("""
+Zulip's [keyboard shortcuts](#keyboard-shortcuts) let you navigate the app
+quickly and efficiently.
+
+Press `?` any time to see a [cheat sheet](#keyboard-shortcuts).
+""")
+    elif human_response_lower in ["formatting", "message formatting"]:
+        return _("""
+You can **format** *your* `message` using the handy formatting buttons, or by
+typing your formatting with Markdown.
+
+Check out the [cheat sheet](#message-formatting) to learn about spoilers, global
+times, and more.
+""")
+    elif human_response_lower in ["help", "?"]:
+        return _("""
+Here are a few messages I understand: {bot_commands}
+
+Check out our [Getting started guide](/help/getting-started-with-zulip),
+or browse the [help center](/help/) to learn more!
+""").format(bot_commands=bot_commands(no_help_command=True))
+    else:
+        return _("""
+You can chat with me as much as you like! To
+get help, try one of the following messages: {bot_commands}
+""").format(bot_commands=bot_commands())
+
+
+def send_welcome_bot_response(send_request: SendMessageRequest) -> None:
+    """Given the send_request object for a direct message from the user
+    to welcome-bot, trigger the welcome-bot reply."""
+    welcome_bot = get_system_bot(settings.WELCOME_BOT, send_request.realm.id)
+    human_response_lower = send_request.message.content.lower()
+    assert send_request.message.recipient.type == Recipient.DIRECT_MESSAGE_GROUP
+    conversation_recipient_id = send_request.message.recipient.id
+    content = select_welcome_bot_response(human_response_lower)
+    realm_id = send_request.realm.id
+    commands = bot_commands()
+    if (
+        commands in content
+        and Message.objects.filter(
+            realm_id=realm_id,
+            sender_id=welcome_bot.id,
+            recipient_id=conversation_recipient_id,
+            content__icontains=commands,
+        ).exists()
+        # Uses index 'zerver_message_realm_sender_recipient'
+    ):
+        # If the bot has already sent bot commands to this user and
+        # if the bot does not understand the current message sent by this user then
+        # do not send any message
+        return
+
+    internal_send_private_message(
+        welcome_bot,
+        send_request.message.sender,
+        remove_single_newlines(content),
+        # Note: Welcome bot doesn't trigger email/push notifications,
+        # as this is intended to be seen contextually in the application.
+        disable_external_notifications=True,
+    )
+
+
+class OnboardingMessageTypeEnum(Enum):
+    moving_messages = 1
+    welcome_to_zulip = 2
+    start_conversation = 3
+    experiments = 4
+    greetings = 5
+
+
+@transaction.atomic(savepoint=False)
+def send_initial_realm_messages(
+    realm: Realm,
+    override_channel_name_map: dict[OnboardingMessageTypeEnum, str | None] | None = None,
+) -> None:
+    """
+    override_channel_name_map allows the caller to customize to which channels to send
+    specific categories of the initial messages. In this override dict, every category
+    from OnboardingMessageTypeEnum must be specified.
+
+    The caller can disable the sending of a specific category by mapping it to None
+    in override_channel_name_map.
+    """
+
+    if override_channel_name_map is not None:
+        channel_name_map = override_channel_name_map
+    else:
+        channel_name_map = {
+            OnboardingMessageTypeEnum.moving_messages: str(Realm.ZULIP_DISCUSSION_CHANNEL_NAME),
+            OnboardingMessageTypeEnum.welcome_to_zulip: str(Realm.ZULIP_DISCUSSION_CHANNEL_NAME),
+            OnboardingMessageTypeEnum.start_conversation: str(Realm.ZULIP_SANDBOX_CHANNEL_NAME),
+            OnboardingMessageTypeEnum.experiments: str(Realm.ZULIP_SANDBOX_CHANNEL_NAME),
+            OnboardingMessageTypeEnum.greetings: str(Realm.DEFAULT_NOTIFICATION_STREAM_NAME),
+        }
+    assert set(channel_name_map.keys()) == set(OnboardingMessageTypeEnum)
+
+    # Sends the initial messages for a new organization.
+    #
+    # Technical note: Each stream created in the realm creation
+    # process should have at least one message declared in this
+    # function, to enforce the pseudo-invariant that every stream has
+    # at least one message.
+    welcome_bot = get_system_bot(settings.WELCOME_BOT, realm.id)
+
+    # Content is declared here to apply translation properly.
+    #
+    # remove_single_newlines needs to be called on any multiline
+    # strings for them to render properly.
+    content1_of_moving_messages_topic_name = (
+        _("""
+If anything is out of place, it’s easy to [move messages]({move_content_another_topic_help_url}),
+[rename]({rename_topic_help_url}) and [split]({move_content_another_topic_help_url}) topics,
+or even move a topic [to a different channel]({move_content_another_channel_help_url}).
+""")
+    ).format(
+        move_content_another_topic_help_url="/help/move-content-to-another-topic",
+        rename_topic_help_url="/help/rename-a-topic",
+        move_content_another_channel_help_url="/help/move-content-to-another-channel",
+    )
+
+    content2_of_moving_messages_topic_name = _("""
+:point_right: Try moving this message to another topic and back.
+""")
+
+    content1_of_welcome_to_zulip_topic_name = _("""
+Zulip is organized to help you communicate more efficiently. Conversations are
+labeled with topics, which summarize what the conversation is about.
+
+For example, this message is in the “{topic_name}” topic in the
+#**{zulip_discussion_channel_name}** channel, as you can see in the left sidebar
+and above.
+""").format(
+        zulip_discussion_channel_name=channel_name_map[OnboardingMessageTypeEnum.welcome_to_zulip],
+        topic_name=_("welcome to Zulip!"),
+    )
+
+    content2_of_welcome_to_zulip_topic_name = _("""
+You can read Zulip one conversation at a time, seeing each message in context,
+no matter how many other conversations are going on.
+""")
+
+    content3_of_welcome_to_zulip_topic_name = _("""
+:point_right: When you're ready, check out your [Inbox](/#inbox) for other
+conversations with unread messages.
+""")
+
+    content1_of_start_conversation_topic_name = _("""
+To kick off a new conversation, pick a channel in the left sidebar, and click
+the `+` button next to its name.
+""")
+
+    content2_of_start_conversation_topic_name = _("""
+Label your conversation with a topic. Think about finishing the sentence: “Hey,
+can we chat about…?”
+""")
+
+    content3_of_start_conversation_topic_name = _("""
+:point_right: Try starting a new conversation in this channel.
+""")
+
+    content1_of_experiments_topic_name = (
+        _("""
+:point_right:  Use this topic to try out [Zulip's messaging features]({format_message_help_url}).
+""")
+    ).format(format_message_help_url="/help/format-your-message-using-markdown")
+
+    content2_of_experiments_topic_name = (
+        _("""
+```spoiler Want to see some examples?
+
+````python
+
+print("code blocks")
+
+````
+
+- bulleted
+- lists
+
+Link to a conversation: #**{zulip_discussion_channel_name}>{topic_name}**
+
+```
+""")
+    ).format(
+        zulip_discussion_channel_name=channel_name_map[OnboardingMessageTypeEnum.welcome_to_zulip],
+        topic_name=_("welcome to Zulip!"),
+    )
+
+    content1_of_greetings_topic_name = _("""
+This **greetings** topic is a great place to say “hi” :wave: to your teammates.
+""")
+
+    content2_of_greetings_topic_name = _("""
+:point_right: Click on this message to start a new message in the same conversation.
+""")
+
+    welcome_messages: list[dict[str, str]] = []
+
+    # Messages added to the "welcome messages" list last will be most
+    # visible to users, since welcome messages will likely be browsed
+    # via the right sidebar or recent conversations view, both of
+    # which are sorted newest-first.
+    #
+    # Initial messages are configured below.
+
+    # Advertising moving messages.
+    if (
+        moving_messages_channel_name := channel_name_map[OnboardingMessageTypeEnum.moving_messages]
+    ) is not None:
+        welcome_messages += [
+            {
+                "channel_name": moving_messages_channel_name,
+                "topic_name": _("moving messages"),
+                "content": content,
+            }
+            for content in [
+                content1_of_moving_messages_topic_name,
+                content2_of_moving_messages_topic_name,
+            ]
+        ]
+
+    # Suggestion to test messaging features.
+    # Dependency on knowing how to send messages.
+    if (
+        experiments_channel_name := channel_name_map[OnboardingMessageTypeEnum.experiments]
+    ) is not None:
+        welcome_messages += [
+            {
+                "channel_name": experiments_channel_name,
+                "topic_name": _("experiments"),
+                "content": content,
+            }
+            for content in [content1_of_experiments_topic_name, content2_of_experiments_topic_name]
+        ]
+
+    if (
+        start_conversation_channel_name := channel_name_map[
+            OnboardingMessageTypeEnum.start_conversation
+        ]
+    ) is not None:
+        # Suggestion to start your first new conversation.
+        welcome_messages += [
+            {
+                "channel_name": start_conversation_channel_name,
+                "topic_name": _("start a conversation"),
+                "content": content,
+            }
+            for content in [
+                content1_of_start_conversation_topic_name,
+                content2_of_start_conversation_topic_name,
+                content3_of_start_conversation_topic_name,
+            ]
+        ]
+
+    # Suggestion to send first message as a hi to your team.
+    if (
+        greetings_channel_name := channel_name_map[OnboardingMessageTypeEnum.greetings]
+    ) is not None:
+        welcome_messages += [
+            {
+                "channel_name": greetings_channel_name,
+                "topic_name": _("greetings"),
+                "content": content,
+            }
+            for content in [content1_of_greetings_topic_name, content2_of_greetings_topic_name]
+        ]
+
+    if (
+        welcome_to_zulip_channel_name := channel_name_map[
+            OnboardingMessageTypeEnum.welcome_to_zulip
+        ]
+    ) is not None:
+        # Main welcome message, this should be last.
+        welcome_messages += [
+            {
+                "channel_name": welcome_to_zulip_channel_name,
+                "topic_name": _("welcome to Zulip!"),
+                "content": content,
+            }
+            for content in [
+                content1_of_welcome_to_zulip_topic_name,
+                content2_of_welcome_to_zulip_topic_name,
+                content3_of_welcome_to_zulip_topic_name,
+            ]
+        ]
+
+    # End of message declarations; now we actually send them.
+
+    messages = [
+        internal_prep_stream_message_by_name(
+            realm,
+            welcome_bot,
+            message["channel_name"],
+            message["topic_name"],
+            remove_single_newlines(message["content"]),
+        )
+        for message in welcome_messages
+    ]
+    message_ids = [
+        sent_message_result.message_id for sent_message_result in do_send_messages(messages)
+    ]
+
+    seen_topics = set()
+    onboarding_topics_first_message_ids = set()
+    for index, message in enumerate(welcome_messages):
+        topic_name = message["topic_name"]
+        if topic_name not in seen_topics:
+            onboarding_topics_first_message_ids.add(message_ids[index])
+            seen_topics.add(topic_name)
+
+    onboarding_user_messages = []
+    for message_id in message_ids:
+        flags = OnboardingUserMessage.flags.historical
+        if message_id in onboarding_topics_first_message_ids:
+            flags |= OnboardingUserMessage.flags.starred
+        onboarding_user_messages.append(
+            OnboardingUserMessage(realm=realm, message_id=message_id, flags=flags)
+        )
+
+    OnboardingUserMessage.objects.bulk_create(onboarding_user_messages)
+
+    # We find the one of our just-sent greetings messages, and react to it.
+    # This is a bit hacky, but works and is kinda a 1-off thing.
+    greetings_message = (
+        Message.objects.select_for_update(no_key=True)
+        .filter(
+            id__in=message_ids, content=remove_single_newlines(content1_of_greetings_topic_name)
+        )
+        .first()
+    )
+    assert greetings_message is not None
+    emoji_data = get_emoji_data(realm.id, "wave")
+    do_add_reaction(
+        welcome_bot, greetings_message, "wave", emoji_data.emoji_code, emoji_data.reaction_type
+    )

zerver/lib/onboarding_steps.py

@@ -0,0 +1,110 @@
+# See https://zulip.readthedocs.io/en/latest/subsystems/onboarding-steps.html
+# for documentation on this subsystem.
+from dataclasses import dataclass
+
+from django.conf import settings
+
+from zerver.models import OnboardingStep, UserProfile
+
+
+@dataclass
+class APIOnboardingStep:
+    type: str
+    name: str
+
+
+@dataclass
+class OneTimeNotice:
+    name: str
+
+    def to_dict(self) -> APIOnboardingStep:
+        return APIOnboardingStep(
+            type="one_time_notice",
+            name=self.name,
+        )
+
+
+@dataclass
+class OneTimeAction:
+    name: str
+
+    def to_dict(self) -> APIOnboardingStep:
+        return APIOnboardingStep(
+            type="one_time_action",
+            name=self.name,
+        )
+
+
+ONE_TIME_NOTICES: list[OneTimeNotice] = [
+    OneTimeNotice(
+        name="visibility_policy_banner",
+    ),
+    OneTimeNotice(
+        name="intro_inbox_view_modal",
+    ),
+    OneTimeNotice(
+        name="intro_recent_view_modal",
+    ),
+    OneTimeNotice(
+        name="first_stream_created_banner",
+    ),
+    OneTimeNotice(
+        name="jump_to_conversation_banner",
+    ),
+    OneTimeNotice(
+        name="non_interleaved_view_messages_fading",
+    ),
+    OneTimeNotice(
+        name="interleaved_view_messages_fading",
+    ),
+    OneTimeNotice(
+        name="intro_resolve_topic",
+    ),
+    OneTimeNotice(
+        name="navigation_tour_video",
+    ),
+    OneTimeNotice(
+        name="intro_go_to_conversation_tooltip",
+    ),
+]
+
+ONE_TIME_ACTIONS = [OneTimeAction(name="narrow_to_dm_with_welcome_bot_new_user")]
+
+ALL_ONBOARDING_STEPS: list[OneTimeNotice | OneTimeAction] = ONE_TIME_NOTICES + ONE_TIME_ACTIONS
+
+
+def get_next_onboarding_steps(user: UserProfile) -> list[APIOnboardingStep]:
+    # If a Zulip server has disabled the tutorial, never send any
+    # onboarding steps.
+    if not settings.TUTORIAL_ENABLED:
+        return []
+
+    seen_onboarding_steps: list[str] = list(
+        OnboardingStep.objects.filter(user=user).values_list("onboarding_step", flat=True)
+    )
+    if settings.NAVIGATION_TOUR_VIDEO_URL is None:
+        # Server admin disabled navigation tour video, treat it as seen.
+        seen_onboarding_steps.append("navigation_tour_video")
+    seen_onboarding_steps_set = frozenset(seen_onboarding_steps)
+
+    onboarding_steps: list[APIOnboardingStep] = []
+    for onboarding_step in ALL_ONBOARDING_STEPS:
+        if onboarding_step.name in seen_onboarding_steps_set:
+            continue
+        onboarding_steps.append(onboarding_step.to_dict())
+
+    return onboarding_steps
+
+
+def copy_onboarding_steps(source_profile: UserProfile, target_profile: UserProfile) -> None:
+    source_steps = OnboardingStep.objects.filter(user=source_profile)
+    OnboardingStep.objects.bulk_create(
+        [
+            OnboardingStep(
+                user=target_profile,
+                onboarding_step=step.onboarding_step,
+                timestamp=step.timestamp,
+            )
+            for step in source_steps
+        ]
+    )

zerver/lib/outgoing_http.py

@@ -0,0 +1,77 @@
+from typing import Any
+
+import requests
+from django.conf import settings
+from typing_extensions import override
+from urllib3.util import Retry
+
+from zproject.config import get_config
+
+
+class OutgoingSession(requests.Session):
+    """Reasonable outgoing HTTP request defaults.
+
+    This class exists to force requests through the Smokescreen
+    proxy, enforce that requests have timeouts set, and to provide an
+    easy way for them to include retries.  The 'Role' is currently
+    unused -- Smokescreen does not currently consume the
+    X-Smokescreen-Role header, but may do so in the future for
+    additional ACL'ing of destinations and/or logging.
+
+    """
+
+    def __init__(
+        self,
+        role: str,
+        timeout: float,
+        headers: dict[str, str] | None = None,
+        max_retries: int | Retry | None = None,
+        proxies: dict[str, str] | None = None,
+    ) -> None:
+        super().__init__()
+        retry: Retry | None = Retry(total=0)
+        if max_retries is not None:
+            if isinstance(max_retries, Retry):
+                retry = max_retries
+            else:
+                retry = Retry(total=max_retries, backoff_factor=1)
+        outgoing_adapter = OutgoingHTTPAdapter(role=role, timeout=timeout, max_retries=retry)
+        self.mount("http://", outgoing_adapter)
+        self.mount("https://", outgoing_adapter)
+        if headers:
+            self.headers.update(headers)
+
+        if proxies is None and not settings.DEVELOPMENT:
+            proxy_host = get_config("http_proxy", "host", "localhost")
+            proxy_port = get_config("http_proxy", "port", "4750")
+            proxy = ""
+            if proxy_host != "" and proxy_port != "":
+                proxy = f"http://{proxy_host}:{proxy_port}"
+                self.proxies.update(
+                    {
+                        "http": proxy,
+                        "https": proxy,
+                    }
+                )
+        elif proxies is not None:
+            self.proxies.update(proxies)
+
+
+class OutgoingHTTPAdapter(requests.adapters.HTTPAdapter):
+    role: str
+    timeout: float
+
+    def __init__(self, role: str, timeout: float, max_retries: Retry | None) -> None:
+        self.role = role
+        self.timeout = timeout
+        super().__init__(max_retries=max_retries)
+
+    @override
+    def send(self, *args: Any, **kwargs: Any) -> requests.Response:
+        if kwargs.get("timeout") is None:
+            kwargs["timeout"] = self.timeout
+        return super().send(*args, **kwargs)
+
+    @override
+    def proxy_headers(self, proxy: str) -> dict[str, str]:
+        return {"X-Smokescreen-Role": self.role}

zerver/lib/outgoing_webhook.py

@@ -0,0 +1,432 @@
+import abc
+import json
+import logging
+from contextlib import suppress
+from dataclasses import dataclass
+from time import perf_counter
+from typing import Any, AnyStr
+
+import requests
+from django.conf import settings
+from django.utils.translation import gettext as _
+from requests import Response
+from typing_extensions import override
+
+from version import ZULIP_VERSION
+from zerver.actions.message_send import check_send_message
+from zerver.lib.exceptions import JsonableError, StreamDoesNotExistError
+from zerver.lib.message_cache import MessageDict
+from zerver.lib.outgoing_http import OutgoingSession
+from zerver.lib.queue import retry_event
+from zerver.lib.topic import get_topic_from_message_info
+from zerver.lib.url_encoding import message_link_url
+from zerver.lib.users import check_can_access_user, check_user_can_access_all_users
+from zerver.models import Realm, Service, UserProfile
+from zerver.models.bots import GENERIC_INTERFACE, SLACK_INTERFACE
+from zerver.models.clients import get_client
+from zerver.models.users import get_user_profile_by_id
+
+
+@dataclass
+class OutgoingWebhookResult:
+    content: str
+    widget_content: str | None = None
+
+
+class OutgoingWebhookServiceInterface(abc.ABC):
+    def __init__(self, token: str, user_profile: UserProfile, service_name: str) -> None:
+        self.token: str = token
+        self.user_profile: UserProfile = user_profile
+        self.service_name: str = service_name
+        self.session: requests.Session = OutgoingSession(
+            role="webhook",
+            timeout=settings.OUTGOING_WEBHOOK_TIMEOUT_SECONDS,
+            headers={"User-Agent": "ZulipOutgoingWebhook/" + ZULIP_VERSION},
+        )
+
+    @abc.abstractmethod
+    def make_request(self, base_url: str, event: dict[str, Any], realm: Realm) -> Response | None:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def process_success(self, response_json: dict[str, Any]) -> OutgoingWebhookResult | None:
+        raise NotImplementedError
+
+
+class GenericOutgoingWebhookService(OutgoingWebhookServiceInterface):
+    @override
+    def make_request(self, base_url: str, event: dict[str, Any], realm: Realm) -> Response | None:
+        """
+        We send a simple version of the message to outgoing
+        webhooks, since most of them really only need
+        `content` and a few other fields.  We may eventually
+        allow certain bots to get more information, but
+        that's not a high priority.  We do send the gravatar
+        info to the clients (so they don't have to compute
+        it themselves).
+        """
+        message_dict = MessageDict.finalize_payload(
+            event["message"],
+            apply_markdown=False,
+            client_gravatar=False,
+            allow_empty_topic_name=True,
+            keep_rendered_content=True,
+            can_access_sender=check_user_can_access_all_users(self.user_profile)
+            or check_can_access_user(
+                get_user_profile_by_id(event["message"]["sender_id"]), self.user_profile
+            ),
+            realm_host=realm.host,
+        )
+
+        request_data = {
+            "data": event["command"],
+            "message": message_dict,
+            "bot_email": self.user_profile.email,
+            "bot_full_name": self.user_profile.full_name,
+            "token": self.token,
+            "trigger": event["trigger"],
+        }
+
+        return self.session.post(base_url, json=request_data)
+
+    @override
+    def process_success(self, response_json: dict[str, Any]) -> OutgoingWebhookResult | None:
+        if response_json.get("response_not_required", False):
+            return None
+
+        if "response_string" in response_json:
+            # We are deprecating response_string.
+            content = str(response_json["response_string"])
+            return OutgoingWebhookResult(content=content)
+
+        if "content" in response_json:
+            content = str(response_json["content"])
+            widget_content_raw = response_json.get("widget_content")
+            widget_content = (
+                json.dumps(widget_content_raw) if widget_content_raw is not None else None
+            )
+            return OutgoingWebhookResult(content=content, widget_content=widget_content)
+
+        return None
+
+
+class SlackOutgoingWebhookService(OutgoingWebhookServiceInterface):
+    @override
+    def make_request(self, base_url: str, event: dict[str, Any], realm: Realm) -> Response | None:
+        if event["message"]["type"] == "private":
+            failure_message = "Slack outgoing webhooks don't support direct messages."
+            fail_with_message(event, failure_message)
+            return None
+
+        # https://api.slack.com/legacy/custom-integrations/outgoing-webhooks#legacy-info__post-data
+        # documents the Slack outgoing webhook format:
+        #
+        # token=XXXXXXXXXXXXXXXXXX
+        # team_id=T0001
+        # team_domain=example
+        # channel_id=C2147483705
+        # channel_name=test
+        # thread_ts=1504640714.003543
+        # timestamp=1504640775.000005
+        # user_id=U2147483697
+        # user_name=Steve
+        # text=googlebot: What is the air-speed velocity of an unladen swallow?
+        # trigger_word=googlebot:
+
+        request_data = [
+            ("token", self.token),
+            ("team_id", f"T{realm.id}"),
+            ("team_domain", realm.host),
+            ("channel_id", f"C{event['message']['stream_id']}"),
+            ("channel_name", event["message"]["display_recipient"]),
+            ("thread_ts", event["message"]["timestamp"]),
+            ("timestamp", event["message"]["timestamp"]),
+            ("user_id", f"U{event['message']['sender_id']}"),
+            ("user_name", event["message"]["sender_full_name"]),
+            ("text", event["command"]),
+            ("trigger_word", event["trigger"]),
+            ("service_id", event["user_profile_id"]),
+        ]
+        return self.session.post(base_url, data=request_data)
+
+    @override
+    def process_success(self, response_json: dict[str, Any]) -> OutgoingWebhookResult | None:
+        if "text" in response_json:
+            content = response_json["text"]
+            return OutgoingWebhookResult(content=content)
+
+        return None
+
+
+AVAILABLE_OUTGOING_WEBHOOK_INTERFACES: dict[str, Any] = {
+    GENERIC_INTERFACE: GenericOutgoingWebhookService,
+    SLACK_INTERFACE: SlackOutgoingWebhookService,
+}
+
+
+def get_service_interface_class(interface: str) -> Any:
+    if interface not in AVAILABLE_OUTGOING_WEBHOOK_INTERFACES:
+        return AVAILABLE_OUTGOING_WEBHOOK_INTERFACES[GENERIC_INTERFACE]
+    else:
+        return AVAILABLE_OUTGOING_WEBHOOK_INTERFACES[interface]
+
+
+def get_outgoing_webhook_service_handler(service: Service) -> Any:
+    service_interface_class = get_service_interface_class(service.interface_name())
+    service_interface = service_interface_class(
+        token=service.token, user_profile=service.user_profile, service_name=service.name
+    )
+    return service_interface
+
+
+def send_response_message(
+    bot_id: int, message_info: dict[str, Any], response_data: dict[str, Any]
+) -> None:
+    """
+    bot_id is the user_id of the bot sending the response
+
+    message_info is used to address the message and should have these fields:
+        type - "stream" or "private"
+        display_recipient - like we have in other message events
+        topic - see get_topic_from_message_info
+
+    response_data is what the bot wants to send back and has these fields:
+        content - raw Markdown content for Zulip to render
+
+    WARNING: This function sends messages bypassing the stream access check
+    for the bot - so use with caution to not call this in codepaths
+    that might let someone send arbitrary messages to any stream through this.
+    """
+
+    recipient_type_name = message_info["type"]
+    display_recipient = message_info["display_recipient"]
+    try:
+        topic_name: str | None = get_topic_from_message_info(message_info)
+    except KeyError:
+        topic_name = None
+
+    bot_user = get_user_profile_by_id(bot_id)
+    realm = bot_user.realm
+    client = get_client("OutgoingWebhookResponse")
+
+    content = response_data.get("content")
+    assert content
+
+    widget_content = response_data.get("widget_content")
+
+    if recipient_type_name == "stream":
+        message_to = [display_recipient]
+    elif recipient_type_name == "private":
+        message_to = [recipient["email"] for recipient in display_recipient]
+    else:
+        raise JsonableError(_("Invalid message type"))
+
+    check_send_message(
+        sender=bot_user,
+        client=client,
+        recipient_type_name=recipient_type_name,
+        message_to=message_to,
+        topic_name=topic_name,
+        message_content=content,
+        widget_content=widget_content,
+        realm=realm,
+        skip_stream_access_check=True,
+    )
+
+
+def fail_with_message(event: dict[str, Any], failure_message: str) -> None:
+    bot_id = event["user_profile_id"]
+    message_info = event["message"]
+    content = "Failure! " + failure_message
+    response_data = dict(content=content)
+    # If the stream has vanished while we were failing, there's no
+    # reasonable place to report the error.
+    with suppress(StreamDoesNotExistError):
+        send_response_message(bot_id=bot_id, message_info=message_info, response_data=response_data)
+
+
+def get_message_url(event: dict[str, Any]) -> str:
+    bot_user = get_user_profile_by_id(event["user_profile_id"])
+    message = event["message"]
+    realm = bot_user.realm
+
+    return message_link_url(
+        realm=realm,
+        message=message,
+    )
+
+
+def notify_bot_owner(
+    event: dict[str, Any],
+    status_code: int | None = None,
+    response_content: AnyStr | None = None,
+    failure_message: str | None = None,
+    exception: Exception | None = None,
+) -> None:
+    message_url = get_message_url(event)
+    bot_id = event["user_profile_id"]
+    bot = get_user_profile_by_id(bot_id)
+    bot_owner = bot.bot_owner
+    assert bot_owner is not None
+
+    notification_message = f"[A message]({message_url}) to your bot @_**{bot.full_name}** triggered an outgoing webhook."
+    if exception:
+        notification_message += (
+            "\nWhen trying to send a request to the webhook service, an exception "
+            f"of type {type(exception).__name__} occurred:\n```\n{exception}\n```"
+        )
+    elif failure_message:
+        notification_message += "\n" + failure_message
+    elif status_code == 407:
+        notification_message += (
+            "\nThe URL configured for the webhook is for a private or disallowed network."
+        )
+    elif status_code:
+        notification_message += f"\nThe webhook got a response with status code *{status_code}*."
+
+    if response_content:
+        notification_message += (
+            f"\nThe response contains the following payload:\n```\n{response_content!r}\n```"
+        )
+
+    message_info = dict(
+        type="private",
+        display_recipient=[dict(email=bot_owner.email)],
+    )
+    response_data = dict(content=notification_message)
+    send_response_message(bot_id=bot_id, message_info=message_info, response_data=response_data)
+
+
+def request_retry(event: dict[str, Any], failure_message: str | None = None) -> None:
+    def failure_processor(event: dict[str, Any]) -> None:
+        """
+        The name of the argument is 'event' on purpose. This argument will hide
+        the 'event' argument of the request_retry function. Keeping the same name
+        results in a smaller diff.
+        """
+        bot_user = get_user_profile_by_id(event["user_profile_id"])
+        fail_with_message(event, "Bot is unavailable")
+        notify_bot_owner(event, failure_message=failure_message)
+        logging.warning(
+            "Maximum retries exceeded for trigger:%s event:%s",
+            bot_user.email,
+            event["command"],
+        )
+
+    retry_event("outgoing_webhooks", event, failure_processor)
+
+
+def process_success_response(
+    event: dict[str, Any], service_handler: Any, response: Response
+) -> None:
+    try:
+        response_json = json.loads(response.text)
+    except json.JSONDecodeError:
+        raise JsonableError(_("Invalid JSON in response"))
+
+    if response_json == "":
+        # Versions of zulip_botserver before 2021-05 used
+        # json.dumps("") as their "no response required" success
+        # response; handle that for backwards-compatibility.
+        return
+
+    if not isinstance(response_json, dict):
+        raise JsonableError(_("Invalid response format"))
+
+    success_data = service_handler.process_success(response_json)
+
+    if success_data is None:
+        return
+
+    content = success_data.content
+
+    if content is None or content.strip() == "":
+        return
+
+    widget_content = success_data.widget_content
+    bot_id = event["user_profile_id"]
+    message_info = event["message"]
+    response_data = dict(content=content, widget_content=widget_content)
+    send_response_message(bot_id=bot_id, message_info=message_info, response_data=response_data)
+
+
+def do_rest_call(
+    base_url: str,
+    event: dict[str, Any],
+    service_handler: OutgoingWebhookServiceInterface,
+) -> Response | None:
+    """Returns response of call if no exception occurs."""
+    try:
+        start_time = perf_counter()
+        bot_profile = service_handler.user_profile
+        response = service_handler.make_request(
+            base_url,
+            event,
+            bot_profile.realm,
+        )
+        logging.info(
+            "Outgoing webhook request from %s@%s took %f seconds",
+            bot_profile.id,
+            bot_profile.realm.string_id,
+            perf_counter() - start_time,
+        )
+        if response is None:
+            return None
+        if str(response.status_code).startswith("2"):
+            try:
+                process_success_response(event, service_handler, response)
+            except JsonableError as e:
+                response_message = e.msg
+                logging.info("Outhook trigger failed:", stack_info=True)
+                fail_with_message(event, response_message)
+                response_message = f"The outgoing webhook server attempted to send a message in Zulip, but that request resulted in the following error:\n> {e}"
+                notify_bot_owner(
+                    event, response_content=response.text, failure_message=response_message
+                )
+                return None
+        else:
+            logging.warning(
+                "Message %(message_url)s triggered an outgoing webhook, returning status "
+                'code %(status_code)s.\n Content of response (in quotes): "'
+                '%(response)s"',
+                {
+                    "message_url": get_message_url(event),
+                    "status_code": response.status_code,
+                    "response": response.text,
+                },
+            )
+            failure_message = f"Third party responded with {response.status_code}"
+            fail_with_message(event, failure_message)
+            notify_bot_owner(event, response.status_code, response.content)
+        return response
+    except requests.exceptions.Timeout:
+        logging.info(
+            "Trigger event %s on %s timed out. Retrying",
+            event["command"],
+            event["service_name"],
+        )
+        failure_message = (
+            f"Request timed out after {settings.OUTGOING_WEBHOOK_TIMEOUT_SECONDS} seconds."
+        )
+        request_retry(event, failure_message=failure_message)
+        return None
+
+    except (requests.exceptions.ConnectionError, requests.exceptions.ChunkedEncodingError):
+        logging.info(
+            "Trigger event %s on %s resulted in a connection error. Retrying",
+            event["command"],
+            event["service_name"],
+        )
+        failure_message = "A connection error occurred. Is my bot server down?"
+        request_retry(event, failure_message=failure_message)
+        return None
+
+    except requests.exceptions.RequestException as e:
+        response_message = (
+            f"An exception of type *{type(e).__name__}* occurred for message `{event['command']}`! "
+            "See the Zulip server logs for more information."
+        )
+        logging.exception("Outhook trigger failed:", stack_info=True)
+        fail_with_message(event, response_message)
+        notify_bot_owner(event, exception=e)
+        return None

zerver/lib/parallel.py

@@ -0,0 +1,127 @@
+import logging
+from collections.abc import Callable, Iterable, Iterator
+from concurrent.futures import BrokenExecutor, Future, ProcessPoolExecutor
+from contextlib import contextmanager
+from multiprocessing import current_process
+from typing import Any, TypeVar
+
+import bmemcached
+from django.conf import settings
+from django.core.cache import cache
+from django.db import connection
+
+from zerver.lib.partial import partial
+from zerver.lib.queue import get_queue_client
+
+ParallelRecordType = TypeVar("ParallelRecordType")
+
+
+def _disconnect() -> None:
+    # Close our database, cache, and RabbitMQ connections, so our
+    # forked children do not share them.  Django will transparently
+    # re-open them as needed.
+    connection.close()
+    _cache = cache._cache  # type: ignore[attr-defined] # not in stubs
+    if isinstance(_cache, bmemcached.Client):  # nocoverage
+        # In tests, this is an OrderedDict
+        _cache.disconnect_all()
+
+    if settings.USING_RABBITMQ:  # nocoverage
+        rabbitmq_client = get_queue_client()
+        if rabbitmq_client.connection and rabbitmq_client.connection.is_open:
+            rabbitmq_client.close()
+
+
+def func_with_catch(func: Callable[[ParallelRecordType], None], item: ParallelRecordType) -> None:
+    try:
+        return func(item)
+    except Exception:
+        logging.exception("Error processing item: %s", item)
+
+
+def run_parallel(
+    func: Callable[[ParallelRecordType], None],
+    records: Iterable[ParallelRecordType],
+    processes: int,
+    *,
+    initializer: Callable[..., None] | None = None,
+    initargs: tuple[Any, ...] = tuple(),
+    catch: bool = False,
+    report_every: int = 1000,
+    report: Callable[[int], None] | None = None,
+) -> None:
+    with run_parallel_queue(
+        func,
+        processes,
+        initializer=initializer,
+        initargs=initargs,
+        catch=catch,
+        report_every=report_every,
+        report=report,
+    ) as submit:
+        for record in records:
+            submit(record)
+
+
+@contextmanager
+def run_parallel_queue(
+    func: Callable[[ParallelRecordType], None],
+    processes: int,
+    *,
+    initializer: Callable[..., None] | None = None,
+    initargs: tuple[Any, ...] = tuple(),
+    catch: bool = False,
+    report_every: int = 1000,
+    report: Callable[[int], None] | None = None,
+) -> Iterator[Callable[[ParallelRecordType], None]]:
+    assert processes > 0
+    if settings.TEST_SUITE and current_process().daemon:  # nocoverage
+        assert processes == 1, "Only one process possible under parallel tests"
+
+    wrapped_func = partial(func_with_catch, func) if catch else func
+
+    completed = 0
+    if processes == 1:
+
+        def func_with_notify(item: ParallelRecordType) -> None:
+            wrapped_func(item)
+            nonlocal completed
+            completed += 1
+            if report is not None and completed % report_every == 0:
+                report(completed)
+
+        if initializer is not None:
+            initializer(*initargs)
+        yield func_with_notify
+        return
+
+    else:  # nocoverage
+        _disconnect()
+
+        exceptions = []
+        try:
+            with ProcessPoolExecutor(
+                max_workers=processes, initializer=initializer, initargs=initargs
+            ) as executor:
+
+                def report_callback(future: Future[None]) -> None:
+                    if exc := future.exception():
+                        exceptions.append(exc)
+                        return
+
+                    nonlocal completed
+                    completed += 1
+                    if report is not None and completed % report_every == 0:
+                        report(completed)
+
+                def future_with_notify(item: ParallelRecordType) -> None:
+                    if exceptions:
+                        executor.shutdown(cancel_futures=True)
+                        raise BrokenExecutor
+                    future = executor.submit(wrapped_func, item)
+                    future.add_done_callback(report_callback)
+
+                yield future_with_notify
+        finally:
+            if exceptions:
+                raise exceptions[0]

zerver/lib/partial.py

@@ -0,0 +1,39 @@
+# Workaround for missing functools.partial support in mypy
+# (https://github.com/python/mypy/issues/1484).
+
+from collections.abc import Callable
+from typing import TYPE_CHECKING, TypeVar, overload
+
+if TYPE_CHECKING:
+    from typing import Concatenate
+
+    from typing_extensions import ParamSpec
+
+    P = ParamSpec("P")
+    T1 = TypeVar("T1")
+    T2 = TypeVar("T2")
+    T3 = TypeVar("T3")
+    T4 = TypeVar("T4")
+    R = TypeVar("R")
+
+    @overload
+    def partial(func: Callable[P, R], /) -> Callable[P, R]: ...
+    @overload
+    def partial(func: Callable[Concatenate[T1, P], R], arg1: T1, /) -> Callable[P, R]: ...
+    @overload
+    def partial(
+        func: Callable[Concatenate[T1, T2, P], R], arg1: T1, arg2: T2, /
+    ) -> Callable[P, R]: ...
+    @overload
+    def partial(
+        func: Callable[Concatenate[T1, T2, T3, P], R], arg1: T1, arg2: T2, arg3: T3, /
+    ) -> Callable[P, R]: ...
+    @overload
+    def partial(
+        func: Callable[Concatenate[T1, T2, T3, T4, P], R], arg1: T1, arg2: T2, arg3: T3, arg4: T4, /
+    ) -> Callable[P, R]: ...
+
+    def partial(func: Callable[..., R], /, *args: object) -> Callable[..., R]: ...
+
+else:
+    from functools import partial as partial

zerver/lib/per_request_cache.py

@@ -0,0 +1,40 @@
+import functools
+from collections.abc import Callable, Hashable
+from typing import Any, TypeVar
+
+ReturnT = TypeVar("ReturnT")
+
+FUNCTION_NAME_TO_PER_REQUEST_RESULT: dict[str, dict[Any, Any]] = {}
+
+
+def cache_for_current_request(f: Callable[..., ReturnT]) -> Callable[..., ReturnT]:
+    """Cache the return value for each distinct set of arguments
+    during the current request.  Unlike
+    @cache_for_current_request, this supports functions
+    with any hashable arguments, not just a single int key."""
+    cache_key = f.__name__
+
+    assert cache_key not in FUNCTION_NAME_TO_PER_REQUEST_RESULT
+    FUNCTION_NAME_TO_PER_REQUEST_RESULT[cache_key] = {}
+
+    @functools.wraps(f)
+    def wrapper(*args: Hashable, **kwargs: Hashable) -> ReturnT:
+        cache = FUNCTION_NAME_TO_PER_REQUEST_RESULT[cache_key]
+        key = (args, frozenset(kwargs.items()))
+        if key in cache:
+            return cache[key]
+        result = f(*args, **kwargs)
+        cache[key] = result
+        return result
+
+    return wrapper
+
+
+def flush_per_request_cache(cache_key: str) -> None:
+    if cache_key in FUNCTION_NAME_TO_PER_REQUEST_RESULT:
+        FUNCTION_NAME_TO_PER_REQUEST_RESULT[cache_key] = {}
+
+
+def flush_per_request_caches() -> None:
+    for cache_key in FUNCTION_NAME_TO_PER_REQUEST_RESULT:
+        FUNCTION_NAME_TO_PER_REQUEST_RESULT[cache_key] = {}

zerver/lib/presence.py

@@ -0,0 +1,279 @@
+import time
+from collections import defaultdict
+from collections.abc import Mapping, Sequence
+from datetime import datetime, timedelta
+from typing import Any
+
+from django.conf import settings
+from django.utils.timezone import now as timezone_now
+
+from zerver.lib.timestamp import datetime_to_timestamp
+from zerver.lib.users import check_user_can_access_all_users, get_accessible_user_ids
+from zerver.models import Realm, UserPresence, UserProfile
+
+
+def get_presence_dicts_for_rows(
+    all_rows: Sequence[Mapping[str, Any]], slim_presence: bool
+) -> dict[str, dict[str, Any]]:
+    # This function takes the presence data fetched from the database and
+    # turn it into an appropriate format for the API to return to clients.
+    # Used by the endpoints that fetch presence data:
+    # 1) `POST /users/me/presence`: https://zulip.com/api/update-presence
+    # 2) `POST /register` when presence data is requested:
+    #    https://zulip.com/api/register-queue
+    # 3) `GET /users/{user_id_or_email}/presence`: https://zulip.com/api/get-user-presence
+    if slim_presence:
+        # Stringify user_id here, since it's gonna be turned
+        # into a string anyway by JSON, and it keeps mypy happy.
+        get_user_key = lambda row: str(row["user_profile_id"])
+        get_user_presence_info = get_modern_user_presence_info
+    else:
+        get_user_key = lambda row: row["user_profile__email"]
+        get_user_presence_info = get_legacy_user_presence_info
+
+    user_statuses: dict[str, dict[str, Any]] = {}
+
+    for presence_row in all_rows:
+        user_key = get_user_key(presence_row)
+
+        last_active_time = user_presence_datetime_with_date_joined_default(
+            presence_row["last_active_time"], presence_row["user_profile__date_joined"]
+        )
+        last_connected_time = user_presence_datetime_with_date_joined_default(
+            presence_row["last_connected_time"], presence_row["user_profile__date_joined"]
+        )
+
+        info = get_user_presence_info(
+            last_active_time,
+            last_connected_time,
+        )
+        user_statuses[user_key] = info
+
+    return user_statuses
+
+
+def user_presence_datetime_with_date_joined_default(
+    dt: datetime | None, date_joined: datetime
+) -> datetime:
+    """
+    Our data models support UserPresence objects not having None
+    values for last_active_time/last_connected_time. The legacy API
+    however has always sent timestamps, so for backward
+    compatibility we cannot send such values through the API and need
+    to default to a sane
+
+    This helper functions expects to take a last_active_time or
+    last_connected_time value and the date_joined of the user, which
+    will serve as the default value if the first argument is None.
+    """
+    if dt is None:
+        return date_joined
+
+    return dt
+
+
+def get_modern_user_presence_info(
+    last_active_time: datetime, last_connected_time: datetime
+) -> dict[str, Any]:
+    # TODO: Do further bandwidth optimizations to this structure.
+    result = {}
+    result["active_timestamp"] = datetime_to_timestamp(last_active_time)
+    result["idle_timestamp"] = datetime_to_timestamp(last_connected_time)
+    return result
+
+
+def get_legacy_user_presence_info(
+    last_active_time: datetime, last_connected_time: datetime
+) -> dict[str, Any]:
+    """
+    Reformats the modern UserPresence data structure so that legacy
+    API clients can still access presence data.
+    We expect this code to remain mostly unchanged until we can delete it.
+    """
+
+    # Now we put things together in the legacy presence format with
+    # one client + an `aggregated` field.
+    #
+    # TODO: Look at whether we can drop to just the "aggregated" field
+    # if no clients look at the rest.
+    most_recent_info = format_legacy_presence_dict(last_active_time, last_connected_time)
+
+    result = {}
+
+    # The word "aggregated" here is possibly misleading.
+    # It's really just the most recent client's info.
+    result["aggregated"] = dict(
+        client=most_recent_info["client"],
+        status=most_recent_info["status"],
+        timestamp=most_recent_info["timestamp"],
+    )
+
+    result["website"] = most_recent_info
+
+    return result
+
+
+def format_legacy_presence_dict(
+    last_active_time: datetime, last_connected_time: datetime
+) -> dict[str, Any]:
+    """
+    This function assumes it's being called right after the presence object was updated,
+    and is not meant to be used on old presence data.
+    """
+    if (
+        last_active_time
+        + timedelta(seconds=settings.PRESENCE_LEGACY_EVENT_OFFSET_FOR_ACTIVITY_SECONDS)
+        >= last_connected_time
+    ):
+        status = UserPresence.LEGACY_STATUS_ACTIVE
+        timestamp = datetime_to_timestamp(last_active_time)
+    else:
+        status = UserPresence.LEGACY_STATUS_IDLE
+        timestamp = datetime_to_timestamp(last_connected_time)
+
+    # This field was never used by clients of the legacy API, so we
+    # just set it to a fixed value for API format compatibility.
+    pushable = False
+
+    return dict(client="website", status=status, timestamp=timestamp, pushable=pushable)
+
+
+def get_presence_for_user(
+    user_profile_id: int, slim_presence: bool = False
+) -> dict[str, dict[str, Any]]:
+    query = UserPresence.objects.filter(user_profile_id=user_profile_id).values(
+        "last_active_time",
+        "last_connected_time",
+        "user_profile__email",
+        "user_profile_id",
+        "user_profile__enable_offline_push_notifications",
+        "user_profile__date_joined",
+    )
+    presence_rows = list(query)
+
+    return get_presence_dicts_for_rows(presence_rows, slim_presence)
+
+
+def get_presence_dict_by_realm(
+    realm: Realm,
+    slim_presence: bool = False,
+    last_update_id_fetched_by_client: int | None = None,
+    history_limit_days: int | None = None,
+    requesting_user_profile: UserProfile | None = None,
+) -> tuple[dict[str, dict[str, Any]], int]:
+    now = timezone_now()
+    if history_limit_days is not None:
+        fetch_since_datetime = now - timedelta(days=history_limit_days)
+    else:
+        # The original behavior for this API was to return last two weeks
+        # of data at most, so we preserve that when the history_limit_days
+        # param is not provided.
+        fetch_since_datetime = now - timedelta(days=14)
+
+    kwargs: dict[str, object] = dict()
+    if last_update_id_fetched_by_client is not None:
+        kwargs["last_update_id__gt"] = last_update_id_fetched_by_client
+
+    if last_update_id_fetched_by_client is None or last_update_id_fetched_by_client <= 0:
+        # If the client already has fetched some presence data, as indicated by
+        # last_update_id_fetched_by_client, then filtering by last_connected_time
+        # is redundant, as it shouldn't affect the results.
+        kwargs["last_connected_time__gte"] = fetch_since_datetime
+
+    if history_limit_days != 0:
+        query = UserPresence.objects.filter(
+            realm_id=realm.id,
+            user_profile__is_active=True,
+            user_profile__is_bot=False,
+            **kwargs,
+        )
+    else:
+        # If history_limit_days is 0, the client doesn't want any presence data.
+        # Explicitly return an empty QuerySet to avoid a query or races which
+        # might cause a UserPresence row to get fetched if it gets updated
+        # during the execution of this function.
+        query = UserPresence.objects.none()
+
+    if settings.CAN_ACCESS_ALL_USERS_GROUP_LIMITS_PRESENCE and not check_user_can_access_all_users(
+        requesting_user_profile
+    ):
+        assert requesting_user_profile is not None
+        accessible_user_ids = get_accessible_user_ids(requesting_user_profile)
+        query = query.filter(user_profile_id__in=accessible_user_ids)
+
+    presence_rows = list(
+        query.values(
+            "last_active_time",
+            "last_connected_time",
+            "user_profile__email",
+            "user_profile_id",
+            "user_profile__enable_offline_push_notifications",
+            "user_profile__date_joined",
+            "last_update_id",
+        )
+    )
+    # Get max last_update_id from the list.
+    if presence_rows:
+        last_update_id_fetched_by_server: int | None = max(
+            row["last_update_id"] for row in presence_rows
+        )
+    elif last_update_id_fetched_by_client is not None:
+        # If there are no results, that means that are no new updates to presence
+        # since what the client has last seen. Therefore, returning the same
+        # last_update_id that the client provided is correct.
+        last_update_id_fetched_by_server = last_update_id_fetched_by_client
+    else:
+        # If the client didn't specify a last_update_id, we return -1 to indicate
+        # the lack of any data fetched, while sticking to the convention of
+        # returning an integer.
+        last_update_id_fetched_by_server = -1
+
+    assert last_update_id_fetched_by_server is not None
+    return get_presence_dicts_for_rows(
+        presence_rows, slim_presence
+    ), last_update_id_fetched_by_server
+
+
+def get_presences_for_realm(
+    realm: Realm,
+    slim_presence: bool,
+    last_update_id_fetched_by_client: int | None,
+    history_limit_days: int | None,
+    requesting_user_profile: UserProfile,
+) -> tuple[dict[str, dict[str, dict[str, Any]]], int]:
+    if realm.presence_disabled:  # nocoverage
+        # Return an empty dict if presence is disabled in this realm
+        return defaultdict(dict), -1
+
+    return get_presence_dict_by_realm(
+        realm,
+        slim_presence,
+        last_update_id_fetched_by_client,
+        history_limit_days,
+        requesting_user_profile=requesting_user_profile,
+    )
+
+
+def get_presence_response(
+    requesting_user_profile: UserProfile,
+    slim_presence: bool,
+    last_update_id_fetched_by_client: int | None = None,
+    history_limit_days: int | None = None,
+) -> dict[str, Any]:
+    realm = requesting_user_profile.realm
+    server_timestamp = time.time()
+    presences, last_update_id_fetched_by_server = get_presences_for_realm(
+        realm,
+        slim_presence,
+        last_update_id_fetched_by_client,
+        history_limit_days,
+        requesting_user_profile=requesting_user_profile,
+    )
+
+    response_dict = dict(
+        presences=presences,
+        server_timestamp=server_timestamp,
+        presence_last_update_id=last_update_id_fetched_by_server,
+    )
+
+    return response_dict

zerver/lib/profile.py

@@ -0,0 +1,38 @@
+import cProfile
+from collections.abc import Callable
+from functools import wraps
+from typing import TypeVar
+
+from typing_extensions import ParamSpec
+
+ParamT = ParamSpec("ParamT")
+ReturnT = TypeVar("ReturnT")
+
+
+def profiled(func: Callable[ParamT, ReturnT]) -> Callable[ParamT, ReturnT]:
+    """
+    This decorator should obviously be used only in a dev environment.
+    It works best when surrounding a function that you expect to be
+    called once.  One strategy is to write a backend test and wrap the
+    test case with the profiled decorator.
+
+    You can run a single test case like this:
+
+        # edit zerver/tests/test_external.py and place @profiled above the test case below
+        ./tools/test-backend zerver.tests.test_external.RateLimitTests.test_ratelimit_decrease
+
+    Then view the results like this:
+
+        ./tools/show-profile-results test_ratelimit_decrease.profile
+
+    """
+
+    @wraps(func)
+    def wrapped_func(*args: ParamT.args, **kwargs: ParamT.kwargs) -> ReturnT:
+        fn = func.__name__ + ".profile"
+        prof = cProfile.Profile()
+        retval = prof.runcall(func, *args, **kwargs)
+        prof.dump_stats(fn)
+        return retval
+
+    return wrapped_func

zerver/lib/push_registration.py

@@ -0,0 +1,220 @@
+import base64
+import binascii
+import hashlib
+import logging
+from typing import TypedDict
+
+from django.conf import settings
+from django.utils.translation import gettext as _
+
+from zerver.lib.devices import b64decode_token_id_base64
+from zerver.lib.exceptions import (
+    InvalidBouncerPublicKeyError,
+    InvalidEncryptedPushRegistrationError,
+    JsonableError,
+    MissingRemoteRealmError,
+    RequestExpiredError,
+)
+from zerver.lib.push_notifications import remove_push_device_token
+from zerver.lib.remote_server import (
+    PushNotificationBouncerError,
+    PushNotificationBouncerRetryLaterError,
+    PushNotificationBouncerServerError,
+    send_to_push_bouncer,
+)
+from zerver.models.devices import Device
+from zerver.models.push_notifications import PushDeviceToken
+from zerver.models.users import UserProfile, get_user_profile_by_id
+from zerver.tornado.django_api import send_event_on_commit
+
+if settings.ZILENCER_ENABLED:
+    from zilencer.views import do_register_remote_push_device
+
+logger = logging.getLogger(__name__)
+
+
+class RegisterPushDeviceToBouncerQueueItem(TypedDict):
+    user_profile_id: int
+    device_id: int
+    bouncer_public_key: str
+    encrypted_push_registration: str
+    token_id_base64: str
+
+
+def handle_registration_to_bouncer_failure(
+    user_profile: UserProfile, device_id: int, error_code: str
+) -> None:
+    """Handles a failed registration request to the bouncer by
+    notifying or preparing to notify clients.
+
+    * Sends a `device` event to notify online clients immediately.
+
+    * Stores the `error_code` in the `Device` table. This is later
+      used, along with other metadata, to notify offline clients the
+      next time they call `/register`. See the `devices` field in
+      the `/register` response.
+    """
+    Device.objects.filter(id=device_id).update(push_registration_error_code=error_code)
+    event = dict(
+        type="device",
+        op="update",
+        device_id=device_id,
+        push_registration_error_code=error_code,
+    )
+    send_event_on_commit(user_profile.realm, event, [user_profile.id])
+
+    # Report the `REQUEST_EXPIRED_ERROR` to the server admins as it indicates
+    # a long-lasting outage somewhere between the server and the bouncer,
+    # most likely in either the server or its local network configuration.
+    if error_code == Device.PushRegistrationErrorCode.REQUEST_EXPIRED:
+        logging.error("Push registration request for device_id=%s expired.", device_id)
+
+
+def compute_token_id_int(token: str) -> int:
+    hash_bytes = hashlib.sha256(token.encode()).digest()
+    return b64decode_token_id_base64(base64.b64encode(hash_bytes[0:8]).decode())
+
+
+def cleanup_legacy_push_device_token(user_profile: UserProfile, token_id_int: int) -> None:
+    MAX_REQUEST_RETRIES = 3
+    legacy_tokens = PushDeviceToken.objects.filter(user=user_profile)
+
+    for legacy_token in legacy_tokens:
+        # APNs treats its tokens (hex strings) as case-insensitive, and
+        # different client versions have sent them in different cases; the
+        # rest of the legacy code path accordingly normalizes to lowercase
+        # (see `remove_push_device_token`).  Match on the stored casing
+        # as well as its lower- and upper-case forms so that E2EE
+        # registration reliably cleans up the matching legacy token
+        # regardless of which case the client used.
+        token = legacy_token.token
+        if legacy_token.kind == PushDeviceToken.APNS:
+            candidate_token_ids = {
+                compute_token_id_int(token),
+                compute_token_id_int(token.lower()),
+                compute_token_id_int(token.upper()),
+            }
+        else:
+            candidate_token_ids = {compute_token_id_int(token)}
+
+        if token_id_int not in candidate_token_ids:
+            continue
+
+        for attempt in range(MAX_REQUEST_RETRIES):
+            try:
+                remove_push_device_token(user_profile, legacy_token.token, legacy_token.kind)
+                return
+            except (
+                PushNotificationBouncerRetryLaterError,
+                PushNotificationBouncerServerError,
+            ) as e:
+                # Network error or 5xx error response from bouncer server.
+                # TODO: Retry with backoff. Implement it while working on #35853.
+                if attempt == MAX_REQUEST_RETRIES - 1:
+                    logger.warning(
+                        "Failed to remove legacy push token from bouncer "
+                        "for user %s after retries: %s",
+                        user_profile.id,
+                        e,
+                    )
+                    return
+            except PushNotificationBouncerError as e:
+                # Invalid credentials or unexpected status code
+                logger.error(
+                    "Bouncer error cleaning up legacy push token for user %s: %s",
+                    user_profile.id,
+                    e,
+                )
+                return
+            except JsonableError:
+                # Token doesn't exist i.e. already cleaned up.
+                return
+
+
+def handle_register_push_device_to_bouncer(
+    queue_item: RegisterPushDeviceToBouncerQueueItem,
+) -> None:
+    user_profile_id = queue_item["user_profile_id"]
+    user_profile = get_user_profile_by_id(user_profile_id)
+    device_id = queue_item["device_id"]
+    bouncer_public_key = queue_item["bouncer_public_key"]
+    encrypted_push_registration = queue_item["encrypted_push_registration"]
+    token_id_base64 = queue_item["token_id_base64"]
+
+    try:
+        if settings.ZILENCER_ENABLED:
+            do_register_remote_push_device(
+                bouncer_public_key,
+                encrypted_push_registration,
+                token_id_base64,
+                realm=user_profile.realm,
+            )
+        else:
+            post_data: dict[str, str | int] = {
+                "realm_uuid": str(user_profile.realm.uuid),
+                "token_id": token_id_base64,
+                "encrypted_push_registration": encrypted_push_registration,
+                "bouncer_public_key": bouncer_public_key,
+            }
+            send_to_push_bouncer("POST", "push/e2ee/register", post_data)
+    except (
+        PushNotificationBouncerRetryLaterError,
+        PushNotificationBouncerServerError,
+    ) as e:  # nocoverage
+        # Network error or 5xx error response from bouncer server.
+        # Keep retrying to register until `RequestExpiredError` is raised.
+        raise PushNotificationBouncerRetryLaterError(e.msg)
+    except (
+        # Need to resubmit realm info - `manage.py register_server`
+        MissingRemoteRealmError,
+        # Invalid credentials or unexpected status code
+        PushNotificationBouncerError,
+    ):
+        # Server admins need to fix these set of errors, report them.
+        # Server should keep retrying to register until `RequestExpiredError` is raised.
+        error_msg = f"Push device registration request for device_id={device_id} failed."
+        logging.error(error_msg)
+        raise PushNotificationBouncerRetryLaterError(error_msg)
+    except (
+        InvalidBouncerPublicKeyError,
+        InvalidEncryptedPushRegistrationError,
+        RequestExpiredError,
+        # Any future or unexpected exceptions that we add.
+        JsonableError,
+    ) as e:
+        handle_registration_to_bouncer_failure(
+            user_profile, device_id, error_code=e.__class__.code.name
+        )
+        return
+
+    # Registration successful.
+    token_id_int = b64decode_token_id_base64(token_id_base64)
+    Device.objects.filter(id=device_id).update(
+        push_token_id=token_id_int, pending_push_token_id=None
+    )
+    event = dict(
+        type="device",
+        op="update",
+        device_id=device_id,
+        push_token_id=token_id_base64,
+        pending_push_token_id=None,
+    )
+    send_event_on_commit(user_profile.realm, event, [user_profile.id])
+
+    # When a client successfully registers a token for E2EE notifications,
+    # clear out any existing legacy registration on that account for the same token.
+    cleanup_legacy_push_device_token(user_profile, token_id_int)
+
+
+def check_push_key(push_key_str: str) -> bytes:
+    error_message = _("Invalid `push_key`")
+
+    try:
+        push_key_bytes = base64.b64decode(push_key_str, validate=True)
+    except binascii.Error:
+        raise JsonableError(error_message)
+
+    if len(push_key_bytes) != 33 or push_key_bytes[0] != 0x31:
+        raise JsonableError(error_message)
+
+    return push_key_bytes

zerver/lib/pysa.py

@@ -0,0 +1,7 @@
+from typing import TypeVar
+
+T = TypeVar("T")
+
+
+def mark_sanitized(arg: T) -> T:
+    return arg

zerver/lib/query_helpers.py

@@ -0,0 +1,30 @@
+from typing import TypeVar
+
+from django.db import models
+from django.db.models import QuerySet
+
+ModelT = TypeVar("ModelT", bound=models.Model)
+RowT = TypeVar("RowT")
+
+
+def query_for_ids(
+    query: QuerySet[ModelT, RowT],
+    user_ids: list[int],
+    field: str,
+) -> QuerySet[ModelT, RowT]:
+    """
+    This function optimizes searches of the form
+    `user_profile_id in (1, 2, 3, 4)` by quickly
+    building the where clauses.  Profiling shows significant
+    speedups over the normal Django-based approach.
+
+    Use this very carefully!  Also, the caller should
+    guard against empty lists of user_ids.
+    """
+    assert user_ids
+    clause = f"{field} IN %s"
+    query = query.extra(  # noqa: S610
+        where=[clause],
+        params=(tuple(user_ids),),
+    )
+    return query

zerver/lib/queue.py

@@ -0,0 +1,485 @@
+import logging
+import random
+import ssl
+import threading
+import time
+from abc import ABC, abstractmethod
+from collections import defaultdict
+from collections.abc import Callable, Mapping
+from typing import Any, Generic, TypeAlias, TypeVar
+
+import orjson
+import pika
+import pika.adapters.tornado_connection
+import pika.connection
+import pika.exceptions
+from django.conf import settings
+from django.db import transaction
+from pika.adapters.blocking_connection import BlockingChannel
+from pika.channel import Channel
+from pika.spec import Basic
+from tornado import ioloop
+from typing_extensions import override
+
+from zerver.lib.utils import assert_is_not_none
+
+MAX_REQUEST_RETRIES = 3
+ChannelT = TypeVar("ChannelT", Channel, BlockingChannel)
+Consumer: TypeAlias = Callable[[ChannelT, Basic.Deliver, pika.BasicProperties, bytes], None]
+
+
+# This simple queuing library doesn't expose much of the power of
+# RabbitMQ/Pika's queuing system; its purpose is to just provide an
+# interface for external files to put things into queues and take them
+# out from bots without having to import pika code all over our codebase.
+class QueueClient(ABC, Generic[ChannelT]):
+    def __init__(
+        self,
+        # Disable RabbitMQ heartbeats by default because BlockingConnection can't process them
+        rabbitmq_heartbeat: int | None = 0,
+        prefetch: int = 0,
+    ) -> None:
+        self.log = logging.getLogger("zulip.queue")
+        self.queues: set[str] = set()
+        self.channel: ChannelT | None = None
+        self.prefetch = prefetch
+        self.consumers: dict[str, set[Consumer[ChannelT]]] = defaultdict(set)
+        self.rabbitmq_heartbeat = rabbitmq_heartbeat
+        self.is_consuming = False
+        self._connect()
+
+    @abstractmethod
+    def _connect(self) -> None:
+        raise NotImplementedError
+
+    @abstractmethod
+    def _reconnect(self) -> None:
+        raise NotImplementedError
+
+    def _get_parameters(self) -> pika.ConnectionParameters:
+        credentials = pika.PlainCredentials(
+            settings.RABBITMQ_USERNAME, assert_is_not_none(settings.RABBITMQ_PASSWORD)
+        )
+
+        # With BlockingConnection, we are passed
+        # self.rabbitmq_heartbeat=0, which asks to explicitly disable
+        # the RabbitMQ heartbeat feature.  This is correct since that
+        # heartbeat doesn't make sense with BlockingConnection (we do
+        # need it for TornadoConnection).
+        #
+        # Where we've disabled RabbitMQ's heartbeat, the only
+        # keepalive on this connection is the TCP keepalive (defaults:
+        # `/proc/sys/net/ipv4/tcp_keepalive_*`).  On most Linux
+        # systems, the default is to start sending keepalive packets
+        # after TCP_KEEPIDLE (7200 seconds) of inactivity; after that
+        # point, it send them every TCP_KEEPINTVL (typically 75s).
+        # Some Kubernetes / Docker Swarm networks can kill "idle" TCP
+        # connections after as little as ~15 minutes of inactivity.
+        # To avoid this killing our RabbitMQ connections, we set
+        # TCP_KEEPIDLE to something significantly below 15 minutes.
+        tcp_options = None
+        if self.rabbitmq_heartbeat == 0:
+            tcp_options = dict(TCP_KEEPIDLE=60 * 5)
+
+        ssl_options: type[pika.ConnectionParameters._DEFAULT] | pika.SSLOptions = (
+            pika.ConnectionParameters._DEFAULT
+        )
+        if settings.RABBITMQ_USE_TLS:
+            ssl_options = pika.SSLOptions(context=ssl.create_default_context())
+
+        return pika.ConnectionParameters(
+            settings.RABBITMQ_HOST,
+            port=settings.RABBITMQ_PORT,
+            virtual_host=settings.RABBITMQ_VHOST,
+            heartbeat=self.rabbitmq_heartbeat,
+            tcp_options=tcp_options,
+            ssl_options=ssl_options,
+            credentials=credentials,
+        )
+
+    def _generate_ctag(self, queue_name: str) -> str:
+        return f"{queue_name}_{random.getrandbits(16)}"
+
+    def _reconnect_consumer_callback(self, queue: str, consumer: Consumer[ChannelT]) -> None:
+        self.log.info("Queue reconnecting saved consumer %r to queue %s", consumer, queue)
+        self.ensure_queue(
+            queue,
+            lambda channel: channel.basic_consume(
+                queue,
+                consumer,
+                consumer_tag=self._generate_ctag(queue),
+            ),
+        )
+
+    def _reconnect_consumer_callbacks(self) -> None:
+        for queue, consumers in self.consumers.items():
+            for consumer in consumers:
+                self._reconnect_consumer_callback(queue, consumer)
+
+    def ready(self) -> bool:
+        return self.channel is not None
+
+    @abstractmethod
+    def ensure_queue(self, queue_name: str, callback: Callable[[ChannelT], object]) -> None:
+        raise NotImplementedError
+
+    def publish(self, queue_name: str, body: bytes) -> None:
+        def do_publish(channel: ChannelT) -> None:
+            channel.basic_publish(
+                exchange="",
+                routing_key=queue_name,
+                properties=pika.BasicProperties(delivery_mode=2),
+                body=body,
+            )
+
+        self.ensure_queue(queue_name, do_publish)
+
+    def json_publish(self, queue_name: str, body: Mapping[str, Any]) -> None:
+        data = orjson.dumps(body)
+        try:
+            self.publish(queue_name, data)
+            return
+        except pika.exceptions.AMQPConnectionError:
+            self.log.warning("Failed to send to rabbitmq, trying to reconnect and send again")
+
+        self._reconnect()
+        self.publish(queue_name, data)
+
+
+class SimpleQueueClient(QueueClient[BlockingChannel]):
+    connection: pika.BlockingConnection | None
+
+    @override
+    def _connect(self) -> None:
+        start = time.time()
+        self.connection = pika.BlockingConnection(self._get_parameters())
+        self.channel = self.connection.channel()
+        self.channel.basic_qos(prefetch_count=self.prefetch)
+        self.log.info("SimpleQueueClient connected (connecting took %.3fs)", time.time() - start)
+
+    @override
+    def _reconnect(self) -> None:
+        self.connection = None
+        self.channel = None
+        self.queues = set()
+        self._connect()
+
+    def close(self) -> None:
+        if self.connection is not None:
+            self.connection.close()
+
+    @override
+    def ensure_queue(self, queue_name: str, callback: Callable[[BlockingChannel], object]) -> None:
+        """Ensure that a given queue has been declared, and then call
+        the callback with no arguments."""
+        if self.connection is None or not self.connection.is_open:
+            self._connect()
+            assert self.channel is not None
+        else:
+            assert self.channel is not None
+
+        if queue_name not in self.queues:
+            self.channel.queue_declare(queue=queue_name, durable=True)
+            self.queues.add(queue_name)
+
+        callback(self.channel)
+
+    def start_json_consumer(
+        self,
+        queue_name: str,
+        callback: Callable[[list[dict[str, Any]]], None],
+        batch_size: int = 1,
+        timeout: int | None = None,
+    ) -> None:
+        if batch_size == 1:
+            timeout = None
+
+        def do_consume(channel: BlockingChannel) -> None:
+            events: list[dict[str, Any]] = []
+            last_process = time.time()
+            max_processed: int | None = None
+            self.is_consuming = True
+
+            # This iterator technique will iteratively collect up to
+            # batch_size events from the RabbitMQ queue (if present)
+            # before calling the callback with the batch.  If not
+            # enough events are present, it will sleep for at most
+            # timeout seconds before calling the callback with the
+            # batch of events it has.
+            for method, properties, body in channel.consume(queue_name, inactivity_timeout=timeout):
+                if body is not None:
+                    assert method is not None
+                    events.append(orjson.loads(body))
+                    max_processed = method.delivery_tag
+                now = time.time()
+                if len(events) >= batch_size or (timeout and now >= last_process + timeout):
+                    if events:
+                        assert max_processed is not None
+                        try:
+                            callback(events)
+                            channel.basic_ack(max_processed, multiple=True)
+                        except BaseException:
+                            if channel.is_open:
+                                channel.basic_nack(max_processed, multiple=True)
+                            raise
+                        events = []
+                    last_process = now
+                if not self.is_consuming:
+                    break
+
+        self.ensure_queue(queue_name, do_consume)
+
+    def local_queue_size(self) -> int:
+        assert self.channel is not None
+        return self.channel.get_waiting_message_count() + len(
+            self.channel._pending_events  # type: ignore[attr-defined] # private member missing from stubs
+        )
+
+    def stop_consuming(self) -> None:
+        assert self.channel is not None
+        assert self.is_consuming
+        self.is_consuming = False
+        self.channel.stop_consuming()
+
+
+# Patch pika.adapters.tornado_connection.TornadoConnection so that a socket error doesn't
+# throw an exception and disconnect the tornado process from the rabbitmq
+# queue. Instead, just re-connect as usual
+class ExceptionFreeTornadoConnection(pika.adapters.tornado_connection.TornadoConnection):
+    def _adapter_disconnect(self) -> None:
+        try:
+            super()._adapter_disconnect()  # type: ignore[misc]  # private method missing from stubs
+        except (
+            pika.exceptions.ProbableAuthenticationError,
+            pika.exceptions.ProbableAccessDeniedError,
+            pika.exceptions.IncompatibleProtocolError,
+        ):
+            logging.warning(
+                "Caught exception in ExceptionFreeTornadoConnection when \
+calling _adapter_disconnect, ignoring",
+                exc_info=True,
+            )
+
+
+class TornadoQueueClient(QueueClient[Channel]):
+    connection: ExceptionFreeTornadoConnection | None
+
+    # Based on:
+    # https://pika.readthedocs.io/en/0.9.8/examples/asynchronous_consumer_example.html
+    def __init__(self) -> None:
+        super().__init__(
+            # TornadoConnection can process heartbeats, so enable them.
+            rabbitmq_heartbeat=None,
+            # Only ask for 100 un-acknowledged messages at once from
+            # the server, rather than an unbounded number.
+            prefetch=100,
+        )
+        self._on_open_cbs: list[Callable[[Channel], None]] = []
+        self._connection_failure_count = 0
+
+    @override
+    def _connect(self) -> None:
+        self.log.info("Beginning TornadoQueueClient connection")
+        self.connection = ExceptionFreeTornadoConnection(
+            self._get_parameters(),
+            on_open_callback=self._on_open,
+            on_open_error_callback=self._on_connection_open_error,
+            on_close_callback=self._on_connection_closed,
+        )
+
+    @override
+    def _reconnect(self) -> None:
+        self.connection = None
+        self.channel = None
+        self.queues = set()
+        self.log.warning("TornadoQueueClient attempting to reconnect to RabbitMQ")
+        self._connect()
+
+    CONNECTION_RETRY_SECS = 2
+
+    # When the RabbitMQ server is restarted, it's normal for it to
+    # take a few seconds to come back; we'll retry a few times and all
+    # will be well.  So for the first few failures, we report only at
+    # "warning" level, avoiding an email to the server admin.
+    #
+    # A loss of an existing connection starts a retry loop just like a
+    # failed connection attempt, so it counts as the first failure.
+    #
+    # On an unloaded test system, a RabbitMQ restart takes about 6s,
+    # potentially causing 4 failures.  We add some headroom above that.
+    CONNECTION_FAILURES_BEFORE_NOTIFY = 10
+
+    def _on_connection_open_error(
+        self, connection: pika.connection.Connection, reason: str | Exception
+    ) -> None:
+        self._connection_failure_count += 1
+        retry_secs = self.CONNECTION_RETRY_SECS
+        self.log.log(
+            (
+                logging.CRITICAL
+                if self._connection_failure_count > self.CONNECTION_FAILURES_BEFORE_NOTIFY
+                else logging.WARNING
+            ),
+            "TornadoQueueClient couldn't connect to RabbitMQ, retrying in %d secs...",
+            retry_secs,
+        )
+        ioloop.IOLoop.current().call_later(retry_secs, self._reconnect)
+
+    def _on_connection_closed(
+        self, connection: pika.connection.Connection, reason: Exception
+    ) -> None:
+        if self.connection is None:
+            return
+        self._connection_failure_count = 1
+        retry_secs = self.CONNECTION_RETRY_SECS
+        self.log.warning(
+            "TornadoQueueClient lost connection to RabbitMQ, reconnecting in %d secs...",
+            retry_secs,
+        )
+        ioloop.IOLoop.current().call_later(retry_secs, self._reconnect)
+
+    def _on_open(self, connection: pika.connection.Connection) -> None:
+        assert self.connection is not None
+        self._connection_failure_count = 0
+        try:
+            self.connection.channel(on_open_callback=self._on_channel_open)
+        except pika.exceptions.ConnectionClosed:
+            # The connection didn't stay open long enough for this code to get to it.
+            # Let _on_connection_closed deal with trying again.
+            self.log.warning("TornadoQueueClient couldn't open channel: connection already closed")
+
+    def _on_channel_open(self, channel: Channel) -> None:
+        self.channel = channel
+        for callback in self._on_open_cbs:
+            callback(channel)
+        self._reconnect_consumer_callbacks()
+        self.log.info("TornadoQueueClient connected")
+
+    def close(self) -> None:
+        if self.connection is not None:
+            self.connection.close()
+            self.connection = None
+
+    @override
+    def ensure_queue(self, queue_name: str, callback: Callable[[Channel], object]) -> None:
+        def set_qos(frame: Any) -> None:
+            assert self.channel is not None
+            self.queues.add(queue_name)
+            self.channel.basic_qos(prefetch_count=self.prefetch, callback=finish)
+
+        def finish(frame: Any) -> None:
+            assert self.channel is not None
+            callback(self.channel)
+
+        if queue_name not in self.queues:
+            # If we're not connected yet, send this message
+            # once we have created the channel
+            if not self.ready():
+                self._on_open_cbs.append(lambda channel: self.ensure_queue(queue_name, callback))
+                return
+
+            assert self.channel is not None
+            self.channel.queue_declare(queue=queue_name, durable=True, callback=set_qos)
+        else:
+            assert self.channel is not None
+            callback(self.channel)
+
+    def start_json_consumer(
+        self,
+        queue_name: str,
+        callback: Callable[[list[dict[str, Any]]], None],
+        batch_size: int = 1,
+        timeout: int | None = None,
+    ) -> None:
+        def wrapped_consumer(
+            ch: Channel,
+            method: Basic.Deliver,
+            properties: pika.BasicProperties,
+            body: bytes,
+        ) -> None:
+            assert method.delivery_tag is not None
+            callback([orjson.loads(body)])
+            ch.basic_ack(delivery_tag=method.delivery_tag)
+
+        assert batch_size == 1
+        assert timeout is None
+        self.consumers[queue_name].add(wrapped_consumer)
+
+        if not self.ready():
+            return
+
+        self.ensure_queue(
+            queue_name,
+            lambda channel: channel.basic_consume(
+                queue_name,
+                wrapped_consumer,
+                consumer_tag=self._generate_ctag(queue_name),
+            ),
+        )
+
+
+thread_data = threading.local()
+
+
+def get_queue_client() -> SimpleQueueClient | TornadoQueueClient:
+    if not hasattr(thread_data, "queue_client"):
+        if not settings.USING_RABBITMQ:
+            raise RuntimeError("Cannot get a queue client without USING_RABBITMQ")
+        thread_data.queue_client = SimpleQueueClient()
+
+    return thread_data.queue_client
+
+
+def set_queue_client(queue_client: SimpleQueueClient | TornadoQueueClient) -> None:
+    thread_data.queue_client = queue_client
+
+
+# One should generally use `queue_event_on_commit` unless there's a strong
+# reason to use `queue_json_publish_rollback_unsafe` directly, as it doesn't
+# wait for the db transaction (within which it gets called, if any) to commit
+# and sends event irrespective of commit or rollback.
+def queue_json_publish_rollback_unsafe(
+    queue_name: str,
+    event: dict[str, Any],
+    processor: Callable[[Any], None] | None = None,
+) -> None:
+    if settings.USING_RABBITMQ:
+        get_queue_client().json_publish(queue_name, event)
+    elif processor:
+        # Round-trip through orjson to simulate what RabbitMQ does.
+        processor(orjson.loads(orjson.dumps(event)))
+    else:
+        # The else branch is only hit during tests, where rabbitmq is not enabled.
+        # Must be imported here: A top section import leads to circular imports
+        from zerver.worker.queue_processors import get_worker
+
+        # As above, we round-trip the event through orjson to emulate
+        # what happens with RabbitMQ enqueueing and dequeueing the
+        # event.  This ensures that we don't rely on non-JSON'able
+        # datatypes in the events.
+        get_worker(queue_name, disable_timeout=True).consume_single_event(
+            orjson.loads(orjson.dumps(event))
+        )
+
+
+def queue_event_on_commit(queue_name: str, event: dict[str, Any]) -> None:
+    transaction.on_commit(lambda: queue_json_publish_rollback_unsafe(queue_name, event))
+
+
+def mobile_notifications_queue_name(user_id: int) -> str:
+    if settings.MOBILE_NOTIFICATIONS_SHARDS > 1:
+        shard_id = user_id % settings.MOBILE_NOTIFICATIONS_SHARDS + 1
+        return f"missedmessage_mobile_notifications_shard{shard_id}"
+    return "missedmessage_mobile_notifications"
+
+
+def retry_event(
+    queue_name: str, event: dict[str, Any], failure_processor: Callable[[dict[str, Any]], None]
+) -> None:
+    if "failed_tries" not in event:
+        event["failed_tries"] = 0
+    event["failed_tries"] += 1
+    if event["failed_tries"] > MAX_REQUEST_RETRIES:
+        failure_processor(event)
+    else:
+        queue_json_publish_rollback_unsafe(queue_name, event)

zerver/lib/rate_limiter.lua

@@ -0,0 +1,100 @@
+-- GCRA (Generic Cell Rate Algorithm) rate limiter.
+-- Atomically checks all rules and updates TATs in a single round trip.
+--
+-- KEYS[1] = block_key, KEYS[2] = gcra_key
+-- ARGV[1] = num_rules, ARGV[2..2*num_rules+1] = w1, l1, w2, l2, ...
+-- ARGV[2*num_rules+2] (optional) = override timestamp for tests only
+--
+-- Returns a 4-element list of strings:
+--   [ratelimited, secs_to_freedom, calls_remaining, secs_to_reset]
+
+local block_key = KEYS[1]
+local gcra_key = KEYS[2]
+local num_rules = tonumber(ARGV[1])
+
+-- Use Redis server time so that all app servers see a consistent
+-- clock, avoiding false positives from inter-machine clock drift.
+-- Tests may pass an explicit timestamp as the last argument to get
+-- deterministic behavior without wall-clock sleeps.
+local now
+local test_time_arg = ARGV[2 * num_rules + 2]
+if test_time_arg then
+    now = tonumber(test_time_arg)
+else
+    local time_result = redis.call('TIME')
+    now = tonumber(time_result[1]) + tonumber(time_result[2]) / 1000000
+end
+
+-- Check for manual block (set by block_access).
+if redis.call('EXISTS', block_key) == 1 then
+    local ttl = redis.call('PTTL', block_key)
+    if ttl < 0 then ttl = 1000 end
+    return {'1', string.format('%.17g', ttl / 1000.0), '0', '0'}
+end
+
+-- For each rule, compute the new TAT (Theoretical Arrival Time)
+-- without writing anything yet.  If any rule would be violated,
+-- we reject the request and leave all TATs unchanged.
+local fields = {}
+local new_tats = {}
+local max_ttl = 0
+local limited = false
+local secs_to_freedom = 0
+
+for i = 1, num_rules do
+    local window = tonumber(ARGV[1 + (i - 1) * 2 + 1])
+    local limit = tonumber(ARGV[1 + (i - 1) * 2 + 2])
+    local field = window .. ':' .. limit
+    -- Time between ideally-spaced requests.
+    local emission_interval = window / limit
+
+    local stored = redis.call('HGET', gcra_key, field)
+    -- If the stored TAT is in the past, treat it as now (the
+    -- bucket has fully drained).
+    local tat = math.max(stored and tonumber(stored) or now, now)
+    local new_tat = tat + emission_interval
+
+    -- The bucket overflows when new_tat exceeds the window
+    -- horizon: the request arrived faster than the drain rate
+    -- can sustain.
+    if new_tat > now + window then
+        limited = true
+        local freedom = new_tat - window - now
+        if freedom > secs_to_freedom then
+            secs_to_freedom = freedom
+        end
+    end
+
+    fields[i] = field
+    new_tats[i] = new_tat
+    local remaining = new_tat - now
+    if remaining > max_ttl then
+        max_ttl = remaining
+    end
+end
+
+-- All-or-nothing: if any rule triggered, reject without updating.
+if limited then
+    return {'1', string.format('%.17g', secs_to_freedom), '0', '0'}
+end
+
+-- Request allowed — persist all new TATs atomically.
+for i = 1, num_rules do
+    redis.call('HSET', gcra_key, fields[i], string.format('%.17g', new_tats[i]))
+end
+local ttl_seconds = math.ceil(max_ttl)
+if ttl_seconds > 0 then
+    redis.call('EXPIRE', gcra_key, ttl_seconds)
+end
+
+-- Compute remaining quota for the max (outermost) rule, used for
+-- the X-RateLimit-Remaining response header.
+local max_window = tonumber(ARGV[1 + (num_rules - 1) * 2 + 1])
+local max_limit = tonumber(ARGV[1 + (num_rules - 1) * 2 + 2])
+local max_tat = new_tats[num_rules]
+local calls_remaining = math.max(0, math.floor(
+    (now + max_window - max_tat) * max_limit / max_window
+))
+local secs_to_reset = max_tat - now
+
+return {'0', '0', tostring(calls_remaining), string.format('%.17g', secs_to_reset)}