snapshot of https://github.com/zulip/zulip.git @ 48c69911c7dd4e03b20552eb3bd77ad8151c1357 (2026-09-15)

zerver/actions/alert_words.py

@@ -0,0 +1,24 @@
+from collections.abc import Iterable, Sequence
+
+from django.db import transaction
+
+from zerver.lib.alert_words import add_user_alert_words, remove_user_alert_words
+from zerver.models import UserProfile
+from zerver.tornado.django_api import send_event_on_commit
+
+
+def notify_alert_words(user_profile: UserProfile, words: Sequence[str]) -> None:
+    event = dict(type="alert_words", alert_words=words)
+    send_event_on_commit(user_profile.realm, event, [user_profile.id])
+
+
+@transaction.atomic(durable=True)
+def do_add_alert_words(user_profile: UserProfile, alert_words: Iterable[str]) -> None:
+    words = add_user_alert_words(user_profile, alert_words)
+    notify_alert_words(user_profile, words)
+
+
+@transaction.atomic(durable=True)
+def do_remove_alert_words(user_profile: UserProfile, alert_words: Iterable[str]) -> None:
+    words = remove_user_alert_words(user_profile, alert_words)
+    notify_alert_words(user_profile, words)

zerver/actions/bots.py

@@ -0,0 +1,188 @@
+from django.db import transaction
+from django.utils.timezone import now as timezone_now
+
+from zerver.actions.create_user import created_bot_event
+from zerver.models import RealmAuditLog, Stream, UserProfile
+from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.models.users import active_user_ids, bot_owner_user_ids
+from zerver.tornado.django_api import send_event_on_commit
+
+
+def send_bot_owner_update_events(
+    user_profile: UserProfile, bot_owner: UserProfile, previous_owner: UserProfile | None
+) -> None:
+    # Since `bot_owner_id` is included in the user profile dict we need
+    # to update the users dict with the new bot owner id
+    event = dict(
+        type="realm_user",
+        op="update",
+        person=dict(
+            user_id=user_profile.id,
+            bot_owner_id=bot_owner.id,
+        ),
+    )
+    send_event_on_commit(user_profile.realm, event, active_user_ids(user_profile.realm_id))
+
+    # Delete the bot from previous owner's bot data.
+    if previous_owner and not previous_owner.is_realm_admin:
+        delete_event = dict(
+            type="realm_bot",
+            op="delete",
+            bot=dict(
+                user_id=user_profile.id,
+            ),
+        )
+        previous_owner_id = previous_owner.id
+        send_event_on_commit(
+            user_profile.realm,
+            delete_event,
+            {previous_owner_id},
+        )
+
+    # Notify the new owner that the bot has been added.
+    if not bot_owner.is_realm_admin:
+        add_event = created_bot_event(user_profile)
+        send_event_on_commit(user_profile.realm, add_event, {bot_owner.id})
+
+
+@transaction.atomic(durable=True)
+def do_change_bot_owner(
+    user_profile: UserProfile, bot_owner: UserProfile, acting_user: UserProfile | None
+) -> None:
+    previous_owner = user_profile.bot_owner
+    user_profile.bot_owner = bot_owner
+    user_profile.save()  # Can't use update_fields because of how the foreign key works.
+    event_time = timezone_now()
+    RealmAuditLog.objects.create(
+        realm=user_profile.realm,
+        acting_user=acting_user,
+        modified_user=user_profile,
+        event_type=AuditLogEventType.USER_BOT_OWNER_CHANGED,
+        event_time=event_time,
+        extra_data={
+            RealmAuditLog.OLD_VALUE: None if previous_owner is None else previous_owner.id,
+            RealmAuditLog.NEW_VALUE: bot_owner.id,
+        },
+    )
+
+    send_bot_owner_update_events(user_profile, bot_owner, previous_owner)
+
+
+@transaction.atomic(durable=True)
+def do_change_default_sending_stream(
+    user_profile: UserProfile, stream: Stream | None, *, acting_user: UserProfile | None
+) -> None:
+    old_value = user_profile.default_sending_stream_id
+    user_profile.default_sending_stream = stream
+    user_profile.save(update_fields=["default_sending_stream"])
+
+    event_time = timezone_now()
+    RealmAuditLog.objects.create(
+        realm=user_profile.realm,
+        event_type=AuditLogEventType.USER_DEFAULT_SENDING_STREAM_CHANGED,
+        event_time=event_time,
+        modified_user=user_profile,
+        acting_user=acting_user,
+        extra_data={
+            RealmAuditLog.OLD_VALUE: old_value,
+            RealmAuditLog.NEW_VALUE: None if stream is None else stream.id,
+        },
+    )
+
+    if user_profile.is_bot:
+        if stream:
+            stream_name: str | None = stream.name
+        else:
+            stream_name = None
+        event = dict(
+            type="realm_bot",
+            op="update",
+            bot=dict(
+                user_id=user_profile.id,
+                default_sending_stream=stream_name,
+            ),
+        )
+        send_event_on_commit(
+            user_profile.realm,
+            event,
+            bot_owner_user_ids(user_profile),
+        )
+
+
+@transaction.atomic(durable=True)
+def do_change_default_events_register_stream(
+    user_profile: UserProfile, stream: Stream | None, *, acting_user: UserProfile | None
+) -> None:
+    old_value = user_profile.default_events_register_stream_id
+    user_profile.default_events_register_stream = stream
+    user_profile.save(update_fields=["default_events_register_stream"])
+
+    event_time = timezone_now()
+    RealmAuditLog.objects.create(
+        realm=user_profile.realm,
+        event_type=AuditLogEventType.USER_DEFAULT_REGISTER_STREAM_CHANGED,
+        event_time=event_time,
+        modified_user=user_profile,
+        acting_user=acting_user,
+        extra_data={
+            RealmAuditLog.OLD_VALUE: old_value,
+            RealmAuditLog.NEW_VALUE: None if stream is None else stream.id,
+        },
+    )
+
+    if user_profile.is_bot:
+        if stream:
+            stream_name: str | None = stream.name
+        else:
+            stream_name = None
+
+        event = dict(
+            type="realm_bot",
+            op="update",
+            bot=dict(
+                user_id=user_profile.id,
+                default_events_register_stream=stream_name,
+            ),
+        )
+        send_event_on_commit(
+            user_profile.realm,
+            event,
+            bot_owner_user_ids(user_profile),
+        )
+
+
+@transaction.atomic(durable=True)
+def do_change_default_all_public_streams(
+    user_profile: UserProfile, value: bool, *, acting_user: UserProfile | None
+) -> None:
+    old_value = user_profile.default_all_public_streams
+    user_profile.default_all_public_streams = value
+    user_profile.save(update_fields=["default_all_public_streams"])
+
+    event_time = timezone_now()
+    RealmAuditLog.objects.create(
+        realm=user_profile.realm,
+        event_type=AuditLogEventType.USER_DEFAULT_ALL_PUBLIC_STREAMS_CHANGED,
+        event_time=event_time,
+        modified_user=user_profile,
+        acting_user=acting_user,
+        extra_data={
+            RealmAuditLog.OLD_VALUE: old_value,
+            RealmAuditLog.NEW_VALUE: value,
+        },
+    )
+
+    if user_profile.is_bot:
+        event = dict(
+            type="realm_bot",
+            op="update",
+            bot=dict(
+                user_id=user_profile.id,
+                default_all_public_streams=user_profile.default_all_public_streams,
+            ),
+        )
+        send_event_on_commit(
+            user_profile.realm,
+            event,
+            bot_owner_user_ids(user_profile),
+        )

zerver/actions/channel_folders.py

@@ -0,0 +1,159 @@
+from dataclasses import asdict
+
+from django.db import transaction
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+
+from zerver.lib.channel_folders import get_channel_folder_data, render_channel_folder_description
+from zerver.lib.exceptions import JsonableError
+from zerver.models import ChannelFolder, Realm, RealmAuditLog, UserProfile
+from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.models.users import active_user_ids
+from zerver.tornado.django_api import send_event_on_commit
+
+
+@transaction.atomic(durable=True)
+def check_add_channel_folder(
+    realm: Realm, name: str, description: str, *, acting_user: UserProfile
+) -> ChannelFolder:
+    rendered_description = render_channel_folder_description(
+        description, realm, acting_user=acting_user
+    )
+    channel_folder = ChannelFolder.objects.create(
+        realm=realm,
+        name=name,
+        description=description,
+        rendered_description=rendered_description,
+        creator_id=acting_user.id,
+    )
+    channel_folder.order = channel_folder.id
+    channel_folder.save(update_fields=["order"])
+
+    creation_time = timezone_now()
+    RealmAuditLog.objects.create(
+        realm=realm,
+        acting_user=acting_user,
+        event_type=AuditLogEventType.CHANNEL_FOLDER_CREATED,
+        event_time=creation_time,
+        modified_channel_folder=channel_folder,
+    )
+
+    event = dict(
+        type="channel_folder",
+        op="add",
+        channel_folder=asdict(get_channel_folder_data(channel_folder)),
+    )
+    send_event_on_commit(realm, event, active_user_ids(realm.id))
+
+    return channel_folder
+
+
+@transaction.atomic(durable=True)
+def try_reorder_realm_channel_folders(realm: Realm, order: list[int]) -> None:
+    order_mapping = {_[1]: _[0] for _ in enumerate(order)}
+    channel_folders = ChannelFolder.objects.filter(realm=realm)
+    for channel_folder in channel_folders:
+        if channel_folder.id not in order_mapping:
+            raise JsonableError(_("Invalid order mapping."))
+    for channel_folder in channel_folders:
+        channel_folder.order = order_mapping[channel_folder.id]
+        channel_folder.save(update_fields=["order"])
+
+    event = dict(
+        type="channel_folder",
+        op="reorder",
+        order=order,
+    )
+    send_event_on_commit(realm, event, active_user_ids(realm.id))
+
+
+def do_send_channel_folder_update_event(
+    channel_folder: ChannelFolder, data: dict[str, str | bool]
+) -> None:
+    realm = channel_folder.realm
+    event = dict(type="channel_folder", op="update", channel_folder_id=channel_folder.id, data=data)
+    send_event_on_commit(realm, event, active_user_ids(realm.id))
+
+
+@transaction.atomic(durable=True)
+def do_change_channel_folder_name(
+    channel_folder: ChannelFolder, name: str, *, acting_user: UserProfile
+) -> None:
+    old_value = channel_folder.name
+    channel_folder.name = name
+    channel_folder.save(update_fields=["name"])
+
+    RealmAuditLog.objects.create(
+        realm=acting_user.realm,
+        acting_user=acting_user,
+        event_type=AuditLogEventType.CHANNEL_FOLDER_NAME_CHANGED,
+        event_time=timezone_now(),
+        modified_channel_folder=channel_folder,
+        extra_data={
+            RealmAuditLog.OLD_VALUE: old_value,
+            RealmAuditLog.NEW_VALUE: name,
+        },
+    )
+
+    do_send_channel_folder_update_event(channel_folder, dict(name=name))
+
+
+@transaction.atomic(durable=True)
+def do_change_channel_folder_description(
+    channel_folder: ChannelFolder, description: str, *, acting_user: UserProfile
+) -> None:
+    old_value = channel_folder.description
+    rendered_description = render_channel_folder_description(
+        description, acting_user.realm, acting_user=acting_user
+    )
+    channel_folder.description = description
+    channel_folder.rendered_description = rendered_description
+    channel_folder.save(update_fields=["description", "rendered_description"])
+
+    RealmAuditLog.objects.create(
+        realm=acting_user.realm,
+        acting_user=acting_user,
+        event_type=AuditLogEventType.CHANNEL_FOLDER_DESCRIPTION_CHANGED,
+        event_time=timezone_now(),
+        modified_channel_folder=channel_folder,
+        extra_data={
+            RealmAuditLog.OLD_VALUE: old_value,
+            RealmAuditLog.NEW_VALUE: description,
+        },
+    )
+
+    do_send_channel_folder_update_event(
+        channel_folder, dict(description=description, rendered_description=rendered_description)
+    )
+
+
+@transaction.atomic(durable=True)
+def do_archive_channel_folder(channel_folder: ChannelFolder, *, acting_user: UserProfile) -> None:
+    channel_folder.is_archived = True
+    channel_folder.save(update_fields=["is_archived"])
+
+    RealmAuditLog.objects.create(
+        realm=acting_user.realm,
+        acting_user=acting_user,
+        event_type=AuditLogEventType.CHANNEL_FOLDER_ARCHIVED,
+        event_time=timezone_now(),
+        modified_channel_folder=channel_folder,
+    )
+
+    do_send_channel_folder_update_event(channel_folder, dict(is_archived=True))
+
+
+@transaction.atomic(durable=True)
+def do_unarchive_channel_folder(channel_folder: ChannelFolder, *, acting_user: UserProfile) -> None:
+    channel_folder.is_archived = False
+    channel_folder.save(update_fields=["is_archived"])
+
+    RealmAuditLog.objects.create(
+        realm=acting_user.realm,
+        acting_user=acting_user,
+        event_type=AuditLogEventType.CHANNEL_FOLDER_UNARCHIVED,
+        event_time=timezone_now(),
+        modified_channel_folder=channel_folder,
+    )
+
+    do_send_channel_folder_update_event(channel_folder, dict(is_archived=False))

zerver/actions/create_realm.py

@@ -0,0 +1,404 @@
+import logging
+from datetime import datetime, timedelta
+from typing import Any
+
+from django.conf import settings
+from django.db import transaction
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+from django.utils.translation import override as override_language
+
+from confirmation import settings as confirmation_settings
+from zerver.actions.realm_settings import (
+    do_add_deactivated_redirect,
+    do_change_realm_plan_type,
+    do_deactivate_realm,
+)
+from zerver.lib.bulk_create import create_users
+from zerver.lib.push_notifications import sends_notifications_directly
+from zerver.lib.realm_description import render_realm_description
+from zerver.lib.remote_server import maybe_enqueue_audit_log_upload
+from zerver.lib.server_initialization import create_internal_realm, server_initialized
+from zerver.lib.sessions import delete_realm_user_sessions
+from zerver.lib.streams import ensure_stream
+from zerver.lib.user_groups import (
+    create_system_user_groups_for_realm,
+    get_role_based_system_groups_dict,
+)
+from zerver.lib.zulip_update_announcements import get_latest_zulip_update_announcements_level
+from zerver.models import (
+    DefaultStream,
+    PreregistrationRealm,
+    Realm,
+    RealmAuditLog,
+    RealmAuthenticationMethod,
+    RealmUserDefault,
+    UserProfile,
+)
+from zerver.models.groups import SystemGroups
+from zerver.models.presence import PresenceSequence
+from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.models.scheduled_jobs import ScheduledMessage
+from zerver.models.users import get_system_bot
+from zproject.backends import all_default_backend_names
+
+DEFAULT_EMAIL_ADDRESS_VISIBILITY_FOR_REALM = RealmUserDefault.EMAIL_ADDRESS_VISIBILITY_ADMINS
+
+
+def do_change_realm_subdomain(
+    realm: Realm,
+    new_subdomain: str,
+    *,
+    acting_user: UserProfile | None,
+    add_deactivated_redirect: bool = True,
+) -> None:
+    """Changing a realm's subdomain is a highly disruptive operation,
+    because all existing clients will need to be updated to point to
+    the new URL.  Further, requests to fetch data from existing event
+    queues will fail with an authentication error when this change
+    happens (because the old subdomain is no longer associated with
+    the realm), making it hard for us to provide a graceful update
+    experience for clients.
+    """
+    old_subdomain = realm.subdomain
+    old_url = realm.url
+    was_demo_organization = realm.demo_organization_scheduled_deletion_date is not None
+    # If the realm had been a demo organization scheduled for
+    # deleting, clear that state.
+    realm.demo_organization_scheduled_deletion_date = None
+    realm.string_id = new_subdomain
+    with transaction.atomic(durable=True):
+        realm.save(update_fields=["string_id", "demo_organization_scheduled_deletion_date"])
+        RealmAuditLog.objects.create(
+            realm=realm,
+            event_type=AuditLogEventType.REALM_SUBDOMAIN_CHANGED,
+            event_time=timezone_now(),
+            acting_user=acting_user,
+            # Old RealmAuditLog entries for this used "old_subdomain" and
+            # "new_subdomain" keys to store this data.
+            extra_data={
+                RealmAuditLog.OLD_VALUE: old_subdomain,
+                RealmAuditLog.NEW_VALUE: new_subdomain,
+                "was_demo_organization": was_demo_organization,
+            },
+        )
+
+        # If a realm if being renamed multiple times, we should find all the placeholder
+        # realms and reset their deactivated_redirect field to point to the new realm url
+        placeholder_realms = Realm.objects.filter(deactivated_redirect=old_url, deactivated=True)
+        for placeholder_realm in placeholder_realms:
+            do_add_deactivated_redirect(placeholder_realm, realm.url)
+
+        # If a realm was a demo organization delete any scheduled messages about the
+        # automatic deletion of the demo organization.
+        if was_demo_organization:
+            sender = get_system_bot(settings.NOTIFICATION_BOT, realm.id)
+            scheduled_notification_bot_messages = ScheduledMessage.objects.filter(
+                sender=sender,
+                delivery_type=ScheduledMessage.SEND_LATER,
+                realm=realm,
+                delivered=False,
+                failed=False,
+            )
+            for scheduled_message in scheduled_notification_bot_messages:
+                scheduled_message.delete()
+
+    # The below block isn't executed in a transaction with the earlier code due to
+    # the functions called below being complex and potentially sending events,
+    # which we don't want to do in atomic blocks.
+    # When we change a realm's subdomain the realm with old subdomain is basically
+    # deactivated. We are creating a deactivated realm using old subdomain and setting
+    # it's deactivated redirect to new_subdomain so that we can tell the users that
+    # the realm has been moved to a new subdomain.
+    if add_deactivated_redirect:
+        placeholder_realm = do_create_realm(old_subdomain, realm.name)
+        do_deactivate_realm(
+            placeholder_realm,
+            acting_user=None,
+            deactivation_reason="subdomain_change",
+            email_owners=False,
+        )
+        do_add_deactivated_redirect(placeholder_realm, realm.url)
+
+    # Sessions can't be deleted inside a transaction.
+    delete_realm_user_sessions(realm)
+
+
+@transaction.atomic(savepoint=False)
+def set_default_for_realm_permission_group_settings(
+    realm: Realm, group_settings_defaults_for_org_types: dict[str, dict[int, str]] | None = None
+) -> None:
+    system_groups_dict = get_role_based_system_groups_dict(realm)
+
+    for setting_name, permission_configuration in Realm.REALM_PERMISSION_GROUP_SETTINGS.items():
+        group_name = permission_configuration.default_group_name
+
+        # Below code updates the group_name if the setting default depends on org_type.
+        if (
+            group_settings_defaults_for_org_types is not None
+            and setting_name in group_settings_defaults_for_org_types
+        ):
+            setting_org_type_defaults = group_settings_defaults_for_org_types[setting_name]
+            if realm.org_type in setting_org_type_defaults:
+                group_name = setting_org_type_defaults[realm.org_type]
+
+        setattr(realm, setting_name, system_groups_dict[group_name].usergroup_ptr)
+
+    realm.save(update_fields=list(Realm.REALM_PERMISSION_GROUP_SETTINGS.keys()))
+
+
+def setup_realm_internal_bots(realm: Realm) -> None:
+    """Create this realm's internal bots.
+
+    This function is idempotent; it does nothing for a bot that
+    already exists.
+    """
+    internal_bots = [
+        (bot["name"], bot["email_template"] % (settings.INTERNAL_BOT_DOMAIN,))
+        for bot in settings.REALM_INTERNAL_BOTS
+    ]
+    create_users(realm, internal_bots, bot_type=UserProfile.DEFAULT_BOT)
+    bots = UserProfile.objects.filter(
+        realm=realm,
+        email__in=[bot_info[1] for bot_info in internal_bots],
+        bot_owner__isnull=True,
+    )
+    for bot in bots:
+        bot.bot_owner = bot
+        bot.save()
+
+
+def get_email_address_visibility_default(org_type: int | None = None) -> int:
+    # For the majority of organization types, the default email address visibility
+    # setting for new users should initially be admins only.
+    realm_default_email_address_visibility = DEFAULT_EMAIL_ADDRESS_VISIBILITY_FOR_REALM
+    if org_type in (
+        Realm.ORG_TYPES["education_nonprofit"]["id"],
+        Realm.ORG_TYPES["education"]["id"],
+    ):
+        # Email address of users should be initially visible to moderators and admins.
+        realm_default_email_address_visibility = (
+            RealmUserDefault.EMAIL_ADDRESS_VISIBILITY_MODERATORS
+        )
+    elif org_type == Realm.ORG_TYPES["business"]["id"]:
+        # Email address of users can be visible to all users for business organizations.
+        realm_default_email_address_visibility = RealmUserDefault.EMAIL_ADDRESS_VISIBILITY_EVERYONE
+
+    return realm_default_email_address_visibility
+
+
+def do_create_realm(
+    string_id: str,
+    name: str,
+    *,
+    emails_restricted_to_domains: bool | None = None,
+    description: str | None = None,
+    invite_required: bool | None = None,
+    plan_type: int | None = None,
+    org_type: int | None = None,
+    default_language: str | None = None,
+    date_created: datetime | None = None,
+    is_demo_organization: bool = False,
+    enable_read_receipts: bool | None = None,
+    enable_spectator_access: bool | None = None,
+    prereg_realm: PreregistrationRealm | None = None,
+    how_realm_creator_found_zulip: str | None = None,
+    how_realm_creator_found_zulip_extra_context: str | None = None,
+) -> Realm:
+    if string_id in [settings.SOCIAL_AUTH_SUBDOMAIN, settings.SELF_HOSTING_MANAGEMENT_SUBDOMAIN]:
+        raise AssertionError(
+            "Creating a realm on SOCIAL_AUTH_SUBDOMAIN or SELF_HOSTING_MANAGEMENT_SUBDOMAIN is not allowed!"
+        )
+    if Realm.objects.filter(string_id=string_id).exists():
+        raise AssertionError(f"Realm {string_id} already exists!")
+    if not server_initialized():
+        logging.info("Server not yet initialized. Creating the internal realm first.")
+        create_internal_realm()
+
+    kwargs: dict[str, Any] = {}
+    if emails_restricted_to_domains is not None:
+        kwargs["emails_restricted_to_domains"] = emails_restricted_to_domains
+    if description is not None:
+        kwargs["description"] = description
+        rendered_description, version = render_realm_description(description, realm=None)
+        kwargs["rendered_description"] = rendered_description
+        kwargs["rendered_description_version"] = version
+    if invite_required is not None:
+        kwargs["invite_required"] = invite_required
+    if plan_type is not None:
+        kwargs["plan_type"] = plan_type
+    if org_type is not None:
+        kwargs["org_type"] = org_type
+    if default_language is not None:
+        kwargs["default_language"] = default_language
+    if enable_spectator_access is not None:
+        if enable_spectator_access:
+            # Realms with LIMITED plan cannot have spectators enabled.
+            assert plan_type != Realm.PLAN_TYPE_LIMITED
+            assert plan_type is not None or not settings.BILLING_ENABLED
+        kwargs["enable_spectator_access"] = enable_spectator_access
+
+    if date_created is not None:
+        # The date_created parameter is intended only for use by test
+        # suites that want to backdate the date of a realm's creation.
+        assert not settings.PRODUCTION
+        kwargs["date_created"] = date_created
+
+    if is_demo_organization:
+        # To enable creating demo organizations, a deadline of the number
+        # of days after creation that a demo organization should be deleted
+        # needs to be set on the server.
+        assert settings.DEMO_ORG_DEADLINE_DAYS is not None
+
+    # Generally, closed organizations like companies want read
+    # receipts, whereas it's unclear what an open organization's
+    # preferences will be. We enable the setting by default only for
+    # closed organizations.
+    if enable_read_receipts is not None:
+        kwargs["enable_read_receipts"] = enable_read_receipts
+    else:
+        # Hacky: The default of invited_required is True, so we need
+        # to check for None too.
+        kwargs["enable_read_receipts"] = (
+            invite_required is None or invite_required is True or emails_restricted_to_domains
+        )
+    # Initialize this property correctly in the case that no network activity
+    # is required to do so correctly.
+    kwargs["push_notifications_enabled"] = sends_notifications_directly()
+
+    with transaction.atomic(durable=True):
+        realm = Realm(string_id=string_id, name=name, **kwargs)
+        if is_demo_organization:
+            assert settings.DEMO_ORG_DEADLINE_DAYS is not None
+            realm.demo_organization_scheduled_deletion_date = realm.date_created + timedelta(
+                days=settings.DEMO_ORG_DEADLINE_DAYS
+            )
+
+        # For now a dummy value of -1 is given to groups fields which
+        # is changed later before the transaction is committed.
+        for setting_name in Realm.REALM_PERMISSION_GROUP_SETTINGS:
+            setattr(realm, setting_name + "_id", -1)
+
+        realm.save()
+
+        RealmAuditLog.objects.create(
+            # acting_user will be set as the initial realm owner inside
+            # do_create_user(..., realm_creation=True).
+            acting_user=None,
+            realm=realm,
+            event_type=AuditLogEventType.REALM_CREATED,
+            event_time=realm.date_created,
+            extra_data={
+                "how_realm_creator_found_zulip": how_realm_creator_found_zulip,
+                "how_realm_creator_found_zulip_extra_context": how_realm_creator_found_zulip_extra_context,
+            },
+        )
+
+        RealmUserDefault.objects.create(
+            realm=realm,
+            email_address_visibility=get_email_address_visibility_default(realm.org_type),
+        )
+
+        create_system_user_groups_for_realm(realm)
+
+        group_settings_defaults_for_org_types = {
+            "can_add_subscribers_group": {
+                Realm.ORG_TYPES["education_nonprofit"]["id"]: SystemGroups.MODERATORS,
+                Realm.ORG_TYPES["education"]["id"]: SystemGroups.MODERATORS,
+            },
+            "can_create_public_channel_group": {
+                Realm.ORG_TYPES["education_nonprofit"]["id"]: SystemGroups.ADMINISTRATORS,
+                Realm.ORG_TYPES["education"]["id"]: SystemGroups.ADMINISTRATORS,
+            },
+            "can_create_groups": {
+                Realm.ORG_TYPES["education_nonprofit"]["id"]: SystemGroups.MODERATORS,
+                Realm.ORG_TYPES["education"]["id"]: SystemGroups.MODERATORS,
+            },
+            "can_invite_users_group": {
+                Realm.ORG_TYPES["education_nonprofit"]["id"]: SystemGroups.ADMINISTRATORS,
+                Realm.ORG_TYPES["education"]["id"]: SystemGroups.ADMINISTRATORS,
+            },
+            "can_move_messages_between_channels_group": {
+                Realm.ORG_TYPES["education_nonprofit"]["id"]: SystemGroups.MODERATORS,
+                Realm.ORG_TYPES["education"]["id"]: SystemGroups.MODERATORS,
+            },
+        }
+        set_default_for_realm_permission_group_settings(
+            realm, group_settings_defaults_for_org_types
+        )
+
+        RealmAuthenticationMethod.objects.bulk_create(
+            [
+                RealmAuthenticationMethod(name=backend_name, realm=realm)
+                for backend_name in all_default_backend_names()
+            ]
+        )
+
+        PresenceSequence.objects.create(realm=realm, last_update_id=0)
+
+        maybe_enqueue_audit_log_upload(realm)
+
+    # Create channels once Realm object has been saved
+    with override_language(realm.default_language):
+        zulip_discussion_channel = ensure_stream(
+            realm,
+            str(Realm.ZULIP_DISCUSSION_CHANNEL_NAME),
+            stream_description=_("Questions and discussion about using Zulip."),
+            acting_user=None,
+        )
+        zulip_sandbox_channel = ensure_stream(
+            realm,
+            str(Realm.ZULIP_SANDBOX_CHANNEL_NAME),
+            stream_description=_("Experiment with Zulip here. :test_tube:"),
+            acting_user=None,
+        )
+        new_stream_announcements_stream = ensure_stream(
+            realm,
+            str(Realm.DEFAULT_NOTIFICATION_STREAM_NAME),
+            stream_description=_("For team-wide conversations"),
+            acting_user=None,
+        )
+
+    # By default, 'New stream' & 'Zulip update' announcements are sent to the same stream.
+    realm.new_stream_announcements_stream = new_stream_announcements_stream
+    realm.zulip_update_announcements_stream = new_stream_announcements_stream
+
+    # With the current initial streams situation, the public channels are
+    # 'zulip_discussion_channel', 'zulip_sandbox_channel', 'new_stream_announcements_stream'.
+    public_channels = [
+        DefaultStream(stream=zulip_discussion_channel, realm=realm),
+        DefaultStream(stream=zulip_sandbox_channel, realm=realm),
+        DefaultStream(stream=new_stream_announcements_stream, realm=realm),
+    ]
+    DefaultStream.objects.bulk_create(public_channels)
+
+    # New realm is initialized with the latest zulip update announcements
+    # level as it shouldn't receive a bunch of old updates.
+    realm.zulip_update_announcements_level = get_latest_zulip_update_announcements_level()
+
+    realm.save(
+        update_fields=[
+            "new_stream_announcements_stream",
+            "zulip_update_announcements_stream",
+            "zulip_update_announcements_level",
+        ]
+    )
+
+    if plan_type is None and settings.BILLING_ENABLED:
+        # We use acting_user=None for setting the initial plan type.
+        do_change_realm_plan_type(realm, Realm.PLAN_TYPE_LIMITED, acting_user=None)
+
+    if prereg_realm is not None:
+        prereg_realm.status = confirmation_settings.STATUS_USED
+        prereg_realm.created_realm = realm
+        prereg_realm.save(update_fields=["status", "created_realm"])
+
+    if settings.CORPORATE_ENABLED:
+        # Send a notification to the admin realm when a new organization registers.
+        from corporate.lib.stripe import RealmBillingSession
+
+        billing_session = RealmBillingSession(user=None, realm=realm)
+        billing_session.send_realm_created_internal_admin_message(is_demo_organization)
+
+    setup_realm_internal_bots(realm)
+    return realm

zerver/actions/create_user.py

@@ -0,0 +1,839 @@
+from collections import defaultdict
+from collections.abc import Iterable, Sequence
+from contextlib import suppress
+from typing import Any
+
+from django.conf import settings
+from django.db import IntegrityError, transaction
+from django.db.models import F
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+from django.utils.translation import override as override_language
+
+from confirmation import settings as confirmation_settings
+from zerver.actions.message_send import (
+    internal_send_group_direct_message,
+    internal_send_private_message,
+    internal_send_stream_message,
+)
+from zerver.actions.streams import bulk_add_subscriptions, send_peer_subscriber_events
+from zerver.actions.user_groups import (
+    bulk_add_members_to_user_groups,
+    do_send_user_group_members_update_event,
+)
+from zerver.actions.users import (
+    change_user_is_active,
+    get_service_dicts_for_bot,
+    send_update_events_for_anonymous_group_settings,
+)
+from zerver.lib.avatar import generate_and_upload_jdenticon_avatar
+from zerver.lib.create_user import create_user
+from zerver.lib.default_streams import get_slim_realm_default_streams
+from zerver.lib.email_notifications import enqueue_welcome_emails, send_account_registered_email
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.invites import notify_invites_changed
+from zerver.lib.mention import silent_mention_syntax_for_user
+from zerver.lib.remote_server import maybe_enqueue_audit_log_upload
+from zerver.lib.send_email import clear_scheduled_invitation_emails
+from zerver.lib.streams import can_access_stream_history
+from zerver.lib.subscription_info import bulk_get_subscriber_peer_info
+from zerver.lib.user_counts import realm_user_count, realm_user_count_by_role
+from zerver.lib.user_groups import get_system_user_group_for_user
+from zerver.lib.users import (
+    can_access_delivery_email,
+    format_user_row,
+    get_data_for_inaccessible_user,
+    get_user_ids_who_can_access_user,
+    user_access_restricted_in_realm,
+    user_profile_to_user_row,
+)
+from zerver.lib.utils import assert_is_not_none
+from zerver.models import (
+    DefaultStreamGroup,
+    Message,
+    NamedUserGroup,
+    OnboardingStep,
+    OnboardingUserMessage,
+    PreregistrationRealm,
+    PreregistrationUser,
+    Realm,
+    RealmAuditLog,
+    Recipient,
+    Stream,
+    Subscription,
+    UserGroupMembership,
+    UserMessage,
+    UserProfile,
+)
+from zerver.models.groups import SystemGroups
+from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.models.streams import StreamTopicsPolicyEnum
+from zerver.models.users import ExternalAuthID, active_user_ids, bot_owner_user_ids, get_system_bot
+from zerver.tornado.django_api import send_event_on_commit
+
+MAX_NUM_RECENT_MESSAGES = 1000
+MAX_NUM_RECENT_UNREAD_MESSAGES = 20
+
+
+def send_message_to_signup_notification_stream(
+    sender: UserProfile, realm: Realm, message: str
+) -> None:
+    signup_announcements_stream = realm.signup_announcements_stream
+    if signup_announcements_stream is None:
+        return
+
+    with override_language(realm.default_language):
+        topic_name = _("signups")
+        if (
+            signup_announcements_stream.topics_policy
+            == StreamTopicsPolicyEnum.empty_topic_only.value
+        ):
+            topic_name = ""
+
+    internal_send_stream_message(sender, signup_announcements_stream, topic_name, message)
+
+
+def send_group_direct_message_to_admins(sender: UserProfile, realm: Realm, content: str) -> None:
+    administrators = list(realm.get_human_admin_users())
+    internal_send_group_direct_message(
+        realm,
+        sender,
+        content,
+        recipient_users=administrators,
+    )
+
+
+def notify_new_user(user_profile: UserProfile) -> None:
+    user_count = realm_user_count(user_profile.realm)
+    sender_email = settings.NOTIFICATION_BOT
+    sender = get_system_bot(sender_email, user_profile.realm_id)
+
+    is_first_user = user_count == 1
+    if not is_first_user:
+        with override_language(user_profile.realm.default_language):
+            message = _("{user} joined this organization.").format(
+                user=silent_mention_syntax_for_user(user_profile), user_count=user_count
+            )
+            send_message_to_signup_notification_stream(sender, user_profile.realm, message)
+
+        if settings.BILLING_ENABLED:
+            from corporate.lib.registration import generate_licenses_low_warning_message_if_required
+
+            licenses_low_warning_message = generate_licenses_low_warning_message_if_required(
+                user_profile.realm
+            )
+            if licenses_low_warning_message is not None:
+                message += "\n"
+                message += licenses_low_warning_message
+                send_group_direct_message_to_admins(sender, user_profile.realm, message)
+
+
+def set_up_streams_and_groups_for_new_human_user(
+    *,
+    user_profile: UserProfile,
+    prereg_user: PreregistrationUser | None = None,
+    default_stream_groups: Sequence[DefaultStreamGroup] = [],
+    add_initial_stream_subscriptions: bool = True,
+    realm_creation: bool = False,
+) -> None:
+    realm = user_profile.realm
+
+    if prereg_user is not None:
+        streams: list[Stream] = list(prereg_user.streams.all())
+        user_groups: list[NamedUserGroup] = list(prereg_user.groups.all())
+        acting_user: UserProfile | None = prereg_user.referred_by
+
+        # A PregistrationUser should not be used for another UserProfile
+        assert prereg_user.created_user is None, "PregistrationUser should not be reused"
+    else:
+        streams = []
+        user_groups = []
+        acting_user = None
+
+    if add_initial_stream_subscriptions:
+        # If prereg_user.include_realm_default_subscriptions is true, we
+        # add the default streams for the realm to the list of streams.
+        # Note that we are fine with "slim" Stream objects for calling
+        # bulk_add_subscriptions and add_new_user_history, which we verify
+        # in StreamSetupTest tests that check query counts.
+        if prereg_user is None or prereg_user.include_realm_default_subscriptions:
+            default_streams = get_slim_realm_default_streams(realm.id)
+            streams = list(set(streams) | default_streams)
+
+        for default_stream_group in default_stream_groups:
+            default_stream_group_streams = default_stream_group.streams.all()
+            for stream in default_stream_group_streams:
+                if stream not in streams:
+                    streams.append(stream)
+    else:
+        streams = []
+
+    bulk_add_subscriptions(
+        realm,
+        streams,
+        [user_profile],
+        from_user_creation=True,
+        acting_user=acting_user,
+    )
+
+    bulk_add_members_to_user_groups(
+        user_groups,
+        [user_profile.id],
+        acting_user=acting_user,
+    )
+
+    add_new_user_history(user_profile, streams, realm_creation=realm_creation)
+
+
+def add_new_user_history(
+    user_profile: UserProfile,
+    streams: Iterable[Stream],
+    *,
+    realm_creation: bool = False,
+) -> None:
+    """
+    Give the user some messages in their feed, so that they can learn
+    how to use the home view in a realistic way.
+
+    For realms having older onboarding messages, mark the very
+    most recent messages as unread. Otherwise, ONLY mark the
+    messages tracked in 'OnboardingUserMessage' as unread.
+    """
+
+    realm = user_profile.realm
+    # Find recipient ids for the user's streams, limiting to just
+    # those where we can access the streams' full history.
+    #
+    # TODO: This will do database queries in a loop if many private
+    # streams are involved.
+    recipient_ids = [
+        assert_is_not_none(stream.recipient_id)
+        for stream in streams
+        if can_access_stream_history(user_profile, stream)
+    ]
+
+    # Start by finding recent messages matching those recipients.
+    recent_message_ids = set(
+        Message.objects.filter(
+            # Uses index: zerver_message_realm_recipient_id
+            realm_id=realm.id,
+            recipient_id__in=recipient_ids,
+        )
+        .order_by("-id")
+        .values_list("id", flat=True)[0:MAX_NUM_RECENT_MESSAGES]
+    )
+
+    tracked_onboarding_message_ids = set()
+    message_id_to_onboarding_user_message = {}
+    onboarding_user_messages_queryset = OnboardingUserMessage.objects.filter(realm_id=realm.id)
+    for onboarding_user_message in onboarding_user_messages_queryset:
+        tracked_onboarding_message_ids.add(onboarding_user_message.message_id)
+        message_id_to_onboarding_user_message[onboarding_user_message.message_id] = (
+            onboarding_user_message
+        )
+    tracked_onboarding_messages_exist = len(tracked_onboarding_message_ids) > 0
+
+    message_history_ids = recent_message_ids.union(tracked_onboarding_message_ids)
+
+    if len(message_history_ids) > 0:
+        # Handle the race condition where a message arrives between
+        # bulk_add_subscriptions above and the recent message query just above
+        already_used_ids = set(
+            UserMessage.objects.filter(
+                message_id__in=recent_message_ids, user_profile=user_profile
+            ).values_list("message_id", flat=True)
+        )
+
+        # Exclude the already-used ids and sort them.
+        backfill_message_ids = sorted(message_history_ids - already_used_ids)
+
+        # Find which message ids we should mark as read.
+        # (We don't want too many unread messages.)
+        older_message_ids = set()
+        if not tracked_onboarding_messages_exist:
+            older_message_ids = set(backfill_message_ids[:-MAX_NUM_RECENT_UNREAD_MESSAGES])
+
+        # Create UserMessage rows for the backfill.
+        ums_to_create = []
+        for message_id in backfill_message_ids:
+            um = UserMessage(user_profile=user_profile, message_id=message_id)
+            # Only onboarding messages are available for realm creator.
+            # They are not marked as historical.
+            if not realm_creation:
+                um.flags = UserMessage.flags.historical
+            if tracked_onboarding_messages_exist:
+                if message_id not in tracked_onboarding_message_ids:
+                    um.flags |= UserMessage.flags.read
+                elif message_id_to_onboarding_user_message[message_id].flags.starred.is_set:
+                    um.flags |= UserMessage.flags.starred
+            elif message_id in older_message_ids:
+                um.flags |= UserMessage.flags.read
+            ums_to_create.append(um)
+
+        UserMessage.objects.bulk_create(ums_to_create)
+
+
+# Does the processing for a new user account:
+# * Subscribes to default/invitation streams
+# * Adds to initial user groups
+# * Fills in some recent historical messages
+# * Notifies other users in realm and Zulip about the signup
+# * Deactivates PreregistrationUser objects
+# * Mark 'visibility_policy_banner' as read
+def process_new_human_user(
+    user_profile: UserProfile,
+    prereg_user: PreregistrationUser | None = None,
+    default_stream_groups: Sequence[DefaultStreamGroup] = [],
+    realm_creation: bool = False,
+    add_initial_stream_subscriptions: bool = True,
+) -> None:
+    # subscribe to default/invitation streams, add to groups and
+    # fill in some recent historical messages
+    set_up_streams_and_groups_for_new_human_user(
+        user_profile=user_profile,
+        prereg_user=prereg_user,
+        default_stream_groups=default_stream_groups,
+        add_initial_stream_subscriptions=add_initial_stream_subscriptions,
+        realm_creation=realm_creation,
+    )
+
+    realm = user_profile.realm
+
+    if (
+        prereg_user is not None
+        and prereg_user.referred_by is not None
+        and prereg_user.referred_by.is_active
+        and prereg_user.notify_referrer_on_join
+    ):
+        # This is a cross-realm direct message.
+        with override_language(prereg_user.referred_by.default_language):
+            internal_send_private_message(
+                get_system_bot(settings.NOTIFICATION_BOT, prereg_user.referred_by.realm_id),
+                prereg_user.referred_by,
+                _("{user} accepted your invitation to join Zulip!").format(
+                    user=silent_mention_syntax_for_user(user_profile)
+                ),
+            )
+
+    # For the sake of tracking the history of UserProfiles,
+    # we want to tie the newly created user to the PreregistrationUser
+    # it was created from.
+    if prereg_user is not None:
+        prereg_user.status = confirmation_settings.STATUS_USED
+        prereg_user.created_user = user_profile
+        prereg_user.save(update_fields=["status", "created_user"])
+
+    # Mark any other PreregistrationUsers in the realm that are STATUS_USED as
+    # inactive so we can keep track of the PreregistrationUser we
+    # actually used for analytics.
+    if prereg_user is not None:
+        PreregistrationUser.objects.filter(
+            email__iexact=user_profile.delivery_email, realm=user_profile.realm
+        ).exclude(id=prereg_user.id).update(status=confirmation_settings.STATUS_REVOKED)
+    else:
+        PreregistrationUser.objects.filter(
+            email__iexact=user_profile.delivery_email, realm=user_profile.realm
+        ).update(status=confirmation_settings.STATUS_REVOKED)
+
+    if prereg_user is not None and prereg_user.referred_by is not None:
+        notify_invites_changed(user_profile.realm, changed_invite_referrer=prereg_user.referred_by)
+
+    notify_new_user(user_profile)
+    # Clear any scheduled invitation emails to prevent them
+    # from being sent after the user is created.
+    clear_scheduled_invitation_emails(user_profile.delivery_email)
+    if realm.send_welcome_emails:
+        enqueue_welcome_emails(user_profile, realm_creation)
+
+    # Schedule an initial email with the user's
+    # new account details and log-in information.
+    send_account_registered_email(user_profile, realm_creation)
+
+    # We have an import loop here; it's intentional, because we want
+    # to keep all the onboarding code in zerver/lib/onboarding.py.
+    from zerver.lib.onboarding import send_initial_direct_messages_to_user
+
+    welcome_message_custom_text = realm.welcome_message_custom_text
+    if prereg_user is not None and prereg_user.welcome_message_custom_text is not None:
+        welcome_message_custom_text = prereg_user.welcome_message_custom_text
+    initial_direct_message_ids = send_initial_direct_messages_to_user(
+        user_profile,
+        realm_creation=realm_creation,
+        welcome_message_custom_text=welcome_message_custom_text,
+    )
+    message_id_list = [initial_direct_message_ids.welcome_bot_intro_message_id]
+    if initial_direct_message_ids.welcome_bot_custom_message_id is not None:
+        message_id_list.append(initial_direct_message_ids.welcome_bot_custom_message_id)
+
+    UserMessage.objects.filter(user_profile=user_profile, message_id__in=message_id_list).update(
+        flags=F("flags").bitor(UserMessage.flags.starred)
+    )
+
+    # The 'visibility_policy_banner' is only displayed to existing users.
+    # Mark it as read for a new user.
+    #
+    # If the new user opted to import settings from an existing account, and
+    # 'visibility_policy_banner' is already marked as read for the existing account,
+    # 'copy_onboarding_steps' function already did the needed copying.
+    # Simply ignore the IntegrityError in that case.
+    #
+    # The extremely brief nature of this subtransaction makes a savepoint safe.
+    # See https://postgres.ai/blog/20210831-postgresql-subtransactions-considered-harmful
+    # for context on risks around savepoints.
+    with suppress(IntegrityError), transaction.atomic(savepoint=True):
+        OnboardingStep.objects.create(user=user_profile, onboarding_step="visibility_policy_banner")
+
+
+def notify_created_user(user_profile: UserProfile, notify_user_ids: list[int]) -> None:
+    user_row = user_profile_to_user_row(user_profile)
+
+    format_user_row_kwargs: dict[str, Any] = {
+        "realm_id": user_profile.realm_id,
+        "row": user_row,
+        # Since we don't know what the client
+        # supports at this point in the code, we
+        # just assume client_gravatar and
+        # user_avatar_url_field_optional = False :(
+        "client_gravatar": False,
+        "user_avatar_url_field_optional": False,
+        # We assume there's no custom profile
+        # field data for a new user; initial
+        # values are expected to be added in a
+        # later event.
+        "custom_profile_field_data": {},
+    }
+
+    user_ids_without_access_to_created_user: list[int] = []
+    users_with_access_to_created_users: list[UserProfile] = []
+
+    if notify_user_ids:
+        # This is currently used to send creation event when a guest
+        # gains access to a user, so we depend on the caller to make
+        # sure that only accessible users receive the user data.
+        users_with_access_to_created_users = list(
+            user_profile.realm.get_active_users().filter(id__in=notify_user_ids)
+        )
+    else:
+        active_realm_users = list(user_profile.realm.get_active_users())
+
+        if user_access_restricted_in_realm(user_profile):
+            for user in active_realm_users:
+                if user.is_guest:
+                    # This logic assumes that can_access_all_users_group
+                    # setting can only be set to EVERYONE or MEMBERS.
+                    user_ids_without_access_to_created_user.append(user.id)
+                else:
+                    users_with_access_to_created_users.append(user)
+        else:
+            users_with_access_to_created_users = active_realm_users
+
+    user_ids_with_real_email_access = []
+    user_ids_without_real_email_access = []
+
+    person_for_real_email_access_users = None
+    person_for_without_real_email_access_users = None
+    for recipient_user in users_with_access_to_created_users:
+        if can_access_delivery_email(
+            recipient_user, user_profile.id, user_row["email_address_visibility"]
+        ):
+            user_ids_with_real_email_access.append(recipient_user.id)
+            if person_for_real_email_access_users is None:
+                # This caller assumes that "format_user_row" only depends on
+                # specific value of "acting_user" among users in a realm in
+                # email_address_visibility.
+                person_for_real_email_access_users = format_user_row(
+                    **format_user_row_kwargs,
+                    acting_user=recipient_user,
+                )
+        else:
+            user_ids_without_real_email_access.append(recipient_user.id)
+            if person_for_without_real_email_access_users is None:
+                person_for_without_real_email_access_users = format_user_row(
+                    **format_user_row_kwargs,
+                    acting_user=recipient_user,
+                )
+
+    if user_ids_with_real_email_access:
+        assert person_for_real_email_access_users is not None
+        event: dict[str, Any] = dict(
+            type="realm_user", op="add", person=person_for_real_email_access_users
+        )
+        send_event_on_commit(user_profile.realm, event, user_ids_with_real_email_access)
+
+    if user_ids_without_real_email_access:
+        assert person_for_without_real_email_access_users is not None
+        event = dict(type="realm_user", op="add", person=person_for_without_real_email_access_users)
+        send_event_on_commit(user_profile.realm, event, user_ids_without_real_email_access)
+
+    if user_ids_without_access_to_created_user:
+        event = dict(
+            type="realm_user",
+            op="add",
+            person=get_data_for_inaccessible_user(user_profile.realm, user_profile.id),
+            inaccessible_user=True,
+        )
+        send_event_on_commit(user_profile.realm, event, user_ids_without_access_to_created_user)
+
+
+def created_bot_event(user_profile: UserProfile) -> dict[str, Any]:
+    def stream_name(stream: Stream | None) -> str | None:
+        if not stream:
+            return None
+        return stream.name
+
+    default_sending_stream_name = stream_name(user_profile.default_sending_stream)
+    default_events_register_stream_name = stream_name(user_profile.default_events_register_stream)
+
+    bot = dict(
+        user_id=user_profile.id,
+        default_sending_stream=default_sending_stream_name,
+        default_events_register_stream=default_events_register_stream_name,
+        default_all_public_streams=user_profile.default_all_public_streams,
+        services=get_service_dicts_for_bot(user_profile.id),
+    )
+
+    return dict(type="realm_bot", op="add", bot=bot)
+
+
+def notify_created_bot(user_profile: UserProfile) -> None:
+    event = created_bot_event(user_profile)
+    send_event_on_commit(user_profile.realm, event, bot_owner_user_ids(user_profile))
+
+
+@transaction.atomic(savepoint=False)
+def do_create_user(
+    email: str,
+    password: str | None,
+    realm: Realm,
+    full_name: str,
+    bot_type: int | None = None,
+    role: int | None = None,
+    bot_owner: UserProfile | None = None,
+    tos_version: str | None = None,
+    timezone: str = "",
+    avatar_source: str | None = None,
+    default_language: str | None = None,
+    default_sending_stream: Stream | None = None,
+    default_events_register_stream: Stream | None = None,
+    default_all_public_streams: bool | None = None,
+    prereg_user: PreregistrationUser | None = None,
+    prereg_realm: PreregistrationRealm | None = None,
+    default_stream_groups: Sequence[DefaultStreamGroup] = [],
+    source_profile: UserProfile | None = None,
+    realm_creation: bool = False,
+    *,
+    acting_user: UserProfile | None,
+    enable_marketing_emails: bool = True,
+    email_address_visibility: int | None = None,
+    add_initial_stream_subscriptions: bool = True,
+    external_auth_id_dict: dict[str, str] | None = None,
+) -> UserProfile:
+    if settings.BILLING_ENABLED:
+        from corporate.lib.stripe import RealmBillingSession
+
+    user_profile = create_user(
+        email=email,
+        password=password,
+        realm=realm,
+        full_name=full_name,
+        role=role,
+        bot_type=bot_type,
+        bot_owner=bot_owner,
+        tos_version=tos_version,
+        timezone=timezone,
+        avatar_source=avatar_source,
+        default_language=default_language,
+        default_sending_stream=default_sending_stream,
+        default_events_register_stream=default_events_register_stream,
+        default_all_public_streams=default_all_public_streams,
+        source_profile=source_profile,
+        enable_marketing_emails=enable_marketing_emails,
+        email_address_visibility=email_address_visibility,
+    )
+
+    if user_profile.avatar_source == UserProfile.AVATAR_FROM_JDENTICON:
+        generate_and_upload_jdenticon_avatar(user_profile, str(realm.uuid), future=False)
+
+    event_time = user_profile.date_joined
+    if not acting_user:
+        acting_user = user_profile
+    if realm_creation:
+        # If this user just created a realm, make sure they are
+        # properly tagged as the creator of the realm.
+        realm_creation_audit_log = (
+            RealmAuditLog.objects.filter(event_type=AuditLogEventType.REALM_CREATED, realm=realm)
+            .order_by("id")
+            .last()
+        )
+        assert realm_creation_audit_log is not None
+        realm_creation_audit_log.acting_user = user_profile
+        realm_creation_audit_log.save(update_fields=["acting_user"])
+
+    system_user_group = get_system_user_group_for_user(user_profile)
+    UserGroupMembership.objects.create(user_profile=user_profile, user_group=system_user_group)
+    is_new_full_member = (
+        user_profile.role == UserProfile.ROLE_MEMBER and not user_profile.is_provisional_member
+    )
+    if is_new_full_member:
+        full_members_system_group = NamedUserGroup.objects.get(
+            name=SystemGroups.FULL_MEMBERS,
+            realm_for_sharding=user_profile.realm,
+            is_system_group=True,
+        )
+        UserGroupMembership.objects.create(
+            user_profile=user_profile, user_group=full_members_system_group
+        )
+
+    # These are recorded only once the system group memberships above
+    # have been created, because realm_user_count_by_role counts the
+    # workplace users from memberships of realm.workplace_users_group,
+    # for cases where we cannot compute it directly from role counts.
+    RealmAuditLog.objects.create(
+        realm=user_profile.realm,
+        acting_user=acting_user,
+        modified_user=user_profile,
+        event_type=AuditLogEventType.USER_CREATED,
+        event_time=event_time,
+        extra_data={
+            RealmAuditLog.ROLE_COUNT: realm_user_count_by_role(user_profile.realm),
+        },
+    )
+    RealmAuditLog.objects.create(
+        realm=user_profile.realm,
+        modified_user=user_profile,
+        modified_user_group=system_user_group,
+        event_type=AuditLogEventType.USER_GROUP_DIRECT_USER_MEMBERSHIP_ADDED,
+        event_time=event_time,
+        acting_user=acting_user,
+    )
+    if is_new_full_member:
+        RealmAuditLog.objects.create(
+            realm=user_profile.realm,
+            modified_user=user_profile,
+            modified_user_group=full_members_system_group,
+            event_type=AuditLogEventType.USER_GROUP_DIRECT_USER_MEMBERSHIP_ADDED,
+            event_time=event_time,
+            acting_user=acting_user,
+        )
+    maybe_enqueue_audit_log_upload(user_profile.realm)
+
+    if settings.BILLING_ENABLED:
+        billing_session = RealmBillingSession(user=user_profile, realm=user_profile.realm)
+        billing_session.update_license_ledger_if_needed(event_time)
+
+    # Note that for bots, the caller will send an additional event
+    # with bot-specific info like services.
+    notify_created_user(user_profile, [])
+
+    do_send_user_group_members_update_event("add_members", system_user_group, [user_profile.id])
+    if is_new_full_member:
+        do_send_user_group_members_update_event(
+            "add_members", full_members_system_group, [user_profile.id]
+        )
+
+    if external_auth_id_dict:
+        for (
+            external_auth_method_name,
+            external_auth_id,
+        ) in external_auth_id_dict.items():
+            ExternalAuthID.objects.create(
+                user=user_profile,
+                realm=user_profile.realm,
+                external_auth_method_name=external_auth_method_name,
+                external_auth_id=external_auth_id,
+            )
+
+    if prereg_realm is not None:
+        prereg_realm.created_user = user_profile
+        prereg_realm.save(update_fields=["created_user"])
+
+    if realm_creation:
+        from zerver.lib.onboarding import send_initial_realm_messages
+
+        with override_language(realm.default_language):
+            send_initial_realm_messages(realm)
+
+    if bot_type is None:
+        process_new_human_user(
+            user_profile,
+            prereg_user=prereg_user,
+            default_stream_groups=default_stream_groups,
+            realm_creation=realm_creation,
+            add_initial_stream_subscriptions=add_initial_stream_subscriptions,
+        )
+
+    return user_profile
+
+
+def do_activate_mirror_dummy_user(
+    user_profile: UserProfile, *, acting_user: UserProfile | None
+) -> None:
+    """Called to have a user "take over" a "mirror dummy" user
+    (i.e. is_mirror_dummy=True) account when they sign up with the
+    same email address.
+
+    Essentially, the result should be as though we had created the
+    UserProfile just now with do_create_user, except that the mirror
+    dummy user may appear as the recipient or sender of messages from
+    before their account was fully created.
+
+    TODO: This function likely has bugs resulting from this being a
+    parallel code path to do_create_user; e.g. it likely does not
+    handle preferences or default streams properly.
+    """
+    assert user_profile.is_mirror_dummy
+
+    if settings.BILLING_ENABLED:
+        from corporate.lib.stripe import RealmBillingSession
+
+    with transaction.atomic(savepoint=False):
+        change_user_is_active(user_profile, True)
+        user_profile.is_mirror_dummy = False
+        user_profile.set_unusable_password()
+        user_profile.date_joined = timezone_now()
+        user_profile.tos_version = settings.TERMS_OF_SERVICE_VERSION
+        user_profile.save(
+            update_fields=["date_joined", "password", "is_mirror_dummy", "tos_version"]
+        )
+
+        event_time = user_profile.date_joined
+        RealmAuditLog.objects.create(
+            realm=user_profile.realm,
+            modified_user=user_profile,
+            acting_user=acting_user,
+            event_type=AuditLogEventType.USER_ACTIVATED,
+            event_time=event_time,
+            extra_data={
+                RealmAuditLog.ROLE_COUNT: realm_user_count_by_role(user_profile.realm),
+            },
+        )
+        maybe_enqueue_audit_log_upload(user_profile.realm)
+        if settings.BILLING_ENABLED:
+            billing_session = RealmBillingSession(user=user_profile, realm=user_profile.realm)
+            billing_session.update_license_ledger_if_needed(event_time)
+
+    notify_created_user(user_profile, [])
+
+
+@transaction.atomic(savepoint=False)
+def do_reactivate_user(user_profile: UserProfile, *, acting_user: UserProfile | None) -> None:
+    """Reactivate a user that had previously been deactivated"""
+    if user_profile.is_deleted:
+        raise JsonableError(_("Cannot reactivate a deleted user"))
+
+    if user_profile.is_mirror_dummy:
+        raise JsonableError(
+            _("Cannot activate a placeholder account; ask the user to sign up, instead.")
+        )
+
+    change_user_is_active(user_profile, True)
+
+    event_time = timezone_now()
+    RealmAuditLog.objects.create(
+        realm=user_profile.realm,
+        modified_user=user_profile,
+        acting_user=acting_user,
+        event_type=AuditLogEventType.USER_REACTIVATED,
+        event_time=event_time,
+        extra_data={
+            RealmAuditLog.ROLE_COUNT: realm_user_count_by_role(user_profile.realm),
+        },
+    )
+    maybe_enqueue_audit_log_upload(user_profile.realm)
+
+    bot_owner_changed = False
+    if (
+        user_profile.is_bot
+        and user_profile.bot_owner is not None
+        and not user_profile.bot_owner.is_active
+        and acting_user is not None
+    ):
+        previous_owner = user_profile.bot_owner
+        user_profile.bot_owner = acting_user
+        user_profile.save()  # Can't use update_fields because of how the foreign key works.
+        RealmAuditLog.objects.create(
+            realm=user_profile.realm,
+            acting_user=acting_user,
+            modified_user=user_profile,
+            event_type=AuditLogEventType.USER_BOT_OWNER_CHANGED,
+            event_time=event_time,
+            extra_data={
+                RealmAuditLog.OLD_VALUE: previous_owner.id,
+                RealmAuditLog.NEW_VALUE: acting_user.id,
+            },
+        )
+        bot_owner_changed = True
+
+    if settings.BILLING_ENABLED:
+        from corporate.lib.stripe import RealmBillingSession
+
+        billing_session = RealmBillingSession(user=user_profile, realm=user_profile.realm)
+        billing_session.update_license_ledger_if_needed(event_time)
+
+    event = dict(
+        type="realm_user", op="update", person=dict(user_id=user_profile.id, is_active=True)
+    )
+    send_event_on_commit(user_profile.realm, event, get_user_ids_who_can_access_user(user_profile))
+
+    if not user_profile.is_bot:
+        realm_export_consent_event = dict(
+            type="realm_export_consent",
+            user_id=user_profile.id,
+            consented=user_profile.allow_private_data_export,
+        )
+        send_event_on_commit(
+            user_profile.realm,
+            realm_export_consent_event,
+            list(user_profile.realm.get_human_admin_users().values_list("id", flat=True)),
+        )
+
+    if user_profile.is_bot and bot_owner_changed:
+        from zerver.actions.bots import send_bot_owner_update_events
+
+        assert acting_user is not None
+        send_bot_owner_update_events(user_profile, acting_user, previous_owner)
+
+    subscribed_recipient_ids = Subscription.objects.filter(
+        user_profile_id=user_profile.id, active=True, recipient__type=Recipient.STREAM
+    ).values_list("recipient__type_id", flat=True)
+    subscribed_streams = Stream.objects.filter(id__in=subscribed_recipient_ids, deactivated=False)
+    subscriber_peer_info = bulk_get_subscriber_peer_info(
+        realm=user_profile.realm,
+        streams=subscribed_streams,
+    )
+
+    altered_user_dict: dict[int, set[int]] = defaultdict(set)
+    for stream in subscribed_streams:
+        altered_user_dict[stream.id] = {user_profile.id}
+
+    stream_dict = {stream.id: stream for stream in subscribed_streams}
+
+    send_peer_subscriber_events(
+        op="peer_add",
+        realm=user_profile.realm,
+        altered_user_dict=altered_user_dict,
+        stream_dict=stream_dict,
+        subscriber_peer_info=subscriber_peer_info,
+    )
+
+    member_user_groups = user_profile.direct_groups.select_related("named_user_group").order_by(
+        "id"
+    )
+    named_user_groups = []
+    setting_user_groups = []
+    for group in member_user_groups:
+        if hasattr(group, "named_user_group"):
+            named_user_groups.append(group)
+        else:
+            setting_user_groups.append(group)
+
+    for user_group in named_user_groups:
+        do_send_user_group_members_update_event(
+            "add_members", user_group.named_user_group, [user_profile.id]
+        )
+
+    if setting_user_groups:
+        notify_user_ids = active_user_ids(user_profile.realm_id)
+        send_update_events_for_anonymous_group_settings(
+            setting_user_groups, user_profile.realm, list(notify_user_ids)
+        )

zerver/actions/custom_profile_fields.py

@@ -0,0 +1,323 @@
+from collections.abc import Iterable
+
+import orjson
+from django.db import transaction
+from django.utils.translation import gettext as _
+
+from zerver.actions.message_send import send_user_profile_update_notification
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.external_accounts import DEFAULT_EXTERNAL_ACCOUNTS
+from zerver.lib.mention import silent_mention_syntax_for_user
+from zerver.lib.streams import render_stream_description
+from zerver.lib.types import ProfileDataElementUpdateDict, ProfileFieldData, UserProfileChangeDict
+from zerver.lib.users import get_user_ids_who_can_access_user
+from zerver.models import CustomProfileField, CustomProfileFieldValue, Realm, UserProfile
+from zerver.models.custom_profile_fields import custom_profile_fields_for_realm
+from zerver.models.users import active_user_ids
+from zerver.tornado.django_api import send_event_on_commit
+
+
+def notify_realm_custom_profile_fields(realm: Realm) -> None:
+    fields = custom_profile_fields_for_realm(realm.id)
+    event = dict(type="custom_profile_fields", fields=[f.as_dict() for f in fields])
+    send_event_on_commit(realm, event, active_user_ids(realm.id))
+
+
+@transaction.atomic(durable=True)
+def try_add_realm_default_custom_profile_field(
+    realm: Realm,
+    field_subtype: str,
+    display_in_profile_summary: bool = False,
+    required: bool = False,
+    editable_by_user: bool = True,
+    use_for_user_matching: bool = False,
+) -> CustomProfileField:
+    field_data = DEFAULT_EXTERNAL_ACCOUNTS[field_subtype]
+    custom_profile_field = CustomProfileField(
+        realm=realm,
+        name=str(field_data.name),
+        field_type=CustomProfileField.EXTERNAL_ACCOUNT,
+        hint=field_data.hint,
+        field_data=orjson.dumps(dict(subtype=field_subtype)).decode(),
+        display_in_profile_summary=display_in_profile_summary,
+        required=required,
+        editable_by_user=editable_by_user,
+        use_for_user_matching=use_for_user_matching,
+    )
+    custom_profile_field.save()
+    custom_profile_field.order = custom_profile_field.id
+    custom_profile_field.save(update_fields=["order"])
+    notify_realm_custom_profile_fields(realm)
+    return custom_profile_field
+
+
+@transaction.atomic(durable=True)
+def try_add_realm_custom_profile_field(
+    realm: Realm,
+    name: str,
+    field_type: int,
+    hint: str = "",
+    field_data: ProfileFieldData | None = None,
+    display_in_profile_summary: bool = False,
+    required: bool = False,
+    editable_by_user: bool = True,
+    use_for_user_matching: bool = False,
+) -> CustomProfileField:
+    custom_profile_field = CustomProfileField(
+        realm=realm,
+        name=name,
+        field_type=field_type,
+        display_in_profile_summary=display_in_profile_summary,
+        required=required,
+        editable_by_user=editable_by_user,
+        use_for_user_matching=use_for_user_matching,
+    )
+    custom_profile_field.hint = hint
+    if custom_profile_field.field_type in (
+        CustomProfileField.DROPDOWN,
+        CustomProfileField.EXTERNAL_ACCOUNT,
+    ):
+        custom_profile_field.field_data = orjson.dumps(field_data or {}).decode()
+
+    custom_profile_field.save()
+    custom_profile_field.order = custom_profile_field.id
+    custom_profile_field.save(update_fields=["order"])
+    notify_realm_custom_profile_fields(realm)
+    return custom_profile_field
+
+
+@transaction.atomic(durable=True)
+def do_remove_realm_custom_profile_field(realm: Realm, field: CustomProfileField) -> None:
+    """
+    Deleting a field will also delete the user profile data
+    associated with it in CustomProfileFieldValue model.
+    """
+    field.delete()
+    notify_realm_custom_profile_fields(realm)
+
+
+def do_remove_realm_custom_profile_fields(realm: Realm) -> None:
+    CustomProfileField.objects.filter(realm=realm).delete()
+
+
+def remove_custom_profile_field_value_if_required(
+    field: CustomProfileField, field_data: ProfileFieldData
+) -> None:
+    old_values = set(orjson.loads(field.field_data).keys())
+    new_values = set(field_data.keys())
+    removed_values = old_values - new_values
+
+    if not removed_values:
+        return
+
+    values_to_delete = CustomProfileFieldValue.objects.filter(
+        field=field, value__in=removed_values
+    ).select_related("user_profile")
+
+    updated_users = [field_value.user_profile for field_value in values_to_delete]
+
+    values_to_delete.delete()
+
+    for user_profile in updated_users:
+        notify_user_update_custom_profile_data(
+            user_profile,
+            {
+                "id": field.id,
+                "value": None,
+                "rendered_value": None,
+                "type": field.field_type,
+            },
+        )
+
+
+@transaction.atomic(durable=True)
+def try_update_realm_custom_profile_field(
+    realm: Realm,
+    field: CustomProfileField,
+    name: str | None = None,
+    hint: str | None = None,
+    field_data: ProfileFieldData | None = None,
+    display_in_profile_summary: bool | None = None,
+    required: bool | None = None,
+    editable_by_user: bool | None = None,
+    use_for_user_matching: bool | None = None,
+) -> None:
+    if name is not None:
+        field.name = name
+    if hint is not None:
+        field.hint = hint
+    if required is not None:
+        field.required = required
+    if editable_by_user is not None:
+        field.editable_by_user = editable_by_user
+    if display_in_profile_summary is not None:
+        field.display_in_profile_summary = display_in_profile_summary
+    if use_for_user_matching is not None:
+        field.use_for_user_matching = use_for_user_matching
+
+    if field.field_type in (
+        CustomProfileField.DROPDOWN,
+        CustomProfileField.EXTERNAL_ACCOUNT,
+    ):
+        # If field_data is None, field_data is unchanged and there is no need for
+        # comparing field_data values.
+        if field_data is not None and field.field_type == CustomProfileField.DROPDOWN:
+            remove_custom_profile_field_value_if_required(field, field_data)
+
+        # If field.field_data is the default empty string, we will set field_data
+        # to an empty dict.
+        if field_data is not None or field.field_data == "":
+            field.field_data = orjson.dumps(field_data or {}).decode()
+    field.save()
+    notify_realm_custom_profile_fields(realm)
+
+
+@transaction.atomic(durable=True)
+def try_reorder_realm_custom_profile_fields(realm: Realm, order: Iterable[int]) -> None:
+    order_mapping = {_[1]: _[0] for _ in enumerate(order)}
+    custom_profile_fields = CustomProfileField.objects.filter(realm=realm)
+    for custom_profile_field in custom_profile_fields:
+        if custom_profile_field.id not in order_mapping:
+            raise JsonableError(_("Invalid order mapping."))
+    for custom_profile_field in custom_profile_fields:
+        custom_profile_field.order = order_mapping[custom_profile_field.id]
+        custom_profile_field.save(update_fields=["order"])
+    notify_realm_custom_profile_fields(realm)
+
+
+def notify_user_update_custom_profile_data(
+    user_profile: UserProfile, field: dict[str, int | str | list[int] | None]
+) -> None:
+    data = dict(id=field["id"], value=field["value"])
+
+    if field["rendered_value"]:
+        data["rendered_value"] = field["rendered_value"]
+    payload = dict(user_id=user_profile.id, custom_profile_field=data)
+    event = dict(type="realm_user", op="update", person=payload)
+    send_event_on_commit(user_profile.realm, event, get_user_ids_who_can_access_user(user_profile))
+
+
+@transaction.atomic(savepoint=False)
+def do_update_user_custom_profile_data_if_changed(
+    user_profile: UserProfile,
+    data: list[ProfileDataElementUpdateDict],
+    acting_user: UserProfile | None,
+    notify: bool,
+) -> None:
+    changes: list[UserProfileChangeDict] = []
+
+    for custom_profile_field in data:
+        field_value, created = CustomProfileFieldValue.objects.get_or_create(
+            user_profile=user_profile, field_id=custom_profile_field["id"]
+        )
+
+        # field_value.value is a TextField() so we need to have field["value"]
+        # in string form to correctly make comparisons and assignments.
+        if isinstance(custom_profile_field["value"], str):
+            custom_profile_field_value_string = custom_profile_field["value"]
+        else:
+            custom_profile_field_value_string = orjson.dumps(custom_profile_field["value"]).decode()
+
+        if not created and field_value.value == custom_profile_field_value_string:
+            # If the field value isn't actually being changed to a different one,
+            # we have nothing to do here for this field.
+            continue
+
+        old_value = get_custom_profile_field_display_value(field_value)
+
+        field_value.value = custom_profile_field_value_string
+        if field_value.field.is_renderable():
+            field_value.rendered_value = render_stream_description(
+                custom_profile_field_value_string, user_profile.realm
+            )
+            field_value.save(update_fields=["value", "rendered_value"])
+        else:
+            field_value.save(update_fields=["value"])
+        notify_user_update_custom_profile_data(
+            user_profile,
+            {
+                "id": field_value.field_id,
+                "value": field_value.value,
+                "rendered_value": field_value.rendered_value,
+                "type": field_value.field.field_type,
+            },
+        )
+
+        new_value = get_custom_profile_field_display_value(field_value)
+
+        changes.append(
+            UserProfileChangeDict(
+                field_name=field_value.field.name,
+                old_value=old_value,
+                new_value=new_value,
+            )
+        )
+
+    if changes and notify:
+        send_user_profile_update_notification(
+            user_profile=user_profile, acting_user=acting_user, changes=changes
+        )
+
+
+def get_custom_profile_field_display_value(field_value: CustomProfileFieldValue) -> str:
+    """Convert custom profile field value to human-readable string based on field type."""
+    if not field_value.value:
+        return ""
+    type = field_value.field.field_type
+
+    if type == CustomProfileField.DROPDOWN:
+        field_data_dict = orjson.loads(field_value.field.field_data)
+        value_key = field_value.value
+        return field_data_dict[value_key]["text"]
+
+    if type == CustomProfileField.USER:
+        user_ids = orjson.loads(field_value.value)
+        users = UserProfile.objects.filter(id__in=user_ids).only("id", "full_name")
+        users_mentions = [silent_mention_syntax_for_user(user) for user in users]
+        return (", ").join(users_mentions)
+
+    return field_value.value
+
+
+@transaction.atomic(savepoint=False)
+def check_remove_custom_profile_field_value(
+    user_profile: UserProfile, field_id: int, acting_user: UserProfile, notify: bool
+) -> None:
+    try:
+        custom_profile_field = CustomProfileField.objects.get(realm=user_profile.realm, id=field_id)
+        if not acting_user.is_realm_admin and not custom_profile_field.editable_by_user:
+            raise JsonableError(
+                _(
+                    "You are not allowed to change this field. Contact an administrator to update it."
+                )
+            )
+
+        field_value = CustomProfileFieldValue.objects.get(
+            field=custom_profile_field, user_profile=user_profile
+        )
+        old_value = get_custom_profile_field_display_value(field_value)
+        field_value.delete()
+        notify_user_update_custom_profile_data(
+            user_profile,
+            {
+                "id": field_id,
+                "value": None,
+                "rendered_value": None,
+                "type": custom_profile_field.field_type,
+            },
+        )
+        if notify:
+            changes: list[UserProfileChangeDict] = [
+                UserProfileChangeDict(
+                    field_name=custom_profile_field.name,
+                    old_value=old_value,
+                    new_value="",
+                )
+            ]
+            send_user_profile_update_notification(
+                user_profile=user_profile, acting_user=acting_user, changes=changes
+            )
+    except CustomProfileField.DoesNotExist:
+        raise JsonableError(_("Field id {id} not found.").format(id=field_id))
+    except CustomProfileFieldValue.DoesNotExist:
+        pass

zerver/actions/data_import.py

@@ -0,0 +1,235 @@
+import glob
+import logging
+import os
+import shutil
+import tempfile
+from datetime import timedelta
+from typing import Any
+
+from django.conf import settings
+from django.db import IntegrityError
+from django.utils.timezone import now as timezone_now
+
+from confirmation import settings as confirmation_settings
+from zerver.actions.create_realm import get_email_address_visibility_default
+from zerver.actions.create_user import do_reactivate_user
+from zerver.actions.realm_settings import do_delete_realm
+from zerver.actions.users import do_change_user_role
+from zerver.context_processors import is_realm_import_enabled
+from zerver.data_import.slack import do_convert_zipfile
+from zerver.lib.exceptions import SlackImportInvalidFileError
+from zerver.lib.import_realm import do_import_realm
+from zerver.lib.upload import save_attachment_contents
+from zerver.models.prereg_users import PreregistrationRealm
+from zerver.models.realms import Realm, get_realm
+from zerver.models.users import RealmUserDefault, UserProfile, get_user_by_delivery_email
+
+logger = logging.getLogger("zulip.registration")
+
+
+def is_string_id_unique_violation(e: BaseException) -> bool:
+    """True when e is the IntegrityError from violating the unique constraint
+    on Realm.string_id -- i.e. the subdomain is already taken by another
+    realm. Other IntegrityErrors (e.g. genuine bugs elsewhere in the import)
+    return False, so they get the normal failure handling rather than being
+    mistaken for a taken subdomain."""
+    if not isinstance(e, IntegrityError):
+        return False
+    # Django wraps the underlying psycopg2 error as __cause__; its
+    # diagnostics carry the Postgres constraint name.
+    diag = getattr(e.__cause__, "diag", None)
+    return getattr(diag, "constraint_name", None) == "zerver_realm_string_id_key"
+
+
+def import_slack_data(event: dict[str, Any]) -> None:
+    # This is only possible if data imports were enqueued before the
+    # setting was turned off.
+    assert is_realm_import_enabled()
+
+    preregistration_realm = PreregistrationRealm.objects.get(id=event["preregistration_realm_id"])
+    string_id = preregistration_realm.string_id
+
+    if preregistration_realm.created_realm is not None:
+        # A prior delivery of this event already finished the import. The
+        # worker can die after a successful import but before the event is
+        # acked, which redelivers the event; there is nothing left to do.
+        logger.info("(%s) Slack import already completed; skipping redelivery", string_id)
+        return
+
+    try:
+        existing_realm: Realm | None = get_realm(string_id)
+    except Realm.DoesNotExist:
+        existing_realm = None
+    if existing_realm is not None:
+        if preregistration_realm.data_import_metadata.get("import_created_realm_id") == (
+            existing_realm.id
+        ):
+            # A prior attempt of *this* import created the realm and then the
+            # worker died before finishing, and the event was redelivered.
+            # The realm is our own half-imported orphan, so delete it and
+            # re-import below, rather than treating it as a foreign conflict.
+            logger.warning(
+                "(%s) Cleaning up our own orphaned realm from a prior import attempt", string_id
+            )
+            orphan = Realm.objects.get(id=existing_realm.id)
+            assert orphan.date_created >= timezone_now() - timedelta(hours=24)
+            do_delete_realm(orphan)
+        else:
+            # The subdomain is taken by an earlier or concurrent import of the
+            # same subdomain that won the race to create the realm (string_id
+            # is unique). Importing would fail on that unique constraint, and
+            # we must not touch the other import's realm, so abort before the
+            # expensive conversion work and surface the conflict to the user.
+            logger.error("(%s) Aborting Slack import: subdomain is already in use", string_id)
+            preregistration_realm.data_import_metadata["is_import_work_queued"] = False
+            preregistration_realm.data_import_metadata["subdomain_unavailable"] = True
+            preregistration_realm.save(update_fields=["data_import_metadata"])
+            return
+
+    output_dir = tempfile.mkdtemp(
+        prefix=f"import-{preregistration_realm.id}-converted-",
+        dir=settings.IMPORT_TMPFILE_DIRECTORY,
+    )
+    logger.info(
+        "(%s) Starting Slack import from %s / %s",
+        string_id,
+        event["filename"],
+        event["slack_access_token"],
+    )
+
+    try:
+        with tempfile.NamedTemporaryFile(
+            prefix=f"import-{preregistration_realm.id}-slack-",
+            suffix=".zip",
+            dir=settings.IMPORT_TMPFILE_DIRECTORY,
+        ) as fh:
+            save_attachment_contents(event["filename"], fh)
+            fh.flush()
+            logger.info(
+                "(%s) Wrote %d bytes of %s to %s", string_id, fh.tell(), event["filename"], fh.name
+            )
+            do_convert_zipfile(
+                fh.name,
+                output_dir,
+                event["slack_access_token"],
+                convert_slack_threads=True,
+            )
+            attachment_size = sum(
+                os.path.getsize(os.path.join(dirpath, filename))
+                for dirpath, dirnames, filenames in os.walk(f"{output_dir}/uploads/")
+                for filename in filenames
+            )
+            message_count = len(glob.glob(f"{output_dir}/messages-*.json")) * 1000
+            logger.info(
+                "(%s) Completed conversion into %s (%d MB attachments, <= %d messages)",
+                string_id,
+                output_dir,
+                attachment_size / 1024 / 1024,
+                message_count,
+            )
+
+            def record_created_realm(new_realm: Realm) -> None:
+                # Stamp the registration with the realm it owns, atomically
+                # with the realm's creation, so a redelivered import (after a
+                # crash) can recognize and clean up its own half-imported realm
+                # instead of mistaking it for a foreign conflict.
+                preregistration_realm.data_import_metadata["import_created_realm_id"] = new_realm.id
+                preregistration_realm.save(update_fields=["data_import_metadata"])
+
+            realm = do_import_realm(output_dir, string_id, on_realm_created=record_created_realm)
+            logger.info("(%s) Completed import, performing post-import steps", string_id)
+            realm.org_type = preregistration_realm.org_type
+            realm.default_language = preregistration_realm.default_language
+            realm.save()
+
+            realm_user_default = RealmUserDefault.objects.get(realm=realm)
+            realm_user_default.email_address_visibility = (
+                preregistration_realm.data_import_metadata.get(
+                    "email_address_visibility",
+                    get_email_address_visibility_default(realm.org_type),
+                )
+            )
+            realm_user_default.save(update_fields=["email_address_visibility"])
+
+            # Set email address visibility for all users in the realm.
+            UserProfile.objects.filter(realm=realm, is_bot=False).update(
+                email_address_visibility=realm_user_default.email_address_visibility
+            )
+
+            # Try finding the user who imported this realm and make them owner.
+            try:
+                importing_user: UserProfile | None = get_user_by_delivery_email(
+                    preregistration_realm.email, realm
+                )
+            except UserProfile.DoesNotExist:
+                importing_user = None
+
+            if importing_user is None or importing_user.is_bot or importing_user.is_mirror_dummy:
+                # The email address that the importing user validated
+                # with Zulip either does not appear in the data export,
+                # or maps to an account that cannot own the
+                # organization (a bot or a placeholder/mirror-dummy
+                # account). Prompt them to pick which account is theirs.
+                preregistration_realm.data_import_metadata["need_select_realm_owner"] = True
+            else:
+                if not importing_user.is_active:
+                    # The importer's account was deactivated in the
+                    # export; reactivate it so they can own the realm.
+                    do_reactivate_user(importing_user, acting_user=None)
+                if importing_user.role != UserProfile.ROLE_REALM_OWNER:
+                    do_change_user_role(
+                        importing_user,
+                        UserProfile.ROLE_REALM_OWNER,
+                        acting_user=importing_user,
+                        notify=False,
+                    )
+                preregistration_realm.status = confirmation_settings.STATUS_USED
+
+            preregistration_realm.created_realm = realm
+            preregistration_realm.data_import_metadata["is_import_work_queued"] = False
+            preregistration_realm.save()
+            logger.info("(%s) All done!", string_id)
+    except Exception as e:
+        logger.exception("Error during Slack import")
+        try:
+            preregistration_realm.created_realm = None
+            preregistration_realm.data_import_metadata["is_import_work_queued"] = False
+            if type(e) is SlackImportInvalidFileError:
+                # Store the error to be displayed to the user.
+                preregistration_realm.data_import_metadata["invalid_file_error_message"] = str(e)
+
+            # do_import_realm creates the realm with this string_id. If a
+            # concurrent or earlier import of the same subdomain won the
+            # race to create it, our realm insert violates the unique
+            # constraint on Realm.string_id and do_import_realm raises that
+            # IntegrityError. The existing realm belongs to the other
+            # import, so we must not clean it up: deleting it would destroy
+            # their data and can deadlock against their in-progress work on
+            # zerver_userprofile. We record the conflict for the status poll
+            # to surface instead.
+            subdomain_taken = is_string_id_unique_violation(e)
+            if subdomain_taken:
+                preregistration_realm.data_import_metadata["subdomain_unavailable"] = True
+            preregistration_realm.save()
+
+            # For any other failure -- including IntegrityErrors from
+            # genuine bugs on some other constraint -- delete the realm
+            # this import created, if it got that far. A failure before
+            # do_import_realm creates the realm (e.g. an invalid-file error
+            # during conversion) leaves no such realm, so the get() below
+            # raises Realm.DoesNotExist, which we swallow. Otherwise the
+            # realm with this string_id is the (possibly half-imported) one
+            # we created. The age assertion is a last line of defense
+            # against a bug ever pointing this deletion at an established
+            # realm; a Slack import's realm is created at conversion time,
+            # so it is always far younger than a day by the time we get
+            # here.
+            if not subdomain_taken:
+                realm = Realm.objects.get(string_id=string_id)
+                assert realm.date_created >= timezone_now() - timedelta(hours=24)
+                do_delete_realm(realm)
+        except Realm.DoesNotExist:
+            pass
+        raise
+    finally:
+        shutil.rmtree(output_dir)

zerver/actions/default_streams.py

@@ -0,0 +1,196 @@
+from collections.abc import Iterable
+from typing import Any
+
+from django.db import transaction
+from django.utils.translation import gettext as _
+
+from zerver.lib.default_streams import get_default_stream_ids_for_realm
+from zerver.lib.exceptions import JsonableError
+from zerver.models import DefaultStream, DefaultStreamGroup, Realm, Stream
+from zerver.models.streams import get_default_stream_groups
+from zerver.models.users import active_non_guest_user_ids
+from zerver.tornado.django_api import send_event_on_commit
+
+
+def check_default_stream_group_name(group_name: str) -> None:
+    if group_name.strip() == "":
+        raise JsonableError(
+            _("Invalid default channel group name '{group_name}'").format(group_name=group_name)
+        )
+    if len(group_name) > DefaultStreamGroup.MAX_NAME_LENGTH:
+        raise JsonableError(
+            _("Default channel group name too long (limit: {max_length} characters)").format(
+                max_length=DefaultStreamGroup.MAX_NAME_LENGTH,
+            )
+        )
+    for i in group_name:
+        if ord(i) == 0:
+            raise JsonableError(
+                _(
+                    "Default channel group name '{group_name}' contains NULL (0x00) characters."
+                ).format(
+                    group_name=group_name,
+                )
+            )
+
+
+def lookup_default_stream_groups(
+    default_stream_group_names: list[str], realm: Realm
+) -> list[DefaultStreamGroup]:
+    groups_by_name = {
+        group.name: group
+        for group in DefaultStreamGroup.objects.filter(
+            realm=realm, name__in=default_stream_group_names
+        )
+    }
+    default_stream_groups = []
+    for group_name in default_stream_group_names:
+        if group_name not in groups_by_name:
+            raise JsonableError(
+                _("Invalid default channel group {group_name}").format(group_name=group_name)
+            )
+        default_stream_groups.append(groups_by_name[group_name])
+    return default_stream_groups
+
+
+def notify_default_streams(realm: Realm) -> None:
+    event = dict(
+        type="default_streams",
+        default_streams=list(get_default_stream_ids_for_realm(realm.id)),
+    )
+    send_event_on_commit(realm, event, active_non_guest_user_ids(realm.id))
+
+
+def notify_default_stream_groups(realm: Realm) -> None:
+    event = dict(
+        type="default_stream_groups",
+        default_stream_groups=default_stream_groups_to_dicts_sorted(
+            get_default_stream_groups(realm)
+        ),
+    )
+    send_event_on_commit(realm, event, active_non_guest_user_ids(realm.id))
+
+
+def do_add_default_stream(stream: Stream) -> None:
+    realm_id = stream.realm_id
+    stream_id = stream.id
+    if not DefaultStream.objects.filter(realm_id=realm_id, stream_id=stream_id).exists():
+        DefaultStream.objects.create(realm_id=realm_id, stream_id=stream_id)
+        notify_default_streams(stream.realm)
+
+
+@transaction.atomic(savepoint=False)
+def do_remove_default_stream(stream: Stream) -> None:
+    realm_id = stream.realm_id
+    stream_id = stream.id
+    DefaultStream.objects.filter(realm_id=realm_id, stream_id=stream_id).delete()
+    notify_default_streams(stream.realm)
+
+
+def do_create_default_stream_group(
+    realm: Realm, group_name: str, description: str, streams: list[Stream]
+) -> None:
+    default_stream_ids = get_default_stream_ids_for_realm(realm.id)
+    for stream in streams:
+        if stream.id in default_stream_ids:
+            raise JsonableError(
+                _(
+                    "'{channel_name}' is a default channel and cannot be added to '{group_name}'",
+                ).format(channel_name=stream.name, group_name=group_name)
+            )
+
+    check_default_stream_group_name(group_name)
+    (group, created) = DefaultStreamGroup.objects.get_or_create(
+        name=group_name, realm=realm, description=description
+    )
+    if not created:
+        raise JsonableError(
+            _(
+                "Default channel group '{group_name}' already exists",
+            ).format(group_name=group_name)
+        )
+
+    group.streams.set(streams)
+    notify_default_stream_groups(realm)
+
+
+def do_add_streams_to_default_stream_group(
+    realm: Realm, group: DefaultStreamGroup, streams: list[Stream]
+) -> None:
+    default_stream_ids = get_default_stream_ids_for_realm(realm.id)
+    for stream in streams:
+        if stream.id in default_stream_ids:
+            raise JsonableError(
+                _(
+                    "'{channel_name}' is a default channel and cannot be added to '{group_name}'",
+                ).format(channel_name=stream.name, group_name=group.name)
+            )
+        if stream in group.streams.all():
+            raise JsonableError(
+                _(
+                    "Channel '{channel_name}' is already present in default channel group '{group_name}'",
+                ).format(channel_name=stream.name, group_name=group.name)
+            )
+        group.streams.add(stream)
+
+    group.save()
+    notify_default_stream_groups(realm)
+
+
+def do_remove_streams_from_default_stream_group(
+    realm: Realm, group: DefaultStreamGroup, streams: list[Stream]
+) -> None:
+    group_stream_ids = {stream.id for stream in group.streams.all()}
+    for stream in streams:
+        if stream.id not in group_stream_ids:
+            raise JsonableError(
+                _(
+                    "Channel '{channel_name}' is not present in default channel group '{group_name}'",
+                ).format(channel_name=stream.name, group_name=group.name)
+            )
+
+    delete_stream_ids = {stream.id for stream in streams}
+
+    group.streams.remove(*delete_stream_ids)
+    notify_default_stream_groups(realm)
+
+
+def do_change_default_stream_group_name(
+    realm: Realm, group: DefaultStreamGroup, new_group_name: str
+) -> None:
+    if group.name == new_group_name:
+        raise JsonableError(
+            _("This default channel group is already named '{group_name}'").format(
+                group_name=new_group_name
+            )
+        )
+
+    if DefaultStreamGroup.objects.filter(name=new_group_name, realm=realm).exists():
+        raise JsonableError(
+            _("Default channel group '{group_name}' already exists").format(
+                group_name=new_group_name
+            )
+        )
+
+    group.name = new_group_name
+    group.save()
+    notify_default_stream_groups(realm)
+
+
+def do_change_default_stream_group_description(
+    realm: Realm, group: DefaultStreamGroup, new_description: str
+) -> None:
+    group.description = new_description
+    group.save()
+    notify_default_stream_groups(realm)
+
+
+def do_remove_default_stream_group(realm: Realm, group: DefaultStreamGroup) -> None:
+    group.delete()
+    notify_default_stream_groups(realm)
+
+
+def default_stream_groups_to_dicts_sorted(
+    groups: Iterable[DefaultStreamGroup],
+) -> list[dict[str, Any]]:
+    return sorted((group.to_dict() for group in groups), key=lambda elt: elt["name"])

zerver/actions/devices.py

@@ -0,0 +1,27 @@
+from zerver.lib.devices import check_device_id
+from zerver.models.devices import Device
+from zerver.models.users import UserProfile
+from zerver.tornado.django_api import send_event_on_commit
+
+
+def do_register_device(user_profile: UserProfile) -> int:
+    device = Device.objects.create(user=user_profile)
+    event = dict(
+        type="device",
+        op="add",
+        device_id=device.id,
+    )
+    send_event_on_commit(user_profile.realm, event, [user_profile.id])
+    return device.id
+
+
+def do_remove_device(user_profile: UserProfile, device_id: int) -> None:
+    device = check_device_id(device_id, user_profile.id)
+    device.delete()
+
+    event = dict(
+        type="device",
+        op="remove",
+        device_id=device_id,
+    )
+    send_event_on_commit(user_profile.realm, event, [user_profile.id])

zerver/actions/invites.py

@@ -0,0 +1,580 @@
+import logging
+from collections.abc import Collection, Sequence
+from dataclasses import asdict, dataclass
+from datetime import datetime, timedelta
+from email.utils import format_datetime as email_format_datetime
+from typing import Any
+
+from django.conf import settings
+from django.contrib.contenttypes.models import ContentType
+from django.db import transaction
+from django.db.models import Q, QuerySet, Sum
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+from zxcvbn import zxcvbn
+
+from analytics.lib.counts import COUNT_STATS, do_increment_logging_stat
+from analytics.models import RealmCount
+from confirmation import settings as confirmation_settings
+from confirmation.models import (
+    Confirmation,
+    confirmation_url_for,
+    create_confirmation_link,
+    create_confirmation_object,
+)
+from zerver.context_processors import common_context
+from zerver.lib.email_validation import (
+    get_existing_user_errors,
+    get_realm_email_validator,
+    validate_email_is_valid,
+)
+from zerver.lib.exceptions import InvitationError
+from zerver.lib.invites import notify_invites_changed
+from zerver.lib.queue import queue_event_on_commit
+from zerver.lib.send_email import FromAddress, clear_scheduled_invitation_emails, send_future_email
+from zerver.lib.timestamp import datetime_to_timestamp
+from zerver.lib.types import Invitee
+from zerver.lib.utils import assert_is_not_none
+from zerver.models import (
+    Message,
+    MultiuseInvite,
+    NamedUserGroup,
+    PreregistrationUser,
+    Realm,
+    RealmAuditLog,
+    Stream,
+    UserProfile,
+)
+from zerver.models.prereg_users import filter_to_valid_prereg_users
+from zerver.models.realm_audit_logs import AuditLogEventType
+
+
+def estimate_recent_invites(realms: Collection[Realm] | QuerySet[Realm], *, days: int) -> int:
+    """An upper bound on the number of invites sent in the last `days` days"""
+    recent_invites = RealmCount.objects.filter(
+        realm__in=realms,
+        property="invites_sent::day",
+        # It's important to filter on this even though the count
+        # doesn't use subgroup, so that we use the index.
+        subgroup=None,
+        end_time__gte=timezone_now() - timedelta(days=days),
+    ).aggregate(Sum("value"))["value__sum"]
+    if recent_invites is None:
+        return 0
+    return recent_invites
+
+
+def too_many_recent_realm_invites(realm: Realm, num_invitees: int) -> bool:
+    # Basic check that we're blow the realm-set limit
+    recent_invites = estimate_recent_invites([realm], days=1)
+    if num_invitees + recent_invites > realm.max_invites:
+        return True
+
+    if realm.plan_type != Realm.PLAN_TYPE_LIMITED:
+        return False
+    if realm.max_invites != settings.INVITES_DEFAULT_REALM_DAILY_MAX:
+        return False
+
+    # If they're a non-paid plan with default invitation limits,
+    # we further limit how many invitations can be sent in a day
+    # as a function of how many current users they have. The
+    # allowed ratio has some heuristics to lock down likely-spammy
+    # realms.  This ratio likely only matters for the first
+    # handful of invites; if those users accept, then the realm is
+    # unlikely to hit these limits.  If a real realm hits them,
+    # the resulting message suggests that they contact support if
+    # they have a real use case.
+    warning_flags = []
+    if zxcvbn(realm.string_id)["score"] == 4:
+        # Very high entropy realm names are suspicious
+        warning_flags.append("random-realm-name")
+
+    if not realm.description:
+        warning_flags.append("no-realm-description")
+
+    if realm.icon_source == Realm.ICON_FROM_GRAVATAR:
+        warning_flags.append("no-realm-icon")
+
+    if realm.date_created >= timezone_now() - timedelta(hours=1):
+        warning_flags.append("realm-created-in-last-hour")
+
+    current_user_count = UserProfile.objects.filter(
+        realm=realm, is_bot=False, is_active=True
+    ).count()
+    if current_user_count == 1:
+        warning_flags.append("only-one-user")
+
+    estimated_sent = RealmCount.objects.filter(
+        realm=realm, property="messages_sent:message_type:day"
+    ).aggregate(messages=Sum("value"))
+    if (
+        not estimated_sent["messages"]
+        # Only after we've done the rough-estimate check, take the
+        # time to do the exact check:
+        and not Message.objects.filter(
+            # Uses index: zerver_message_realm_sender_recipient (prefix)
+            realm=realm,
+            sender__is_bot=False,
+        ).exists()
+    ):
+        warning_flags.append("no-messages-sent")
+
+    if len(warning_flags) == 6:
+        permitted_ratio = 2
+    elif len(warning_flags) >= 3:
+        permitted_ratio = 3
+    else:
+        permitted_ratio = 5
+
+    ratio = (num_invitees + recent_invites) / current_user_count
+    logging.log(
+        logging.WARNING if ratio > permitted_ratio else logging.INFO,
+        "%s (!: %s) inviting %d more, have %d recent, but only %d current users.  Ratio %.1f, %d allowed",
+        realm.string_id,
+        ",".join(warning_flags),
+        num_invitees,
+        recent_invites,
+        current_user_count,
+        ratio,
+        permitted_ratio,
+    )
+    return ratio > permitted_ratio
+
+
+def check_invite_limit(realm: Realm, num_invitees: int) -> None:
+    """Discourage using invitation emails as a vector for carrying spam."""
+    msg = _(
+        "To protect users, Zulip limits the number of invitations you can send in one day. Because you have reached the limit, no invitations were sent."
+    )
+    if not settings.OPEN_REALM_CREATION:
+        return
+
+    if too_many_recent_realm_invites(realm, num_invitees):
+        raise InvitationError(
+            msg,
+            [],
+            sent_invitations=False,
+            daily_limit_reached=True,
+        )
+
+    default_max = settings.INVITES_DEFAULT_REALM_DAILY_MAX
+    newrealm_age = timedelta(days=settings.INVITES_NEW_REALM_DAYS)
+    if realm.date_created <= timezone_now() - newrealm_age:
+        # If this isn't a "newly-created" realm, we're done. The
+        # remaining code applies an aggregate limit across all
+        # "new" realms, to address sudden bursts of spam realms.
+        return
+
+    if realm.max_invites > default_max:
+        # If a user is on a realm where we've bumped up
+        # max_invites, then we exempt them from invite limits.
+        return
+
+    new_realms = Realm.objects.filter(
+        date_created__gte=timezone_now() - newrealm_age,
+        _max_invites__lte=default_max,
+    ).all()
+
+    for days, count in settings.INVITES_NEW_REALM_LIMIT_DAYS:
+        recent_invites = estimate_recent_invites(new_realms, days=days)
+        if num_invitees + recent_invites > count:
+            raise InvitationError(
+                msg,
+                [],
+                sent_invitations=False,
+                daily_limit_reached=True,
+            )
+
+
+@transaction.atomic(durable=True)
+def do_invite_users(
+    user_profile: UserProfile,
+    invitees: Collection[Invitee],
+    streams: Collection[Stream],
+    notify_referrer_on_join: bool = True,
+    user_groups: Collection[NamedUserGroup] = [],
+    *,
+    invite_expires_in_minutes: int | None,
+    include_realm_default_subscriptions: bool,
+    invite_as: int = PreregistrationUser.INVITE_AS["MEMBER"],
+    welcome_message_custom_text: str | None = None,
+) -> list[tuple[str, str, bool]]:
+    num_invites = len(invitees)
+
+    # Lock the realm, since we need to not race with other invitations
+    realm = Realm.objects.select_for_update(no_key=True).get(id=user_profile.realm_id)
+    check_invite_limit(realm, num_invites)
+
+    if settings.BILLING_ENABLED:
+        from corporate.lib.registration import check_spare_licenses_available_for_inviting_new_users
+
+        if invite_as == PreregistrationUser.INVITE_AS["GUEST_USER"]:
+            check_spare_licenses_available_for_inviting_new_users(
+                realm, extra_guests_count=num_invites
+            )
+        else:
+            check_spare_licenses_available_for_inviting_new_users(
+                realm, extra_non_guests_count=num_invites
+            )
+
+    if not realm.invite_required:
+        # Inhibit joining an open realm to send spam invitations.
+        min_age = timedelta(days=settings.INVITES_MIN_USER_AGE_DAYS)
+        if user_profile.date_joined > timezone_now() - min_age and not user_profile.is_realm_admin:
+            raise InvitationError(
+                _(
+                    "Your account is too new to send invites for this organization. "
+                    "Ask an organization admin, or a more experienced user."
+                ),
+                [],
+                sent_invitations=False,
+            )
+
+    good_invitees: set[Invitee] = set()
+    errors: list[tuple[str, str, bool]] = []
+    validate_email_allowed_in_realm = get_realm_email_validator(realm)
+    for invitee in invitees:
+        email_error = validate_email_is_valid(
+            invitee.email,
+            validate_email_allowed_in_realm,
+        )
+
+        if email_error:
+            errors.append((invitee.email, email_error, False))
+        else:
+            good_invitees.add(invitee)
+
+    """
+    good_invitees are invitee names and emails that look ok so far,
+    but we still need to make sure they're not
+    gonna conflict with existing users
+    """
+    error_dict = get_existing_user_errors(
+        realm, {invitee.email for invitee in good_invitees}, allow_inactive_mirror_dummies=True
+    )
+
+    skipped: list[tuple[str, str, bool]] = []
+    for email in error_dict:
+        msg, deactivated = error_dict[email]
+        skipped.append((email, msg, deactivated))
+        good_invitees = {item for item in good_invitees if item.email != email}
+
+    validated_invitees = list(good_invitees)
+
+    if errors:
+        raise InvitationError(
+            _("Some emails did not validate, so we didn't send any invitations."),
+            errors + skipped,
+            sent_invitations=False,
+        )
+
+    if skipped and len(skipped) == len(invitees):
+        # All e-mails were skipped, so we didn't actually invite anyone.
+        raise InvitationError(
+            _("We weren't able to invite anyone."), skipped, sent_invitations=False
+        )
+
+    # Now that we are past all the possible errors, we actually create
+    # the PreregistrationUser objects and trigger the email invitations.
+    for validated_invitee in validated_invitees:
+        # The logged in user is the referrer.
+        prereg_user = PreregistrationUser(
+            email=validated_invitee.email,
+            # We include a name if specified in the invitation request, but leave full_name_validated=False,
+            # so the receiving user can spell their name as they like after accepting the invitation.
+            full_name=validated_invitee.full_name,
+            referred_by=user_profile,
+            invited_as=invite_as,
+            realm=realm,
+            include_realm_default_subscriptions=include_realm_default_subscriptions,
+            notify_referrer_on_join=notify_referrer_on_join,
+            welcome_message_custom_text=welcome_message_custom_text,
+        )
+        prereg_user.save()
+        stream_ids = [stream.id for stream in streams]
+        prereg_user.streams.set(stream_ids)
+        groups_ids = [user_group.id for user_group in user_groups]
+        prereg_user.groups.set(groups_ids)
+
+        confirmation = create_confirmation_object(
+            prereg_user, Confirmation.INVITATION, validity_in_minutes=invite_expires_in_minutes
+        )
+        do_send_user_invite_email(
+            prereg_user,
+            confirmation=confirmation,
+            invite_expires_in_minutes=invite_expires_in_minutes,
+        )
+
+    notify_invites_changed(realm, changed_invite_referrer=user_profile)
+
+    return skipped
+
+
+def get_invitation_expiry_date(confirmation_obj: Confirmation) -> int | None:
+    expiry_date = confirmation_obj.expiry_date
+    if expiry_date is None:
+        return expiry_date
+    return datetime_to_timestamp(expiry_date)
+
+
+@dataclass
+class PreregistrationInviteData:
+    email: str
+    invited_by_user_id: int
+    invited: int
+    expiry_date: int | None
+    id: int
+    invited_as: int
+    is_multiuse: bool
+    notify_referrer_on_join: bool
+
+
+@dataclass
+class MultiuseInviteData:
+    invited_by_user_id: int
+    invited: int
+    expiry_date: int | None
+    id: int
+    link_url: str
+    invited_as: int
+    is_multiuse: bool
+
+
+def do_get_invites_controlled_by_user(user_profile: UserProfile) -> list[dict[str, Any]]:
+    """
+    Returns a list of dicts representing invitations that can be controlled by user_profile.
+    This isn't necessarily the same as all the invitations generated by the user, as administrators
+    can control also invitations that they did not themselves create.
+    """
+    if user_profile.is_realm_admin:
+        prereg_users = filter_to_valid_prereg_users(
+            PreregistrationUser.objects.filter(realm=user_profile.realm, referred_by__isnull=False)
+        )
+    else:
+        prereg_users = filter_to_valid_prereg_users(
+            PreregistrationUser.objects.filter(referred_by=user_profile)
+        )
+
+    invites: list[PreregistrationInviteData | MultiuseInviteData] = []
+
+    for invitee in prereg_users:
+        assert invitee.referred_by is not None
+        invites.append(
+            PreregistrationInviteData(
+                email=invitee.email,
+                invited_by_user_id=invitee.referred_by.id,
+                invited=datetime_to_timestamp(invitee.invited_at),
+                expiry_date=get_invitation_expiry_date(invitee.confirmation.get()),
+                id=invitee.id,
+                invited_as=invitee.invited_as,
+                is_multiuse=False,
+                notify_referrer_on_join=invitee.notify_referrer_on_join,
+            )
+        )
+
+    if user_profile.is_realm_admin:
+        multiuse_invite_ids = (
+            MultiuseInvite.objects.filter(realm=user_profile.realm)
+            .exclude(status=confirmation_settings.STATUS_REVOKED)
+            .values_list("id", flat=True)
+        )
+        multiuse_confirmation_objs = Confirmation.objects.filter(
+            realm=user_profile.realm,
+            type=Confirmation.MULTIUSE_INVITE,
+            content_type=ContentType.objects.get_for_model(MultiuseInvite),
+            object_id__in=multiuse_invite_ids,
+        ).filter(Q(expiry_date__gte=timezone_now()) | Q(expiry_date=None))
+    else:
+        multiuse_invite_ids = (
+            MultiuseInvite.objects.filter(referred_by=user_profile)
+            .exclude(status=confirmation_settings.STATUS_REVOKED)
+            .values_list("id", flat=True)
+        )
+        multiuse_confirmation_objs = Confirmation.objects.filter(
+            type=Confirmation.MULTIUSE_INVITE,
+            content_type=ContentType.objects.get_for_model(MultiuseInvite),
+            object_id__in=multiuse_invite_ids,
+        ).filter(Q(expiry_date__gte=timezone_now()) | Q(expiry_date=None))
+
+    for confirmation_obj in multiuse_confirmation_objs:
+        invite = confirmation_obj.content_object
+        assert invite is not None
+
+        assert invite.status != confirmation_settings.STATUS_REVOKED
+        invites.append(
+            MultiuseInviteData(
+                invited_by_user_id=invite.referred_by.id,
+                invited=datetime_to_timestamp(confirmation_obj.date_sent),
+                expiry_date=get_invitation_expiry_date(confirmation_obj),
+                id=invite.id,
+                link_url=confirmation_url_for(confirmation_obj),
+                invited_as=invite.invited_as,
+                is_multiuse=True,
+            )
+        )
+    return [asdict(invite) for invite in invites]
+
+
+@transaction.atomic(durable=True)
+def do_create_multiuse_invite_link(
+    referred_by: UserProfile,
+    invited_as: int,
+    invite_expires_in_minutes: int | None,
+    include_realm_default_subscriptions: bool,
+    streams: Sequence[Stream] = [],
+    user_groups: Sequence[NamedUserGroup] = [],
+    welcome_message_custom_text: str | None = None,
+) -> str:
+    realm = referred_by.realm
+    invite = MultiuseInvite.objects.create(
+        realm=realm,
+        referred_by=referred_by,
+        include_realm_default_subscriptions=include_realm_default_subscriptions,
+        welcome_message_custom_text=welcome_message_custom_text,
+    )
+    if streams:
+        invite.streams.set(streams)
+    if user_groups:
+        invite.groups.set(user_groups)
+    invite.invited_as = invited_as
+    invite.save()
+    notify_invites_changed(referred_by.realm, changed_invite_referrer=referred_by)
+    return create_confirmation_link(
+        invite, Confirmation.MULTIUSE_INVITE, validity_in_minutes=invite_expires_in_minutes
+    )
+
+
+@transaction.atomic(durable=True)
+def do_revoke_user_invite(
+    prereg_user: PreregistrationUser, *, acting_user: UserProfile | None = None
+) -> None:
+    email = prereg_user.email
+    realm = prereg_user.realm
+    assert realm is not None
+
+    prereg_user.status = confirmation_settings.STATUS_REVOKED
+    prereg_user.save(update_fields=["status"])
+    clear_scheduled_invitation_emails(email)
+
+    confirmation = Confirmation.objects.get(
+        type=Confirmation.INVITATION,
+        content_type=ContentType.objects.get_for_model(PreregistrationUser),
+        object_id=prereg_user.id,
+    )
+    # Defensive assert.
+    assert confirmation.content_object == prereg_user
+
+    RealmAuditLog.objects.create(
+        realm=realm,
+        event_type=AuditLogEventType.INVITATION_REVOKED,
+        event_time=timezone_now(),
+        acting_user=acting_user,
+        extra_data={
+            "confirmation_id": confirmation.id,
+            "invitation_type": Confirmation.INVITATION,
+            "invitation_object_id": prereg_user.id,
+        },
+    )
+
+    notify_invites_changed(realm, changed_invite_referrer=prereg_user.referred_by)
+
+
+@transaction.atomic(durable=True)
+def do_revoke_multi_use_invite(
+    multiuse_invite: MultiuseInvite, *, acting_user: UserProfile | None = None
+) -> None:
+    realm = multiuse_invite.referred_by.realm
+
+    multiuse_invite.status = confirmation_settings.STATUS_REVOKED
+    multiuse_invite.save(update_fields=["status"])
+
+    confirmation = Confirmation.objects.get(
+        type=Confirmation.MULTIUSE_INVITE,
+        content_type=ContentType.objects.get_for_model(MultiuseInvite),
+        object_id=multiuse_invite.id,
+    )
+    # Defensive assert.
+    assert confirmation.content_object == multiuse_invite
+
+    RealmAuditLog.objects.create(
+        realm=realm,
+        event_type=AuditLogEventType.INVITATION_REVOKED,
+        event_time=timezone_now(),
+        acting_user=acting_user,
+        extra_data={
+            "confirmation_id": confirmation.id,
+            "invitation_type": Confirmation.MULTIUSE_INVITE,
+            "invitation_object_id": multiuse_invite.id,
+        },
+    )
+
+    notify_invites_changed(realm, changed_invite_referrer=multiuse_invite.referred_by)
+
+
+@transaction.atomic(savepoint=False)
+def do_send_user_invite_email(
+    prereg_user: PreregistrationUser,
+    *,
+    confirmation: Confirmation | None = None,
+    event_time: datetime | None = None,
+    invite_expires_in_minutes: float | None = None,
+) -> None:
+    # Take a lock on the realm, so we can check for invitation limits without races
+    realm_id = assert_is_not_none(prereg_user.realm_id)
+    realm = Realm.objects.select_for_update(no_key=True).get(id=realm_id)
+    check_invite_limit(realm, 1)
+    referrer = assert_is_not_none(prereg_user.referred_by)
+
+    if event_time is None:
+        event_time = prereg_user.invited_at
+    do_increment_logging_stat(realm, COUNT_STATS["invites_sent::day"], None, event_time)
+
+    if confirmation is None:
+        confirmation = prereg_user.confirmation.get()
+
+    event = {
+        "template_prefix": "zerver/emails/invitation",
+        "to_emails": [prereg_user.email],
+        "from_address": FromAddress.tokenized_no_reply_address(),
+        "language": realm.default_language,
+        "context": {
+            "referrer_full_name": referrer.full_name,
+            "referrer_email": referrer.delivery_email,
+            "activate_url": confirmation_url_for(confirmation),
+            "referrer_realm_name": realm.name,
+            "corporate_enabled": settings.CORPORATE_ENABLED,
+        },
+        "remove_suppressed_destination": True,
+        "realm_id": realm.id,
+        "date": email_format_datetime(event_time),
+    }
+    queue_event_on_commit("email_senders", event)
+
+    clear_scheduled_invitation_emails(prereg_user.email)
+    if invite_expires_in_minutes is None and confirmation.expiry_date is not None:
+        # Pull the remaining time from the confirmation object
+        invite_expires_in_minutes = (confirmation.expiry_date - event_time).total_seconds() / 60
+
+    if invite_expires_in_minutes is None or invite_expires_in_minutes < 4 * 24 * 60:
+        # We do not queue reminder email for never expiring
+        # invitations. This is probably a low importance bug; it
+        # would likely be more natural to send a reminder after 7
+        # days.
+        return
+
+    context = common_context(referrer)
+    context.update(
+        activate_url=confirmation_url_for(confirmation),
+        referrer_name=referrer.full_name,
+        referrer_email=referrer.delivery_email,
+        referrer_realm_name=realm.name,
+    )
+    send_future_email(
+        "zerver/emails/invitation_reminder",
+        realm,
+        to_emails=[prereg_user.email],
+        from_address=FromAddress.tokenized_no_reply_placeholder,
+        language=realm.default_language,
+        context=context,
+        delay=timedelta(minutes=invite_expires_in_minutes - (2 * 24 * 60)),
+    )

zerver/actions/message_delete.py

@@ -0,0 +1,128 @@
+from collections.abc import Iterable
+from typing import TypedDict
+
+from django.db.models import Q
+from pydantic import BaseModel
+
+from zerver.lib import retention
+from zerver.lib.retention import move_messages_to_archive
+from zerver.lib.streams import get_public_streams_queryset
+from zerver.models import Message, Realm, Stream, UserProfile
+from zerver.models.recipients import Recipient
+from zerver.tornado.django_api import send_event_on_commit
+
+
+class DeleteMessagesEvent(TypedDict, total=False):
+    type: str
+    message_ids: list[int]
+    message_type: str
+    topic: str
+    stream_id: int
+
+
+class DeactivateUserActions(BaseModel):
+    delete_profile: bool = False
+    delete_public_channel_messages: bool = False
+    delete_private_channel_messages: bool = False
+    delete_direct_messages: bool = False
+
+
+def check_update_first_message_id(
+    realm: Realm, stream: Stream, message_ids: list[int], users_to_notify: Iterable[int]
+) -> None:
+    # This will not update the `first_message_id` of streams where the
+    # first message was deleted prior to the implementation of this function.
+    assert stream.recipient_id is not None
+    if stream.first_message_id not in message_ids:
+        return
+    current_first_message_id = (
+        Message.objects.filter(realm_id=realm.id, recipient_id=stream.recipient_id)
+        .values_list("id", flat=True)
+        .order_by("id")
+        .first()
+    )
+
+    stream.first_message_id = current_first_message_id
+    stream.save(update_fields=["first_message_id"])
+
+    stream_event = dict(
+        type="stream",
+        op="update",
+        property="first_message_id",
+        value=stream.first_message_id,
+        stream_id=stream.id,
+        name=stream.name,
+    )
+    send_event_on_commit(realm, stream_event, users_to_notify)
+
+
+def do_delete_messages(
+    realm: Realm, messages: Iterable[Message], *, acting_user: UserProfile | None
+) -> None:
+    message_ids = [message.id for message in messages]
+    move_messages_to_archive(
+        message_ids,
+        realm=realm,
+        chunk_size=retention.STREAM_MESSAGE_BATCH_SIZE,
+        acting_user=acting_user,
+    )
+
+
+def do_delete_messages_by_sender(user: UserProfile, skip_notify: bool = False) -> None:
+    message_ids = list(
+        # Uses index: zerver_message_realm_sender_recipient (prefix)
+        Message.objects.filter(realm_id=user.realm_id, sender=user)
+        .values_list("id", flat=True)
+        .order_by("id")
+    )
+    move_messages_to_archive(
+        message_ids,
+        user.realm,
+        chunk_size=retention.STREAM_MESSAGE_BATCH_SIZE,
+        skip_notify=skip_notify,
+    )
+
+
+def delete_deactivated_user_messages(
+    realm: Realm,
+    user_profile: UserProfile,
+    deactivate_user_actions: DeactivateUserActions,
+    acting_user: UserProfile | None,
+) -> None:
+    delete_public_channel_messages = deactivate_user_actions.delete_public_channel_messages
+    delete_private_channel_messages = deactivate_user_actions.delete_private_channel_messages
+    delete_direct_messages = deactivate_user_actions.delete_direct_messages
+
+    if not (
+        delete_public_channel_messages or delete_private_channel_messages or delete_direct_messages
+    ):
+        return
+
+    message_filter_query = Q()
+    message_exclude_query = Q()
+
+    if delete_direct_messages:
+        message_filter_query |= Q(recipient__type=Recipient.DIRECT_MESSAGE_GROUP)
+
+    public_channel_ids = get_public_streams_queryset(realm).values_list("id", flat=True)
+    if delete_public_channel_messages and delete_private_channel_messages:
+        message_filter_query |= Q(recipient__type=Recipient.STREAM)
+    elif delete_public_channel_messages:
+        message_filter_query |= Q(
+            recipient__type=Recipient.STREAM,
+            recipient__type_id__in=list(public_channel_ids),
+        )
+    elif delete_private_channel_messages:
+        message_filter_query |= Q(recipient__type=Recipient.STREAM)
+        message_exclude_query |= Q(
+            recipient__type=Recipient.STREAM,
+            recipient__type_id__in=list(public_channel_ids),
+        )
+
+    messages = list(
+        Message.objects.filter(sender=user_profile, realm_id=user_profile.realm_id)
+        .filter(message_filter_query)
+        .exclude(message_exclude_query)
+        .select_related("recipient")
+    )
+    do_delete_messages(user_profile.realm, messages, acting_user=acting_user)

zerver/actions/message_flags.py

@@ -0,0 +1,461 @@
+import time
+from collections import defaultdict
+from dataclasses import asdict, dataclass, field
+
+from django.db import transaction
+from django.db.models import F
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+
+from analytics.lib.counts import COUNT_STATS, do_increment_logging_stat
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.message import (
+    bulk_access_messages,
+    format_unread_message_details,
+    get_raw_unread_data,
+)
+from zerver.lib.queue import mobile_notifications_queue_name, queue_event_on_commit
+from zerver.lib.stream_subscription import get_subscribed_stream_recipient_ids_for_user
+from zerver.lib.topic import filter_by_topic_name_via_message
+from zerver.lib.user_message import DEFAULT_HISTORICAL_FLAGS, create_historical_user_messages
+from zerver.models import Device, Message, PushDeviceToken, Recipient, UserMessage, UserProfile
+from zerver.tornado.django_api import send_event_on_commit, send_event_rollback_unsafe
+
+
+@dataclass
+class ReadMessagesEvent:
+    messages: list[int]
+    all: bool
+    type: str = field(default="update_message_flags", init=False)
+    op: str = field(default="add", init=False)
+    operation: str = field(default="add", init=False)
+    flag: str = field(default="read", init=False)
+
+
+def do_mark_all_as_read(user_profile: UserProfile, *, timeout: float | None = None) -> int | None:
+    start_time = time.monotonic()
+
+    # First, we clear mobile push notifications.  This is safer in the
+    # event that the below logic times out and we're killed.
+    all_push_message_ids = list(
+        UserMessage.objects.filter(
+            user_profile=user_profile,
+        )
+        .extra(  # noqa: S610
+            where=[UserMessage.where_active_push_notification()],
+        )
+        .values_list("message_id", flat=True)[0:10000]
+    )
+    do_clear_mobile_push_notifications_for_ids([user_profile.id], all_push_message_ids)
+
+    batch_size = 2000
+    count = 0
+    while True:
+        if timeout is not None and time.monotonic() >= start_time + timeout:
+            return None
+
+        with transaction.atomic(durable=True):
+            query = (
+                UserMessage.select_for_update_query()
+                .filter(user_profile=user_profile)
+                .extra(where=[UserMessage.where_unread()])[:batch_size]  # noqa: S610
+            )
+            # This updated_count is the same as the number of UserMessage
+            # rows selected, because due to the FOR UPDATE lock, we're guaranteed
+            # that all the selected rows will indeed be updated.
+            # UPDATE queries don't support LIMIT, so we have to use a subquery
+            # to do batching.
+            updated_count = UserMessage.objects.filter(id__in=query).update(
+                flags=F("flags").bitor(UserMessage.flags.read),
+            )
+
+            event_time = timezone_now()
+            do_increment_logging_stat(
+                user_profile,
+                COUNT_STATS["messages_read::hour"],
+                None,
+                event_time,
+                increment=updated_count,
+            )
+            do_increment_logging_stat(
+                user_profile,
+                COUNT_STATS["messages_read_interactions::hour"],
+                None,
+                event_time,
+                increment=min(1, updated_count),
+            )
+
+            count += updated_count
+            if updated_count < batch_size:
+                break
+
+    event = asdict(
+        ReadMessagesEvent(
+            messages=[],  # we don't send messages, since the client reloads anyway
+            all=True,
+        )
+    )
+    send_event_rollback_unsafe(user_profile.realm, event, [user_profile.id])
+
+    return count
+
+
+@transaction.atomic(durable=True)
+def do_mark_stream_messages_as_read(
+    user_profile: UserProfile, stream_recipient_id: int, topic_name: str | None = None
+) -> int:
+    query = (
+        UserMessage.select_for_update_query()
+        .filter(
+            user_profile=user_profile,
+            message__recipient_id=stream_recipient_id,
+        )
+        .extra(  # noqa: S610
+            where=[UserMessage.where_unread()],
+        )
+    )
+
+    if topic_name:
+        query = filter_by_topic_name_via_message(
+            query=query,
+            topic_name=topic_name,
+        )
+
+    message_ids = list(query.values_list("message_id", flat=True))
+
+    if len(message_ids) == 0:
+        return 0
+
+    count = query.update(
+        flags=F("flags").bitor(UserMessage.flags.read),
+    )
+
+    event = asdict(
+        ReadMessagesEvent(
+            messages=message_ids,
+            all=False,
+        )
+    )
+    event_time = timezone_now()
+
+    send_event_on_commit(user_profile.realm, event, [user_profile.id])
+    do_clear_mobile_push_notifications_for_ids([user_profile.id], message_ids)
+
+    do_increment_logging_stat(
+        user_profile, COUNT_STATS["messages_read::hour"], None, event_time, increment=count
+    )
+    do_increment_logging_stat(
+        user_profile,
+        COUNT_STATS["messages_read_interactions::hour"],
+        None,
+        event_time,
+        increment=min(1, count),
+    )
+    return count
+
+
+@transaction.atomic(savepoint=False)
+def do_mark_muted_user_messages_as_read(
+    user_profile: UserProfile,
+    muted_user: UserProfile,
+) -> int:
+    query = (
+        UserMessage.select_for_update_query()
+        .filter(
+            user_profile=user_profile,
+            message__sender=muted_user,
+            # Filtering by message.realm_id is redundant, but required to use index:
+            # zerver_message_realm_sender_recipient
+            message__realm_id=user_profile.realm_id,
+        )
+        .extra(where=[UserMessage.where_unread()])  # noqa: S610
+    )
+    message_ids = list(query.values_list("message_id", flat=True))
+
+    if len(message_ids) == 0:
+        return 0
+
+    count = query.update(
+        flags=F("flags").bitor(UserMessage.flags.read),
+    )
+
+    event = asdict(
+        ReadMessagesEvent(
+            messages=message_ids,
+            all=False,
+        )
+    )
+    event_time = timezone_now()
+
+    send_event_on_commit(user_profile.realm, event, [user_profile.id])
+    do_clear_mobile_push_notifications_for_ids([user_profile.id], message_ids)
+
+    do_increment_logging_stat(
+        user_profile, COUNT_STATS["messages_read::hour"], None, event_time, increment=count
+    )
+    do_increment_logging_stat(
+        user_profile,
+        COUNT_STATS["messages_read_interactions::hour"],
+        None,
+        event_time,
+        increment=min(1, count),
+    )
+    return count
+
+
+def do_update_mobile_push_notification(
+    message: Message,
+    prior_mention_user_ids: set[int],
+    mentions_user_ids: set[int],
+    stream_push_user_ids: set[int],
+) -> None:
+    # Called during the message edit code path to remove mobile push
+    # notifications for users who are no longer mentioned following
+    # the edit.  See #15428 for details.
+    #
+    # A perfect implementation would also support updating the message
+    # in a sent notification if a message was edited to mention a
+    # group rather than a user (or vice versa), though it is likely
+    # not worth the effort to do such a change.
+    if not message.is_channel_message:
+        return
+
+    remove_notify_users = prior_mention_user_ids - mentions_user_ids - stream_push_user_ids
+    do_clear_mobile_push_notifications_for_ids(list(remove_notify_users), [message.id])
+
+
+def do_clear_mobile_push_notifications_for_ids(
+    user_profile_ids: list[int] | None, message_ids: list[int]
+) -> None:
+    if len(message_ids) == 0:
+        return
+
+    if user_profile_ids is not None:
+        # This block gets executed in the following cases:
+        # * Message(s) marked as read by a user
+        # * A message edited to remove mention(s)
+        if len(user_profile_ids) == 0:
+            return
+
+        # This supports clearing notifications for several users only for
+        # the message-edit use case where we'll have a single message_id.
+        assert len(user_profile_ids) == 1 or len(message_ids) == 1
+
+        notifications_to_update = (
+            UserMessage.objects.filter(
+                message_id__in=message_ids,
+                user_profile_id__in=user_profile_ids,
+            )
+            .extra(  # noqa: S610
+                where=[UserMessage.where_active_push_notification()],
+            )
+            .values_list("id", "user_profile_id", "message_id")
+        )
+    else:
+        # This block handles clearing notifications when message(s) get deleted.
+        notifications_to_update = (
+            # Uses index: zerver_usermessage_message_active_mobile_push_notification_idx
+            UserMessage.objects.filter(
+                message_id__in=message_ids,
+            )
+            .extra(  # noqa: S610
+                where=[UserMessage.where_active_push_notification()],
+            )
+            .values_list("id", "user_profile_id", "message_id")
+        )
+
+    messages_by_user = defaultdict(list)
+    for unused, user_id, message_id in notifications_to_update:
+        messages_by_user[user_id].append(message_id)
+
+    clear_notifications_user_ids = set(messages_by_user.keys())
+    push_device_registered_user_ids = set(
+        PushDeviceToken.objects.filter(user_id__in=clear_notifications_user_ids)
+        .values_list("user_id", flat=True)
+        .union(
+            # Uses index "zerver_device_user_push_token_id_idx".
+            Device.objects.filter(
+                user_id__in=clear_notifications_user_ids, push_token_id__isnull=False
+            ).values_list("user_id", flat=True)
+        )
+    )
+    push_device_not_registered_user_ids = (
+        clear_notifications_user_ids - push_device_registered_user_ids
+    )
+
+    usermessages_to_update_ids = []
+    for um_id, user_id, unused in notifications_to_update:
+        if user_id in push_device_not_registered_user_ids:
+            usermessages_to_update_ids.append(um_id)
+
+    if usermessages_to_update_ids:
+        BATCH_SIZE = 1000
+        for i in range(0, len(usermessages_to_update_ids), BATCH_SIZE):
+            with transaction.atomic(savepoint=False):
+                UserMessage.select_for_update_query().filter(
+                    id__in=usermessages_to_update_ids[i : i + BATCH_SIZE]
+                ).update(
+                    flags=F("flags").bitand(~UserMessage.flags.active_mobile_push_notification)
+                )
+
+    for user_profile_id, event_message_ids in messages_by_user.items():
+        if user_profile_id in push_device_not_registered_user_ids:
+            continue
+
+        notice = {
+            "type": "remove",
+            "user_profile_id": user_profile_id,
+            "message_ids": event_message_ids,
+        }
+        queue_event_on_commit(mobile_notifications_queue_name(user_profile_id), notice)
+
+
+def do_update_message_flags(
+    user_profile: UserProfile, operation: str, flag: str, messages: list[int]
+) -> tuple[int, list[int]]:
+    valid_flags = [item for item in UserMessage.flags if item not in UserMessage.NON_API_FLAGS]
+    if flag not in valid_flags:
+        raise JsonableError(_("Invalid flag: '{flag}'").format(flag=flag))
+    if flag in UserMessage.NON_EDITABLE_FLAGS:
+        raise JsonableError(_("Flag not editable: '{flag}'").format(flag=flag))
+    if operation not in ("add", "remove"):
+        raise JsonableError(
+            _("Invalid message flag operation: '{operation}'").format(operation=operation)
+        )
+    is_adding = operation == "add"
+    flagattr = getattr(UserMessage.flags, flag)
+    flag_target = flagattr if is_adding else 0
+
+    ignored_because_not_subscribed_channels = []
+    with transaction.atomic(durable=True):
+        if flag == "read" and not is_adding:
+            # We have an invariant that all stream messages marked as
+            # unread must be in streams the user is subscribed to.
+            #
+            # When marking as unread, we enforce this invariant by
+            # ignoring any messages in streams the user is not
+            # currently subscribed to.
+            subscribed_recipient_ids = get_subscribed_stream_recipient_ids_for_user(user_profile)
+
+            messages_in_unsubscribed_streams = set(
+                # Uses index: zerver_message_pkey
+                Message.objects.select_related("recipient")
+                .filter(id__in=messages, recipient__type=Recipient.STREAM)
+                .exclude(recipient_id__in=subscribed_recipient_ids)
+                .values_list("id", "recipient__type_id")
+            )
+
+            message_ids_in_unsubscribed_streams = {
+                message[0] for message in messages_in_unsubscribed_streams
+            }
+
+            ignored_because_not_subscribed_channels = list(
+                {message[1] for message in messages_in_unsubscribed_streams}
+            )
+
+            messages = [
+                message_id
+                for message_id in messages
+                if message_id not in message_ids_in_unsubscribed_streams
+            ]
+
+        ums = {
+            um.message_id: um
+            for um in UserMessage.select_for_update_query().filter(
+                user_profile=user_profile, message_id__in=messages
+            )
+        }
+
+        # Filter out rows that already have the desired flag.  We do
+        # this here, rather than in the original database query,
+        # because not all flags have database indexes and we want to
+        # bound the cost of this operation.
+        messages = [
+            message_id
+            for message_id in messages
+            if (int(ums[message_id].flags) if message_id in ums else DEFAULT_HISTORICAL_FLAGS)
+            & flagattr
+            != flag_target
+        ]
+        count = len(messages)
+
+        if DEFAULT_HISTORICAL_FLAGS & flagattr != flag_target:
+            # When marking messages as read, creating "historical"
+            # UserMessage rows would be a waste of storage, because
+            # `flags.read | flags.historical` is exactly the flags we
+            # simulate when processing a message for which a user has
+            # access but no UserMessage row.
+            #
+            # Users can mutate flags for messages that don't have a
+            # UserMessage yet.  Validate that the user is even allowed
+            # to access these message_ids; if so, we will create
+            # "historical" UserMessage rows for the messages in question.
+            #
+            # See create_historical_user_messages for a more detailed
+            # explanation.
+            historical_message_ids = set(messages) - set(ums.keys())
+            historical_messages = bulk_access_messages(
+                user_profile,
+                list(
+                    # Uses index: zerver_message_pkey
+                    Message.objects.filter(id__in=historical_message_ids).prefetch_related(
+                        "recipient"
+                    )
+                ),
+                is_modifying_message=False,
+            )
+            if len(historical_messages) != len(historical_message_ids):
+                raise JsonableError(_("Invalid message(s)"))
+
+            create_historical_user_messages(
+                user_id=user_profile.id,
+                message_ids=list(historical_message_ids),
+                flagattr=flagattr,
+                flag_target=flag_target,
+            )
+
+        to_update = UserMessage.objects.filter(
+            user_profile=user_profile, message_id__in=set(messages) & set(ums.keys())
+        )
+        if is_adding:
+            to_update.update(flags=F("flags").bitor(flagattr))
+        else:
+            to_update.update(flags=F("flags").bitand(~flagattr))
+
+        event = {
+            "type": "update_message_flags",
+            "op": operation,
+            "operation": operation,
+            "flag": flag,
+            "messages": messages,
+            "all": False,
+        }
+
+        if flag == "read" and not is_adding:
+            # When removing the read flag (i.e. marking messages as
+            # unread), extend the event with an additional object with
+            # details on the messages required to update the client's
+            # `unread_msgs` data structure.
+            raw_unread_data = get_raw_unread_data(user_profile, messages)
+            event["message_details"] = format_unread_message_details(
+                user_profile.id, raw_unread_data
+            )
+
+        send_event_on_commit(user_profile.realm, event, [user_profile.id])
+
+        if flag == "read" and is_adding:
+            event_time = timezone_now()
+            do_clear_mobile_push_notifications_for_ids([user_profile.id], messages)
+
+            do_increment_logging_stat(
+                user_profile, COUNT_STATS["messages_read::hour"], None, event_time, increment=count
+            )
+            do_increment_logging_stat(
+                user_profile,
+                COUNT_STATS["messages_read_interactions::hour"],
+                None,
+                event_time,
+                increment=min(1, count),
+            )
+
+    return (count, ignored_because_not_subscribed_channels)

zerver/actions/message_summary.py

@@ -0,0 +1,215 @@
+import time
+from typing import Any, Literal
+
+import orjson
+from django.conf import settings
+from django.utils.timezone import now as timezone_now
+from openai import OpenAI
+from openai.types.chat import ChatCompletionMessageParam
+
+from analytics.lib.counts import COUNT_STATS, do_increment_logging_stat
+from zerver.lib.markdown import markdown_convert
+from zerver.lib.message import messages_for_ids
+from zerver.lib.narrow import (
+    LARGER_THAN_MAX_MESSAGE_ID,
+    AnchorInfo,
+    NarrowParameter,
+    clean_narrow_for_message_fetch,
+    fetch_messages,
+)
+from zerver.models import UserProfile
+from zerver.models.realms import MessageEditHistoryVisibilityPolicyEnum
+
+# Maximum number of messages that can be summarized in a single request.
+MAX_MESSAGES_SUMMARIZED = 100
+
+ai_time_start = 0.0
+ai_total_time = 0.0
+ai_total_requests = 0
+
+
+def get_ai_time() -> float:
+    return ai_total_time
+
+
+def ai_stats_start() -> None:
+    global ai_time_start
+    ai_time_start = time.time()
+
+
+def get_ai_requests() -> int:
+    return ai_total_requests
+
+
+def ai_stats_finish() -> None:
+    global ai_total_time, ai_total_requests
+    ai_total_requests += 1
+    ai_total_time += time.time() - ai_time_start
+
+
+def format_zulip_messages_for_model(zulip_messages: list[dict[str, Any]]) -> str:
+    # Format the Zulip messages for processing by the model.
+    #
+    # - We don't need to encode the recipient, since that's the same for
+    #   every message in the conversation.
+    # - We use full names to reference senders, since we want the
+    #   model to refer to users by name. We may want to experiment
+    #   with using silent-mention syntax for users if we move to
+    #   Markdown-rendering what the model returns.
+    # - We don't include timestamps, since experiments with current models
+    #   suggest they do not make relevant use of them.
+    # - We haven't figured out a useful way to include reaction metadata (either
+    #   the emoji themselves or just the counter).
+    # - Polls/TODO widgets are currently sent to the model as empty messages,
+    #   since this logic doesn't inspect SubMessage objects.
+    zulip_messages_list = [
+        {"sender": message["sender_full_name"], "content": message["content"]}
+        for message in zulip_messages
+    ]
+    return orjson.dumps(zulip_messages_list).decode()
+
+
+def make_message(
+    content: str, role: Literal["user", "system"] = "user"
+) -> ChatCompletionMessageParam:
+    if role == "system":
+        return {"content": content, "role": "system"}
+    return {"content": content, "role": "user"}
+
+
+def get_max_summary_length(conversation_length: int) -> int:
+    # Longer summaries work better for longer conversation.
+    # TODO: Test more with message content length.
+    return min(6, 4 + int((conversation_length - 10) / 10))
+
+
+def do_summarize_narrow(
+    user_profile: UserProfile,
+    narrow: list[NarrowParameter] | None,
+) -> str | None:
+    model = settings.TOPIC_SUMMARIZATION_MODEL
+    if model is None:  # nocoverage
+        return None
+
+    # TODO: This implementation does not attempt to make use of
+    # caching previous summaries of the same conversation or rolling
+    # summaries. Doing so correctly will require careful work around
+    # invalidation of caches when messages are edited, moved, or sent.
+    narrow = clean_narrow_for_message_fetch(narrow, user_profile.realm, user_profile)
+    query_info = fetch_messages(
+        narrow=narrow,
+        user_profile=user_profile,
+        realm=user_profile.realm,
+        is_web_public_query=False,
+        anchor_info=AnchorInfo(type="message_id", value=LARGER_THAN_MAX_MESSAGE_ID),
+        include_anchor=True,
+        num_before=MAX_MESSAGES_SUMMARIZED,
+        num_after=0,
+    )
+
+    if len(query_info.rows) == 0:  # nocoverage
+        return None
+
+    result_message_ids: list[int] = []
+    user_message_flags: dict[int, list[str]] = {}
+    for row in query_info.rows:
+        message_id = row[0]
+        result_message_ids.append(message_id)
+        # We skip populating flags, since they would be ignored below anyway.
+        user_message_flags[message_id] = []
+
+    message_list = messages_for_ids(
+        message_ids=result_message_ids,
+        user_message_flags=user_message_flags,
+        search_fields={},
+        # We currently prefer the plain-text content of messages to
+        apply_markdown=False,
+        # Avoid wasting resources computing gravatars.
+        client_gravatar=True,
+        allow_empty_topic_name=False,
+        # Avoid fetching edit history, which won't be passed to the model.
+        message_edit_history_visibility_policy=MessageEditHistoryVisibilityPolicyEnum.none.value,
+        user_profile=user_profile,
+        realm=user_profile.realm,
+    )
+
+    # IDEA: We could consider translating input and output text to
+    # English to improve results when using a summarization model that
+    # is primarily trained on English.
+    conversation_length = len(message_list)
+    max_summary_length = get_max_summary_length(conversation_length)
+    intro = "The following is a chat conversation in the Zulip team chat app."
+    topic: str | None = None
+    channel: str | None = None
+    if narrow and len(narrow) == 2:
+        for term in narrow:
+            assert not term.negated
+            if term.operator == "channel":
+                channel = term.operand
+            if term.operator == "topic":
+                topic = term.operand
+    if channel:
+        intro += f" channel: {channel}"
+    if topic:
+        intro += f", topic: {topic}"
+
+    formatted_conversation = format_zulip_messages_for_model(message_list)
+    prompt = (
+        f"Succinctly summarize this conversation based only on the information provided, "
+        f"in up to {max_summary_length} sentences, for someone who is familiar with the context. "
+        f"Mention key conclusions and actions, if any. Refer to specific people as appropriate. "
+        f"Don't use an intro phrase. You can use Zulip's CommonMark based formatting."
+    )
+    messages = [
+        make_message(intro, "system"),
+        make_message(formatted_conversation),
+        make_message(prompt),
+    ]
+
+    # Stats for database queries are tracked separately.
+    ai_stats_start()
+
+    # TODO when implementing user plans:
+    # - Before querying the model, check whether we've enough tokens left using
+    # an estimated token count.
+    # - Then increase the `LoggingCountStat` using the estimated token count.
+    # (These first two steps should be a short database transaction that
+    # locks the `LoggingCountStat` row).
+    # - Then query the model.
+    # - Then adjust the `LoggingCountStat` by `(actual - estimated)`,
+    # being careful to avoid doing this to the next day if the query
+    # happened milliseconds before midnight; changing the
+    # `LoggingCountStat` we added the estimate to.
+    # That way, you can't easily get extra tokens by sending
+    # 25 requests all at once when you're just below the limit.
+
+    client = OpenAI(
+        api_key=settings.TOPIC_SUMMARIZATION_API_KEY,
+        base_url=settings.TOPIC_SUMMARIZATION_API_BASE,
+    )
+    response = client.chat.completions.create(
+        model=model,
+        messages=messages,
+        **settings.TOPIC_SUMMARIZATION_PARAMETERS,
+    )
+    assert response.usage is not None
+    input_tokens = response.usage.prompt_tokens
+    output_tokens = response.usage.completion_tokens
+
+    # Divide by 1 billion to get actual cost in USD.
+    credits_used = (output_tokens * settings.OUTPUT_COST_PER_GIGATOKEN) + (
+        input_tokens * settings.INPUT_COST_PER_GIGATOKEN
+    )
+    ai_stats_finish()
+
+    do_increment_logging_stat(
+        user_profile, COUNT_STATS["ai_credit_usage::day"], None, timezone_now(), credits_used
+    )
+
+    summary = response.choices[0].message.content
+    assert summary is not None
+    # TODO: This may want to fetch `MentionData`, in order to be able
+    # to process channel or user mentions that might be in the
+    # content. Requires a prompt that supports it.
+    rendered_summary = markdown_convert(summary, message_realm=user_profile.realm).rendered_content
+    return rendered_summary

zerver/actions/muted_users.py

@@ -0,0 +1,51 @@
+from datetime import datetime
+
+from django.db import transaction
+from django.utils.timezone import now as timezone_now
+
+from zerver.actions.message_flags import do_mark_muted_user_messages_as_read
+from zerver.lib.muted_users import add_user_mute, get_user_mutes
+from zerver.models import MutedUser, RealmAuditLog, UserProfile
+from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.tornado.django_api import send_event_on_commit
+
+
+@transaction.atomic(durable=True)
+def do_mute_user(
+    user_profile: UserProfile,
+    muted_user: UserProfile,
+    date_muted: datetime | None = None,
+) -> None:
+    if date_muted is None:
+        date_muted = timezone_now()
+    add_user_mute(user_profile, muted_user, date_muted)
+    do_mark_muted_user_messages_as_read(user_profile, muted_user)
+    event = dict(type="muted_users", muted_users=get_user_mutes(user_profile))
+    send_event_on_commit(user_profile.realm, event, [user_profile.id])
+
+    RealmAuditLog.objects.create(
+        realm=user_profile.realm,
+        acting_user=user_profile,
+        modified_user=user_profile,
+        event_type=AuditLogEventType.USER_MUTED,
+        event_time=date_muted,
+        extra_data={"muted_user_id": muted_user.id},
+    )
+
+
+@transaction.atomic(durable=True)
+def do_unmute_user(mute_object: MutedUser) -> None:
+    user_profile = mute_object.user_profile
+    muted_user = mute_object.muted_user
+    mute_object.delete()
+    event = dict(type="muted_users", muted_users=get_user_mutes(user_profile))
+    send_event_on_commit(user_profile.realm, event, [user_profile.id])
+
+    RealmAuditLog.objects.create(
+        realm=user_profile.realm,
+        acting_user=user_profile,
+        modified_user=user_profile,
+        event_type=AuditLogEventType.USER_UNMUTED,
+        event_time=timezone_now(),
+        extra_data={"unmuted_user_id": muted_user.id},
+    )

zerver/actions/navigation_views.py

@@ -0,0 +1,121 @@
+from django.db import transaction
+from django.utils.timezone import now as timezone_now
+
+from zerver.lib.navigation_views import get_navigation_view_dict
+from zerver.models import NavigationView, RealmAuditLog, UserProfile
+from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.tornado.django_api import send_event_on_commit
+
+
+@transaction.atomic(durable=True)
+def do_add_navigation_view(
+    user: UserProfile,
+    fragment: str,
+    is_pinned: bool,
+    name: str | None = None,
+) -> NavigationView:
+    navigation_view = NavigationView.objects.create(
+        user=user,
+        fragment=fragment,
+        is_pinned=is_pinned,
+        name=name,
+    )
+
+    RealmAuditLog.objects.create(
+        realm=user.realm,
+        acting_user=user,
+        modified_user=user,
+        event_type=AuditLogEventType.NAVIGATION_VIEW_CREATED,
+        event_time=timezone_now(),
+        extra_data={"fragment": fragment},
+    )
+
+    event = {
+        "type": "navigation_view",
+        "op": "add",
+        "navigation_view": get_navigation_view_dict(navigation_view),
+    }
+    send_event_on_commit(user.realm, event, [user.id])
+    return navigation_view
+
+
+@transaction.atomic(durable=True)
+def do_update_navigation_view(
+    user: UserProfile,
+    navigation_view: NavigationView,
+    is_pinned: bool | None,
+    name: str | None = None,
+) -> None:
+    update_dict: dict[str, str | bool] = {}
+    audit_logs_extra_data: list[dict[str, str | bool | None]] = []
+    if name is not None:
+        old_name = navigation_view.name
+        navigation_view.name = name
+        update_dict["name"] = name
+        audit_logs_extra_data.append(
+            {
+                "fragment": navigation_view.fragment,
+                RealmAuditLog.OLD_VALUE: old_name,
+                RealmAuditLog.NEW_VALUE: name,
+                "property": "name",
+            }
+        )
+
+    if is_pinned is not None:
+        old_is_pinned_value = navigation_view.is_pinned
+        navigation_view.is_pinned = is_pinned
+        update_dict["is_pinned"] = is_pinned
+        audit_logs_extra_data.append(
+            {
+                "fragment": navigation_view.fragment,
+                RealmAuditLog.OLD_VALUE: old_is_pinned_value,
+                RealmAuditLog.NEW_VALUE: is_pinned,
+                "property": "is_pinned",
+            }
+        )
+
+    navigation_view.save(update_fields=["name", "is_pinned"])
+
+    now = timezone_now()
+    for audit_log_extra_data in audit_logs_extra_data:
+        RealmAuditLog.objects.create(
+            realm=user.realm,
+            acting_user=user,
+            modified_user=user,
+            event_type=AuditLogEventType.NAVIGATION_VIEW_UPDATED,
+            event_time=now,
+            extra_data=audit_log_extra_data,
+        )
+
+    event = {
+        "type": "navigation_view",
+        "op": "update",
+        "fragment": navigation_view.fragment,
+        "data": update_dict,
+    }
+    send_event_on_commit(user.realm, event, [user.id])
+
+
+@transaction.atomic(durable=True)
+def do_remove_navigation_view(
+    user: UserProfile,
+    navigation_view: NavigationView,
+) -> None:
+    fragment = navigation_view.fragment
+    navigation_view.delete()
+
+    RealmAuditLog.objects.create(
+        realm=user.realm,
+        acting_user=user,
+        modified_user=user,
+        event_type=AuditLogEventType.NAVIGATION_VIEW_DELETED,
+        event_time=timezone_now(),
+        extra_data={"fragment": fragment},
+    )
+
+    event = {
+        "type": "navigation_view",
+        "op": "remove",
+        "fragment": navigation_view.fragment,
+    }
+    send_event_on_commit(user.realm, event, [user.id])

zerver/actions/onboarding_steps.py

@@ -0,0 +1,17 @@
+from dataclasses import asdict
+
+from django.db import transaction
+
+from zerver.lib.onboarding_steps import get_next_onboarding_steps
+from zerver.models import OnboardingStep, UserProfile
+from zerver.tornado.django_api import send_event_on_commit
+
+
+@transaction.atomic(durable=True)
+def do_mark_onboarding_step_as_read(user: UserProfile, onboarding_step: str) -> None:
+    OnboardingStep.objects.get_or_create(user=user, onboarding_step=onboarding_step)
+    event = dict(
+        type="onboarding_steps",
+        onboarding_steps=[asdict(step) for step in get_next_onboarding_steps(user)],
+    )
+    send_event_on_commit(user.realm, event, [user.id])

zerver/actions/presence.py

@@ -0,0 +1,300 @@
+import logging
+import time
+from datetime import datetime, timedelta
+
+from django.conf import settings
+from django.db import connection, transaction
+from psycopg2 import sql
+
+from zerver.actions.user_activity import update_user_activity_interval
+from zerver.lib.presence import (
+    format_legacy_presence_dict,
+    get_modern_user_presence_info,
+    user_presence_datetime_with_date_joined_default,
+)
+from zerver.lib.users import get_user_ids_who_can_access_user
+from zerver.models import Client, UserPresence, UserProfile
+from zerver.models.clients import get_client
+from zerver.models.users import active_user_ids
+from zerver.tornado.django_api import send_event_rollback_unsafe
+
+logger = logging.getLogger(__name__)
+
+
+def send_presence_changed(
+    user_profile: UserProfile, presence: UserPresence, *, force_send_update: bool = False
+) -> None:
+    # Most presence data is sent to clients in the main presence
+    # endpoint in response to the user's own presence; this results
+    # data that is 1-2 minutes stale for who is online.  The flaw with
+    # this plan is when a user comes back online and then immediately
+    # sends a message, recipients may still see that user as offline!
+    # We solve that by sending an immediate presence update clients.
+    #
+    # The API documentation explains this interaction in more detail.
+    if settings.CAN_ACCESS_ALL_USERS_GROUP_LIMITS_PRESENCE:
+        user_ids = get_user_ids_who_can_access_user(user_profile)
+    else:
+        user_ids = active_user_ids(user_profile.realm_id)
+
+    if (
+        len(user_ids) > settings.USER_LIMIT_FOR_SENDING_PRESENCE_UPDATE_EVENTS
+        and not force_send_update
+    ):
+        # These immediate presence generate quadratic work for Tornado
+        # (linear number of users in each event and the frequency of
+        # users coming online grows linearly with userbase too).  In
+        # organizations with thousands of users, this can overload
+        # Tornado, especially if much of the realm comes online at the
+        # same time.
+        #
+        # The utility of these live-presence updates goes down as
+        # organizations get bigger (since one is much less likely to
+        # be paying attention to the sidebar); so beyond a limit, we
+        # stop sending them at all.
+        return
+
+    last_active_time = user_presence_datetime_with_date_joined_default(
+        presence.last_active_time, user_profile.date_joined
+    )
+    last_connected_time = user_presence_datetime_with_date_joined_default(
+        presence.last_connected_time, user_profile.date_joined
+    )
+
+    # The mobile app handles these events so we need to use the old format.
+    # The format of the event should also account for the slim_presence
+    # API parameter when this becomes possible in the future.
+    legacy_presence_dict = format_legacy_presence_dict(last_active_time, last_connected_time)
+    modern_presence_dict = get_modern_user_presence_info(last_active_time, last_connected_time)
+    event = dict(
+        type="presence",
+        email=user_profile.email,
+        user_id=user_profile.id,
+        server_timestamp=time.time(),
+        legacy_presence={legacy_presence_dict["client"]: legacy_presence_dict},
+        modern_presence=modern_presence_dict,
+    )
+    send_event_rollback_unsafe(user_profile.realm, event, user_ids)
+
+
+def consolidate_client(client: Client) -> Client:
+    # The web app reports a client as 'website'
+    # The desktop app reports a client as ZulipDesktop
+    # due to it setting a custom user agent. We want both
+    # to count as web users
+
+    # Alias ZulipDesktop to website
+    if client.name == "ZulipDesktop":
+        return get_client("website")
+    else:
+        return client
+
+
+# This function takes a very hot lock on the PresenceSequence row for the user's realm.
+# Since all presence updates in the realm all compete for this lock, we need to be
+# maximally efficient and only hold it as briefly as possible.
+# For that reason, we need durable=True to ensure we're not running inside a larger
+# transaction, which may stay alive longer than we'd like, holding the lock.
+@transaction.atomic(durable=True)
+def do_update_user_presence(
+    user_profile: UserProfile,
+    client: Client,
+    log_time: datetime,
+    status: int,
+    *,
+    force_send_update: bool = False,
+) -> None:
+    # This function requires some careful handling around setting the
+    # last_update_id field when updatng UserPresence objects. See the
+    # PresenceSequence model and the comments throughout the code for more details.
+
+    client = consolidate_client(client)
+
+    # If the user doesn't have a UserPresence row yet, we create one with
+    # sensible defaults. If we're getting a presence update, clearly the user
+    # at least connected, so last_connected_time should be set. last_active_time
+    # will depend on whether the status sent is idle or active.
+    defaults = dict(
+        last_active_time=None,
+        last_connected_time=log_time,
+        realm_id=user_profile.realm_id,
+    )
+    if status == UserPresence.LEGACY_STATUS_ACTIVE_INT:
+        defaults["last_active_time"] = log_time
+
+    try:
+        presence = UserPresence.objects.select_for_update(no_key=True).get(
+            user_profile=user_profile
+        )
+        creating = False
+    except UserPresence.DoesNotExist:
+        # We're not ready to write until we know the next last_update_id value.
+        # We don't want to hold the lock on PresenceSequence for too long,
+        # so we defer that until the last moment.
+        # Create the presence object in-memory only for now.
+        presence = UserPresence(**defaults, user_profile=user_profile)
+        creating = True
+
+    # We initialize these values as a large delta so that if the user
+    # was never active, we always treat the user as newly online.
+    time_since_last_active_for_comparison = timedelta(days=1)
+    time_since_last_connected_for_comparison = timedelta(days=1)
+    if presence.last_active_time is not None:
+        time_since_last_active_for_comparison = log_time - presence.last_active_time
+    if presence.last_connected_time is not None:
+        time_since_last_connected_for_comparison = log_time - presence.last_connected_time
+
+    assert (3 * settings.PRESENCE_PING_INTERVAL_SECS + 20) <= settings.OFFLINE_THRESHOLD_SECS
+    now_online = time_since_last_active_for_comparison > timedelta(
+        # Here, we decide whether the user is newly online, and we need to consider
+        # sending an immediate presence update via the events system that this user is now online,
+        # rather than waiting for other clients to poll the presence update.
+        # Sending these presence update events adds load to the system, so we only want to do this
+        # if the user has missed a couple regular presence check-ins
+        # (so their state is at least 2 * PRESENCE_PING_INTERVAL_SECS + 10 old),
+        # and also is under the risk of being shown by clients as offline before the next regular presence check-in
+        # (so at least `settings.OFFLINE_THRESHOLD_SECS - settings.PRESENCE_PING_INTERVAL_SECS - 10`).
+        # These two values happen to be the same in the default configuration.
+        seconds=settings.OFFLINE_THRESHOLD_SECS - settings.PRESENCE_PING_INTERVAL_SECS - 10
+    )
+    became_online = status == UserPresence.LEGACY_STATUS_ACTIVE_INT and now_online
+
+    update_fields = []
+
+    # This check is to prevent updating `last_connected_time` several
+    # times per minute with multiple connected browser windows.
+    # We also need to be careful not to wrongly "update" the timestamp if we actually already
+    # have newer presence than the reported log_time.
+    if not creating and time_since_last_connected_for_comparison > timedelta(
+        seconds=settings.PRESENCE_UPDATE_MIN_FREQ_SECONDS
+    ):
+        presence.last_connected_time = log_time
+        update_fields.append("last_connected_time")
+    if (
+        not creating
+        and status == UserPresence.LEGACY_STATUS_ACTIVE_INT
+        and time_since_last_active_for_comparison
+        > timedelta(seconds=settings.PRESENCE_UPDATE_MIN_FREQ_SECONDS)
+    ):
+        presence.last_active_time = log_time
+        update_fields.append("last_active_time")
+        if presence.last_connected_time is None or log_time > presence.last_connected_time:
+            # Update last_connected_time as well to ensure
+            # last_connected_time >= last_active_time.
+            presence.last_connected_time = log_time
+            update_fields.append("last_connected_time")
+
+    # WARNING: Delicate, performance-sensitive block.
+
+    # It's time to determine last_update_id and update the presence object in the database.
+    # This briefly takes the crucial lock on the PresenceSequence row for the user's realm.
+    # We're doing this in a single SQL query to avoid any unnecessary overhead, in particular
+    # database round-trips.
+    # We're also intentionally doing this at the very end of the function, at the last step
+    # before the transaction commits. This ensures the lock is held for the shortest
+    # time possible.
+    # Note: The lock isn't acquired explicitly via something like SELECT FOR UPDATE,
+    # but rather we rely on the UPDATE statement taking an implicit row lock.
+
+    # Equivalent Python code:
+    # if creating or len(update_fields) > 0:
+    #     presence_sequence = PresenceSequence.objects.select_for_update(no_key=True).get(realm_id=user_profile.realm_id)
+    #     new_last_update_id = presence_sequence.last_update_id + 1
+    #     presence_sequence.last_update_id = new_last_update_id
+    #     if creating:
+    #         presence.last_update_id = new_last_update_id
+    #         presence.save()
+    #     elif len(update_fields) > 0:
+    #         presence.last_update_id = new_last_update_id
+    #         presence.save(update_fields=[*update_fields, "last_update_id"])
+    #     presence_sequence.save(update_fields=["last_update_id"])
+    # But let's do it in a single, direct SQL query instead.
+
+    if creating or len(update_fields) > 0:
+        query = sql.SQL("""
+            WITH new_last_update_id AS (
+                UPDATE zerver_presencesequence
+                SET last_update_id = last_update_id + 1
+                WHERE realm_id = {realm_id}
+                RETURNING last_update_id
+            )
+        """).format(realm_id=sql.Literal(user_profile.realm_id))
+
+        if creating:
+            # There's a small possibility of a race where a different process may have
+            # already created a row for this user. Given the extremely close timing
+            # of these events, there's no clear reason to prefer one over the other,
+            # so we choose the simplest option of DO NOTHING here and let the other
+            # concurrent transaction win.
+            # This also allows us to avoid sending a spurious presence update event,
+            # by checking if the row was actually created.
+            query += sql.SQL("""
+                INSERT INTO zerver_userpresence (user_profile_id, last_active_time, last_connected_time, realm_id, last_update_id)
+                VALUES ({user_profile_id}, {last_active_time}, {last_connected_time}, {realm_id}, (SELECT last_update_id FROM new_last_update_id))
+                ON CONFLICT (user_profile_id) DO NOTHING
+                """).format(
+                user_profile_id=sql.Literal(user_profile.id),
+                last_active_time=sql.Literal(presence.last_active_time),
+                last_connected_time=sql.Literal(presence.last_connected_time),
+                realm_id=sql.Literal(user_profile.realm_id),
+            )
+        else:
+            assert len(update_fields) > 0
+            update_fields_segment = sql.SQL(", ").join(
+                sql.SQL("{field} = {value}  ").format(
+                    field=sql.Identifier(field), value=sql.Literal(getattr(presence, field))
+                )
+                for field in update_fields
+            )
+            query += sql.SQL("""
+                UPDATE zerver_userpresence
+                SET {update_fields_segment}, last_update_id = (SELECT last_update_id FROM new_last_update_id)
+                WHERE id = {presence_id}
+            """).format(
+                update_fields_segment=update_fields_segment, presence_id=sql.Literal(presence.id)
+            )
+
+        with connection.cursor() as cursor:
+            cursor.execute(query)
+            if creating:
+                # Check if the row was actually created or if we
+                # hit the ON CONFLICT DO NOTHING case.
+                actually_created = cursor.rowcount > 0
+
+    if creating and not actually_created:
+        # If we ended up doing nothing due to something else creating the row
+        # in the meantime, then we shouldn't send an event here.
+        logger.info("UserPresence row already created for %s, returning.", user_profile.id)
+        return
+
+    if force_send_update or (
+        not user_profile.realm.presence_disabled and (creating or became_online)
+    ):
+        # We do the transaction.on_commit here, rather than inside
+        # send_presence_changed, to help keep presence transactions
+        # brief; the active_user_ids call there is more expensive than
+        # this whole function.
+        transaction.on_commit(
+            lambda: send_presence_changed(
+                user_profile, presence, force_send_update=force_send_update
+            )
+        )
+
+
+def update_user_presence(
+    user_profile: UserProfile,
+    client: Client,
+    log_time: datetime,
+    status: int,
+    new_user_input: bool,
+) -> None:
+    logger.debug(
+        "Processing presence update for user %s, client %s, status %s",
+        user_profile.id,
+        client,
+        status,
+    )
+    if user_profile.presence_enabled:
+        do_update_user_presence(user_profile, client, log_time, status)
+    if new_user_input:
+        update_user_activity_interval(user_profile, log_time)

zerver/actions/push_notifications.py

@@ -0,0 +1,94 @@
+from django.utils.timezone import now as timezone_now
+
+from zerver.lib.timestamp import datetime_to_timestamp
+from zerver.models import UserProfile
+from zerver.models.devices import Device
+from zerver.tornado.django_api import send_event_on_commit
+
+
+def do_register_push_device(
+    user_profile: UserProfile,
+    device: Device,
+    *,
+    token_kind: str,
+    push_key_bytes: bytes,
+    push_key_id: int,
+    token_id_int: int,
+    token_id_base64: str,
+) -> None:
+    registered_at = timezone_now()
+    device.push_key = push_key_bytes
+    device.push_key_id = push_key_id
+    device.pending_push_token_id = token_id_int
+    device.push_token_kind = token_kind
+    device.push_token_last_updated_timestamp = registered_at
+    device.push_registration_error_code = None
+    device.save(
+        update_fields=[
+            "push_key",
+            "push_key_id",
+            "pending_push_token_id",
+            "push_token_kind",
+            "push_token_last_updated_timestamp",
+            "push_registration_error_code",
+        ]
+    )
+
+    event = dict(
+        type="device",
+        op="update",
+        device_id=device.id,
+        push_key_id=device.push_key_id,
+        pending_push_token_id=token_id_base64,
+        push_token_last_updated_timestamp=datetime_to_timestamp(registered_at),
+        push_registration_error_code=None,
+    )
+    send_event_on_commit(user_profile.realm, event, [user_profile.id])
+
+
+def do_rotate_push_key(
+    user_profile: UserProfile, device: Device, push_key_bytes: bytes, push_key_id: int
+) -> None:
+    if (
+        device.push_key_id == push_key_id
+        and device.push_key is not None
+        and bytes(device.push_key) == push_key_bytes
+    ):
+        return
+
+    device.push_key = push_key_bytes
+    device.push_key_id = push_key_id
+    device.save(update_fields=["push_key", "push_key_id"])
+
+    event = dict(
+        type="device",
+        op="update",
+        device_id=device.id,
+        push_key_id=device.push_key_id,
+    )
+    send_event_on_commit(user_profile.realm, event, [user_profile.id])
+
+
+def do_rotate_token(
+    user_profile: UserProfile, device: Device, token_id_int: int, token_id_base64: str
+) -> None:
+    token_updated_at = timezone_now()
+    device.pending_push_token_id = token_id_int
+    device.push_token_last_updated_timestamp = token_updated_at
+    device.push_registration_error_code = None
+    device.save(
+        update_fields=[
+            "pending_push_token_id",
+            "push_token_last_updated_timestamp",
+            "push_registration_error_code",
+        ]
+    )
+    event = dict(
+        type="device",
+        op="update",
+        device_id=device.id,
+        pending_push_token_id=token_id_base64,
+        push_token_last_updated_timestamp=datetime_to_timestamp(token_updated_at),
+        push_registration_error_code=None,
+    )
+    send_event_on_commit(user_profile.realm, event, [user_profile.id])

zerver/actions/reactions.py

@@ -0,0 +1,190 @@
+from typing import Literal
+
+from typing_extensions import TypedDict
+
+from zerver.actions.user_topics import do_set_user_topic_visibility_policy
+from zerver.lib.emoji import check_emoji_request, get_emoji_data
+from zerver.lib.exceptions import ReactionExistsError
+from zerver.lib.message import (
+    access_message_and_usermessage,
+    event_recipient_ids_for_action_on_messages,
+    set_visibility_policy_possible,
+    should_change_visibility_policy,
+    visibility_policy_for_participation,
+)
+from zerver.lib.message_cache import update_message_cache
+from zerver.lib.streams import access_stream_by_id
+from zerver.lib.user_message import create_historical_user_messages
+from zerver.models import Message, Reaction, UserProfile
+from zerver.tornado.django_api import send_event_on_commit
+
+
+class ReactionEvent(TypedDict):
+    type: Literal["reaction"]
+    op: Literal["add", "remove"]
+    user_id: int
+    message_id: int
+    emoji_name: str
+    emoji_code: str
+    reaction_type: str
+
+
+def notify_reaction_update(
+    user_profile: UserProfile,
+    message: Message,
+    reaction: Reaction,
+    op: Literal["add", "remove"],
+) -> None:
+    event: ReactionEvent = {
+        "type": "reaction",
+        "op": op,
+        "user_id": user_profile.id,
+        "message_id": message.id,
+        "emoji_name": reaction.emoji_name,
+        "emoji_code": reaction.emoji_code,
+        "reaction_type": reaction.reaction_type,
+    }
+
+    # Update the cached message since new reaction is added.
+    update_message_cache([message])
+
+    user_ids = event_recipient_ids_for_action_on_messages([message.id], message.is_channel_message)
+    send_event_on_commit(user_profile.realm, event, list(user_ids))
+
+
+def do_add_reaction(
+    user_profile: UserProfile,
+    message: Message,
+    emoji_name: str,
+    emoji_code: str,
+    reaction_type: str,
+) -> None:
+    """Should be called while holding a SELECT FOR UPDATE lock
+    (e.g. via access_message(..., lock_message=True)) on the
+    Message row, to prevent race conditions.
+    """
+
+    reaction = Reaction(
+        user_profile=user_profile,
+        message=message,
+        emoji_name=emoji_name,
+        emoji_code=emoji_code,
+        reaction_type=reaction_type,
+    )
+
+    reaction.save()
+
+    # Determine and set the visibility_policy depending on 'automatically_follow_topics_policy'
+    # and 'automatically_unmute_topics_in_muted_streams_policy'.
+    if set_visibility_policy_possible(
+        user_profile, message
+    ) and UserProfile.AUTOMATICALLY_CHANGE_VISIBILITY_POLICY_ON_PARTICIPATION in [
+        user_profile.automatically_follow_topics_policy,
+        user_profile.automatically_unmute_topics_in_muted_streams_policy,
+    ]:
+        stream_id = message.recipient.type_id
+        (stream, sub) = access_stream_by_id(user_profile, stream_id)
+        assert stream is not None
+        if sub:
+            new_visibility_policy = visibility_policy_for_participation(user_profile, sub.is_muted)
+            if new_visibility_policy and should_change_visibility_policy(
+                new_visibility_policy,
+                user_profile,
+                stream_id,
+                topic_name=message.topic_name(),
+            ):
+                do_set_user_topic_visibility_policy(
+                    user_profile=user_profile,
+                    stream=stream,
+                    topic_name=message.topic_name(),
+                    visibility_policy=new_visibility_policy,
+                )
+
+    notify_reaction_update(user_profile, message, reaction, "add")
+
+
+def check_add_reaction(
+    user_profile: UserProfile,
+    message_id: int,
+    emoji_name: str,
+    emoji_code: str | None,
+    reaction_type: str | None,
+) -> None:
+    message, user_message = access_message_and_usermessage(
+        user_profile, message_id, lock_message=True, is_modifying_message=True
+    )
+
+    if emoji_code is None or reaction_type is None:
+        emoji_data = get_emoji_data(message.realm_id, emoji_name)
+
+        if emoji_code is None:
+            # The emoji_code argument is only required for rare corner
+            # cases discussed in the long block comment below.  For simple
+            # API clients, we allow specifying just the name, and just
+            # look up the code using the current name->code mapping.
+            emoji_code = emoji_data.emoji_code
+
+        if reaction_type is None:
+            reaction_type = emoji_data.reaction_type
+
+    if Reaction.objects.filter(
+        user_profile=user_profile,
+        message=message,
+        emoji_code=emoji_code,
+        reaction_type=reaction_type,
+    ).exists():
+        raise ReactionExistsError
+
+    query = Reaction.objects.filter(
+        message=message, emoji_code=emoji_code, reaction_type=reaction_type
+    )
+    if query.exists():
+        # If another user has already reacted to this message with
+        # same emoji code, we treat the new reaction as a vote for the
+        # existing reaction.  So the emoji name used by that earlier
+        # reaction takes precedence over whatever was passed in this
+        # request.  This is necessary to avoid a message having 2
+        # "different" emoji reactions with the same emoji code (and
+        # thus same image) on the same message, which looks ugly.
+        #
+        # In this "voting for an existing reaction" case, we shouldn't
+        # check whether the emoji code and emoji name match, since
+        # it's possible that the (emoji_type, emoji_name, emoji_code)
+        # triple for this existing reaction may not pass validation
+        # now (e.g. because it is for a realm emoji that has been
+        # since deactivated).  We still want to allow users to add a
+        # vote any old reaction they see in the UI even if that is a
+        # deactivated custom emoji, so we just use the emoji name from
+        # the existing reaction with no further validation.
+        reaction = query.first()
+        assert reaction is not None
+        emoji_name = reaction.emoji_name
+    else:
+        # Otherwise, use the name provided in this request, but verify
+        # it is valid in the user's realm (e.g. not a deactivated
+        # realm emoji).
+        check_emoji_request(user_profile.realm, emoji_name, emoji_code, reaction_type)
+
+    if user_message is None:
+        # See called function for more context.
+        create_historical_user_messages(user_id=user_profile.id, message_ids=[message.id])
+
+    do_add_reaction(user_profile, message, emoji_name, emoji_code, reaction_type)
+
+
+def do_remove_reaction(
+    user_profile: UserProfile, message: Message, emoji_code: str, reaction_type: str
+) -> None:
+    """Should be called while holding a SELECT FOR UPDATE lock
+    (e.g. via access_message(..., lock_message=True)) on the
+    Message row, to prevent race conditions.
+    """
+    reaction = Reaction.objects.filter(
+        user_profile=user_profile,
+        message=message,
+        emoji_code=emoji_code,
+        reaction_type=reaction_type,
+    ).get()
+    reaction.delete()
+
+    notify_reaction_update(user_profile, message, reaction, "remove")

zerver/actions/realm_domains.py

@@ -0,0 +1,104 @@
+from django.db import transaction
+from django.utils.timezone import now as timezone_now
+
+from zerver.actions.realm_settings import do_set_realm_property
+from zerver.models import Realm, RealmAuditLog, RealmDomain, UserProfile
+from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.models.realms import RealmDomainDict, get_realm_domains
+from zerver.models.users import active_user_ids
+from zerver.tornado.django_api import send_event_on_commit
+
+
+@transaction.atomic(durable=True)
+def do_add_realm_domain(
+    realm: Realm, domain: str, allow_subdomains: bool, *, acting_user: UserProfile | None
+) -> RealmDomain:
+    realm_domain = RealmDomain.objects.create(
+        realm=realm, domain=domain, allow_subdomains=allow_subdomains
+    )
+    added_domain = RealmDomainDict(domain=domain, allow_subdomains=allow_subdomains)
+
+    RealmAuditLog.objects.create(
+        realm=realm,
+        acting_user=acting_user,
+        event_type=AuditLogEventType.REALM_DOMAIN_ADDED,
+        event_time=timezone_now(),
+        extra_data={
+            "realm_domains": get_realm_domains(realm),
+            "added_domain": added_domain,
+        },
+    )
+
+    event = dict(
+        type="realm_domains",
+        op="add",
+        realm_domain=RealmDomainDict(
+            domain=realm_domain.domain, allow_subdomains=realm_domain.allow_subdomains
+        ),
+    )
+    send_event_on_commit(realm, event, active_user_ids(realm.id))
+
+    return realm_domain
+
+
+@transaction.atomic(durable=True)
+def do_change_realm_domain(
+    realm_domain: RealmDomain, allow_subdomains: bool, *, acting_user: UserProfile | None
+) -> None:
+    realm_domain.allow_subdomains = allow_subdomains
+    realm_domain.save(update_fields=["allow_subdomains"])
+    changed_domain = RealmDomainDict(
+        domain=realm_domain.domain,
+        allow_subdomains=realm_domain.allow_subdomains,
+    )
+
+    RealmAuditLog.objects.create(
+        realm=realm_domain.realm,
+        acting_user=acting_user,
+        event_type=AuditLogEventType.REALM_DOMAIN_CHANGED,
+        event_time=timezone_now(),
+        extra_data={
+            "realm_domains": get_realm_domains(realm_domain.realm),
+            "changed_domain": changed_domain,
+        },
+    )
+
+    event = dict(
+        type="realm_domains",
+        op="change",
+        realm_domain=dict(
+            domain=realm_domain.domain, allow_subdomains=realm_domain.allow_subdomains
+        ),
+    )
+    send_event_on_commit(realm_domain.realm, event, active_user_ids(realm_domain.realm_id))
+
+
+@transaction.atomic(durable=True)
+def do_remove_realm_domain(realm_domain: RealmDomain, *, acting_user: UserProfile | None) -> None:
+    realm = realm_domain.realm
+    domain = realm_domain.domain
+    realm_domain.delete()
+    removed_domain = RealmDomainDict(
+        domain=realm_domain.domain,
+        allow_subdomains=realm_domain.allow_subdomains,
+    )
+
+    RealmAuditLog.objects.create(
+        realm=realm,
+        acting_user=acting_user,
+        event_type=AuditLogEventType.REALM_DOMAIN_REMOVED,
+        event_time=timezone_now(),
+        extra_data={
+            "realm_domains": get_realm_domains(realm),
+            "removed_domain": removed_domain,
+        },
+    )
+
+    if not RealmDomain.objects.filter(realm=realm).exists() and realm.emails_restricted_to_domains:
+        # If this was the last realm domain, we mark the realm as no
+        # longer restricted to domain, because the feature doesn't do
+        # anything if there are no domains, and this is probably less
+        # confusing than the alternative.
+        do_set_realm_property(realm, "emails_restricted_to_domains", False, acting_user=acting_user)
+    event = dict(type="realm_domains", op="remove", domain=domain)
+    send_event_on_commit(realm, event, active_user_ids(realm.id))

zerver/actions/realm_emoji.py

@@ -0,0 +1,97 @@
+from typing import IO
+
+from django.core.exceptions import ValidationError
+from django.db import transaction
+from django.db.utils import IntegrityError
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+
+from zerver.lib.emoji import get_emoji_file_name
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.mime_types import INLINE_MIME_TYPES, bare_content_type
+from zerver.lib.thumbnail import THUMBNAIL_ACCEPT_IMAGE_TYPES, BadImageError
+from zerver.lib.upload import upload_emoji_image
+from zerver.models import Realm, RealmAuditLog, RealmEmoji, UserProfile
+from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.models.realm_emoji import emoji_as_emojiinfo, get_all_custom_emoji_for_realm
+from zerver.models.users import active_user_ids
+from zerver.tornado.django_api import send_event_on_commit
+
+
+@transaction.atomic(durable=True)
+def check_add_realm_emoji(
+    realm: Realm, name: str, author: UserProfile, image_file: IO[bytes], content_type: str
+) -> RealmEmoji:
+    content_type = bare_content_type(content_type)
+    try:
+        realm_emoji = RealmEmoji(realm=realm, name=name, author=author)
+        realm_emoji.full_clean()
+        realm_emoji.save()
+    except (IntegrityError, ValidationError):
+        # This exists to handle races; upload_emoji checked prior to
+        # calling, and also hand-validated the name, but we're now in
+        # a transaction.  The error string should match the one in
+        # upload_emoji.
+        raise JsonableError(_("A custom emoji with this name already exists."))
+
+    # This mirrors the check in upload_emoji_image because we want to
+    # have some reasonable guarantees on the content-type before
+    # deriving the file extension from it.
+    if content_type not in THUMBNAIL_ACCEPT_IMAGE_TYPES or content_type not in INLINE_MIME_TYPES:
+        raise BadImageError(_("Invalid image format"))
+
+    emoji_file_name = get_emoji_file_name(content_type, realm_emoji.id)
+    is_animated = upload_emoji_image(image_file, emoji_file_name, author, content_type)
+
+    realm_emoji.file_name = emoji_file_name
+    realm_emoji.is_animated = is_animated
+    realm_emoji.save(update_fields=["file_name", "is_animated"])
+
+    new_emoji = emoji_as_emojiinfo(realm_emoji)
+    RealmAuditLog.objects.create(
+        realm=realm,
+        acting_user=author,
+        event_type=AuditLogEventType.REALM_EMOJI_ADDED,
+        event_time=timezone_now(),
+        extra_data={
+            "added_emoji": new_emoji,
+        },
+    )
+    # TODO/compatibility: Remove realm_emoji_dict once we can require that clients support
+    # the individual_emoji_changes client capability.
+    realm_emoji_dict = get_all_custom_emoji_for_realm(realm.id)
+    event = dict(type="realm_emoji", op="add", emoji=new_emoji, realm_emoji=realm_emoji_dict)
+    send_event_on_commit(realm, event, active_user_ids(realm.id))
+    return realm_emoji
+
+
+@transaction.atomic(durable=True)
+def do_remove_realm_emoji(realm: Realm, name: str, *, acting_user: UserProfile | None) -> None:
+    realm_emoji = RealmEmoji.objects.get(realm=realm, name=name, deactivated=False)
+    realm_emoji.deactivated = True
+    realm_emoji.save(update_fields=["deactivated"])
+
+    deactivated_emoji = emoji_as_emojiinfo(realm_emoji)
+    RealmAuditLog.objects.create(
+        realm=realm,
+        acting_user=acting_user,
+        event_type=AuditLogEventType.REALM_EMOJI_REMOVED,
+        event_time=timezone_now(),
+        extra_data={
+            "deactivated_emoji": deactivated_emoji,
+        },
+    )
+
+    # TODO/compatibility: Remove realm_emoji_dict once we can require that clients support
+    # the individual_emoji_changes client capability.
+    realm_emoji_dict = get_all_custom_emoji_for_realm(realm.id)
+    event = dict(
+        type="realm_emoji",
+        op="update_one",
+        emoji_id=str(realm_emoji.id),
+        data=dict(
+            deactivated=True,
+        ),
+        realm_emoji=realm_emoji_dict,
+    )
+    send_event_on_commit(realm, event, active_user_ids(realm.id))

zerver/actions/realm_export.py

@@ -0,0 +1,34 @@
+from django.db import transaction
+from django.utils.timezone import now as timezone_now
+
+from zerver.lib.export import get_realm_exports_serialized
+from zerver.lib.upload import delete_export_tarball
+from zerver.models import Realm, RealmAuditLog, RealmExport, UserProfile
+from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.tornado.django_api import send_event_on_commit
+
+
+def notify_realm_export(realm: Realm) -> None:
+    event = dict(type="realm_export", exports=get_realm_exports_serialized(realm))
+    send_event_on_commit(realm, event, realm.get_human_admin_users().values_list("id", flat=True))
+
+
+@transaction.atomic(durable=True)
+def do_delete_realm_export(export_row: RealmExport, acting_user: UserProfile) -> None:
+    export_path = export_row.export_path
+    assert export_path is not None
+
+    delete_export_tarball(export_path)
+
+    export_row.status = RealmExport.DELETED
+    export_row.date_deleted = timezone_now()
+    export_row.save(update_fields=["status", "date_deleted"])
+    notify_realm_export(export_row.realm)
+
+    RealmAuditLog.objects.create(
+        acting_user=acting_user,
+        realm=export_row.realm,
+        event_type=AuditLogEventType.REALM_EXPORT_DELETED,
+        event_time=export_row.date_deleted,
+        extra_data={"realm_export_id": export_row.id},
+    )

zerver/actions/realm_icon.py

@@ -0,0 +1,38 @@
+from django.db import transaction
+from django.utils.timezone import now as timezone_now
+
+from zerver.lib.realm_icon import realm_icon_url
+from zerver.models import Realm, RealmAuditLog, UserProfile
+from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.models.users import active_user_ids
+from zerver.tornado.django_api import send_event_on_commit
+
+
+@transaction.atomic(durable=True)
+def do_change_icon_source(
+    realm: Realm, icon_source: str, *, acting_user: UserProfile | None
+) -> None:
+    realm.icon_source = icon_source
+    realm.icon_version += 1
+    realm.save(update_fields=["icon_source", "icon_version"])
+
+    event_time = timezone_now()
+    RealmAuditLog.objects.create(
+        realm=realm,
+        event_type=AuditLogEventType.REALM_ICON_SOURCE_CHANGED,
+        extra_data={"icon_source": icon_source, "icon_version": realm.icon_version},
+        event_time=event_time,
+        acting_user=acting_user,
+    )
+
+    event = dict(
+        type="realm",
+        op="update_dict",
+        property="icon",
+        data=dict(icon_source=realm.icon_source, icon_url=realm_icon_url(realm)),
+    )
+    send_event_on_commit(
+        realm,
+        event,
+        active_user_ids(realm.id),
+    )

zerver/actions/realm_linkifiers.py

@@ -0,0 +1,230 @@
+from django.db import transaction
+from django.db.models import Max
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.types import LinkifierDict
+from zerver.models import Realm, RealmAuditLog, RealmFilter, UserProfile
+from zerver.models.linkifiers import flush_linkifiers, linkifiers_for_realm, normalize_optional_text
+from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.models.users import active_user_ids
+from zerver.tornado.django_api import send_event_on_commit
+
+
+def notify_linkifiers(realm: Realm, realm_linkifiers: list[LinkifierDict]) -> None:
+    event: dict[str, object] = dict(type="realm_linkifiers", realm_linkifiers=realm_linkifiers)
+    send_event_on_commit(realm, event, active_user_ids(realm.id))
+
+
+# NOTE: Regexes must be simple enough that they can be easily translated to JavaScript
+# RegExp syntax. In addition to JS-compatible syntax, the following features are available:
+#   * Named groups will be converted to numbered groups automatically
+#   * Inline-regex flags will be stripped, and where possible translated to RegExp-wide flags
+@transaction.atomic(durable=True)
+def do_add_linkifier(
+    realm: Realm,
+    pattern: str,
+    url_template: str,
+    example_input: str | None = None,
+    reverse_template: str | None = None,
+    alternative_url_templates: list[str] | None = None,
+    *,
+    acting_user: UserProfile | None,
+) -> int:
+    pattern = pattern.strip()
+    url_template = url_template.strip()
+    example_input = normalize_optional_text(example_input)
+    reverse_template = normalize_optional_text(reverse_template)
+    if alternative_url_templates is None:
+        alternative_url_templates = []
+    # This makes sure that the new linkifier is always ordered the last modulo
+    # the rare race condition.
+    max_order = RealmFilter.objects.aggregate(Max("order"))["order__max"]
+    if max_order is None:
+        linkifier = RealmFilter(
+            realm=realm,
+            pattern=pattern,
+            url_template=url_template,
+            example_input=example_input,
+            reverse_template=reverse_template,
+            alternative_url_templates=alternative_url_templates,
+        )
+    else:
+        linkifier = RealmFilter(
+            realm=realm,
+            pattern=pattern,
+            url_template=url_template,
+            example_input=example_input,
+            reverse_template=reverse_template,
+            alternative_url_templates=alternative_url_templates,
+            order=max_order + 1,
+        )
+    linkifier.full_clean()
+    linkifier.save()
+
+    realm_linkifiers = linkifiers_for_realm(realm.id)
+    RealmAuditLog.objects.create(
+        realm=realm,
+        acting_user=acting_user,
+        event_type=AuditLogEventType.REALM_LINKIFIER_ADDED,
+        event_time=timezone_now(),
+        extra_data={
+            "realm_linkifiers": realm_linkifiers,
+            "added_linkifier": LinkifierDict(
+                pattern=pattern,
+                url_template=url_template,
+                id=linkifier.id,
+                example_input=example_input,
+                reverse_template=reverse_template,
+                alternative_url_templates=linkifier.alternative_url_templates,
+            ),
+        },
+    )
+    notify_linkifiers(realm, realm_linkifiers)
+
+    return linkifier.id
+
+
+@transaction.atomic(durable=True)
+def do_remove_linkifier(
+    realm: Realm,
+    pattern: str | None = None,
+    id: int | None = None,
+    *,
+    acting_user: UserProfile | None = None,
+) -> None:
+    if pattern is not None:
+        realm_linkifier = RealmFilter.objects.get(realm=realm, pattern=pattern)
+    else:
+        assert id is not None
+        realm_linkifier = RealmFilter.objects.get(realm=realm, id=id)
+
+    pattern = realm_linkifier.pattern
+    url_template = realm_linkifier.url_template
+    realm_linkifier.delete()
+
+    realm_linkifiers = linkifiers_for_realm(realm.id)
+    RealmAuditLog.objects.create(
+        realm=realm,
+        acting_user=acting_user,
+        event_type=AuditLogEventType.REALM_LINKIFIER_REMOVED,
+        event_time=timezone_now(),
+        extra_data={
+            "realm_linkifiers": realm_linkifiers,
+            "removed_linkifier": {
+                "pattern": pattern,
+                "url_template": url_template,
+                "example_input": realm_linkifier.example_input,
+                "reverse_template": realm_linkifier.reverse_template,
+                "alternative_url_templates": realm_linkifier.alternative_url_templates,
+            },
+        },
+    )
+    notify_linkifiers(realm, realm_linkifiers)
+
+
+@transaction.atomic(durable=True)
+def do_update_linkifier(
+    realm: Realm,
+    id: int,
+    pattern: str,
+    url_template: str,
+    example_input: str | None = None,
+    reverse_template: str | None = None,
+    alternative_url_templates: list[str] | None = None,
+    *,
+    acting_user: UserProfile | None,
+) -> None:
+    pattern = pattern.strip()
+    url_template = url_template.strip()
+    linkifier = RealmFilter.objects.get(realm=realm, id=id)
+    linkifier.pattern = pattern
+    linkifier.url_template = url_template
+    linkifier.example_input = normalize_optional_text(example_input)
+    linkifier.reverse_template = normalize_optional_text(reverse_template)
+    if alternative_url_templates is not None:
+        linkifier.alternative_url_templates = alternative_url_templates
+
+    # Validation of fields is done by full_clean.
+    linkifier.full_clean()
+    update_fields = ["pattern", "url_template"]
+    # None means unchanged. Empty string means null, normalize_optional_text
+    # takes care of converting empty string to None.
+    if example_input is not None:
+        update_fields.append("example_input")
+    if reverse_template is not None:
+        update_fields.append("reverse_template")
+    if alternative_url_templates is not None:
+        update_fields.append("alternative_url_templates")
+    linkifier.save(update_fields=update_fields)
+
+    realm_linkifiers = linkifiers_for_realm(realm.id)
+    RealmAuditLog.objects.create(
+        realm=realm,
+        acting_user=acting_user,
+        event_type=AuditLogEventType.REALM_LINKIFIER_CHANGED,
+        event_time=timezone_now(),
+        extra_data={
+            "realm_linkifiers": realm_linkifiers,
+            "changed_linkifier": LinkifierDict(
+                pattern=pattern,
+                url_template=url_template,
+                id=linkifier.id,
+                example_input=linkifier.example_input,
+                reverse_template=linkifier.reverse_template,
+                alternative_url_templates=linkifier.alternative_url_templates,
+            ),
+        },
+    )
+
+    notify_linkifiers(realm, realm_linkifiers)
+
+
+@transaction.atomic(durable=True)
+def check_reorder_linkifiers(
+    realm: Realm, ordered_linkifier_ids: list[int], *, acting_user: UserProfile | None
+) -> None:
+    """ordered_linkifier_ids should contain ids of all existing linkifiers.
+    In the rare situation when any of the linkifier gets deleted that more ids
+    are passed, the checks below are sufficient to detect inconsistencies most of
+    the time."""
+    # Repeated IDs in the user request would collapse into the same key when
+    # constructing the set.
+    linkifier_id_set = set(ordered_linkifier_ids)
+    if len(linkifier_id_set) < len(ordered_linkifier_ids):
+        raise JsonableError(_("The ordered list must not contain duplicated linkifiers"))
+
+    linkifiers = RealmFilter.objects.filter(realm=realm)
+    if {linkifier.id for linkifier in linkifiers} != linkifier_id_set:
+        raise JsonableError(
+            _("The ordered list must enumerate all existing linkifiers exactly once")
+        )
+
+    # After the validation, we are sure that there is nothing to do. Return
+    # early to avoid flushing the cache and populating the audit logs.
+    if len(linkifiers) == 0:
+        return
+
+    id_to_new_order = {
+        linkifier_id: order for order, linkifier_id in enumerate(ordered_linkifier_ids)
+    }
+
+    for linkifier in linkifiers:
+        assert linkifier.id in id_to_new_order
+        linkifier.order = id_to_new_order[linkifier.id]
+    RealmFilter.objects.bulk_update(linkifiers, fields=["order"])
+    flush_linkifiers(instance=linkifiers[0])
+
+    # This roundtrip re-fetches the linkifiers sorted in the new order.
+    realm_linkifiers = linkifiers_for_realm(realm.id)
+    RealmAuditLog.objects.create(
+        realm=realm,
+        acting_user=acting_user,
+        event_type=AuditLogEventType.REALM_LINKIFIERS_REORDERED,
+        event_time=timezone_now(),
+        extra_data={
+            "realm_linkifiers": realm_linkifiers,
+        },
+    )
+    notify_linkifiers(realm, realm_linkifiers)

zerver/actions/realm_logo.py

@@ -0,0 +1,38 @@
+from django.db import transaction
+from django.utils.timezone import now as timezone_now
+
+from zerver.lib.realm_logo import get_realm_logo_data
+from zerver.models import Realm, RealmAuditLog, UserProfile
+from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.models.users import active_user_ids
+from zerver.tornado.django_api import send_event_on_commit
+
+
+@transaction.atomic(durable=True)
+def do_change_logo_source(
+    realm: Realm, logo_source: str, night: bool, *, acting_user: UserProfile | None
+) -> None:
+    if not night:
+        realm.logo_source = logo_source
+        realm.logo_version += 1
+        realm.save(update_fields=["logo_source", "logo_version"])
+
+    else:
+        realm.night_logo_source = logo_source
+        realm.night_logo_version += 1
+        realm.save(update_fields=["night_logo_source", "night_logo_version"])
+
+    RealmAuditLog.objects.create(
+        event_type=AuditLogEventType.REALM_LOGO_CHANGED,
+        realm=realm,
+        event_time=timezone_now(),
+        acting_user=acting_user,
+    )
+
+    event = dict(
+        type="realm",
+        op="update_dict",
+        property="night_logo" if night else "logo",
+        data=get_realm_logo_data(realm, night),
+    )
+    send_event_on_commit(realm, event, active_user_ids(realm.id))

zerver/actions/realm_playgrounds.py

@@ -0,0 +1,87 @@
+from django.core.exceptions import ValidationError
+from django.db import transaction
+from django.utils.timezone import now as timezone_now
+
+from zerver.lib.exceptions import ValidationFailureError
+from zerver.lib.types import RealmPlaygroundDict
+from zerver.models import Realm, RealmAuditLog, RealmPlayground, UserProfile
+from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.models.realm_playgrounds import get_realm_playgrounds
+from zerver.models.users import active_user_ids
+from zerver.tornado.django_api import send_event_on_commit
+
+
+def notify_realm_playgrounds(realm: Realm, realm_playgrounds: list[RealmPlaygroundDict]) -> None:
+    event = dict(type="realm_playgrounds", realm_playgrounds=realm_playgrounds)
+    send_event_on_commit(realm, event, active_user_ids(realm.id))
+
+
+@transaction.atomic(durable=True)
+def check_add_realm_playground(
+    realm: Realm,
+    *,
+    acting_user: UserProfile | None,
+    name: str,
+    pygments_language: str,
+    url_template: str,
+) -> int:
+    realm_playground = RealmPlayground(
+        realm=realm,
+        name=name,
+        pygments_language=pygments_language,
+        url_template=url_template,
+    )
+    # The additional validations using url_template_validation
+    # check_pygments_language, etc are included in full_clean.
+    # Because we want to avoid raising ValidationError from this check_*
+    # function, we do error handling here to turn it into a JsonableError.
+    try:
+        realm_playground.full_clean()
+    except ValidationError as e:
+        raise ValidationFailureError(e)
+    realm_playground.save()
+    realm_playgrounds = get_realm_playgrounds(realm)
+    RealmAuditLog.objects.create(
+        realm=realm,
+        acting_user=acting_user,
+        event_type=AuditLogEventType.REALM_PLAYGROUND_ADDED,
+        event_time=timezone_now(),
+        extra_data={
+            "realm_playgrounds": realm_playgrounds,
+            "added_playground": RealmPlaygroundDict(
+                id=realm_playground.id,
+                name=realm_playground.name,
+                pygments_language=realm_playground.pygments_language,
+                url_template=realm_playground.url_template,
+            ),
+        },
+    )
+    notify_realm_playgrounds(realm, realm_playgrounds)
+    return realm_playground.id
+
+
+@transaction.atomic(durable=True)
+def do_remove_realm_playground(
+    realm: Realm, realm_playground: RealmPlayground, *, acting_user: UserProfile | None
+) -> None:
+    removed_playground = {
+        "name": realm_playground.name,
+        "pygments_language": realm_playground.pygments_language,
+        "url_template": realm_playground.url_template,
+    }
+
+    realm_playground.delete()
+    realm_playgrounds = get_realm_playgrounds(realm)
+
+    RealmAuditLog.objects.create(
+        realm=realm,
+        acting_user=acting_user,
+        event_type=AuditLogEventType.REALM_PLAYGROUND_REMOVED,
+        event_time=timezone_now(),
+        extra_data={
+            "realm_playgrounds": realm_playgrounds,
+            "removed_playground": removed_playground,
+        },
+    )
+
+    notify_realm_playgrounds(realm, realm_playgrounds)

zerver/actions/reminders.py

@@ -0,0 +1,49 @@
+import datetime
+
+from django.db import transaction
+
+from zerver.actions.message_send import check_message
+from zerver.actions.scheduled_messages import do_schedule_messages
+from zerver.lib.addressee import Addressee
+from zerver.lib.message import access_message
+from zerver.lib.reminders import get_reminder_formatted_content, notify_remove_reminder
+from zerver.models import Client, ScheduledMessage, UserProfile
+
+
+def schedule_reminder_for_message(
+    current_user: UserProfile,
+    client: Client,
+    message_id: int,
+    deliver_at: datetime.datetime,
+    note: str,
+) -> int:
+    message = access_message(current_user, message_id, is_modifying_message=False)
+    # Even though reminder will be sent from NOTIFICATION_BOT, we still
+    # set current_user as the sender here to help us make the permission checks easier.
+    addressee = Addressee.for_user_profile(current_user)
+    # This can raise an exception in the unlikely event that the current user cannot DM themself.
+    send_request = check_message(
+        current_user,
+        client,
+        addressee,
+        get_reminder_formatted_content(message, current_user, note),
+        current_user.realm,
+    )
+    send_request.deliver_at = deliver_at
+    send_request.reminder_target_message_id = message_id
+    send_request.reminder_note = note
+
+    return do_schedule_messages(
+        [send_request],
+        current_user,
+        read_by_sender=False,
+        delivery_type=ScheduledMessage.REMIND,
+    )[0]
+
+
+@transaction.atomic(durable=True)
+def do_delete_reminder(user_profile: UserProfile, reminder: ScheduledMessage) -> None:
+    assert reminder.delivery_type == ScheduledMessage.REMIND
+    reminder_id = reminder.id
+    reminder.delete()
+    notify_remove_reminder(user_profile, reminder_id)

zerver/actions/saved_snippets.py

@@ -0,0 +1,91 @@
+from typing import Any
+
+from django.db import transaction
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+
+from zerver.lib.exceptions import ResourceNotFoundError
+from zerver.models import RealmAuditLog, SavedSnippet, UserProfile
+from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.tornado.django_api import send_event_on_commit
+
+
+@transaction.atomic(durable=True)
+def do_create_saved_snippet(
+    title: str,
+    content: str,
+    user_profile: UserProfile,
+) -> SavedSnippet:
+    saved_snippet = SavedSnippet.objects.create(
+        realm=user_profile.realm,
+        user_profile=user_profile,
+        title=title,
+        content=content,
+    )
+
+    RealmAuditLog.objects.create(
+        realm=user_profile.realm,
+        acting_user=user_profile,
+        modified_user=user_profile,
+        event_type=AuditLogEventType.SAVED_SNIPPET_CREATED,
+        event_time=timezone_now(),
+        extra_data={"saved_snippet_id": saved_snippet.id},
+    )
+
+    event = {
+        "type": "saved_snippets",
+        "op": "add",
+        "saved_snippet": saved_snippet.to_api_dict(),
+    }
+    send_event_on_commit(user_profile.realm, event, [user_profile.id])
+
+    return saved_snippet
+
+
+def do_edit_saved_snippet(
+    saved_snippet_id: int,
+    title: str | None,
+    content: str | None,
+    user_profile: UserProfile,
+) -> SavedSnippet:
+    try:
+        saved_snippet = SavedSnippet.objects.get(id=saved_snippet_id, user_profile=user_profile)
+    except SavedSnippet.DoesNotExist:
+        raise ResourceNotFoundError(_("Saved snippet does not exist."))
+
+    if title is not None:
+        saved_snippet.title = title
+    if content is not None:
+        saved_snippet.content = content
+
+    with transaction.atomic(durable=True):
+        saved_snippet.save()
+
+        event = {
+            "type": "saved_snippets",
+            "op": "update",
+            "saved_snippet": saved_snippet.to_api_dict(),
+        }
+        send_event_on_commit(user_profile.realm, event, [user_profile.id])
+
+    return saved_snippet
+
+
+def do_get_saved_snippets(user_profile: UserProfile) -> list[dict[str, Any]]:
+    saved_snippets = SavedSnippet.objects.filter(user_profile=user_profile)
+
+    return [saved_snippet.to_api_dict() for saved_snippet in saved_snippets]
+
+
+def do_delete_saved_snippet(
+    saved_snippet_id: int,
+    user_profile: UserProfile,
+) -> None:
+    try:
+        saved_snippet = SavedSnippet.objects.get(id=saved_snippet_id, user_profile=user_profile)
+    except SavedSnippet.DoesNotExist:
+        raise ResourceNotFoundError(_("Saved snippet does not exist."))
+    saved_snippet.delete()
+
+    event = {"type": "saved_snippets", "op": "remove", "saved_snippet_id": saved_snippet_id}
+    send_event_on_commit(user_profile.realm, event, [user_profile.id])

zerver/actions/scheduled_messages.py

@@ -0,0 +1,500 @@
+import logging
+from collections.abc import Sequence
+from datetime import datetime, timedelta
+
+from django.conf import settings
+from django.db import transaction
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+from django.utils.translation import override as override_language
+
+from zerver.actions.message_send import (
+    check_message,
+    do_send_messages,
+    internal_send_private_message,
+)
+from zerver.actions.uploads import check_attachment_reference_change, do_claim_attachments
+from zerver.lib.addressee import Addressee
+from zerver.lib.display_recipient import get_recipient_ids
+from zerver.lib.exceptions import (
+    DeliveryTimeNotInFutureError,
+    JsonableError,
+    RealmDeactivatedError,
+    UserDeactivatedError,
+)
+from zerver.lib.message import SendMessageRequest, access_message, truncate_topic
+from zerver.lib.recipient_parsing import extract_direct_message_recipient_ids, extract_stream_id
+from zerver.lib.reminders import get_reminder_formatted_content, notify_remove_reminder
+from zerver.lib.scheduled_messages import access_scheduled_message
+from zerver.lib.string_validation import check_stream_topic
+from zerver.lib.timestamp import datetime_to_global_time
+from zerver.models import Client, Realm, ScheduledMessage, Subscription, UserProfile
+from zerver.models.users import get_system_bot, is_cross_realm_bot_email
+from zerver.tornado.django_api import send_event_on_commit
+
+SCHEDULED_MESSAGE_LATE_CUTOFF_MINUTES = 10
+
+
+def check_schedule_message(
+    sender: UserProfile,
+    client: Client,
+    recipient_type_name: str,
+    message_to: list[int],
+    topic_name: str | None,
+    message_content: str,
+    deliver_at: datetime,
+    realm: Realm | None = None,
+    *,
+    read_by_sender: bool | None = None,
+    skip_events: bool = False,
+) -> int:
+    addressee = Addressee.legacy_build(sender, recipient_type_name, message_to, topic_name, realm)
+    send_request = check_message(
+        sender,
+        client,
+        addressee,
+        message_content,
+        realm=realm,
+    )
+    send_request.deliver_at = deliver_at
+
+    if read_by_sender is None:
+        # Legacy default: a scheduled message you sent from a non-API client is
+        # automatically marked as read for yourself, unless it was sent to
+        # yourself only.
+        read_by_sender = client.default_read_by_sender() and not addressee.is_message_to_self(
+            sender
+        )
+
+    return do_schedule_messages(
+        [send_request],
+        sender,
+        read_by_sender=read_by_sender,
+        skip_events=skip_events,
+        delivery_type=ScheduledMessage.SEND_LATER,
+    )[0]
+
+
+def notify_new_scheduled_message(
+    user_profile: UserProfile, scheduled_messages: list[ScheduledMessage]
+) -> None:
+    event = {
+        "type": "scheduled_messages",
+        "op": "add",
+        "scheduled_messages": [
+            scheduled_message.to_dict() for scheduled_message in scheduled_messages
+        ],
+    }
+    send_event_on_commit(user_profile.realm, event, [user_profile.id])
+
+
+def notify_new_reminder(user_profile: UserProfile, reminders: list[ScheduledMessage]) -> None:
+    event = {
+        "type": "reminders",
+        "op": "add",
+        "reminders": [reminder.to_reminder_dict() for reminder in reminders],
+    }
+    send_event_on_commit(user_profile.realm, event, [user_profile.id])
+
+
+def do_schedule_messages(
+    send_message_requests: Sequence[SendMessageRequest],
+    sender: UserProfile,
+    *,
+    read_by_sender: bool = False,
+    skip_events: bool = False,
+    delivery_type: int,
+) -> list[int]:
+    scheduled_messages: list[tuple[ScheduledMessage, SendMessageRequest]] = []
+
+    for send_request in send_message_requests:
+        scheduled_message = ScheduledMessage()
+        scheduled_message.sender = send_request.message.sender
+        scheduled_message.recipient = send_request.message.recipient
+        topic_name = send_request.message.topic_name()
+        scheduled_message.set_topic_name(topic_name=topic_name)
+        scheduled_message.content = send_request.message.content
+        scheduled_message.rendered_content = send_request.rendering_result.rendered_content
+        scheduled_message.sending_client = send_request.message.sending_client
+        scheduled_message.stream = send_request.stream
+        scheduled_message.realm = send_request.realm
+        assert send_request.deliver_at is not None
+        scheduled_message.scheduled_timestamp = send_request.deliver_at
+        scheduled_message.read_by_sender = read_by_sender
+        scheduled_message.delivery_type = delivery_type
+
+        if delivery_type == ScheduledMessage.REMIND:
+            scheduled_message.reminder_target_message_id = send_request.reminder_target_message_id
+            scheduled_message.reminder_note = send_request.reminder_note
+
+        scheduled_messages.append((scheduled_message, send_request))
+
+    with transaction.atomic(durable=True):
+        scheduled_message_objects = [
+            scheduled_message for scheduled_message, ignored in scheduled_messages
+        ]
+        ScheduledMessage.objects.bulk_create(scheduled_message_objects)
+        for scheduled_message, send_request in scheduled_messages:
+            if do_claim_attachments(
+                scheduled_message, send_request.rendering_result.potential_attachment_path_ids
+            ):
+                scheduled_message.has_attachment = True
+                scheduled_message.save(update_fields=["has_attachment"])
+
+        if not skip_events:
+            if delivery_type == ScheduledMessage.REMIND:
+                notify_new_reminder(sender, scheduled_message_objects)
+            else:
+                notify_new_scheduled_message(sender, scheduled_message_objects)
+    return [scheduled_message.id for scheduled_message, ignored in scheduled_messages]
+
+
+def notify_update_scheduled_message(
+    user_profile: UserProfile, scheduled_message: ScheduledMessage
+) -> None:
+    event = {
+        "type": "scheduled_messages",
+        "op": "update",
+        "scheduled_message": scheduled_message.to_dict(),
+    }
+    send_event_on_commit(user_profile.realm, event, [user_profile.id])
+
+
+@transaction.atomic(durable=True)
+def edit_scheduled_message(
+    sender: UserProfile,
+    client: Client,
+    scheduled_message_id: int,
+    recipient_type_name: str | None,
+    message_to: int | list[int] | None,
+    topic_name: str | None,
+    message_content: str | None,
+    deliver_at: datetime | None,
+    realm: Realm,
+) -> None:
+    scheduled_message_object = access_scheduled_message(
+        sender, scheduled_message_id, lock_message=True
+    )
+
+    # We locked the row above, so a concurrent delivery has already
+    # committed and this guard reliably observes its delivered=True.
+    if scheduled_message_object.delivered is True:
+        raise JsonableError(_("Scheduled message was already sent"))
+
+    # If the server failed to send the scheduled message, a new scheduled
+    # delivery timestamp (`deliver_at`) is required.
+    if scheduled_message_object.failed and deliver_at is None:
+        raise DeliveryTimeNotInFutureError
+
+    # Get existing scheduled message's recipient IDs and recipient_type_name.
+    existing_recipient, existing_recipient_type_name = get_recipient_ids(
+        scheduled_message_object.recipient, sender.id
+    )
+
+    send_request: SendMessageRequest | None = None
+    # If any recipient information or message content has been updated,
+    # we check the message again.
+    if recipient_type_name is not None or message_to is not None or message_content is not None:
+        # Update message type if changed.
+        if recipient_type_name is not None:
+            updated_recipient_type_name = recipient_type_name
+        else:
+            updated_recipient_type_name = existing_recipient_type_name
+
+        # Update message recipient if changed.
+        if message_to is not None:
+            if updated_recipient_type_name == "stream":
+                stream_id = extract_stream_id(message_to)
+                updated_recipient = [stream_id]
+            else:
+                updated_recipient = extract_direct_message_recipient_ids(message_to)
+        else:
+            updated_recipient = existing_recipient
+
+        # Update topic name if changed.
+        if topic_name is not None:
+            updated_topic_name = topic_name
+        else:
+            # This will be ignored in Addressee.legacy_build if type
+            # is being changed from stream to direct.
+            updated_topic_name = scheduled_message_object.topic_name()
+
+        # Update message content if changed.
+        if message_content is not None:
+            updated_content = message_content
+        else:
+            updated_content = scheduled_message_object.content
+
+        # Check message again.
+        addressee = Addressee.legacy_build(
+            sender, updated_recipient_type_name, updated_recipient, updated_topic_name
+        )
+        send_request = check_message(
+            sender,
+            client,
+            addressee,
+            updated_content,
+            realm=realm,
+        )
+
+    if recipient_type_name is not None or message_to is not None:
+        assert send_request is not None
+        # User has updated the scheduled message's recipient.
+        scheduled_message_object.recipient = send_request.message.recipient
+        scheduled_message_object.stream = send_request.stream
+        # Update the topic based on the new recipient information.
+        new_topic_name = send_request.message.topic_name()
+        scheduled_message_object.set_topic_name(topic_name=new_topic_name)
+    elif topic_name is not None and existing_recipient_type_name == "stream":
+        # User has updated the scheduled message's topic, but not
+        # the existing recipient information. We ignore topics sent
+        # for scheduled direct messages.
+        check_stream_topic(topic_name)
+        new_topic_name = truncate_topic(topic_name)
+        scheduled_message_object.set_topic_name(topic_name=new_topic_name)
+
+    if message_content is not None:
+        assert send_request is not None
+        # User has updated the scheduled messages's content.
+        scheduled_message_object.content = send_request.message.content
+        scheduled_message_object.rendered_content = send_request.rendering_result.rendered_content
+        attachment_reference_change = check_attachment_reference_change(
+            scheduled_message_object, send_request.rendering_result
+        )
+        scheduled_message_object.has_attachment = attachment_reference_change.did_attachment_change
+
+    if deliver_at is not None:
+        # User has updated the scheduled message's send timestamp.
+        scheduled_message_object.scheduled_timestamp = deliver_at
+
+    # Update for most recent Client information.
+    scheduled_message_object.sending_client = client
+
+    # If the user is editing a scheduled message that the server tried
+    # and failed to send, we need to update the `failed` boolean field
+    # as well as the associated `failure_message` field.
+    if scheduled_message_object.failed:
+        scheduled_message_object.failed = False
+        scheduled_message_object.failure_message = None
+
+    scheduled_message_object.save()
+
+    notify_update_scheduled_message(sender, scheduled_message_object)
+
+
+def notify_remove_scheduled_message(user_profile: UserProfile, scheduled_message_id: int) -> None:
+    event = {
+        "type": "scheduled_messages",
+        "op": "remove",
+        "scheduled_message_id": scheduled_message_id,
+    }
+    send_event_on_commit(user_profile.realm, event, [user_profile.id])
+
+
+@transaction.atomic(durable=True)
+def delete_scheduled_message(user_profile: UserProfile, scheduled_message_id: int) -> None:
+    scheduled_message_object = access_scheduled_message(user_profile, scheduled_message_id)
+    scheduled_message_id = scheduled_message_object.id
+    scheduled_message_object.delete()
+    notify_remove_scheduled_message(user_profile, scheduled_message_id)
+
+
+def send_reminder(scheduled_message: ScheduledMessage) -> None:
+    message_id = scheduled_message.reminder_target_message_id
+    assert message_id is not None
+    current_user = scheduled_message.sender
+    try:
+        message = access_message(current_user, message_id, is_modifying_message=False)
+        content = get_reminder_formatted_content(
+            message, current_user, scheduled_message.reminder_note
+        )
+    except JsonableError:
+        # If we no longer have access to the message, we send the reminder with the
+        # last known message position and content.
+        content = scheduled_message.content
+    # Reminder messages are always sent from the notification bot. We use acting_user
+    # to have appropriate permissions for the messages.
+    message_id = internal_send_private_message(
+        get_system_bot(settings.NOTIFICATION_BOT, scheduled_message.realm.id),
+        current_user,
+        content,
+        acting_user=current_user,
+    )
+    if message_id is None:
+        # internal_send_private_message swallows JsonableError from
+        # check_message and returns None; surface that as a failure so
+        # the worker marks the row failed rather than delivered with a
+        # NULL delivered_message_id.
+        raise JsonableError(_("Reminder could not be sent."))
+    scheduled_message.delivered_message_id = message_id
+    scheduled_message.delivered = True
+    scheduled_message.save(update_fields=["delivered", "delivered_message_id"])
+    notify_remove_reminder(current_user, scheduled_message.id)
+
+
+def send_scheduled_message(scheduled_message: ScheduledMessage) -> None:
+    assert not scheduled_message.delivered
+    assert not scheduled_message.failed
+
+    # Repeat the checks from validate_account_and_subdomain, in case
+    # the state changed since the message was scheduled.
+    if scheduled_message.realm.deactivated:
+        raise RealmDeactivatedError
+
+    if not scheduled_message.sender.is_active:
+        raise UserDeactivatedError
+
+    # Reminders go to the sender's own DMs from Notification Bot, so
+    # the late-cutoff concern (stale messages surprising other
+    # recipients) doesn't apply. Dispatch before the cutoff check so
+    # reminders still fire if the worker is backed up.
+    if scheduled_message.delivery_type == ScheduledMessage.REMIND:
+        send_reminder(scheduled_message)
+        return
+
+    # Limit how late we're willing to send a scheduled message.
+    latest_send_time = scheduled_message.scheduled_timestamp + timedelta(
+        minutes=SCHEDULED_MESSAGE_LATE_CUTOFF_MINUTES
+    )
+    if timezone_now() > latest_send_time:
+        raise JsonableError(_("Message could not be sent at the scheduled time."))
+
+    # Recheck that we have permission to send this message, in case
+    # permissions have changed since the message was scheduled.
+    if scheduled_message.stream is not None:
+        addressee = Addressee.for_stream(scheduled_message.stream, scheduled_message.topic_name())
+    else:
+        subscriber_ids = list(
+            Subscription.objects.filter(recipient=scheduled_message.recipient).values_list(
+                "user_profile_id", flat=True
+            )
+        )
+        addressee = Addressee.for_user_ids(subscriber_ids, scheduled_message.realm)
+
+    # Calling check_message again is important because permissions may
+    # have changed since the message was originally scheduled. This
+    # means that Markdown syntax referencing mutable organization data
+    # (for example, mentioning a user by name) will work (or not) as
+    # if the message was sent at the delivery time, not the sending
+    # time.
+    send_request = check_message(
+        scheduled_message.sender,
+        scheduled_message.sending_client,
+        addressee,
+        scheduled_message.content,
+        scheduled_message.realm,
+    )
+
+    sent_message_result = do_send_messages(
+        [send_request],
+        mark_as_read=[scheduled_message.sender_id] if scheduled_message.read_by_sender else [],
+    )[0]
+    scheduled_message.delivered_message_id = sent_message_result.message_id
+    scheduled_message.delivered = True
+    scheduled_message.save(update_fields=["delivered", "delivered_message_id"])
+    notify_remove_scheduled_message(scheduled_message.sender, scheduled_message.id)
+
+
+def send_failed_scheduled_message_notification(
+    user_profile: UserProfile, scheduled_message_id: int
+) -> None:
+    scheduled_message = access_scheduled_message(user_profile, scheduled_message_id)
+
+    with override_language(user_profile.default_language):
+        error_string = scheduled_message.failure_message
+        delivery_time_markdown = datetime_to_global_time(scheduled_message.scheduled_timestamp)
+
+        content = "".join(
+            [
+                _(
+                    "The message you scheduled for {delivery_datetime} was not sent because of the following error:"
+                ),
+                "\n\n",
+                "> {error_message}",
+                "\n\n",
+                _("[View scheduled messages](#scheduled)"),
+                "\n\n",
+            ]
+        )
+
+    content = content.format(
+        delivery_datetime=delivery_time_markdown,
+        error_message=error_string,
+    )
+
+    internal_send_private_message(
+        get_system_bot(settings.NOTIFICATION_BOT, user_profile.realm_id),
+        user_profile,
+        content,
+    )
+
+
+@transaction.atomic(durable=True)
+def try_deliver_one_scheduled_message() -> bool:
+    # Returns whether there was a scheduled message to attempt
+    # delivery on, regardless of whether delivery succeeded.
+    scheduled_message = (
+        ScheduledMessage.objects.filter(
+            scheduled_timestamp__lte=timezone_now(),
+            delivered=False,
+            failed=False,
+        )
+        .select_for_update(no_key=True)
+        .first()
+    )
+
+    if scheduled_message is None:
+        return False
+
+    logging.info(
+        "Sending scheduled message %s with date %s (sender: %s)",
+        scheduled_message.id,
+        scheduled_message.scheduled_timestamp,
+        scheduled_message.sender_id,
+    )
+
+    with override_language(scheduled_message.sender.default_language):
+        try:
+            send_scheduled_message(scheduled_message)
+        except Exception as e:
+            scheduled_message.refresh_from_db()
+            was_delivered = scheduled_message.delivered
+            scheduled_message.failed = True
+
+            if isinstance(e, JsonableError):
+                scheduled_message.failure_message = e.msg
+                logging.info("Failed with message: %s", e.msg)
+            else:
+                # An unexpected failure; store and send user a generic
+                # internal server error in notification message.
+                scheduled_message.failure_message = _("Internal server error")
+                logging.exception(
+                    "Unexpected error sending scheduled message %s (sent: %s)",
+                    scheduled_message.id,
+                    was_delivered,
+                    stack_info=True,
+                )
+
+            scheduled_message.save(update_fields=["failed", "failure_message"])
+
+            if (
+                not was_delivered
+                # Reminders have their own notification system.
+                and scheduled_message.delivery_type != ScheduledMessage.REMIND
+                # Do not send notification if either the realm or
+                # the sending user account has been deactivated.
+                and not isinstance(e, RealmDeactivatedError)
+                and not isinstance(e, UserDeactivatedError)
+                # Cross-realm system bots (welcome-bot, notification-bot,
+                # emailgateway) have no human behind them to read the
+                # notification, and notifying them leaves a stray
+                # notification-bot <-> system-bot DirectMessageGroup
+                # attached to no realm.
+                and not is_cross_realm_bot_email(scheduled_message.sender.delivery_email)
+            ):
+                notify_update_scheduled_message(scheduled_message.sender, scheduled_message)
+                send_failed_scheduled_message_notification(
+                    scheduled_message.sender, scheduled_message.id
+                )
+
+    return True

zerver/actions/submessage.py

@@ -0,0 +1,98 @@
+from django.utils.translation import gettext as _
+
+from zerver.actions.user_topics import do_set_user_topic_visibility_policy
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.message import (
+    event_recipient_ids_for_action_on_messages,
+    set_visibility_policy_possible,
+    should_change_visibility_policy,
+    visibility_policy_for_participation,
+)
+from zerver.lib.streams import access_stream_by_id
+from zerver.models import Realm, SubMessage, UserProfile
+from zerver.tornado.django_api import send_event_on_commit
+
+
+def verify_submessage_sender(
+    *,
+    message_id: int,
+    message_sender_id: int,
+    submessage_sender_id: int,
+) -> None:
+    """Even though our submessage architecture is geared toward
+    collaboration among all message readers, we still enforce
+    the first person to attach a submessage to the message
+    must be the original sender of the message.
+    """
+
+    if message_sender_id == submessage_sender_id:
+        return
+
+    if SubMessage.objects.filter(
+        message_id=message_id,
+        sender_id=message_sender_id,
+    ).exists():
+        return
+
+    raise JsonableError(_("You cannot attach a submessage to this message."))
+
+
+def do_add_submessage(
+    realm: Realm,
+    sender_id: int,
+    message_id: int,
+    msg_type: str,
+    content: str,
+) -> None:
+    """Should be called while holding a SELECT FOR UPDATE lock
+    (e.g. via access_message(..., lock_message=True)) on the
+    Message row, to prevent race conditions.
+    """
+    submessage = SubMessage(
+        sender_id=sender_id,
+        message_id=message_id,
+        msg_type=msg_type,
+        content=content,
+    )
+    submessage.save()
+
+    # Determine and set the visibility_policy depending on 'automatically_follow_topics_policy'
+    # and 'automatically_unmute_topics_policy'.
+    sender = submessage.sender
+    if set_visibility_policy_possible(
+        sender, submessage.message
+    ) and UserProfile.AUTOMATICALLY_CHANGE_VISIBILITY_POLICY_ON_PARTICIPATION in [
+        sender.automatically_follow_topics_policy,
+        sender.automatically_unmute_topics_in_muted_streams_policy,
+    ]:
+        stream_id = submessage.message.recipient.type_id
+        (stream, sub) = access_stream_by_id(sender, stream_id)
+        assert stream is not None
+        if sub:
+            new_visibility_policy = visibility_policy_for_participation(sender, sub.is_muted)
+            if new_visibility_policy and should_change_visibility_policy(
+                new_visibility_policy,
+                sender,
+                stream_id,
+                topic_name=submessage.message.topic_name(),
+            ):
+                do_set_user_topic_visibility_policy(
+                    user_profile=sender,
+                    stream=stream,
+                    topic_name=submessage.message.topic_name(),
+                    visibility_policy=new_visibility_policy,
+                )
+
+    event = dict(
+        type="submessage",
+        msg_type=msg_type,
+        message_id=message_id,
+        submessage_id=submessage.id,
+        sender_id=sender_id,
+        content=content,
+    )
+    target_user_ids = event_recipient_ids_for_action_on_messages(
+        [submessage.message.id], submessage.message.is_channel_message
+    )
+
+    send_event_on_commit(realm, event, target_user_ids)

zerver/actions/typing.py

@@ -0,0 +1,173 @@
+from typing import Literal
+
+from django.conf import settings
+from django.utils.translation import gettext as _
+
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.stream_subscription import get_active_subscriptions_for_stream_id
+from zerver.models import Realm, Stream, UserProfile
+from zerver.models.users import get_user_by_id_in_realm_including_cross_realm
+from zerver.tornado.django_api import send_event_rollback_unsafe
+
+
+def do_send_typing_notification(
+    realm: Realm, sender: UserProfile, recipient_user_profiles: list[UserProfile], operator: str
+) -> None:
+    sender_dict = {"user_id": sender.id, "email": sender.email}
+
+    # Include a list of recipients in the event body to help identify where the typing is happening
+    recipient_dicts = [
+        {"user_id": profile.id, "email": profile.email} for profile in recipient_user_profiles
+    ]
+    event = dict(
+        type="typing",
+        message_type="direct",
+        op=operator,
+        sender=sender_dict,
+        recipients=recipient_dicts,
+    )
+
+    # Only deliver the notification to active user recipients
+    user_ids_to_notify = [
+        user.id
+        for user in recipient_user_profiles
+        if user.is_active and user.receives_typing_notifications
+    ]
+
+    send_event_rollback_unsafe(realm, event, user_ids_to_notify)
+
+
+# check_send_typing_notification:
+# Checks the typing notification and sends it
+def check_send_typing_notification(sender: UserProfile, user_ids: list[int], operator: str) -> None:
+    realm = sender.realm
+
+    if sender.id not in user_ids:
+        user_ids.append(sender.id)
+
+    # If any of the user_ids being sent in are invalid, we will
+    # just reject the whole request, since a partial list of user_ids
+    # can create confusion related to direct message groups. Plus it's
+    # a good sign that a client is confused (or possibly even malicious)
+    # if we get bad user_ids.
+    user_profiles = []
+    for user_id in user_ids:
+        try:
+            # We include cross-bot realms as possible recipients,
+            # so that clients can know which direct message group
+            # conversation is relevant here.
+            user_profile = get_user_by_id_in_realm_including_cross_realm(user_id, sender.realm)
+        except UserProfile.DoesNotExist:
+            raise JsonableError(_("Invalid user ID {user_id}").format(user_id=user_id))
+        user_profiles.append(user_profile)
+
+    do_send_typing_notification(
+        realm=realm,
+        sender=sender,
+        recipient_user_profiles=user_profiles,
+        operator=operator,
+    )
+
+
+def do_send_stream_typing_notification(
+    sender: UserProfile, operator: str, stream: Stream, topic_name: str
+) -> None:
+    sender_dict = {"user_id": sender.id, "email": sender.email}
+
+    event = dict(
+        type="typing",
+        message_type="stream",
+        op=operator,
+        sender=sender_dict,
+        stream_id=stream.id,
+        topic=topic_name,
+    )
+
+    subscriptions_query = get_active_subscriptions_for_stream_id(
+        stream.id, include_deactivated_users=False
+    )
+
+    total_subscriptions = subscriptions_query.count()
+    if total_subscriptions > settings.MAX_STREAM_SIZE_FOR_TYPING_NOTIFICATIONS:
+        # TODO: Stream typing notifications are disabled in streams
+        # with too many subscribers for performance reasons.
+        return
+
+    user_ids_to_notify = set(
+        subscriptions_query.exclude(user_profile__long_term_idle=True)
+        .exclude(user_profile__receives_typing_notifications=False)
+        .values_list("user_profile_id", flat=True)
+    )
+
+    send_event_rollback_unsafe(sender.realm, event, user_ids_to_notify)
+
+
+def do_send_stream_message_edit_typing_notification(
+    sender: UserProfile,
+    channel_id: int,
+    message_id: int,
+    operator: Literal["start", "stop"],
+    topic_name: str,
+) -> None:
+    event = dict(
+        type="typing_edit_message",
+        op=operator,
+        sender_id=sender.id,
+        message_id=message_id,
+        recipient=dict(
+            type="channel",
+            channel_id=channel_id,
+            topic=topic_name,
+        ),
+    )
+
+    subscriptions_query = get_active_subscriptions_for_stream_id(
+        channel_id, include_deactivated_users=False
+    )
+
+    total_subscriptions = subscriptions_query.count()
+    if total_subscriptions > settings.MAX_STREAM_SIZE_FOR_TYPING_NOTIFICATIONS:
+        # TODO: Stream typing notifications are disabled in streams
+        # with too many subscribers for performance reasons.
+        return
+
+    # We don't notify long_term_idle subscribers.
+    user_ids_to_notify = set(
+        subscriptions_query.exclude(user_profile__long_term_idle=True)
+        .exclude(user_profile__receives_typing_notifications=False)
+        .values_list("user_profile_id", flat=True)
+    )
+
+    send_event_rollback_unsafe(sender.realm, event, user_ids_to_notify)
+
+
+def do_send_direct_message_edit_typing_notification(
+    sender: UserProfile,
+    user_ids: list[int],
+    message_id: int,
+    operator: Literal["start", "stop"],
+) -> None:
+    recipient_user_profiles = []
+    for user_id in user_ids:
+        user_profile = get_user_by_id_in_realm_including_cross_realm(user_id, sender.realm)
+        recipient_user_profiles.append(user_profile)
+
+    # Only deliver the notification to active user recipients
+    user_ids_to_notify = [
+        user.id
+        for user in recipient_user_profiles
+        if user.is_active and user.receives_typing_notifications
+    ]
+
+    event = dict(
+        type="typing_edit_message",
+        op=operator,
+        sender_id=sender.id,
+        message_id=message_id,
+        recipient=dict(
+            type="direct",
+            user_ids=user_ids_to_notify,
+        ),
+    )
+
+    send_event_rollback_unsafe(sender.realm, event, user_ids_to_notify)

zerver/actions/uploads.py

@@ -0,0 +1,143 @@
+import logging
+from dataclasses import dataclass
+from typing import Any
+
+from django.db import transaction
+
+from zerver.lib.attachments import get_old_unclaimed_attachments, validate_attachment_request
+from zerver.lib.markdown import MessageRenderingResult
+from zerver.lib.upload import claim_attachment, delete_message_attachments
+from zerver.models import (
+    ArchivedAttachment,
+    Attachment,
+    ImageAttachment,
+    Message,
+    ScheduledMessage,
+    Stream,
+    UserProfile,
+)
+from zerver.tornado.django_api import send_event_on_commit
+
+
+@dataclass
+class AttachmentChangeResult:
+    did_attachment_change: bool
+    detached_attachments: list[dict[str, Any]]
+
+
+def notify_attachment_update(
+    user_profile: UserProfile, op: str, attachment_dict: dict[str, Any]
+) -> None:
+    event = {
+        "type": "attachment",
+        "op": op,
+        "attachment": attachment_dict,
+        "upload_space_used": user_profile.realm.currently_used_upload_space_bytes(),
+    }
+    send_event_on_commit(user_profile.realm, event, [user_profile.id])
+
+
+def do_claim_attachments(
+    message: Message | ScheduledMessage, potential_path_ids: list[str]
+) -> bool:
+    claimed = False
+    for path_id in potential_path_ids:
+        user_profile = message.sender
+        is_message_realm_public = False
+        is_message_web_public = False
+        if isinstance(message, Message):
+            is_channel_message = message.is_channel_message
+        else:
+            assert isinstance(message, ScheduledMessage)
+            is_channel_message = message.is_channel_message()
+
+        if is_channel_message:
+            stream = Stream.objects.get(id=message.recipient.type_id)
+            is_message_realm_public = stream.is_public()
+            is_message_web_public = (
+                user_profile.realm.web_public_streams_enabled() and stream.is_web_public
+            )
+
+        if not validate_attachment_request(user_profile, path_id)[0]:
+            # Technically, there are 2 cases here:
+            # * The user put something in their message that has the form
+            # of an upload URL, but does not actually correspond to a previously
+            # uploaded file.  validate_attachment_request will return None.
+            # * The user is trying to send a link to a file they don't have permission to
+            # access themselves.  validate_attachment_request will return False.
+            #
+            # Either case is unusual and suggests a UI bug that got
+            # the user in this situation, so we log in these cases.
+            logging.warning(
+                "User %s tried to share upload %s in message %s, but lacks permission",
+                user_profile.id,
+                path_id,
+                message.id,
+            )
+            continue
+
+        claimed = True
+        attachment = claim_attachment(
+            path_id, message, is_message_realm_public, is_message_web_public
+        )
+        if not isinstance(message, ScheduledMessage):
+            # attachment update events don't say anything about scheduled messages,
+            # so sending an event is pointless.
+            notify_attachment_update(user_profile, "update", attachment.to_dict())
+    return claimed
+
+
+@transaction.atomic(durable=True)
+def do_delete_old_unclaimed_attachments(weeks_ago: int) -> None:
+    old_unclaimed_attachments, old_unclaimed_archived_attachments = get_old_unclaimed_attachments(
+        weeks_ago
+    )
+
+    with delete_message_attachments(delete_from=(ImageAttachment, Attachment)) as delete_one:
+        for path_id in (
+            old_unclaimed_attachments.values_list("path_id", flat=True)
+            .select_for_update(of=("self",), no_key=False)
+            .iterator()
+        ):
+            delete_one(path_id)
+    with delete_message_attachments(
+        delete_from=(ImageAttachment, ArchivedAttachment)
+    ) as delete_one:
+        for path_id in (
+            old_unclaimed_archived_attachments.values_list("path_id", flat=True)
+            .select_for_update(of=("self",), no_key=False)
+            .iterator()
+        ):
+            delete_one(path_id)
+
+
+def check_attachment_reference_change(
+    message: Message | ScheduledMessage, rendering_result: MessageRenderingResult
+) -> AttachmentChangeResult:
+    # For an unsaved message edit (message.* has been updated, but not
+    # saved to the database), adjusts Attachment data to correspond to
+    # the new content.
+    prev_attachments = {a.path_id for a in message.attachment_set.all()}
+    new_attachments = set(rendering_result.potential_attachment_path_ids)
+
+    if new_attachments == prev_attachments:
+        return AttachmentChangeResult(bool(prev_attachments), [])
+
+    to_remove = list(prev_attachments - new_attachments)
+    if len(to_remove) > 0:
+        attachments_to_update = Attachment.objects.filter(path_id__in=to_remove).select_for_update(
+            no_key=True
+        )
+        message.attachment_set.remove(*attachments_to_update)
+
+    sender = message.sender
+    detached_attachments_query = Attachment.objects.filter(
+        path_id__in=to_remove, messages__isnull=True, owner=sender
+    )
+    detached_attachments = [attachment.to_dict() for attachment in detached_attachments_query]
+
+    to_add = list(new_attachments - prev_attachments)
+    if len(to_add) > 0:
+        do_claim_attachments(message, to_add)
+
+    return AttachmentChangeResult(message.attachment_set.exists(), detached_attachments)

zerver/actions/user_activity.py

@@ -0,0 +1,31 @@
+from datetime import datetime
+
+from zerver.lib.queue import queue_json_publish_rollback_unsafe
+from zerver.lib.timestamp import datetime_to_timestamp
+from zerver.models import UserActivityInterval, UserProfile
+
+
+def do_update_user_activity_interval(user_profile: UserProfile, log_time: datetime) -> None:
+    effective_end = log_time + UserActivityInterval.MIN_INTERVAL_LENGTH
+    # This code isn't perfect, because with various races we might end
+    # up creating two overlapping intervals, but that shouldn't happen
+    # often, and can be corrected for in post-processing
+    last = UserActivityInterval.objects.filter(user_profile=user_profile).order_by("-end").first()
+    # Two intervals overlap iff each interval ends after the other
+    # begins.  In this case, we just extend the old interval to
+    # include the new interval.
+    if last is not None and log_time <= last.end and effective_end >= last.start:
+        last.end = max(last.end, effective_end)
+        last.start = min(last.start, log_time)
+        last.save(update_fields=["start", "end"])
+        return
+
+    # Otherwise, the intervals don't overlap, so we should make a new one
+    UserActivityInterval.objects.create(
+        user_profile=user_profile, start=log_time, end=effective_end
+    )
+
+
+def update_user_activity_interval(user_profile: UserProfile, log_time: datetime) -> None:
+    event = {"user_profile_id": user_profile.id, "time": datetime_to_timestamp(log_time)}
+    queue_json_publish_rollback_unsafe("user_activity_interval", event)

zerver/actions/user_groups.py

@@ -0,0 +1,777 @@
+from collections.abc import Mapping, Sequence
+from datetime import datetime
+from typing import TypedDict
+
+import django.db.utils
+from django.db import transaction
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.stream_subscription import get_user_ids_for_streams
+from zerver.lib.stream_traffic import get_streams_traffic
+from zerver.lib.streams import (
+    bulk_can_access_stream_metadata_user_ids,
+    get_anonymous_group_membership_dict_for_streams,
+    get_metadata_access_streams_via_group_ids,
+    send_stream_creation_event,
+    send_stream_deletion_event,
+)
+from zerver.lib.timestamp import datetime_to_timestamp
+from zerver.lib.types import UserGroupMembersData, UserGroupMembersDict
+from zerver.lib.user_counts import realm_user_count_by_role
+from zerver.lib.user_groups import (
+    convert_to_user_group_members_dict,
+    get_group_setting_value_for_api,
+    get_group_setting_value_for_audit_log_data,
+    get_recursive_supergroups_union_for_groups,
+    get_role_based_system_groups_dict,
+    set_defaults_for_group_settings,
+)
+from zerver.lib.workplace_users import check_any_group_used_for_workplace_users_group
+from zerver.models import (
+    GroupGroupMembership,
+    NamedUserGroup,
+    Realm,
+    RealmAuditLog,
+    UserGroup,
+    UserGroupMembership,
+    UserProfile,
+)
+from zerver.models.groups import SystemGroups
+from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.models.users import active_user_ids
+from zerver.tornado.django_api import send_event_on_commit
+
+
+class MemberGroupUserDict(TypedDict):
+    id: int
+    role: int
+    date_joined: datetime
+
+
+@transaction.atomic(savepoint=False)
+def create_user_group_in_database(
+    name: str,
+    members: list[UserProfile],
+    realm: Realm,
+    *,
+    acting_user: UserProfile | None,
+    description: str = "",
+    group_settings_map: Mapping[str, UserGroup] = {},
+    is_system_group: bool = False,
+) -> NamedUserGroup:
+    user_group = NamedUserGroup(
+        name=name,
+        realm=realm,
+        description=description,
+        is_system_group=is_system_group,
+        realm_for_sharding=realm,
+        creator=acting_user,
+    )
+
+    for setting_name, setting_value in group_settings_map.items():
+        setattr(user_group, setting_name, setting_value)
+
+    system_groups_name_dict = get_role_based_system_groups_dict(realm)
+    user_group = set_defaults_for_group_settings(
+        user_group, group_settings_map, system_groups_name_dict
+    )
+    user_group.save()
+
+    UserGroupMembership.objects.bulk_create(
+        UserGroupMembership(user_profile=member, user_group=user_group) for member in members
+    )
+
+    creation_time = timezone_now()
+    audit_log_entries = [
+        RealmAuditLog(
+            realm=realm,
+            acting_user=acting_user,
+            event_type=AuditLogEventType.USER_GROUP_CREATED,
+            event_time=creation_time,
+            modified_user_group=user_group,
+        ),
+    ] + [
+        RealmAuditLog(
+            realm=realm,
+            acting_user=acting_user,
+            event_type=AuditLogEventType.USER_GROUP_DIRECT_USER_MEMBERSHIP_ADDED,
+            event_time=creation_time,
+            modified_user=member,
+            modified_user_group=user_group,
+        )
+        for member in members
+    ]
+    RealmAuditLog.objects.bulk_create(audit_log_entries)
+    return user_group
+
+
+@transaction.atomic(savepoint=False)
+def update_users_in_full_members_system_group(
+    realm: Realm,
+    affected_user_ids: Sequence[int] = [],
+    *,
+    acting_user: UserProfile | None,
+    skip_workplace_users_count_audit_log: bool = False,
+) -> None:
+    full_members_system_group = NamedUserGroup.objects.get(
+        realm_for_sharding=realm, name=SystemGroups.FULL_MEMBERS, is_system_group=True
+    )
+    members_system_group = NamedUserGroup.objects.get(
+        realm_for_sharding=realm, name=SystemGroups.MEMBERS, is_system_group=True
+    )
+
+    full_member_group_users: list[MemberGroupUserDict] = list()
+    member_group_users: list[MemberGroupUserDict] = list()
+
+    if affected_user_ids:
+        full_member_group_users = list(
+            full_members_system_group.direct_members.filter(id__in=affected_user_ids).values(
+                "id", "role", "date_joined"
+            )
+        )
+        member_group_users = list(
+            members_system_group.direct_members.filter(id__in=affected_user_ids).values(
+                "id", "role", "date_joined"
+            )
+        )
+    else:
+        full_member_group_users = list(
+            full_members_system_group.direct_members.all().values("id", "role", "date_joined")
+        )
+        member_group_users = list(
+            members_system_group.direct_members.all().values("id", "role", "date_joined")
+        )
+
+    def is_provisional_member(user: MemberGroupUserDict) -> bool:
+        diff = (timezone_now() - user["date_joined"]).days
+        if diff < realm.waiting_period_threshold:
+            return True
+        return False
+
+    old_full_members = [
+        user
+        for user in full_member_group_users
+        if is_provisional_member(user) or user["role"] != UserProfile.ROLE_MEMBER
+    ]
+
+    full_member_group_user_ids = [user["id"] for user in full_member_group_users]
+    members_excluding_full_members = [
+        user for user in member_group_users if user["id"] not in full_member_group_user_ids
+    ]
+
+    new_full_members = [
+        user for user in members_excluding_full_members if not is_provisional_member(user)
+    ]
+
+    old_full_member_ids = [user["id"] for user in old_full_members]
+    new_full_member_ids = [user["id"] for user in new_full_members]
+
+    if len(old_full_members) > 0:
+        bulk_remove_members_from_user_groups(
+            [full_members_system_group],
+            old_full_member_ids,
+            acting_user=acting_user,
+            skip_workplace_users_count_audit_log=skip_workplace_users_count_audit_log,
+        )
+
+    if len(new_full_members) > 0:
+        bulk_add_members_to_user_groups(
+            [full_members_system_group],
+            new_full_member_ids,
+            acting_user=acting_user,
+            skip_workplace_users_count_audit_log=skip_workplace_users_count_audit_log,
+        )
+
+
+def promote_new_full_members() -> None:
+    for realm in Realm.objects.filter(deactivated=False).exclude(waiting_period_threshold=0):
+        update_users_in_full_members_system_group(realm, acting_user=None)
+
+
+def do_send_create_user_group_event(
+    user_group: NamedUserGroup,
+    member_ids: list[int],
+    direct_subgroup_ids: Sequence[int] = [],
+    *,
+    for_reactivation: bool = False,
+) -> None:
+    creator_id = user_group.creator_id
+    assert user_group.date_created is not None
+    date_created = datetime_to_timestamp(user_group.date_created)
+
+    setting_values = {}
+    for setting_name in NamedUserGroup.GROUP_PERMISSION_SETTINGS:
+        setting_values[setting_name] = convert_to_user_group_members_dict(
+            get_group_setting_value_for_api(getattr(user_group, setting_name))
+        )
+
+    event = dict(
+        type="user_group",
+        op="add",
+        group=dict(
+            name=user_group.name,
+            creator_id=creator_id,
+            date_created=date_created,
+            members=member_ids,
+            description=user_group.description,
+            id=user_group.id,
+            is_system_group=user_group.is_system_group,
+            direct_subgroup_ids=direct_subgroup_ids,
+            **setting_values,
+            deactivated=False,
+        ),
+        for_reactivation=for_reactivation,
+    )
+    send_event_on_commit(user_group.realm, event, active_user_ids(user_group.realm_id))
+
+
+def check_add_user_group_core(
+    realm: Realm,
+    name: str,
+    initial_members: list[UserProfile],
+    description: str = "",
+    group_settings_map: Mapping[str, UserGroup] = {},
+    *,
+    acting_user: UserProfile | None,
+) -> NamedUserGroup:
+    user_group = create_user_group_in_database(
+        name,
+        initial_members,
+        realm,
+        description=description,
+        group_settings_map=group_settings_map,
+        acting_user=acting_user,
+    )
+    do_send_create_user_group_event(user_group, [member.id for member in initial_members])
+    return user_group
+
+
+def check_add_user_group(
+    realm: Realm,
+    name: str,
+    initial_members: list[UserProfile],
+    description: str = "",
+    group_settings_map: Mapping[str, UserGroup] = {},
+    *,
+    acting_user: UserProfile | None,
+) -> NamedUserGroup:
+    try:
+        return check_add_user_group_core(
+            realm, name, initial_members, description, group_settings_map, acting_user=acting_user
+        )
+    except django.db.utils.IntegrityError:
+        raise JsonableError(_("User group '{group_name}' already exists.").format(group_name=name))
+
+
+def do_send_user_group_update_event(
+    user_group: NamedUserGroup, data: dict[str, str | int | UserGroupMembersDict]
+) -> None:
+    event = dict(type="user_group", op="update", group_id=user_group.id, data=data)
+    if "name" in data:
+        # This field will be popped eventually before sending the event
+        # to client, but is needed to make sure we do not send the
+        # name update event for deactivated groups to client with
+        # 'include_deactivated_groups' client capability set to false.
+        event["deactivated"] = user_group.deactivated
+    send_event_on_commit(user_group.realm, event, active_user_ids(user_group.realm_id))
+
+
+@transaction.atomic(savepoint=False)
+def do_update_user_group_name(
+    user_group: NamedUserGroup, name: str, *, acting_user: UserProfile | None
+) -> None:
+    try:
+        old_value = user_group.name
+        user_group.name = name
+        user_group.save(update_fields=["name"])
+        RealmAuditLog.objects.create(
+            realm=user_group.realm,
+            modified_user_group=user_group,
+            event_type=AuditLogEventType.USER_GROUP_NAME_CHANGED,
+            event_time=timezone_now(),
+            acting_user=acting_user,
+            extra_data={
+                RealmAuditLog.OLD_VALUE: old_value,
+                RealmAuditLog.NEW_VALUE: name,
+            },
+        )
+    except django.db.utils.IntegrityError:
+        raise JsonableError(_("User group '{group_name}' already exists.").format(group_name=name))
+    do_send_user_group_update_event(user_group, dict(name=name))
+
+
+@transaction.atomic(savepoint=False)
+def do_update_user_group_description(
+    user_group: NamedUserGroup, description: str, *, acting_user: UserProfile | None
+) -> None:
+    old_value = user_group.description
+    user_group.description = description
+    user_group.save(update_fields=["description"])
+    RealmAuditLog.objects.create(
+        realm=user_group.realm,
+        modified_user_group=user_group,
+        event_type=AuditLogEventType.USER_GROUP_DESCRIPTION_CHANGED,
+        event_time=timezone_now(),
+        acting_user=acting_user,
+        extra_data={
+            RealmAuditLog.OLD_VALUE: old_value,
+            RealmAuditLog.NEW_VALUE: description,
+        },
+    )
+    do_send_user_group_update_event(user_group, dict(description=description))
+
+
+def do_send_user_group_members_update_event(
+    event_name: str, user_group: NamedUserGroup, user_ids: list[int]
+) -> None:
+    event = dict(type="user_group", op=event_name, group_id=user_group.id, user_ids=user_ids)
+    send_event_on_commit(user_group.realm, event, active_user_ids(user_group.realm_id))
+
+
+@transaction.atomic(savepoint=False)
+def bulk_add_members_to_user_groups(
+    user_groups: list[NamedUserGroup],
+    user_profile_ids: list[int],
+    *,
+    acting_user: UserProfile | None,
+    skip_workplace_users_count_audit_log: bool = False,
+) -> None:
+    # All intended callers of this function involve a single user
+    # being added to one or more groups, or many users being added to
+    # a single group; but it's easy enough for the implementation to
+    # support both.
+
+    if len(user_groups) == 0 or len(user_profile_ids) == 0:
+        return
+
+    realm = user_groups[0].realm
+    supergroups = get_recursive_supergroups_union_for_groups(
+        [user_group.id for user_group in user_groups]
+    )
+    streams = list(
+        get_metadata_access_streams_via_group_ids([group.id for group in supergroups], realm)
+    )
+    old_stream_metadata_user_ids = bulk_can_access_stream_metadata_user_ids(streams)
+    memberships = [
+        UserGroupMembership(user_group_id=user_group.id, user_profile_id=user_id)
+        for user_id in user_profile_ids
+        for user_group in user_groups
+    ]
+    UserGroupMembership.objects.bulk_create(memberships)
+    now = timezone_now()
+    RealmAuditLog.objects.bulk_create(
+        RealmAuditLog(
+            realm=realm,
+            modified_user_id=user_id,
+            modified_user_group=user_group,
+            event_type=AuditLogEventType.USER_GROUP_DIRECT_USER_MEMBERSHIP_ADDED,
+            event_time=now,
+            acting_user=acting_user,
+        )
+        for user_id in user_profile_ids
+        for user_group in user_groups
+    )
+
+    if not skip_workplace_users_count_audit_log and check_any_group_used_for_workplace_users_group(
+        realm, user_groups
+    ):
+        RealmAuditLog.objects.create(
+            realm=realm,
+            acting_user=acting_user,
+            event_type=AuditLogEventType.WORKPLACE_USERS_COUNT_CHANGED,
+            event_time=now,
+            extra_data={
+                RealmAuditLog.ROLE_COUNT: realm_user_count_by_role(realm),
+                "trigger": "user_membership_changed",
+            },
+        )
+
+        from zerver.lib.remote_server import maybe_enqueue_audit_log_upload
+
+        maybe_enqueue_audit_log_upload(realm)
+
+    subscriber_ids_for_streams = get_user_ids_for_streams({stream.id for stream in streams})
+    new_stream_metadata_user_ids = bulk_can_access_stream_metadata_user_ids(streams)
+    recent_traffic = get_streams_traffic(realm, {stream.id for stream in streams})
+    anonymous_group_membership = get_anonymous_group_membership_dict_for_streams(streams)
+
+    for user_group in user_groups:
+        do_send_user_group_members_update_event("add_members", user_group, user_profile_ids)
+
+    for stream in streams:
+        user_ids_gaining_metadata_access = (
+            new_stream_metadata_user_ids[stream.id] - old_stream_metadata_user_ids[stream.id]
+        )
+        send_stream_creation_event(
+            realm,
+            stream,
+            list(user_ids_gaining_metadata_access),
+            recent_traffic,
+            anonymous_group_membership,
+        )
+        peer_add_event = dict(
+            type="subscription",
+            op="peer_add",
+            stream_ids=[stream.id],
+            user_ids=sorted(subscriber_ids_for_streams[stream.id]),
+        )
+        send_event_on_commit(
+            realm,
+            peer_add_event,
+            user_ids_gaining_metadata_access,
+        )
+
+
+@transaction.atomic(savepoint=False)
+def bulk_remove_members_from_user_groups(
+    user_groups: list[NamedUserGroup],
+    user_profile_ids: list[int],
+    *,
+    acting_user: UserProfile | None,
+    skip_workplace_users_count_audit_log: bool = False,
+) -> None:
+    # All intended callers of this function involve a single user
+    # being added to one or more groups, or many users being added to
+    # a single group; but it's easy enough for the implementation to
+    # support both.
+
+    if len(user_groups) == 0 or len(user_profile_ids) == 0:
+        return
+
+    realm = user_groups[0].realm
+    supergroups = get_recursive_supergroups_union_for_groups(
+        [user_group.id for user_group in user_groups]
+    )
+    streams = list(
+        get_metadata_access_streams_via_group_ids([group.id for group in supergroups], realm)
+    )
+    old_stream_metadata_user_ids = bulk_can_access_stream_metadata_user_ids(streams)
+
+    UserGroupMembership.objects.filter(
+        user_group__in=user_groups, user_profile_id__in=user_profile_ids
+    ).delete()
+    now = timezone_now()
+    RealmAuditLog.objects.bulk_create(
+        RealmAuditLog(
+            realm=user_group.realm,
+            modified_user_id=user_id,
+            modified_user_group=user_group,
+            event_type=AuditLogEventType.USER_GROUP_DIRECT_USER_MEMBERSHIP_REMOVED,
+            event_time=now,
+            acting_user=acting_user,
+        )
+        for user_id in user_profile_ids
+        for user_group in user_groups
+    )
+
+    if not skip_workplace_users_count_audit_log and check_any_group_used_for_workplace_users_group(
+        realm, user_groups
+    ):
+        RealmAuditLog.objects.create(
+            realm=realm,
+            acting_user=acting_user,
+            event_type=AuditLogEventType.WORKPLACE_USERS_COUNT_CHANGED,
+            event_time=now,
+            extra_data={
+                RealmAuditLog.ROLE_COUNT: realm_user_count_by_role(realm),
+                "trigger": "user_membership_changed",
+            },
+        )
+
+        from zerver.lib.remote_server import maybe_enqueue_audit_log_upload
+
+        maybe_enqueue_audit_log_upload(realm)
+
+    for user_group in user_groups:
+        do_send_user_group_members_update_event("remove_members", user_group, user_profile_ids)
+
+    new_stream_metadata_user_ids = bulk_can_access_stream_metadata_user_ids(streams)
+    for stream in streams:
+        user_ids_losing_metadata_access = (
+            old_stream_metadata_user_ids[stream.id] - new_stream_metadata_user_ids[stream.id]
+        )
+        send_stream_deletion_event(
+            realm,
+            list(user_ids_losing_metadata_access),
+            [stream],
+        )
+
+
+def do_send_subgroups_update_event(
+    event_name: str, user_group: NamedUserGroup, subgroup_ids: list[int]
+) -> None:
+    event = dict(
+        type="user_group", op=event_name, group_id=user_group.id, direct_subgroup_ids=subgroup_ids
+    )
+    send_event_on_commit(user_group.realm, event, active_user_ids(user_group.realm_id))
+
+
+@transaction.atomic(savepoint=False)
+def add_subgroups_to_user_group(
+    user_group: NamedUserGroup,
+    subgroups: list[NamedUserGroup],
+    *,
+    acting_user: UserProfile | None,
+) -> None:
+    if len(subgroups) == 0:
+        return
+
+    realm = user_group.realm
+    supergroups = get_recursive_supergroups_union_for_groups([user_group.id])
+    streams = list(
+        get_metadata_access_streams_via_group_ids([group.id for group in supergroups], realm)
+    )
+    old_stream_metadata_user_ids = bulk_can_access_stream_metadata_user_ids(streams)
+
+    group_memberships = [
+        GroupGroupMembership(supergroup=user_group, subgroup=subgroup) for subgroup in subgroups
+    ]
+    GroupGroupMembership.objects.bulk_create(group_memberships)
+
+    subgroup_ids = [subgroup.id for subgroup in subgroups]
+    now = timezone_now()
+    audit_log_entries = [
+        RealmAuditLog(
+            realm=user_group.realm,
+            modified_user_group=user_group,
+            event_type=AuditLogEventType.USER_GROUP_DIRECT_SUBGROUP_MEMBERSHIP_ADDED,
+            event_time=now,
+            acting_user=acting_user,
+            extra_data={"subgroup_ids": subgroup_ids},
+        ),
+        *(
+            RealmAuditLog(
+                realm=user_group.realm,
+                modified_user_group_id=subgroup_id,
+                event_type=AuditLogEventType.USER_GROUP_DIRECT_SUPERGROUP_MEMBERSHIP_ADDED,
+                event_time=now,
+                acting_user=acting_user,
+                extra_data={"supergroup_ids": [user_group.id]},
+            )
+            for subgroup_id in subgroup_ids
+        ),
+    ]
+    RealmAuditLog.objects.bulk_create(audit_log_entries)
+
+    if check_any_group_used_for_workplace_users_group(realm, [user_group]):
+        RealmAuditLog.objects.create(
+            realm=realm,
+            acting_user=acting_user,
+            event_type=AuditLogEventType.WORKPLACE_USERS_COUNT_CHANGED,
+            event_time=now,
+            extra_data={
+                RealmAuditLog.ROLE_COUNT: realm_user_count_by_role(realm),
+                "trigger": "subgroups_changed",
+            },
+        )
+
+        from zerver.lib.remote_server import maybe_enqueue_audit_log_upload
+
+        maybe_enqueue_audit_log_upload(realm)
+
+    subscriber_ids_for_streams = get_user_ids_for_streams({stream.id for stream in streams})
+    new_stream_metadata_user_ids = bulk_can_access_stream_metadata_user_ids(streams)
+    recent_traffic = get_streams_traffic(realm, {stream.id for stream in streams})
+    anonymous_group_membership = get_anonymous_group_membership_dict_for_streams(streams)
+
+    do_send_subgroups_update_event("add_subgroups", user_group, subgroup_ids)
+
+    for stream in streams:
+        user_ids_gaining_metadata_access = (
+            new_stream_metadata_user_ids[stream.id] - old_stream_metadata_user_ids[stream.id]
+        )
+        send_stream_creation_event(
+            realm,
+            stream,
+            list(user_ids_gaining_metadata_access),
+            recent_traffic,
+            anonymous_group_membership,
+        )
+        peer_add_event = dict(
+            type="subscription",
+            op="peer_add",
+            stream_ids=[stream.id],
+            user_ids=sorted(subscriber_ids_for_streams[stream.id]),
+        )
+        send_event_on_commit(
+            realm,
+            peer_add_event,
+            user_ids_gaining_metadata_access,
+        )
+
+
+@transaction.atomic(savepoint=False)
+def remove_subgroups_from_user_group(
+    user_group: NamedUserGroup,
+    subgroups: list[NamedUserGroup],
+    *,
+    acting_user: UserProfile | None,
+) -> None:
+    if len(subgroups) == 0:
+        return
+
+    realm = user_group.realm
+    supergroups = get_recursive_supergroups_union_for_groups([user_group.id])
+    streams = list(
+        get_metadata_access_streams_via_group_ids([group.id for group in supergroups], realm)
+    )
+    old_stream_metadata_user_ids = bulk_can_access_stream_metadata_user_ids(streams)
+
+    GroupGroupMembership.objects.filter(supergroup=user_group, subgroup__in=subgroups).delete()
+
+    subgroup_ids = [subgroup.id for subgroup in subgroups]
+    now = timezone_now()
+    audit_log_entries = [
+        RealmAuditLog(
+            realm=user_group.realm,
+            modified_user_group=user_group,
+            event_type=AuditLogEventType.USER_GROUP_DIRECT_SUBGROUP_MEMBERSHIP_REMOVED,
+            event_time=now,
+            acting_user=acting_user,
+            extra_data={"subgroup_ids": subgroup_ids},
+        ),
+        *(
+            RealmAuditLog(
+                realm=user_group.realm,
+                modified_user_group_id=subgroup_id,
+                event_type=AuditLogEventType.USER_GROUP_DIRECT_SUPERGROUP_MEMBERSHIP_REMOVED,
+                event_time=now,
+                acting_user=acting_user,
+                extra_data={"supergroup_ids": [user_group.id]},
+            )
+            for subgroup_id in subgroup_ids
+        ),
+    ]
+    RealmAuditLog.objects.bulk_create(audit_log_entries)
+
+    if check_any_group_used_for_workplace_users_group(realm, [user_group]):
+        RealmAuditLog.objects.create(
+            realm=realm,
+            acting_user=acting_user,
+            event_type=AuditLogEventType.WORKPLACE_USERS_COUNT_CHANGED,
+            event_time=now,
+            extra_data={
+                RealmAuditLog.ROLE_COUNT: realm_user_count_by_role(realm),
+                "trigger": "subgroups_changed",
+            },
+        )
+
+        from zerver.lib.remote_server import maybe_enqueue_audit_log_upload
+
+        maybe_enqueue_audit_log_upload(realm)
+
+    do_send_subgroups_update_event("remove_subgroups", user_group, subgroup_ids)
+
+    new_stream_metadata_user_ids = bulk_can_access_stream_metadata_user_ids(streams)
+    for stream in streams:
+        user_ids_losing_metadata_access = (
+            old_stream_metadata_user_ids[stream.id] - new_stream_metadata_user_ids[stream.id]
+        )
+        send_stream_deletion_event(
+            realm,
+            list(user_ids_losing_metadata_access),
+            [stream],
+        )
+
+
+@transaction.atomic(savepoint=False)
+def do_deactivate_user_group(
+    user_group: NamedUserGroup, *, acting_user: UserProfile | None
+) -> None:
+    user_group.deactivated = True
+    user_group.save(update_fields=["deactivated"])
+
+    now = timezone_now()
+    RealmAuditLog.objects.create(
+        realm=user_group.realm,
+        modified_user_group_id=user_group.id,
+        event_type=AuditLogEventType.USER_GROUP_DEACTIVATED,
+        event_time=now,
+        acting_user=acting_user,
+    )
+
+    do_send_user_group_update_event(user_group, dict(deactivated=True))
+
+    event = dict(type="user_group", op="remove", group_id=user_group.id)
+    send_event_on_commit(user_group.realm, event, active_user_ids(user_group.realm_id))
+
+
+@transaction.atomic(savepoint=False)
+def do_reactivate_user_group(
+    user_group: NamedUserGroup, *, acting_user: UserProfile | None
+) -> None:
+    user_group.deactivated = False
+    user_group.save(update_fields=["deactivated"])
+
+    now = timezone_now()
+    RealmAuditLog.objects.create(
+        realm=user_group.realm,
+        modified_user_group_id=user_group.id,
+        event_type=AuditLogEventType.USER_GROUP_REACTIVATED,
+        event_time=now,
+        acting_user=acting_user,
+    )
+
+    do_send_user_group_update_event(user_group, dict(deactivated=False))
+
+    direct_member_ids = list(user_group.direct_members.all().values_list("id", flat=True))
+    direct_subgroup_ids = list(user_group.direct_subgroups.all().values_list("id", flat=True))
+    do_send_create_user_group_event(
+        user_group, direct_member_ids, direct_subgroup_ids, for_reactivation=True
+    )
+
+
+@transaction.atomic(savepoint=False)
+def do_change_user_group_permission_setting(
+    user_group: NamedUserGroup,
+    setting_name: str,
+    setting_value_group: UserGroup,
+    *,
+    old_setting_api_value: int | UserGroupMembersData | None = None,
+    acting_user: UserProfile | None,
+) -> None:
+    old_value = getattr(user_group, setting_name)
+    setattr(user_group, setting_name, setting_value_group)
+    user_group.save()
+
+    if old_setting_api_value is None:
+        # Most production callers will have computed this as part of
+        # verifying whether there's an actual change to make, but it
+        # feels quite clumsy to have to pass it from unit tests, so we
+        # compute it here if not provided by the caller.
+        old_setting_api_value = get_group_setting_value_for_api(old_value)
+    new_setting_api_value = get_group_setting_value_for_api(setting_value_group)
+
+    if not hasattr(old_value, "named_user_group") and hasattr(
+        setting_value_group, "named_user_group"
+    ):
+        # We delete the UserGroup which the setting was set to
+        # previously if it does not have any linked NamedUserGroup
+        # object, as it is not used anywhere else. A new UserGroup
+        # object would be created if the setting is later set to
+        # a combination of users and groups.
+        old_value.delete()
+
+    RealmAuditLog.objects.create(
+        realm=user_group.realm,
+        acting_user=acting_user,
+        event_type=AuditLogEventType.USER_GROUP_GROUP_BASED_SETTING_CHANGED,
+        event_time=timezone_now(),
+        modified_user_group=user_group,
+        extra_data={
+            RealmAuditLog.OLD_VALUE: get_group_setting_value_for_audit_log_data(
+                old_setting_api_value
+            ),
+            RealmAuditLog.NEW_VALUE: get_group_setting_value_for_audit_log_data(
+                new_setting_api_value
+            ),
+            "property": setting_name,
+        },
+    )
+
+    event_data_dict: dict[str, str | int | UserGroupMembersDict] = {
+        setting_name: convert_to_user_group_members_dict(new_setting_api_value)
+    }
+    do_send_user_group_update_event(user_group, event_data_dict)

zerver/actions/user_settings.py

@@ -0,0 +1,723 @@
+from collections.abc import Iterable
+from datetime import datetime, timedelta
+from enum import Enum
+
+from django.conf import settings
+from django.db import transaction
+from django.db.models import F
+from django.utils.timezone import now as timezone_now
+
+from confirmation.models import Confirmation, create_confirmation_link
+from confirmation.settings import STATUS_REVOKED, STATUS_USED
+from zerver.actions.message_send import send_user_profile_update_notification
+from zerver.actions.presence import do_update_user_presence
+from zerver.lib.avatar import avatar_url, generate_and_upload_jdenticon_avatar
+from zerver.lib.cache import (
+    bulk_flush_users,
+    cache_delete,
+    delete_user_profile_caches,
+    user_profile_by_api_key_cache_key,
+)
+from zerver.lib.create_user import get_display_email_address
+from zerver.lib.i18n import get_language_name
+from zerver.lib.queue import queue_event_on_commit
+from zerver.lib.send_email import FromAddress, clear_scheduled_emails, send_email
+from zerver.lib.timezone import canonicalize_timezone
+from zerver.lib.types import UserProfileChangeDict
+from zerver.lib.upload import delete_avatar_image
+from zerver.lib.users import (
+    can_access_delivery_email,
+    check_bot_name_available,
+    check_full_name,
+    get_user_ids_who_can_access_user,
+    get_users_with_access_to_real_email,
+    user_access_restricted_in_realm,
+)
+from zerver.lib.utils import generate_api_key
+from zerver.models import (
+    Device,
+    Draft,
+    EmailChangeStatus,
+    Realm,
+    RealmAuditLog,
+    ScheduledEmail,
+    ScheduledMessageNotificationEmail,
+    UserPresence,
+    UserProfile,
+)
+from zerver.models.clients import get_client
+from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.models.users import get_user_profile_by_id
+from zerver.tornado.django_api import send_event_on_commit
+
+
+def send_user_email_update_event(user_profile: UserProfile) -> None:
+    payload = dict(user_id=user_profile.id, new_email=user_profile.email)
+    event = dict(type="realm_user", op="update", person=payload)
+    send_event_on_commit(
+        user_profile.realm,
+        event,
+        get_user_ids_who_can_access_user(user_profile),
+    )
+
+
+def send_delivery_email_update_events(
+    user_profile: UserProfile, old_visibility_setting: int, visibility_setting: int
+) -> None:
+    if not user_access_restricted_in_realm(user_profile):
+        active_users = user_profile.realm.get_active_users()
+    else:
+        # The get_user_ids_who_can_access_user returns user IDs and not
+        # user objects and we instead do one more query for UserProfile
+        # objects. We need complete UserProfile objects only for a couple
+        # of cases and it is not worth to query the whole UserProfile
+        # objects in all the cases and it is fine to do the extra query
+        # wherever needed.
+        user_ids_who_can_access_user = get_user_ids_who_can_access_user(user_profile)
+        active_users = UserProfile.objects.filter(
+            id__in=user_ids_who_can_access_user, is_active=True
+        )
+
+    delivery_email_now_visible_user_ids = []
+    delivery_email_now_invisible_user_ids = []
+
+    for active_user in active_users:
+        could_access_delivery_email_previously = can_access_delivery_email(
+            active_user, user_profile.id, old_visibility_setting
+        )
+        can_access_delivery_email_now = can_access_delivery_email(
+            active_user, user_profile.id, visibility_setting
+        )
+
+        if could_access_delivery_email_previously != can_access_delivery_email_now:
+            if can_access_delivery_email_now:
+                delivery_email_now_visible_user_ids.append(active_user.id)
+            else:
+                delivery_email_now_invisible_user_ids.append(active_user.id)
+
+    if delivery_email_now_visible_user_ids:
+        person = dict(user_id=user_profile.id, delivery_email=user_profile.delivery_email)
+        event = dict(type="realm_user", op="update", person=person)
+        send_event_on_commit(
+            user_profile.realm,
+            event,
+            delivery_email_now_visible_user_ids,
+        )
+    if delivery_email_now_invisible_user_ids:
+        person = dict(user_id=user_profile.id, delivery_email=None)
+        event = dict(type="realm_user", op="update", person=person)
+        send_event_on_commit(
+            user_profile.realm,
+            event,
+            delivery_email_now_invisible_user_ids,
+        )
+
+
+@transaction.atomic(savepoint=False)
+def do_change_user_delivery_email(
+    user_profile: UserProfile, new_email: str, *, acting_user: UserProfile | None
+) -> None:
+    delete_user_profile_caches([user_profile], user_profile.realm_id)
+
+    original_email = user_profile.delivery_email
+
+    user_profile.delivery_email = new_email
+    if user_profile.email_address_is_realm_public():
+        user_profile.email = new_email
+        user_profile.save(update_fields=["email", "delivery_email"])
+    else:
+        user_profile.save(update_fields=["delivery_email"])
+
+    # We notify all the users who have access to delivery email.
+    payload = dict(user_id=user_profile.id, delivery_email=new_email)
+    event = dict(type="realm_user", op="update", person=payload)
+    delivery_email_visible_user_ids = get_users_with_access_to_real_email(user_profile)
+
+    send_event_on_commit(user_profile.realm, event, delivery_email_visible_user_ids)
+
+    if user_profile.avatar_source == UserProfile.AVATAR_FROM_GRAVATAR:
+        # If the user is using Gravatar to manage their email address,
+        # their Gravatar just changed, and we need to notify other
+        # clients.
+        notify_avatar_url_change(user_profile)
+
+    if user_profile.email_address_is_realm_public():
+        # Additionally, if we're also changing the publicly visible
+        # email, we send a new_email event as well.
+        send_user_email_update_event(user_profile)
+
+    event_time = timezone_now()
+    RealmAuditLog.objects.create(
+        realm=user_profile.realm,
+        acting_user=acting_user,
+        modified_user=user_profile,
+        event_type=AuditLogEventType.USER_EMAIL_CHANGED,
+        event_time=event_time,
+        extra_data={RealmAuditLog.OLD_VALUE: original_email, RealmAuditLog.NEW_VALUE: new_email},
+    )
+
+
+def do_start_email_change_process(user_profile: UserProfile, new_email: str) -> None:
+    old_email = user_profile.delivery_email
+    obj = EmailChangeStatus.objects.create(
+        new_email=new_email,
+        old_email=old_email,
+        user_profile=user_profile,
+        realm=user_profile.realm,
+    )
+
+    # Deactivate existing email change requests
+    EmailChangeStatus.objects.filter(realm=user_profile.realm, user_profile=user_profile).exclude(
+        id=obj.id,
+    ).exclude(status=STATUS_USED).update(status=STATUS_REVOKED)
+
+    activation_url = create_confirmation_link(obj, Confirmation.EMAIL_CHANGE)
+    from zerver.context_processors import common_context
+
+    context = common_context(user_profile)
+    context.update(
+        old_email=old_email,
+        new_email=new_email,
+        activate_url=activation_url,
+        organization_host=user_profile.realm.host,
+    )
+    language = user_profile.default_language
+
+    email_template = "zerver/emails/confirm_new_email"
+
+    if old_email == "":
+        # The assertions here are to help document the only circumstance under which
+        # this condition should be possible.
+        assert (
+            user_profile.realm.demo_organization_scheduled_deletion_date is not None
+            and user_profile.is_realm_owner
+        )
+        email_template = "zerver/emails/confirm_demo_organization_email"
+
+    send_email(
+        template_prefix=email_template,
+        to_emails=[new_email],
+        from_name=FromAddress.security_email_from_name(language=language),
+        from_address=FromAddress.tokenized_no_reply_address(),
+        language=language,
+        context=context,
+        realm=user_profile.realm,
+    )
+
+
+def do_change_password(user_profile: UserProfile, password: str, commit: bool = True) -> None:
+    user_profile.set_password(password)
+    if commit:
+        user_profile.save(update_fields=["password"])
+
+    # Imported here to prevent import cycles
+    from zproject.backends import RateLimitedAuthenticationByUsername
+
+    RateLimitedAuthenticationByUsername(user_profile.delivery_email).clear_history()
+    event_time = timezone_now()
+    RealmAuditLog.objects.create(
+        realm=user_profile.realm,
+        acting_user=user_profile,
+        modified_user=user_profile,
+        event_type=AuditLogEventType.USER_PASSWORD_CHANGED,
+        event_time=event_time,
+    )
+
+
+@transaction.atomic(savepoint=False)
+def do_change_full_name(
+    user_profile: UserProfile, full_name: str, acting_user: UserProfile | None, notify: bool
+) -> None:
+    old_name = user_profile.full_name
+    if old_name == full_name:
+        return
+
+    user_profile.full_name = full_name
+    user_profile.save(update_fields=["full_name"])
+    event_time = timezone_now()
+    RealmAuditLog.objects.create(
+        realm=user_profile.realm,
+        acting_user=acting_user,
+        modified_user=user_profile,
+        event_type=AuditLogEventType.USER_FULL_NAME_CHANGED,
+        event_time=event_time,
+        extra_data={RealmAuditLog.OLD_VALUE: old_name, RealmAuditLog.NEW_VALUE: full_name},
+    )
+    payload = dict(user_id=user_profile.id, full_name=user_profile.full_name)
+    send_event_on_commit(
+        user_profile.realm,
+        dict(type="realm_user", op="update", person=payload),
+        get_user_ids_who_can_access_user(user_profile),
+    )
+
+    if notify:
+        changes: list[UserProfileChangeDict] = [
+            UserProfileChangeDict(
+                field_name="full name",
+                old_value=old_name,
+                new_value=full_name,
+            )
+        ]
+
+        send_user_profile_update_notification(user_profile, acting_user, changes)
+
+
+def check_change_full_name(
+    user_profile: UserProfile, full_name_raw: str, acting_user: UserProfile | None
+) -> str:
+    """Verifies that the user's proposed full name is valid.  The caller
+    is responsible for checking check permissions.  Returns the new
+    full name, which may differ from what was passed in (because this
+    function strips whitespace)."""
+    new_full_name = check_full_name(
+        full_name_raw=full_name_raw, user_profile=user_profile, realm=user_profile.realm
+    )
+    do_change_full_name(user_profile, new_full_name, acting_user, notify=True)
+    return new_full_name
+
+
+def check_change_bot_full_name(
+    user_profile: UserProfile, full_name_raw: str, acting_user: UserProfile
+) -> None:
+    new_full_name = check_full_name(
+        full_name_raw=full_name_raw, user_profile=user_profile, realm=user_profile.realm
+    )
+
+    if new_full_name == user_profile.full_name:
+        # Our web app will try to patch full_name even if the user didn't
+        # modify the name in the form.  We just silently ignore those
+        # situations.
+        return
+
+    check_bot_name_available(
+        realm_id=user_profile.realm_id,
+        full_name=new_full_name,
+        is_activation=False,
+    )
+    do_change_full_name(user_profile, new_full_name, acting_user, notify=False)
+
+
+@transaction.atomic(durable=True)
+def do_change_tos_version(user_profile: UserProfile, tos_version: str | None) -> None:
+    old_value = user_profile.tos_version
+    user_profile.tos_version = tos_version
+    user_profile.save(update_fields=["tos_version"])
+    event_time = timezone_now()
+    RealmAuditLog.objects.create(
+        realm=user_profile.realm,
+        acting_user=user_profile,
+        modified_user=user_profile,
+        event_type=AuditLogEventType.USER_TERMS_OF_SERVICE_VERSION_CHANGED,
+        event_time=event_time,
+        extra_data={
+            RealmAuditLog.OLD_VALUE: old_value,
+            RealmAuditLog.NEW_VALUE: tos_version,
+        },
+    )
+
+
+@transaction.atomic(durable=True)
+def do_regenerate_api_key(user_profile: UserProfile, acting_user: UserProfile) -> str:
+    old_api_key = user_profile.api_key
+    new_api_key = generate_api_key()
+    user_profile.api_key = new_api_key
+    user_profile.save(update_fields=["api_key"])
+
+    # We need to explicitly delete the old API key from our caches,
+    # because the on-save handler for flushing the UserProfile object
+    # in zerver/lib/cache.py only has access to the new API key.
+    cache_delete(user_profile_by_api_key_cache_key(old_api_key))
+
+    event_time = timezone_now()
+    RealmAuditLog.objects.create(
+        realm=user_profile.realm,
+        acting_user=acting_user,
+        modified_user=user_profile,
+        event_type=AuditLogEventType.USER_API_KEY_CHANGED,
+        event_time=event_time,
+    )
+
+    event = {"type": "clear_push_device_tokens", "user_profile_id": user_profile.id}
+    queue_event_on_commit("deferred_work", event)
+
+    # Delete all of the user's Device records to stop sending E2EE push notifications.
+    Device.objects.filter(user_id=user_profile.id).delete()
+
+    return new_api_key
+
+
+def bulk_regenerate_api_keys(user_profile_ids: Iterable[int]) -> None:
+    for user_profile_id in user_profile_ids:
+        user_profile = get_user_profile_by_id(user_profile_id)
+        do_regenerate_api_key(user_profile, user_profile)
+
+
+def notify_avatar_url_change(user_profile: UserProfile) -> None:
+    payload = dict(
+        avatar_source=user_profile.avatar_source,
+        avatar_url=avatar_url(user_profile),
+        avatar_url_medium=avatar_url(user_profile, medium=True),
+        avatar_version=user_profile.avatar_version,
+        # Even clients using client_gravatar don't need the email,
+        # since we're sending the URL anyway.
+        user_id=user_profile.id,
+    )
+
+    event = dict(type="realm_user", op="update", person=payload)
+    send_event_on_commit(
+        user_profile.realm,
+        event,
+        get_user_ids_who_can_access_user(user_profile),
+    )
+
+
+@transaction.atomic(savepoint=False)
+def do_change_avatar_fields(
+    user_profile: UserProfile,
+    avatar_source: str,
+    skip_notify: bool = False,
+    *,
+    acting_user: UserProfile | None,
+) -> None:
+    user_profile.avatar_source = avatar_source
+    user_profile.avatar_version += 1
+    user_profile.save(update_fields=["avatar_source", "avatar_version"])
+    event_time = timezone_now()
+    RealmAuditLog.objects.create(
+        realm=user_profile.realm,
+        modified_user=user_profile,
+        event_type=AuditLogEventType.USER_AVATAR_SOURCE_CHANGED,
+        extra_data={"avatar_source": avatar_source},
+        event_time=event_time,
+        acting_user=acting_user,
+    )
+
+    if not skip_notify:
+        notify_avatar_url_change(user_profile)
+
+
+def do_scrub_avatar_images(user: UserProfile, *, acting_user: UserProfile | None) -> None:
+    old_version = user.avatar_version
+    # Note: while scrubbing, no need to generate and upload
+    # avatar when default_avatar_source is jdenticon.
+    do_change_avatar_fields(
+        user, user.realm.default_avatar_source, skip_notify=True, acting_user=acting_user
+    )
+    for version in range(1, old_version + 1):
+        delete_avatar_image(user, version)
+
+
+def set_avatar_to_default(user_profile: UserProfile, *, acting_user: UserProfile | None) -> None:
+    default_avatar_source = user_profile.realm.default_avatar_source
+
+    if user_profile.avatar_source == default_avatar_source:  # nocoverage
+        return
+
+    if default_avatar_source == UserProfile.AVATAR_FROM_JDENTICON:
+        generate_and_upload_jdenticon_avatar(
+            user_profile, realm_uuid=str(user_profile.realm.uuid), future=True
+        )
+    do_change_avatar_fields(user_profile, default_avatar_source, acting_user=acting_user)
+
+
+def update_scheduled_email_notifications_time(
+    user_profile: UserProfile, old_batching_period: int, new_batching_period: int
+) -> None:
+    existing_scheduled_emails = ScheduledMessageNotificationEmail.objects.filter(
+        user_profile=user_profile
+    )
+
+    scheduled_timestamp_change = timedelta(seconds=new_batching_period) - timedelta(
+        seconds=old_batching_period
+    )
+
+    existing_scheduled_emails.update(
+        scheduled_timestamp=F("scheduled_timestamp") + scheduled_timestamp_change
+    )
+
+
+def do_change_user_setting(
+    user_profile: UserProfile,
+    setting_name: str,
+    setting_value: bool | str | int | Enum,
+    *,
+    acting_user: UserProfile | None,
+) -> None:
+    bulk_change_user_setting(
+        user_profile.realm, [user_profile], setting_name, setting_value, acting_user=acting_user
+    )
+
+
+@transaction.atomic(savepoint=False)
+def bulk_change_user_setting(
+    realm: Realm,
+    user_profiles: list[UserProfile],
+    setting_name: str,
+    setting_value: bool | str | int | Enum,
+    *,
+    acting_user: UserProfile | None,
+) -> None:
+    event_time = timezone_now()
+    audit_logs = []
+    old_values = {}
+
+    if isinstance(setting_value, Enum):
+        db_setting_value = setting_value.value
+        event_value: bool | str | int = setting_value.name
+    else:
+        db_setting_value = setting_value
+        event_value = db_setting_value
+
+    if setting_name == "timezone":
+        assert isinstance(db_setting_value, str)
+        assert isinstance(event_value, str)
+        db_setting_value = canonicalize_timezone(db_setting_value)
+    else:
+        property_type = UserProfile.property_types[setting_name]
+        if isinstance(setting_value, Enum):
+            assert isinstance(setting_value, property_type)
+            assert isinstance(event_value, str)
+        else:
+            assert isinstance(db_setting_value, property_type)
+            assert isinstance(event_value, property_type)
+
+    for user in user_profiles:
+        old_value = getattr(user, setting_name)
+        old_values[user.id] = old_value
+
+        setattr(user, setting_name, db_setting_value)
+
+        audit_logs.append(
+            RealmAuditLog(
+                realm=user.realm,
+                event_type=AuditLogEventType.USER_SETTING_CHANGED,
+                event_time=event_time,
+                acting_user=acting_user,
+                modified_user=user,
+                extra_data={
+                    RealmAuditLog.OLD_VALUE: old_value,
+                    RealmAuditLog.NEW_VALUE: db_setting_value,
+                    "property": setting_name,
+                },
+            )
+        )
+
+    UserProfile.objects.bulk_update(user_profiles, [setting_name])
+    RealmAuditLog.objects.bulk_create(audit_logs)
+
+    # Disabling digest emails should clear a user's email queue
+    if setting_name == "enable_digest_emails" and not db_setting_value:
+        assert len(user_profiles) == 1
+        user_profile = user_profiles[0]
+        clear_scheduled_emails([user_profile.id], ScheduledEmail.DIGEST)
+
+    if setting_name == "email_notifications_batching_period_seconds":
+        for user_profile in user_profiles:
+            old_value = old_values[user_profile.id]
+            assert isinstance(old_value, int)
+            assert isinstance(db_setting_value, int)
+            update_scheduled_email_notifications_time(user_profile, old_value, db_setting_value)
+
+    event = {
+        "type": "user_settings",
+        "op": "update",
+        "property": setting_name,
+        "value": event_value,
+    }
+
+    if setting_name == "default_language":
+        assert isinstance(db_setting_value, str)
+        event["language_name"] = get_language_name(db_setting_value)
+
+    transaction.on_commit(lambda: bulk_flush_users(user_profiles=user_profiles, realm=realm))
+
+    user_ids = [u.id for u in user_profiles]
+    send_event_on_commit(realm, event, user_ids)
+
+    if setting_name == "allow_private_data_export":
+        assert len(user_profiles) == 1
+        user_profile = user_profiles[0]
+        realm_export_event = {
+            "type": "realm_export_consent",
+            "user_id": user_profile.id,
+            "consented": event_value,
+        }
+        send_event_on_commit(
+            user_profile.realm,
+            realm_export_event,
+            list(realm.get_human_admin_users().values_list("id", flat=True)),
+        )
+
+    # Updates to the time zone display setting are sent to all users
+    if setting_name == "timezone":
+        assert len(user_profiles) == 1
+        user_profile = user_profiles[0]
+        payload = dict(
+            email=user_profile.email,
+            user_id=user_profile.id,
+            timezone=canonicalize_timezone(user_profile.timezone),
+        )
+        timezone_event = dict(type="realm_user", op="update", person=payload)
+        send_event_on_commit(
+            realm,
+            timezone_event,
+            get_user_ids_who_can_access_user(user_profile),
+        )
+
+    if setting_name == "email_address_visibility":
+        assert len(user_profiles) == 1
+        user_profile = user_profiles[0]
+        old_value = old_values[user_profile.id]
+        send_delivery_email_update_events(
+            user_profile, old_value, user_profile.email_address_visibility
+        )
+
+        if UserProfile.EMAIL_ADDRESS_VISIBILITY_EVERYONE not in [old_value, db_setting_value]:
+            # We use real email addresses on UserProfile.email only if
+            # EMAIL_ADDRESS_VISIBILITY_EVERYONE is configured, so
+            # changes between values that will not require changing
+            # that field, so we can save work and return here.
+            return
+
+        user_profile.email = get_display_email_address(user_profile)
+        user_profile.save(update_fields=["email"])
+
+        send_user_email_update_event(user_profile)
+        notify_avatar_url_change(user_profile)
+
+    if setting_name == "enable_drafts_synchronization" and db_setting_value is False:
+        # Delete all of the drafts from the backend but don't send delete events
+        # for them since all that's happened is that we stopped syncing changes,
+        # not deleted every previously synced draft - to do that use the DELETE
+        # endpoint.
+        Draft.objects.filter(user_profile__in=user_profiles).delete()
+
+    if setting_name == "presence_enabled":
+        # The presence_enabled setting's primary function is to stop
+        # doing presence updates for the user altogether.
+        #
+        # When a user toggles the presence_enabled setting, we
+        # immediately trigger a presence update, so all users see the
+        # user's current presence state as consistent with the new
+        # setting; not doing so can make it look like the settings
+        # change didn't have any effect.
+
+        assert len(user_profiles) == 1
+        user_profile = user_profiles[0]
+
+        if db_setting_value:
+            status = UserPresence.LEGACY_STATUS_ACTIVE_INT
+            presence_time = timezone_now()
+        else:
+            # We want to ensure the user's presence data is such that
+            # they will be treated as offline by correct clients,
+            # There are two cases:
+            #
+            # (1) If the user's presence was current, we backdate it
+            #     so that the user will, for as long as presence
+            #     remains disabled, appear to have been last online a
+            #     few minutes before they disabled presence.
+            #
+            # (2) If the user only has a presence older than what our
+            #     backdate would be, we keep the original value - it
+            #     already guarantees that the user will appear to be
+            #     offline.
+            backdated_presence_time = timezone_now() - timedelta(
+                # We add a small additional offset as a fudge factor in
+                # case of clock skew.
+                seconds=settings.OFFLINE_THRESHOLD_SECS
+                + settings.PRESENCE_UPDATE_MIN_FREQ_SECONDS
+                + 10
+            )
+            minimum_previous_presence_time = backdated_presence_time - timedelta(
+                seconds=settings.PRESENCE_UPDATE_MIN_FREQ_SECONDS + 10
+            )
+
+            try:
+                presence: UserPresence | None = UserPresence.objects.get(user_profile=user_profile)
+                assert presence is not None
+                assert presence.last_connected_time is not None
+
+                original_last_active_time = presence.last_active_time
+                if presence.last_connected_time <= backdated_presence_time:
+                    # This is case (2), so we don't need to do
+                    # anything. presence is already as intended.
+                    # last_active_time <= last_connected_time always holds, so
+                    # last_active_time <= backdated_presence_time is also ensured here.
+                    return
+
+                # do_update_user_presence will only send an event if
+                # the presence event it receives is sufficiently newer
+                # than whatever it had before, so we backdate the
+                # existing presence data if necessary to ensure that
+                # our new update event will be seen as new by clients.
+                presence_time = backdated_presence_time
+                presence.last_connected_time = min(
+                    minimum_previous_presence_time, presence.last_connected_time
+                )
+                update_fields = ["last_connected_time"]
+
+                if (
+                    original_last_active_time is None
+                    or original_last_active_time < backdated_presence_time
+                ):
+                    # If the user's last_active_time was old enough (or nonexistent), we don't want to perturb
+                    # it, as it's already in a correct state. Thus we only do_update_user_presence with an
+                    # IDLE status - which will only update the last_connected_time.
+                    status = UserPresence.LEGACY_STATUS_IDLE_INT
+                else:
+                    assert presence.last_active_time is not None
+                    presence.last_active_time = min(
+                        minimum_previous_presence_time, presence.last_active_time
+                    )
+                    update_fields.append("last_active_time")
+                    status = UserPresence.LEGACY_STATUS_ACTIVE_INT
+                presence.save(update_fields=update_fields)
+
+            except UserPresence.DoesNotExist:
+                # If the user has no presence data at all, that should
+                # logically get consistent treatment with having very
+                # old presence data (case (2) above). We want to leave
+                # their presence state intact - so just return without
+                # doing anything.
+                return
+
+        # do_update_user_presence doesn't allow being run inside another
+        # transaction.atomic block, so we need to use on_commit here to ensure
+        # it gets executed outside of our current transaction.
+        transaction.on_commit(
+            lambda: do_update_user_presence(
+                user_profile,
+                get_client("website"),
+                presence_time,
+                status,
+                force_send_update=True,
+            )
+        )
+
+
+@transaction.atomic(durable=True)
+def do_change_user_date_joined(user_profile: UserProfile, date_joined: datetime) -> None:
+    old_date_joined = user_profile.date_joined
+    user_profile.date_joined = date_joined
+    user_profile.save(update_fields=["date_joined"])
+
+    event_time = timezone_now()
+    RealmAuditLog.objects.create(
+        realm=user_profile.realm,
+        acting_user=user_profile,
+        modified_user=user_profile,
+        event_type=AuditLogEventType.USER_DATE_JOINED_CHANGED,
+        event_time=event_time,
+        extra_data={
+            RealmAuditLog.OLD_VALUE: old_date_joined.isoformat(),
+            RealmAuditLog.NEW_VALUE: date_joined.isoformat(),
+        },
+    )
+
+    payload = dict(user_id=user_profile.id, date_joined=date_joined.isoformat(timespec="minutes"))
+    send_event_on_commit(
+        user_profile.realm,
+        dict(type="realm_user", op="update", person=payload),
+        get_user_ids_who_can_access_user(user_profile),
+    )

zerver/actions/user_status.py

@@ -0,0 +1,55 @@
+from django.db import transaction
+
+from zerver.actions.user_settings import do_change_user_setting
+from zerver.lib.user_status import update_user_status
+from zerver.lib.users import get_user_ids_who_can_access_user
+from zerver.models import UserProfile
+from zerver.tornado.django_api import send_event_on_commit
+
+
+@transaction.atomic(durable=True)
+def do_update_user_status(
+    user_profile: UserProfile,
+    away: bool | None,
+    status_text: str | None,
+    client_id: int,
+    emoji_name: str | None,
+    emoji_code: str | None,
+    reaction_type: str | None,
+) -> None:
+    # Deprecated way for clients to access the user's `presence_enabled`
+    # setting, with away != presence_enabled. Can be removed when clients
+    # migrate "away" (also referred to as "unavailable") feature to directly
+    # use and update the user's presence_enabled setting.
+    if away is not None:
+        user_setting = "presence_enabled"
+        value = not away
+        do_change_user_setting(user_profile, user_setting, value, acting_user=user_profile)
+
+    realm = user_profile.realm
+
+    update_user_status(
+        user_profile_id=user_profile.id,
+        status_text=status_text,
+        client_id=client_id,
+        emoji_name=emoji_name,
+        emoji_code=emoji_code,
+        reaction_type=reaction_type,
+    )
+
+    event = dict(
+        type="user_status",
+        user_id=user_profile.id,
+    )
+
+    if away is not None:
+        event["away"] = away
+
+    if status_text is not None:
+        event["status_text"] = status_text
+
+    if emoji_name is not None:
+        event["emoji_name"] = emoji_name
+        event["emoji_code"] = emoji_code
+        event["reaction_type"] = reaction_type
+    send_event_on_commit(realm, event, get_user_ids_who_can_access_user(user_profile))

zerver/actions/user_topics.py

@@ -0,0 +1,87 @@
+from datetime import datetime
+from typing import Any
+
+from django.db import transaction
+from django.utils.timezone import now as timezone_now
+
+from zerver.lib.timestamp import datetime_to_timestamp
+from zerver.lib.topic import maybe_rename_general_chat_to_empty_topic
+from zerver.lib.user_topics import (
+    bulk_set_user_topic_visibility_policy_in_database,
+    get_topic_mutes,
+)
+from zerver.models import Stream, UserProfile
+from zerver.tornado.django_api import send_event_on_commit
+
+
+@transaction.atomic(savepoint=False)
+def bulk_do_set_user_topic_visibility_policy(
+    user_profiles: list[UserProfile],
+    stream: Stream,
+    topic_name: str,
+    *,
+    visibility_policy: int,
+    last_updated: datetime | None = None,
+    skip_muted_topics_event: bool = False,
+) -> None:
+    if last_updated is None:
+        last_updated = timezone_now()
+
+    topic_name = maybe_rename_general_chat_to_empty_topic(topic_name)
+
+    user_profiles_with_changed_user_topic_rows = bulk_set_user_topic_visibility_policy_in_database(
+        user_profiles,
+        stream.id,
+        topic_name,
+        visibility_policy=visibility_policy,
+        recipient_id=stream.recipient_id,
+        last_updated=last_updated,
+    )
+
+    # Users with requests to set the visibility_policy to its current value
+    # or to delete a UserTopic row that doesn't exist shouldn't
+    # send an unnecessary event.
+    if len(user_profiles_with_changed_user_topic_rows) == 0:
+        return
+
+    for user_profile in user_profiles_with_changed_user_topic_rows:
+        # This first muted_topics event is deprecated and will be removed
+        # once clients are migrated to handle the user_topic event type
+        # instead.
+        if not skip_muted_topics_event:
+            muted_topics_event = dict(
+                type="muted_topics", muted_topics=get_topic_mutes(user_profile)
+            )
+            send_event_on_commit(user_profile.realm, muted_topics_event, [user_profile.id])
+
+        user_topic_event: dict[str, Any] = {
+            "type": "user_topic",
+            "stream_id": stream.id,
+            "topic_name": topic_name,
+            "last_updated": datetime_to_timestamp(last_updated),
+            "visibility_policy": visibility_policy,
+        }
+
+        send_event_on_commit(user_profile.realm, user_topic_event, [user_profile.id])
+
+
+def do_set_user_topic_visibility_policy(
+    user_profile: UserProfile,
+    stream: Stream,
+    topic_name: str,
+    *,
+    visibility_policy: int,
+    last_updated: datetime | None = None,
+    skip_muted_topics_event: bool = False,
+) -> None:
+    # For conciseness, this function should be used when a single
+    # user_profile is involved. In the case of multiple user profiles,
+    # call 'bulk_do_set_user_topic_visibility_policy' directly.
+    bulk_do_set_user_topic_visibility_policy(
+        [user_profile],
+        stream,
+        topic_name,
+        visibility_policy=visibility_policy,
+        last_updated=last_updated,
+        skip_muted_topics_event=skip_muted_topics_event,
+    )

zerver/actions/video_calls.py

@@ -0,0 +1,17 @@
+from django.db import transaction
+
+from zerver.models import UserProfile
+from zerver.tornado.django_api import send_event_on_commit
+
+
+@transaction.atomic(durable=True)
+def do_set_video_call_provider_token(
+    user: UserProfile, token_key: str, /, token: dict[str, object] | None
+) -> None:
+    user.third_party_api_state[token_key] = token
+    user.save(update_fields=["third_party_api_state"])
+    send_event_on_commit(
+        user.realm,
+        dict(type=f"has_{token_key}_token", value=token is not None),
+        [user.id],
+    )

zerver/apps.py

@@ -0,0 +1,89 @@
+import logging
+from typing import Any
+
+import django_stubs_ext
+from django.apps import AppConfig
+from django.conf import settings
+from django.core.cache import cache
+from django.core.checks import register
+from django.db import connection
+from django.db.models.signals import post_migrate, pre_migrate
+from typing_extensions import override
+
+from zerver.checks import (
+    check_auth_settings,
+    check_external_host_setting,
+    check_jitsi_server_url,
+    check_required_settings,
+    check_uploads_settings,
+)
+
+
+def flush_cache(sender: AppConfig | None, **kwargs: Any) -> None:
+    logging.info("Clearing memcached cache after migrations")
+    cache.clear()
+
+
+def create_zulip_schema(sender: AppConfig, **kwargs: Any) -> None:  # nocoverage
+    # We do this here, and not in a zerver migration, because we
+    # cannot be sure that zerver migrations run _first_, of all of the
+    # Django apps.  Updating `SET search_path` is just for the current
+    # connection; the user's default search_path is set in the
+    # zerver/0001 migrations.
+    with connection.cursor() as cursor:
+        cursor.execute("CREATE SCHEMA IF NOT EXISTS zulip")
+        cursor.execute("SET search_path = zulip,public")
+
+
+def skip_smokescreen_proxy_for_boto3() -> None:
+    # https://github.com/boto/botocore/issues/2644: the request boto3
+    # makes to fetch EC2 instance credentials from the IMDS endpoint
+    # always reads the proxy configuration from the environment,
+    # ignoring any per-client setting. Our application processes route
+    # outgoing HTTP through Smokescreen, so that metadata request would
+    # be proxied; and Smokescreen would refuse the internal
+    # IMDS address (169.254.169.254). Disable proxying for boto3 so the
+    # metadata request (and other boto3 traffic) goes direct.
+    #
+    # This must run at process startup so that it takes effect in every
+    # process, including queue workers.
+    import botocore.utils
+
+    botocore.utils.should_bypass_proxies = lambda url: True
+
+
+class ZerverConfig(AppConfig):
+    name: str = "zerver"
+
+    @override
+    def ready(self) -> None:
+        register(check_required_settings)
+        register(check_external_host_setting)
+        register(check_auth_settings)
+        register(check_uploads_settings)
+        register(check_jitsi_server_url)
+
+        if settings.SENTRY_DSN:  # nocoverage
+            from zproject.config import get_config
+            from zproject.sentry import setup_sentry
+
+            setup_sentry(settings.SENTRY_DSN, get_config("machine", "deploy_type", "development"))
+
+        from django.contrib.auth.forms import SetPasswordMixin
+
+        django_stubs_ext.monkeypatch(extra_classes=[SetPasswordMixin])
+
+        if settings.S3_SKIP_PROXY is True and settings.LOCAL_UPLOADS_DIR is None:  # nocoverage
+            skip_smokescreen_proxy_for_boto3()
+
+        # We import zerver.signals here for the side effect of
+        # registering the user_logged_in signal receiver.  This import
+        # needs to be here (rather than e.g. at top-of-file) to avoid
+        # running that code too early in Django's setup process, but
+        # in any case, this is an intentionally unused import.
+        import zerver.signals  # noqa: F401
+
+        if settings.POST_MIGRATION_CACHE_FLUSHING:
+            post_migrate.connect(flush_cache, sender=self)
+
+        pre_migrate.connect(create_zulip_schema)

zerver/checks.py

@@ -0,0 +1,246 @@
+import os
+import re
+from collections.abc import Iterable, Sequence
+from typing import Any
+
+from django.apps.config import AppConfig
+from django.conf import settings
+from django.core import checks
+from django.core.exceptions import ValidationError
+from django.core.validators import URLValidator
+
+
+def get_settings_error_message_strings(setting_name: str) -> tuple[str, str]:
+    # Even in Docker, MANUAL_CONFIGURATION means the admin
+    # manages /etc/zulip/settings.py themselves, so the SETTING_*
+    # environment variables are not where to make changes.
+    if settings.RUNNING_IN_HELM:
+        settings_location = "your Helm values"
+        setting_display_name = "zulip.environment.SETTING_" + setting_name
+    elif settings.RUNNING_IN_DOCKER and os.environ.get("MANUAL_CONFIGURATION") != "True":
+        settings_location = "your Docker environment configuration"
+        setting_display_name = "SETTING_" + setting_name
+    else:
+        settings_location = "/etc/zulip/settings.py"
+        setting_display_name = setting_name
+    return settings_location, setting_display_name
+
+
+def check_required_settings(
+    app_configs: Sequence[AppConfig] | None,
+    databases: Sequence[str] | None,
+    **kwargs: Any,
+) -> Iterable[checks.CheckMessage]:
+    # These are the settings that we will check that the user has filled in for
+    # production deployments before starting the app.  It consists of a series
+    # of pairs of (setting name, default value that it must be changed from)
+    required_settings = [
+        ("EXTERNAL_HOST", "zulip.example.com"),
+        ("ZULIP_ADMINISTRATOR", "zulip-admin@example.com"),
+        # SECRET_KEY doesn't really need to be here, in
+        # that we set it automatically, but just in
+        # case, it seems worth having in this list
+        ("SECRET_KEY", ""),
+        ("AUTHENTICATION_BACKENDS", ()),
+    ]
+    errors = []
+    for setting_name, default in required_settings:
+        value = getattr(settings, setting_name, None)
+        if value and value != default:
+            continue
+
+        settings_location, setting_display_name = get_settings_error_message_strings(setting_name)
+        if value:
+            # The setting is still the example value from the
+            # documentation, which the admin must replace -- saying
+            # "you must set" it would be confusing, as it is set.
+            message = (
+                f"{setting_display_name} is still set to the example value {default!r}; "
+                f"change it in {settings_location}"
+            )
+        else:
+            message = f"You must set {setting_display_name} in {settings_location}"
+        errors.append(
+            checks.Error(
+                message,
+                obj=f"settings.{setting_name}",
+                id="zulip.E001",
+            )
+        )
+    return errors
+
+
+def check_external_host_setting(
+    app_configs: Sequence[AppConfig] | None,
+    databases: Sequence[str] | None,
+    **kwargs: Any,
+) -> Iterable[checks.CheckMessage]:
+    if not hasattr(settings, "EXTERNAL_HOST"):  # nocoverage
+        return []
+
+    errors = []
+    scheme = settings.EXTERNAL_URI_SCHEME
+    if scheme != "https://" and not settings.DEVELOPMENT:
+        errors.append(
+            checks.Error(
+                "Zulip does not support a non-HTTPS external scheme in production",
+                obj="settings.EXTERNAL_URI_SCHEME",
+                hint="Do not override EXTERNAL_URI_SCHEME in production",
+                id="zulip.E004",
+            )
+        )
+
+    hostname = settings.EXTERNAL_HOST
+    if "." not in hostname and os.environ.get("ZULIP_TEST_SUITE") != "true" and settings.PRODUCTION:
+        suggest = ".localdomain" if hostname == "localhost" else ".local"
+        errors.append(
+            checks.Error(
+                f"EXTERNAL_HOST ({hostname}) does not contain a domain part",
+                obj="settings.EXTERNAL_HOST",
+                hint=f"Add {suggest} to the end",
+                id="zulip.E002",
+            )
+        )
+
+    if ":" in hostname:
+        hostname = hostname.split(":")[0]
+
+    if len(hostname) > 255:
+        errors.append(
+            checks.Error(
+                f"EXTERNAL_HOST ({hostname}) is too long to be a valid hostname",
+                obj="settings.EXTERNAL_HOST",
+                id="zulip.E002",
+            )
+        )
+    domain_part = re.compile(r"(?!-)[A-Z\d-]{1,63}(?<!-)$", re.IGNORECASE)
+    if not hostname.isascii():
+        suggestion = ".".join(
+            "xn--" + part.encode("punycode").decode() if not part.isascii() else part
+            for part in hostname.split(".")
+        )
+        errors.append(
+            checks.Error(
+                f"EXTERNAL_HOST ({hostname}) contains non-ASCII characters",
+                hint=f"Switch to punycode: {suggestion}",
+                obj="settings.EXTERNAL_HOST",
+                id="zulip.E002",
+            )
+        )
+    elif not all(domain_part.match(x) for x in hostname.split(".")):
+        errors.append(
+            checks.Error(
+                f"EXTERNAL_HOST ({hostname}) does not validate as a hostname",
+                obj="settings.EXTERNAL_HOST",
+                id="zulip.E002",
+            )
+        )
+    return errors
+
+
+def check_auth_settings(
+    app_configs: Sequence[AppConfig] | None,
+    databases: Sequence[str] | None,
+    **kwargs: Any,
+) -> Iterable[checks.CheckMessage]:
+    errors = []
+    for idp_name, idp_dict in settings.SOCIAL_AUTH_SAML_ENABLED_IDPS.items():
+        if "zulip_groups" in idp_dict.get("extra_attrs", []):
+            errors.append(
+                checks.Error(
+                    "zulip_groups can't be listed in extra_attrs",
+                    obj=f'settings.SOCIAL_AUTH_SAML_ENABLED_IDPS["{idp_name}"]["extra_attrs"]',
+                    id="zulip.E003",
+                )
+            )
+
+    for subdomain, config_dict in settings.SOCIAL_AUTH_SYNC_ATTRS_DICT.items():
+        for auth_name, attrs_map in config_dict.items():
+            for attr_key, attr_value in attrs_map.items():
+                if attr_value == "zulip_groups":
+                    errors.append(
+                        checks.Error(
+                            "zulip_groups can't be listed as a SAML attribute",
+                            obj=f'settings.SOCIAL_AUTH_SYNC_ATTRS_DICT["{subdomain}"]["{auth_name}"]["{attr_key}"]',
+                            id="zulip.E004",
+                        )
+                    )
+    return errors
+
+
+def check_uploads_settings(
+    app_configs: Sequence[AppConfig] | None,
+    databases: Sequence[str] | None,
+    **kwargs: Any,
+) -> Iterable[checks.CheckMessage]:
+    uploads_dir = settings.LOCAL_UPLOADS_DIR
+    if uploads_dir is None:
+        errors = []
+        if settings.S3_AUTH_UPLOADS_BUCKET == "":
+            errors.append(
+                checks.Error(
+                    "Neither settings.LOCAL_UPLOADS_DIR nor settings.S3_AUTH_UPLOADS_BUCKET is set",
+                    obj="settings.S3_AUTH_UPLOADS_BUCKET",
+                    id="zulip.E005",
+                )
+            )
+        if settings.S3_AVATAR_BUCKET == "":
+            errors.append(
+                checks.Error(
+                    "Neither settings.LOCAL_UPLOADS_DIR nor settings.S3_AVATAR_BUCKET is set",
+                    obj="settings.S3_AVATAR_BUCKET",
+                    id="zulip.E005",
+                )
+            )
+        return errors
+
+    if not os.path.isdir(uploads_dir):
+        return [
+            checks.Error(
+                f"settings.LOCAL_UPLOADS_DIR ({uploads_dir}) does not exist",
+                obj="settings.LOCAL_UPLOADS_DIR",
+                id="zulip.E006",
+            )
+        ]
+    elif not os.access(uploads_dir, os.W_OK):
+        return [
+            checks.Error(
+                f"settings.LOCAL_UPLOADS_DIR ({uploads_dir}) is not writable",
+                obj="settings.LOCAL_UPLOADS_DIR",
+                id="zulip.E006",
+            )
+        ]
+    return []
+
+
+def check_jitsi_server_url(
+    app_configs: Sequence[AppConfig] | None,
+    databases: Sequence[str] | None,
+    **kwargs: Any,
+) -> Iterable[checks.CheckMessage]:
+    # As of Zulip 12.0, the web app will not load if a server-level
+    # Jitsi Meet integration has been configured that is not a URL.
+    # A None value is the valid way to disable the integration. By
+    # default, the integration uses the SaaS https://meet.jit.si
+    # server.
+    jitsi_server_url = settings.JITSI_SERVER_URL
+    if jitsi_server_url is None or jitsi_server_url == "https://meet.jit.si":
+        return []
+
+    assert isinstance(jitsi_server_url, str)
+    validate = URLValidator()
+    try:
+        validate(jitsi_server_url)
+        return []
+    except ValidationError:
+        settings_location, setting_display_name = get_settings_error_message_strings(
+            "JITSI_SERVER_URL"
+        )
+        message = f"{setting_display_name} in {settings_location} is not a URL"
+        return [
+            checks.Error(
+                message,
+                obj="settings.JITSI_SERVER_URL",
+                id="zulip.E007",
+            )
+        ]

zerver/context_processors.py

@@ -0,0 +1,322 @@
+from collections.abc import Mapping
+from typing import Any
+from urllib.parse import urljoin
+
+from django.conf import settings
+from django.http import HttpRequest
+from django.utils.html import escape
+from django.utils.safestring import SafeString
+from django.utils.translation import get_language
+from django.utils.translation import override as override_language
+
+from version import (
+    LATEST_MAJOR_VERSION,
+    LATEST_RELEASE_ANNOUNCEMENT,
+    LATEST_RELEASE_VERSION,
+    ZULIP_VERSION,
+)
+from zerver.lib.exceptions import InvalidSubdomainError
+from zerver.lib.i18n import get_language_list
+from zerver.lib.realm_description import get_realm_rendered_description, get_realm_text_description
+from zerver.lib.realm_icon import get_realm_icon_url
+from zerver.lib.request import RequestNotes
+from zerver.lib.send_email import FromAddress
+from zerver.lib.subdomains import get_subdomain, is_root_domain_available
+from zerver.models import PreregistrationRealm, Realm, RealmUserDefault, UserProfile
+from zerver.models.realms import get_realm
+from zproject.backends import (
+    AUTH_BACKEND_NAME_MAP,
+    auth_enabled_helper,
+    get_external_method_dicts,
+    password_auth_enabled,
+    require_email_format_usernames,
+)
+from zproject.config import get_config
+
+DEFAULT_PAGE_PARAMS: Mapping[str, Any] = {
+    "page_type": "default",
+    "development_environment": settings.DEVELOPMENT,
+    # This language is used to initialize i18n.ts to do translations
+    # in the user's language. Callers are expected to replace this
+    # with the user's detected language using get_language wherever
+    # possible.
+    "request_language": "en",
+}
+
+
+def common_context(user: UserProfile) -> dict[str, Any]:
+    """Common context used for things like outgoing emails that don't
+    have a request.
+    """
+    return {
+        "realm_url": user.realm.url,
+        "realm_name": user.realm.name,
+        "root_domain_url": settings.ROOT_DOMAIN_URI,
+        "external_url_scheme": settings.EXTERNAL_URI_SCHEME,
+        "external_host": settings.EXTERNAL_HOST,
+        "user_name": user.full_name,
+        "corporate_enabled": settings.CORPORATE_ENABLED,
+    }
+
+
+def get_realm_from_request(request: HttpRequest) -> Realm | None:
+    request_notes = RequestNotes.get_notes(request)
+    if request.user.is_authenticated:
+        return request.user.realm
+    if not request_notes.has_fetched_realm:
+        # We cache the realm object from this function on the request data,
+        # so that functions that call get_realm_from_request don't
+        # need to do duplicate queries on the same realm while
+        # processing a single request.
+        subdomain = get_subdomain(request)
+        request_notes = RequestNotes.get_notes(request)
+        try:
+            request_notes.realm = get_realm(subdomain)
+        except Realm.DoesNotExist:
+            request_notes.realm = None
+        request_notes.has_fetched_realm = True
+    return request_notes.realm
+
+
+def get_valid_realm_from_request(request: HttpRequest) -> Realm:
+    realm = get_realm_from_request(request)
+    if realm is None:
+        raise InvalidSubdomainError
+    return realm
+
+
+def get_apps_page_url() -> str:
+    if settings.CORPORATE_ENABLED:
+        return "/apps/"
+    return "https://zulip.com/apps/"
+
+
+def is_isolated_page(request: HttpRequest) -> bool:
+    """Accept a GET param `?nav=no` to render an isolated, navless page."""
+    return request.GET.get("nav") == "no"
+
+
+def zulip_default_corporate_context(request: HttpRequest) -> dict[str, Any]:
+    from corporate.lib.decorator import is_self_hosting_management_subdomain
+
+    # Check if view function is in corporate app.
+    if request.resolver_match is not None and not request.resolver_match.func.__module__.startswith(
+        "corporate"
+    ):
+        return {
+            "is_self_hosting_management_page": False,
+        }
+
+    # Add common context variables that are only used on the corporate site.
+    return {
+        "is_self_hosting_management_page": is_self_hosting_management_subdomain(request),
+    }
+
+
+def zulip_default_context(request: HttpRequest) -> dict[str, Any]:
+    """Context available to all Zulip Jinja2 templates that have a request
+    passed in.  Designed to provide the long list of variables at the
+    bottom of this function in a wide range of situations: logged-in
+    or logged-out, subdomains or not, etc.
+
+    The main variable in the below is whether we know what realm the
+    user is trying to interact with.
+    """
+    realm = get_realm_from_request(request)
+
+    if realm is None:
+        realm_url = settings.ROOT_DOMAIN_URI
+        realm_name = None
+        realm_icon = None
+    else:
+        realm_url = realm.url
+        realm_name = realm.name
+        realm_icon = get_realm_icon_url(realm)
+
+    skip_footer = False
+    register_link_disabled = settings.REGISTER_LINK_DISABLED
+    allow_search_engine_indexing = False
+    non_realm_specific_page = False
+
+    if (
+        settings.ROOT_DOMAIN_LANDING_PAGE
+        and get_subdomain(request) == Realm.SUBDOMAIN_FOR_ROOT_DOMAIN
+    ):
+        register_link_disabled = True
+        allow_search_engine_indexing = True
+        non_realm_specific_page = True
+    elif realm is None:
+        register_link_disabled = True
+        skip_footer = True
+        non_realm_specific_page = True
+
+    apps_page_web = settings.ROOT_DOMAIN_URI + "/accounts/go/"
+
+    if settings.DEVELOPMENT:
+        secrets_path = "zproject/dev-secrets.conf"
+        settings_path = "zproject/dev_settings.py"
+        settings_comments_path = "zproject/prod_settings_template.py"
+    else:
+        secrets_path = "/etc/zulip/zulip-secrets.conf"
+        settings_path = "/etc/zulip/settings.py"
+        settings_comments_path = "/etc/zulip/settings.py"
+
+    # Used to remove links to Zulip docs and landing page from footer of self-hosted pages.
+    corporate_enabled = settings.CORPORATE_ENABLED
+
+    support_email = FromAddress.SUPPORT
+    support_email_html_tag = SafeString(
+        f'<a href="mailto:{escape(support_email)}">{escape(support_email)}</a>'
+    )
+
+    # Sync this with default_params_schema in base_page_params.ts.
+    default_page_params: dict[str, Any] = {
+        **DEFAULT_PAGE_PARAMS,
+        "request_language": get_language(),
+    }
+
+    context = {
+        "root_domain_landing_page": settings.ROOT_DOMAIN_LANDING_PAGE,
+        "custom_logo_url": settings.CUSTOM_LOGO_URL,
+        "register_link_disabled": register_link_disabled,
+        "terms_of_service": settings.TERMS_OF_SERVICE_VERSION is not None,
+        "login_url": settings.HOME_NOT_LOGGED_IN,
+        "only_sso": settings.ONLY_SSO,
+        "external_host": settings.EXTERNAL_HOST,
+        "external_url_scheme": settings.EXTERNAL_URI_SCHEME,
+        "realm_url": realm_url,
+        "realm_name": realm_name,
+        "realm_icon": realm_icon,
+        "root_domain_url": settings.ROOT_DOMAIN_URI,
+        "apps_page_url": get_apps_page_url(),
+        "apps_page_web": apps_page_web,
+        "open_realm_creation": settings.OPEN_REALM_CREATION,
+        "development_environment": settings.DEVELOPMENT,
+        "support_email": support_email,
+        "support_email_html_tag": support_email_html_tag,
+        "password_min_length": settings.PASSWORD_MIN_LENGTH,
+        "password_max_length": settings.PASSWORD_MAX_LENGTH,
+        "password_min_guesses": settings.PASSWORD_MIN_GUESSES,
+        "zulip_version": ZULIP_VERSION,
+        "user_is_authenticated": request.user.is_authenticated,
+        "settings_path": settings_path,
+        "secrets_path": secrets_path,
+        "settings_comments_path": settings_comments_path,
+        "platform": RequestNotes.get_notes(request).client_name,
+        "allow_search_engine_indexing": allow_search_engine_indexing,
+        "landing_page_navbar_message": settings.LANDING_PAGE_NAVBAR_MESSAGE,
+        "is_isolated_page": is_isolated_page(request),
+        "skip_footer": skip_footer,
+        "default_page_params": default_page_params,
+        "corporate_enabled": corporate_enabled,
+        "non_realm_specific_page": non_realm_specific_page,
+    }
+
+    if settings.SENTRY_FRONTEND_DSN is not None:
+        sentry_params = {
+            "dsn": settings.SENTRY_FRONTEND_DSN,
+            "environment": get_config("machine", "deploy_type", "development"),
+            "realm_key": "www" if realm is None else realm.string_id or "(root)",
+            "sample_rate": settings.SENTRY_FRONTEND_SAMPLE_RATE,
+            "server_version": ZULIP_VERSION,
+            "trace_rate": settings.SENTRY_FRONTEND_TRACE_RATE,
+        }
+        if request.user.is_authenticated:
+            with override_language(None):
+                sentry_params["user"] = {
+                    "id": request.user.id,
+                    "role": request.user.get_role_name(),
+                }
+        context["sentry_params"] = sentry_params
+
+    context["PAGE_METADATA_URL"] = f"{realm_url}{request.path}"
+    if realm is not None and realm.icon_source == realm.ICON_UPLOADED:
+        context["PAGE_METADATA_IMAGE"] = urljoin(realm_url, realm_icon)
+
+    return context
+
+
+def login_context(request: HttpRequest) -> dict[str, Any]:
+    realm = get_realm_from_request(request)
+
+    if realm is None:
+        realm_description = None
+        realm_invite_required = False
+        realm_web_public_access_enabled = False
+    else:
+        realm_description = get_realm_rendered_description(realm)
+        realm_invite_required = realm.invite_required
+        # We offer web-public access only if the realm has actual web
+        # public streams configured, in addition to having it enabled.
+        realm_web_public_access_enabled = realm.allow_web_public_streams_access()
+
+    # Get realm default emojiset for rendering emojis in organization description
+    if realm is not None:
+        try:
+            realm_user_default = RealmUserDefault.objects.get(realm=realm)
+            realm_default_emojiset = realm_user_default.emojiset
+        except RealmUserDefault.DoesNotExist:  # nocoverage
+            realm_default_emojiset = UserProfile.GOOGLE_EMOJISET
+    else:
+        realm_default_emojiset = UserProfile.GOOGLE_EMOJISET
+
+    context: dict[str, Any] = {
+        "realm_invite_required": realm_invite_required,
+        "realm_description": realm_description,
+        "require_email_format_usernames": require_email_format_usernames(realm),
+        "password_auth_enabled": password_auth_enabled(realm),
+        "two_factor_authentication_enabled": settings.TWO_FACTOR_AUTHENTICATION_ENABLED,
+        "realm_web_public_access_enabled": realm_web_public_access_enabled,
+    }
+
+    if realm is not None and realm.description:
+        context["PAGE_TITLE"] = realm.name
+        context["PAGE_DESCRIPTION"] = get_realm_text_description(realm)
+
+    # Add the keys for our standard authentication backends.
+    no_auth_enabled = True
+    for auth_backend_name in AUTH_BACKEND_NAME_MAP:
+        name_lower = auth_backend_name.lower()
+        key = f"{name_lower}_auth_enabled"
+        is_enabled = auth_enabled_helper([auth_backend_name], realm)
+        context[key] = is_enabled
+        if is_enabled:
+            no_auth_enabled = False
+
+    context["external_authentication_methods"] = get_external_method_dicts(realm)
+    context["no_auth_enabled"] = no_auth_enabled
+
+    # Sync this with login_page_params_schema in base_page_params.ts.
+    context["page_params"] = {
+        **DEFAULT_PAGE_PARAMS,
+        "page_type": "login",
+        "realm_default_emojiset": realm_default_emojiset,
+    }
+
+    return context
+
+
+def latest_info_context() -> dict[str, str]:
+    context = {
+        "latest_release_version": LATEST_RELEASE_VERSION,
+        "latest_major_version": LATEST_MAJOR_VERSION,
+        "latest_release_announcement": LATEST_RELEASE_ANNOUNCEMENT,
+    }
+    return context
+
+
+def is_realm_import_enabled() -> bool:
+    return settings.MAX_WEB_DATA_IMPORT_SIZE_MB != 0
+
+
+def get_realm_create_form_context() -> dict[str, Any]:
+    context = {
+        "language_list": get_language_list(),
+        "MAX_REALM_NAME_LENGTH": str(Realm.MAX_REALM_NAME_LENGTH),
+        "MAX_REALM_SUBDOMAIN_LENGTH": str(Realm.MAX_REALM_SUBDOMAIN_LENGTH),
+        "root_domain_available": is_root_domain_available(),
+        "sorted_realm_types": sorted(Realm.ORG_TYPES.values(), key=lambda d: d["display_order"]),
+        "is_realm_import_enabled": is_realm_import_enabled(),
+        "import_from_choices": PreregistrationRealm.IMPORT_FROM_CHOICES,
+    }
+    return context

zerver/data_import/microsoft_teams.py

@@ -0,0 +1,877 @@
+import logging
+import os
+import re
+from collections import defaultdict
+from collections.abc import Callable, Iterable
+from dataclasses import dataclass
+from datetime import datetime
+from email.headerregistry import Address
+from multiprocessing import Manager
+from multiprocessing.managers import ListProxy
+from typing import Any, Literal, TypeAlias
+from urllib.parse import SplitResult
+
+import regex
+import requests
+from django.conf import settings
+from django.utils.timezone import now as timezone_now
+
+from zerver.data_import.import_util import (
+    AttachmentRecordData,
+    UploadFileRequest,
+    UploadRecordData,
+    ZerverFieldsT,
+    build_message,
+    build_realm,
+    build_recipient,
+    build_stream,
+    build_subscription,
+    build_user_profile,
+    build_usermessages,
+    build_zerver_realm,
+    create_converted_data_files,
+    get_attachment_path_and_content,
+    get_data_file,
+    make_subscriber_map,
+    request_file_stream,
+    scrub_missing_upload_records_after_download,
+    validate_user_emails_for_import,
+    write_response_file_stream_to_path,
+)
+from zerver.data_import.microsoft_teams_message_conversion import (
+    HOSTED_CONTENT_MARKDOWN_IMAGE_SYNTAX_REGEX,
+    convert_microsoft_teams_html_to_markdown,
+)
+from zerver.data_import.sequencer import NEXT_ID
+from zerver.lib.export import MESSAGE_BATCH_CHUNK_SIZE, do_common_export_processes
+from zerver.lib.markdown import get_markdown_image_for_url
+from zerver.lib.mime_types import bare_content_type
+from zerver.lib.parallel import run_parallel_queue
+from zerver.lib.partial import partial
+from zerver.models.recipients import Recipient
+from zerver.models.users import UserProfile
+
+
+@dataclass
+class TeamMetadata:
+    """
+    "team" is equivalent to a Zulip channel
+    """
+
+    description: str
+    display_name: str
+    visibility: Literal["public", "private"]
+    is_archived: bool
+    zulip_channel_id: int
+    zulip_recipient_id: int
+
+
+@dataclass
+class ChannelMetadata:
+    """
+    "channel" is equivalent to Zulip topics.
+    """
+
+    display_name: str
+    is_favourite_by_default: bool
+    is_archived: bool
+    is_favorite_by_default: bool
+    membership_type: str
+    team_id: str
+
+
+@dataclass
+class MessageConversionResult:
+    zerver_messages: list[ZerverFieldsT]
+    zerver_usermessages: list[ZerverFieldsT]
+    upload_records: list[UploadRecordData]
+
+
+@dataclass
+class AttachmentConversionResult:
+    updated_content: str
+    upload_records: list[UploadRecordData]
+
+
+@dataclass
+class ExportMessageAttachmentParameter:
+    realm_id: int
+    message_id: int
+    sender_id: int
+    upload_file_request: UploadFileRequest
+
+
+AddedTeamsT: TypeAlias = dict[str, TeamMetadata]
+TeamIdToZulipRecipientIdT: TypeAlias = dict[str, int]
+MicrosoftTeamsUserIdToZulipUserIdT: TypeAlias = dict[str, int]
+MicrosoftTeamsFieldsT: TypeAlias = dict[str, Any]
+
+MICROSOFT_TEAMS_DEFAULT_ANNOUNCEMENTS_CHANNEL_NAME = "All company"
+
+
+def convert_teams_to_channels(
+    microsoft_teams_user_id_to_zulip_user_id: MicrosoftTeamsUserIdToZulipUserIdT,
+    realm: dict[str, Any],
+    realm_id: int,
+    teams_data_dir: str,
+) -> AddedTeamsT:
+    team_data_folders = []
+    for f in os.listdir(teams_data_dir):
+        path = os.path.join(teams_data_dir, f)
+        if os.path.isdir(path):
+            team_data_folders.append(f)
+
+    logging.info("######### IMPORTING TEAMS STARTED #########\n")
+
+    # Build teams subscription map
+    team_id_to_zulip_subscriber_ids: dict[str, set[int]] = defaultdict(set)
+    for team_id in team_data_folders:
+        team_members_file_name = f"teamMembers_{team_id}.json"
+        team_members_file_path = os.path.join(teams_data_dir, team_id, team_members_file_name)
+        team_members: list[MicrosoftTeamsFieldsT] = get_data_file(team_members_file_path)
+
+        for member in team_members:
+            zulip_user_id = microsoft_teams_user_id_to_zulip_user_id[member["UserId"]]
+            team_id_to_zulip_subscriber_ids[team_id].add(zulip_user_id)
+
+    # Compile teamsSettings.json and teamsList.json and convert
+    # teams to Zulip channels.
+    teams_list = get_data_file(os.path.join(teams_data_dir, "teamsList.json"))
+    teams_settings = get_data_file(os.path.join(teams_data_dir, "teamsSettings.json"))
+    team_dict: dict[str, Any] = {team["GroupsId"]: team for team in teams_list}
+    teams_metadata: AddedTeamsT = {}
+    for team_settings in teams_settings:
+        team_id = team_settings.get("Id")
+
+        assert team_id and team_id in team_dict, (
+            f"Team {team_id} appears in teamsSettings.json but not teamsList.json!"
+        )
+
+        compiled_team_data: MicrosoftTeamsFieldsT = {**team_dict[team_id], **team_settings}
+
+        channel_id = NEXT_ID("channel")
+        recipient_id = NEXT_ID("recipient")
+
+        channel = build_stream(
+            # Microsoft Teams export doesn't include teams creation date.
+            date_created=float(timezone_now().timestamp()),
+            realm_id=realm_id,
+            name=compiled_team_data["Name"],
+            description=compiled_team_data["Description"] or "",
+            stream_id=channel_id,
+            deactivated=compiled_team_data["IsArchived"],
+            invite_only=compiled_team_data["Visibility"] == "private",
+        )
+        realm["zerver_stream"].append(channel)
+
+        recipient = build_recipient(channel_id, recipient_id, Recipient.STREAM)
+        realm["zerver_recipient"].append(recipient)
+
+        for zulip_user_id in team_id_to_zulip_subscriber_ids[team_id]:
+            sub = build_subscription(
+                recipient_id=recipient_id,
+                user_id=zulip_user_id,
+                subscription_id=NEXT_ID("subscription"),
+            )
+            realm["zerver_subscription"].append(sub)
+
+        # If the org uses the "All company" team, set it as the announcements channel.
+        if (
+            compiled_team_data["Name"] == MICROSOFT_TEAMS_DEFAULT_ANNOUNCEMENTS_CHANNEL_NAME
+        ):  # nocoverage
+            realm["zerver_realm"][0]["new_stream_announcements_stream"] = channel_id
+            realm["zerver_realm"][0]["zulip_update_announcements_stream"] = channel_id
+            realm["zerver_realm"][0]["signup_announcements_stream"] = channel_id
+            logging.info("Using the channel 'All company' as default announcements channel.")
+
+        teams_metadata[team_id] = TeamMetadata(
+            description=compiled_team_data["Description"],
+            display_name=compiled_team_data["DisplayName"],
+            visibility=compiled_team_data["Visibility"],
+            is_archived=compiled_team_data["IsArchived"],
+            zulip_channel_id=channel_id,
+            zulip_recipient_id=recipient_id,
+        )
+
+    return teams_metadata
+
+
+@dataclass
+class MicrosoftTeamsUserRoleData:
+    global_administrator_user_ids: set[str]
+    guest_user_ids: set[str]
+
+
+@dataclass
+class ODataQueryParameter:
+    # This is used to compute a request's OData query. It specifies the
+    # amount, type, and order of the data returned for the resource
+    # identified by the URL.
+    # https://learn.microsoft.com/en-us/graph/query-parameters?tabs=http
+    parameter: Literal["$filter", "$search", "$select", "$top"]
+    expression: str
+
+
+MICROSOFT_GRAPH_API_URL = "https://graph.microsoft.com/v1.0{endpoint}"
+
+
+def get_microsoft_graph_api_data(
+    api_url: str,
+    odata_parameters: list[ODataQueryParameter] | None = None,
+    **kwargs: Any,
+) -> Any:
+    if not kwargs.get("token"):
+        raise AssertionError("Microsoft authorization token missing in kwargs")
+    token = kwargs.pop("token")
+    accumulated_result = []
+    parameters = {}
+    if odata_parameters is not None:
+        for parameter in odata_parameters:
+            assert parameter.parameter not in parameters
+            parameters[parameter.parameter] = parameter.expression
+
+    # If a request is paginated, "@odata.nextLink" will be included in the response,
+    # it points to the next page of result. Even if the `$top` query is not specified,
+    # depending on the endpoint and the result size, it may be paged by the server.
+    # https://learn.microsoft.com/en-us/graph/paging?tabs=http#server-side-paging
+    next_link: str | None = api_url
+    while next_link is not None:
+        response = requests.get(
+            next_link, headers={"Authorization": f"Bearer {token}"}, params=parameters
+        )
+
+        if response.status_code != requests.codes.ok:
+            logging.info("HTTP error: %s, Response: %s", response.status_code, response.text)
+            raise Exception("HTTP error accessing the Microsoft Graph API.")
+
+        response_data = response.json()
+        accumulated_result.extend(response_data["value"])
+        next_link = response_data.get("@odata.nextLink")
+
+        # Don't apply initial request's parameters to subsequent paginated requests.
+        if next_link is not None:
+            parameters = {}
+
+    return accumulated_result
+
+
+def get_directory_roles(api_token: str) -> list[MicrosoftTeamsFieldsT]:
+    """
+    https://learn.microsoft.com/en-us/graph/api/directoryrole-list?view=graph-rest-1.0
+    """
+    return get_microsoft_graph_api_data(
+        MICROSOFT_GRAPH_API_URL.format(endpoint="/directoryRoles"), token=api_token
+    )
+
+
+def get_users_with_directory_role_id(
+    directory_role_id: str, api_token: str
+) -> list[MicrosoftTeamsFieldsT]:
+    """
+    https://learn.microsoft.com/en-us/graph/api/directoryrole-list-members?view=graph-rest-1.0
+    """
+    return get_microsoft_graph_api_data(
+        MICROSOFT_GRAPH_API_URL.format(endpoint=f"/directoryRoles/{directory_role_id}/members"),
+        token=api_token,
+    )
+
+
+def get_user_ids_with_member_type(
+    member_type: Literal["Member", "Guest"], api_token: str
+) -> list[MicrosoftTeamsFieldsT]:
+    """
+    https://learn.microsoft.com/en-us/graph/api/user-list?view=graph-rest-1.0
+    """
+    odata_parameter = [
+        ODataQueryParameter(parameter="$filter", expression=f"userType eq '{member_type}'"),
+        ODataQueryParameter(parameter="$select", expression="id"),
+    ]
+    return get_microsoft_graph_api_data(
+        MICROSOFT_GRAPH_API_URL.format(endpoint="/users"),
+        odata_parameter,
+        token=api_token,
+    )
+
+
+def get_user_roles(api_token: str) -> MicrosoftTeamsUserRoleData:
+    """
+    User roles are not included in the export file, so this calls
+    to Microsoft Graph API endpoints for that data. We mainly
+    want to find out who the admins and guests are.
+    """
+    directory_roles: list[MicrosoftTeamsFieldsT] = get_directory_roles(api_token)
+    global_administrator_role_id = None
+    for role in directory_roles:
+        if role["displayName"] == "Global Administrator":
+            global_administrator_role_id = role["id"]
+
+    if global_administrator_role_id is None:
+        raise AssertionError("Could not find Microsoft Teams organization owners/administrators.")
+
+    admin_users_data = get_users_with_directory_role_id(global_administrator_role_id, api_token)
+    guest_users_data = get_user_ids_with_member_type("Guest", api_token)
+
+    return MicrosoftTeamsUserRoleData(
+        global_administrator_user_ids={user_data["id"] for user_data in admin_users_data},
+        guest_user_ids={user_data["id"] for user_data in guest_users_data},
+    )
+
+
+def get_user_email(user: MicrosoftTeamsFieldsT) -> str:
+    if user["Mail"]:
+        return user["Mail"]
+    else:
+        raise AssertionError(f"Could not find email address for Microsoft Teams user {user}")
+
+
+def create_is_mirror_dummy_user(
+    microsoft_team_user_id: str,
+    microsoft_teams_user_id_to_zulip_user_id: MicrosoftTeamsUserIdToZulipUserIdT,
+    realm: dict[str, Any],
+    realm_id: int,
+    domain_name: str,
+) -> None:
+    zulip_user_id = NEXT_ID("user")
+    user_full_name = f"Deleted Teams user {microsoft_team_user_id}"
+    email = Address(username=microsoft_team_user_id, domain=domain_name).addr_spec
+    user_profile_dict = build_user_profile(
+        avatar_source=UserProfile.DEFAULT_AVATAR_SOURCE,
+        date_joined=int(timezone_now().timestamp()),
+        delivery_email=email,
+        email=email,
+        full_name=user_full_name,
+        id=zulip_user_id,
+        is_active=False,
+        role=UserProfile.ROLE_MEMBER,
+        is_mirror_dummy=True,
+        realm_id=realm_id,
+        short_name=user_full_name,
+        timezone="UTC",
+    )
+    realm["zerver_userprofile"].append(user_profile_dict)
+    microsoft_teams_user_id_to_zulip_user_id[microsoft_team_user_id] = zulip_user_id
+
+
+def convert_users(
+    microsoft_teams_user_role_data: MicrosoftTeamsUserRoleData,
+    realm: dict[str, Any],
+    realm_id: int,
+    timestamp: int,
+    users_list: list[MicrosoftTeamsFieldsT],
+) -> MicrosoftTeamsUserIdToZulipUserIdT:
+    zerver_user_profile: list[ZerverFieldsT] = []
+    microsoft_teams_user_id_to_zulip_user_id: MicrosoftTeamsUserIdToZulipUserIdT = defaultdict(int)
+    found_emails: dict[str, int] = {}
+    has_owner = False
+
+    logging.info("######### IMPORTING USERS STARTED #########\n")
+    for user in users_list:
+        microsoft_teams_user_id = user["Id"]
+        user_full_name = user["DisplayName"]
+
+        if microsoft_teams_user_id in microsoft_teams_user_role_data.global_administrator_user_ids:
+            user_role = UserProfile.ROLE_REALM_OWNER
+        elif microsoft_teams_user_id in microsoft_teams_user_role_data.guest_user_ids:
+            user_role = UserProfile.ROLE_GUEST
+        else:
+            user_role = UserProfile.ROLE_MEMBER
+
+        microsoft_teams_user_email = get_user_email(user)
+
+        zulip_user_id = NEXT_ID("user")
+        found_emails[microsoft_teams_user_email.lower()] = zulip_user_id
+
+        user_profile_dict = build_user_profile(
+            avatar_source=UserProfile.DEFAULT_AVATAR_SOURCE,
+            date_joined=timestamp,
+            delivery_email=microsoft_teams_user_email,
+            email=microsoft_teams_user_email,
+            full_name=user_full_name,
+            id=zulip_user_id,
+            # This function only processes user data from `users/usersList.json` which only
+            # lists active users -- no bot or deleted user accounts.
+            is_active=True,
+            role=user_role,
+            is_mirror_dummy=False,
+            realm_id=realm_id,
+            short_name=user_full_name,
+            timezone="UTC",
+        )
+
+        user_profile_dict["realm"] = realm_id
+        zerver_user_profile.append(user_profile_dict)
+        microsoft_teams_user_id_to_zulip_user_id[microsoft_teams_user_id] = zulip_user_id
+
+        if user_role == UserProfile.ROLE_REALM_OWNER:
+            has_owner = True
+
+        logging.info(
+            "%s: %s -> %s",
+            microsoft_teams_user_id,
+            user_full_name,
+            microsoft_teams_user_email,
+        )
+
+    if not has_owner:
+        logging.warning("Converted realm has no owners!")
+
+    validate_user_emails_for_import(list(found_emails))
+    realm["zerver_userprofile"] = zerver_user_profile
+    logging.info("######### IMPORTING USERS FINISHED #########\n")
+    return microsoft_teams_user_id_to_zulip_user_id
+
+
+def get_timestamp_from_message(message: MicrosoftTeamsFieldsT) -> float:
+    return datetime.fromisoformat(message["CreatedDateTime"]).timestamp()
+
+
+def get_microsoft_teams_sender_id_from_message(message: MicrosoftTeamsFieldsT) -> str:
+    return message["From"]["User"]["Id"]
+
+
+def is_microsoft_teams_event_message(message: MicrosoftTeamsFieldsT) -> bool:
+    return message["MessageType"] == "unknownFutureValue" and message["From"] is None
+
+
+def download_and_export_microsoft_teams_upload_file(
+    attachment_records: ListProxy | list[AttachmentRecordData],  # type: ignore[type-arg] # ListProxy is not subscriptable
+    output_dir: str,
+    args: ExportMessageAttachmentParameter,
+) -> None:
+    upload_file_request = args.upload_file_request
+    response = request_file_stream(
+        upload_file_request.request_url,
+        upload_file_request.params,
+        upload_file_request.headers,
+        **upload_file_request.kwargs,
+    )
+
+    # Don't add the file type extension here. The file path will be remapped
+    # during import and must match the updated URL in the message content.
+    # The actual file's content type will be included in its attachment
+    # record.
+    file_output_path = os.path.join(
+        output_dir, "uploads", f"{upload_file_request.output_file_path_id}"
+    )
+
+    os.makedirs(os.path.dirname(file_output_path), exist_ok=True)
+    write_response_file_stream_to_path(response, file_output_path)
+
+    raw_content_type = response.headers.get("Content-Type")
+    assert raw_content_type is not None
+
+    # Can't use build_attachment here since Django doesn't support parallel
+    # connection to the db.
+    attachment_records.append(
+        AttachmentRecordData(
+            content_type=bare_content_type(raw_content_type),
+            create_time=os.path.getmtime(file_output_path),
+            file_name=os.path.basename(file_output_path),
+            # IDs will be assigned later after all uploads have been downloaded.
+            id=0,
+            is_realm_public=True,
+            messages=[args.message_id],
+            owner=args.sender_id,
+            path_id=upload_file_request.output_file_path_id,
+            realm=args.realm_id,
+            scheduled_messages=[],
+            size=os.path.getsize(file_output_path),
+            is_web_public=False,
+        )
+    )
+
+
+def process_hosted_content_attachments(
+    content: str,
+    do_download_and_export_upload_file: Callable[[ExportMessageAttachmentParameter], None],
+    is_direct_message_type: bool,
+    microsoft_graph_api_token: str,
+    realm_id: int,
+    teams_message_id: str,
+    zulip_message_id: int,
+    zulip_user_id: int,
+) -> AttachmentConversionResult:
+    """
+    "Hosted contents" are message file attachments that can be fetched straight
+    from Microsoft Graph API. As of April 2026, there are only 3 known types of
+    items that are categorized as "hosted contents":
+     * images
+     * custom emojis
+     * code snippets
+
+    Since custom emoji and code snippet are formatted as custom HTML blocks,
+    convert_microsoft_teams_html_to_markdown will turn them into empty strings,
+    so this currently only supports processing image hosted contents.
+
+    It's worth checking the API documentation for the hosted content once in a
+    while to verify what file types "hosted content" includes, since the code path
+    for this relies on that being only images:
+        https://learn.microsoft.com/en-us/graph/api/chatmessagehostedcontent-get
+    """
+    upload_records: list[UploadRecordData] = []
+
+    def export_file_and_get_zulip_url(match: regex.Match[str]) -> str:
+        file_name = match.group("file_name")
+        hosted_content_api_url = match.group("api_url")
+        attachment_data = get_attachment_path_and_content(file_name, teams_message_id, realm_id)
+
+        upload_records.append(
+            UploadRecordData(
+                content_type=None,
+                last_modified=float(timezone_now().timestamp()),
+                path=attachment_data.path_id,
+                realm_id=realm_id,
+                s3_path=attachment_data.path_id,
+                size=0,
+                user_profile_id=zulip_user_id,
+            )
+        )
+
+        do_download_and_export_upload_file(
+            ExportMessageAttachmentParameter(
+                realm_id=realm_id,
+                message_id=zulip_message_id,
+                sender_id=zulip_user_id,
+                upload_file_request=UploadFileRequest(
+                    output_file_path_id=attachment_data.path_id,
+                    request_url=hosted_content_api_url,
+                    params=None,
+                    headers={"Authorization": f"Bearer {microsoft_graph_api_token}"},
+                    kwargs={},
+                ),
+            )
+        )
+        # Use inline thumbnail here since MS Teams only supports customizing the hosted
+        # content image's alt tag.
+        return get_markdown_image_for_url(file_name, attachment_data.url)
+
+    if is_direct_message_type:
+        # TODO: hosted content URL for private chats have a different format, we
+        # can process it once we support converting direct messages.
+        return AttachmentConversionResult(updated_content=content, upload_records=upload_records)
+    else:
+        updated_content = regex.sub(
+            HOSTED_CONTENT_MARKDOWN_IMAGE_SYNTAX_REGEX,
+            export_file_and_get_zulip_url,
+            content,
+            flags=re.VERBOSE,
+        )
+        return AttachmentConversionResult(
+            updated_content=updated_content,
+            upload_records=upload_records,
+        )
+
+
+def process_messages(
+    added_teams: dict[str, TeamMetadata],
+    channel_metadata: dict[str, ChannelMetadata] | None,
+    do_download_and_export_upload_file: Callable[[ExportMessageAttachmentParameter], None],
+    domain_name: str,
+    messages: list[MicrosoftTeamsFieldsT],
+    microsoft_graph_api_token: str,
+    microsoft_teams_user_id_to_zulip_user_id: MicrosoftTeamsUserIdToZulipUserIdT,
+    realm: dict[str, Any],
+    realm_id: int,
+    subscriber_map: dict[int, set[int]],
+) -> MessageConversionResult:
+    zerver_usermessage: list[ZerverFieldsT] = []
+    zerver_messages: list[ZerverFieldsT] = []
+    accumulated_batch_upload_records: list[UploadRecordData] = []
+
+    for message in messages:
+        if is_microsoft_teams_event_message(message):
+            continue
+
+        message_content_type = message["Body"]["ContentType"]
+        if message_content_type == "html":
+            try:
+                content = convert_microsoft_teams_html_to_markdown(message["Body"]["Content"])
+            except Exception:  # nocoverage
+                logging.warning(
+                    "Error converting HTML to text for message: '%s'; continuing", content
+                )
+                logging.warning(str(message))
+                continue
+        else:  # nocoverage
+            logging.warning("Unable to convert this message content type: %s", message_content_type)
+            continue
+
+        # Determine message type, private or channel.
+        if message["ChannelIdentity"] is not None:
+            if channel_metadata is None:
+                raise AssertionError("Failed to build channel data.")
+            current_channel = channel_metadata[message["ChannelIdentity"]["ChannelId"]]
+            if current_channel.membership_type == "private":
+                # Don't include private channel messages.
+                continue
+            topic_name = current_channel.display_name
+            is_direct_message_type = False
+            recipient_id = added_teams[message["ChannelIdentity"]["TeamId"]].zulip_recipient_id
+        else:  # nocoverage
+            assert message["ChatId"] is not None
+            # TODO: Converting direct messages is not yet supported. Since
+            # subscription list and recipient map of direct message conversations
+            # are not listed, we have to manually build them as we iterate over
+            # the user messages.
+            continue
+
+        microsoft_teams_sender_id: str = get_microsoft_teams_sender_id_from_message(message)
+
+        if microsoft_teams_sender_id not in microsoft_teams_user_id_to_zulip_user_id:
+            create_is_mirror_dummy_user(
+                microsoft_teams_sender_id,
+                microsoft_teams_user_id_to_zulip_user_id,
+                realm,
+                realm_id,
+                domain_name,
+            )
+
+        message_id = NEXT_ID("message")
+        user_id = microsoft_teams_user_id_to_zulip_user_id[microsoft_teams_sender_id]
+        attachment_result = process_hosted_content_attachments(
+            content=content,
+            do_download_and_export_upload_file=do_download_and_export_upload_file,
+            is_direct_message_type=is_direct_message_type,
+            microsoft_graph_api_token=microsoft_graph_api_token,
+            realm_id=realm_id,
+            teams_message_id=message["Id"],
+            zulip_user_id=user_id,
+            zulip_message_id=message_id,
+        )
+        content = attachment_result.updated_content
+        accumulated_batch_upload_records += attachment_result.upload_records
+
+        zulip_message = build_message(
+            topic_name=topic_name,
+            date_sent=get_timestamp_from_message(message),
+            message_id=message_id,
+            content=content,
+            rendered_content=None,
+            user_id=user_id,
+            recipient_id=recipient_id,
+            realm_id=realm_id,
+            is_channel_message=not is_direct_message_type,
+            # Import will correct the message's has_image and has_link attributes.
+            has_image=False,
+            has_link=False,
+            has_attachment=attachment_result.upload_records != [],
+            is_direct_message_type=is_direct_message_type,
+        )
+        zerver_messages.append(zulip_message)
+
+        (num_created, num_skipped) = build_usermessages(
+            zerver_usermessage=zerver_usermessage,
+            subscriber_map=subscriber_map,
+            recipient_id=recipient_id,
+            mentioned_user_ids=[],
+            message_id=message_id,
+            is_private=is_direct_message_type,
+        )
+
+        logging.debug(
+            "Created %s UserMessages; deferred %s due to long-term idle",
+            num_created,
+            num_skipped,
+        )
+
+    return MessageConversionResult(
+        zerver_messages=zerver_messages,
+        zerver_usermessages=zerver_usermessage,
+        upload_records=accumulated_batch_upload_records,
+    )
+
+
+def get_batched_export_message_data(
+    message_data_paths: list[str], chunk_size: int = MESSAGE_BATCH_CHUNK_SIZE
+) -> Iterable[list[MicrosoftTeamsFieldsT]]:
+    batched_messages: list[MicrosoftTeamsFieldsT] = []
+    for path in message_data_paths:
+        messages = get_data_file(path)
+        # Teams export tool doesn't sort messages in chronological order.
+        # Sort Microsoft Teams messages by their ID, which is their date
+        # sent in unix time.
+        for message in sorted(messages, key=lambda m: int(m["Id"])):
+            if len(batched_messages) == chunk_size:
+                yield batched_messages
+                batched_messages.clear()
+            batched_messages.append(message)
+
+    if batched_messages:
+        yield batched_messages
+
+
+def convert_messages(
+    added_teams: dict[str, TeamMetadata],
+    domain_name: str,
+    microsoft_graph_api_token: str,
+    microsoft_teams_user_id_to_zulip_user_id: MicrosoftTeamsUserIdToZulipUserIdT,
+    output_dir: str,
+    processes: int,
+    realm_id: int,
+    realm: dict[str, Any],
+    teams_data_dir: str,
+    chunk_size: int = MESSAGE_BATCH_CHUNK_SIZE,
+) -> tuple[list[UploadRecordData], list[AttachmentRecordData]]:
+    microsoft_teams_channel_metadata: dict[str, ChannelMetadata] = {}
+    subscriber_map = make_subscriber_map(
+        zerver_subscription=realm["zerver_subscription"],
+    )
+    team_data_folders = []
+    for f in os.listdir(teams_data_dir):
+        path = os.path.join(teams_data_dir, f)
+        if os.path.isdir(path):
+            team_data_folders.append(f)
+
+    message_file_paths = []
+    for team_id in team_data_folders:
+        team_data_folder = os.path.join(teams_data_dir, team_id)
+        team_messages_file_path = os.path.join(team_data_folder, f"messages_{team_id}.json")
+        message_file_paths.append(team_messages_file_path)
+
+        team_channels_list = get_data_file(
+            os.path.join(team_data_folder, f"channels_{team_id}.json")
+        )
+        for team_channel in team_channels_list:
+            microsoft_teams_channel_metadata[team_channel["Id"]] = ChannelMetadata(
+                display_name=team_channel["DisplayName"],
+                is_favourite_by_default=team_channel["IsFavoriteByDefault"],
+                is_archived=team_channel["IsArchived"],
+                is_favorite_by_default=team_channel["IsFavoriteByDefault"],
+                membership_type=team_channel["MembershipType"],
+                team_id=team_id,
+            )
+
+    # Attachment record data is only fully known after the file is
+    # downloaded, so records must be built inside the download
+    # workers. When processes > 1, workers run in separate processes
+    # and can't share memory with the parent, so we use a
+    # multiprocessing-shared list to gather their results.
+    #
+    # A nicer approach worth considering in the future would be to
+    # have run_parallel_queue return futures that the parent can
+    # wait on and collect results from, avoiding this shared-state
+    # coordination entirely.
+    total_attachment_records: list[AttachmentRecordData] | ListProxy = (  # type: ignore[type-arg] # ListProxy is not subscriptable
+        Manager().list() if processes > 1 else []
+    )
+
+    total_accumulated_upload_records: list[UploadRecordData] = []
+    with run_parallel_queue(
+        partial(
+            download_and_export_microsoft_teams_upload_file, total_attachment_records, output_dir
+        ),
+        processes,
+        catch=True,
+        report_every=100,
+        report=lambda count: logging.info("Downloaded %s attachments", count),
+    ) as do_download_and_export_upload_file:
+        dump_file_id = 1
+        for message_chunk in get_batched_export_message_data(message_file_paths, chunk_size):
+            conversion_result = process_messages(
+                added_teams=added_teams,
+                channel_metadata=microsoft_teams_channel_metadata,
+                do_download_and_export_upload_file=do_download_and_export_upload_file,
+                domain_name=domain_name,
+                messages=message_chunk,
+                microsoft_graph_api_token=microsoft_graph_api_token,
+                microsoft_teams_user_id_to_zulip_user_id=microsoft_teams_user_id_to_zulip_user_id,
+                realm=realm,
+                realm_id=realm_id,
+                subscriber_map=subscriber_map,
+            )
+            total_accumulated_upload_records += conversion_result.upload_records
+            create_converted_data_files(
+                dict(
+                    zerver_message=conversion_result.zerver_messages,
+                    zerver_usermessage=conversion_result.zerver_usermessages,
+                ),
+                output_dir,
+                f"/messages-{dump_file_id:06}.json",
+            )
+            dump_file_id += 1
+
+    scrubbed_data = scrub_missing_upload_records_after_download(
+        output_dir,
+        # Attachment record data is built inside download workers only
+        # on successful downloads, so there should be no stale records.
+        zerver_attachments=[],
+        upload_records=total_accumulated_upload_records,
+    )
+    total_accumulated_upload_records = scrubbed_data.upload_records
+
+    def fix_attachment_id(attachment: AttachmentRecordData) -> AttachmentRecordData:
+        attachment.id = NEXT_ID("attachment")
+        return attachment
+
+    return (
+        total_accumulated_upload_records,
+        list(map(fix_attachment_id, total_attachment_records)),
+    )
+
+
+def do_convert_directory(
+    microsoft_teams_dir: str,
+    output_dir: str,
+    microsoft_graph_api_token: str,
+    processes: int = 6,
+) -> None:
+    os.makedirs(output_dir, exist_ok=True)
+    if os.listdir(output_dir):  # nocoverage
+        raise Exception("Output directory should be empty!")
+    users_data_dir = os.path.join(microsoft_teams_dir, "users")
+    if not os.path.isdir(users_data_dir):  # nocoverage
+        raise ValueError("Import does not have the layout we expect from a Microsoft Teams export!")
+
+    realm_id = 0
+    domain_name = SplitResult("", settings.EXTERNAL_HOST, "", "", "").hostname
+    assert isinstance(domain_name, str)
+
+    NOW = float(timezone_now().timestamp())
+    zerver_realm: list[ZerverFieldsT] = build_zerver_realm(realm_id, "", NOW, "Microsoft Teams")
+    realm = build_realm(zerver_realm, realm_id, domain_name, import_source="microsoft_teams")
+    realm["zerver_stream"] = []
+    realm["zerver_defaultstream"] = []
+    realm["zerver_recipient"] = []
+    realm["zerver_subscription"] = []
+
+    microsoft_teams_user_id_to_zulip_user_id = convert_users(
+        microsoft_teams_user_role_data=get_user_roles(microsoft_graph_api_token),
+        realm=realm,
+        realm_id=realm_id,
+        timestamp=int(NOW),
+        users_list=get_data_file(os.path.join(users_data_dir, "usersList.json")),
+    )
+
+    teams_data_dir = os.path.join(microsoft_teams_dir, "teams")
+
+    added_teams = convert_teams_to_channels(
+        microsoft_teams_user_id_to_zulip_user_id=microsoft_teams_user_id_to_zulip_user_id,
+        realm=realm,
+        realm_id=realm_id,
+        teams_data_dir=teams_data_dir,
+    )
+
+    upload_records, attachment_records = convert_messages(
+        added_teams=added_teams,
+        domain_name=domain_name,
+        microsoft_graph_api_token=microsoft_graph_api_token,
+        microsoft_teams_user_id_to_zulip_user_id=microsoft_teams_user_id_to_zulip_user_id,
+        output_dir=output_dir,
+        processes=processes,
+        realm_id=realm_id,
+        realm=realm,
+        teams_data_dir=teams_data_dir,
+    )
+
+    create_converted_data_files(realm, output_dir, "/realm.json")
+    create_converted_data_files(upload_records, output_dir, "/uploads/records.json")
+    attachment: dict[str, list[Any]] = {"zerver_attachment": attachment_records}
+    create_converted_data_files(attachment, output_dir, "/attachment.json")
+    # TODO:
+    create_converted_data_files([], output_dir, "/emoji/records.json")
+    create_converted_data_files([], output_dir, "/avatars/records.json")
+    create_converted_data_files([], output_dir, "/realm_icons/records.json")
+    do_common_export_processes(output_dir)
+
+    logging.info("######### DATA CONVERSION FINISHED #########\n")
+    logging.info("Zulip data dump created at %s", output_dir)

zerver/data_import/microsoft_teams_message_conversion.py

@@ -0,0 +1,41 @@
+import re
+
+from bs4 import Tag
+from typing_extensions import override
+
+from zerver.lib.markdown import get_markdown_image_for_url
+from zerver.lib.markdown.from_html import ZulipMarkdownConverter, convert_html_to_markdown
+
+# https://learn.microsoft.com/en-us/graph/api/chatmessagehostedcontent-get
+HOSTED_CONTENT_GRAPH_API_URL_REGEX = r"https://graph\.microsoft\.com/v1\.0/teams/[^/]+/channels/[^/]+/messages/[^/]+/hostedContents/[^/]+/\$value"
+HOSTED_CONTENT_MARKDOWN_IMAGE_SYNTAX_REGEX = rf"""
+            !\[
+               (?P<file_name>[^\]]+)
+            \]\(
+               (?P<api_url>{HOSTED_CONTENT_GRAPH_API_URL_REGEX})
+            \)
+            """
+
+
+class MicrosoftTeamsToZulipMarkdownConverter(ZulipMarkdownConverter):
+    """HTML-to-Markdown converter for Microsoft Teams exports.
+
+    Hosted-content images are kept in inline `![alt](src)` form, which
+    process_hosted_content_attachments matches to download each image
+    and rewrite it into a Zulip upload link.  Other images get the base
+    class's external-link treatment.
+    """
+
+    @override
+    def convert_img(self, el: Tag, text: str, parent_tags: set[str]) -> str:
+        src = el.get("src", "")
+        alt = el.get("alt", "")
+        assert isinstance(src, str)
+        assert isinstance(alt, str)
+        if re.fullmatch(HOSTED_CONTENT_GRAPH_API_URL_REGEX, src):
+            return get_markdown_image_for_url(alt, src)
+        return super().convert_img(el, text, parent_tags)
+
+
+def convert_microsoft_teams_html_to_markdown(html: str) -> str:
+    return convert_html_to_markdown(html, converter_class=MicrosoftTeamsToZulipMarkdownConverter)

zerver/data_import/sequencer.py

@@ -0,0 +1,74 @@
+from collections.abc import Callable
+from typing import Generic, TypeVar
+
+"""
+This module helps you set up a bunch
+of sequences, similar to how database
+sequences work.
+
+You need to be a bit careful here, since
+you're dealing with a big singleton, but
+for data imports that's usually easy to
+manage.
+"""
+
+T = TypeVar("T")
+
+
+def _seq() -> Callable[[], int]:
+    i = 0
+
+    def next_one() -> int:
+        nonlocal i
+        i += 1
+        return i
+
+    return next_one
+
+
+def sequencer() -> Callable[[str], int]:
+    """
+    Use like this:
+
+    NEXT_ID = sequencer()
+    message_id = NEXT_ID('message')
+    """
+    seq_dict: dict[str, Callable[[], int]] = {}
+
+    def next_one(name: str) -> int:
+        if name not in seq_dict:
+            seq_dict[name] = _seq()
+        seq = seq_dict[name]
+        return seq()
+
+    return next_one
+
+
+"""
+NEXT_ID is a singleton used by an entire process, which is
+almost always reasonable.  If you want to have two parallel
+sequences, just use different `name` values.
+
+This object gets created once and only once during the first
+import of the file.
+"""
+
+NEXT_ID = sequencer()
+
+
+class IdMapper(Generic[T]):
+    def __init__(self) -> None:
+        self.map: dict[T, int] = {}
+        self.cnt = 0
+
+    def has(self, their_id: T) -> bool:
+        return their_id in self.map
+
+    def get(self, their_id: T) -> int:
+        if their_id in self.map:
+            return self.map[their_id]
+
+        self.cnt += 1
+        our_id = self.cnt
+        self.map[their_id] = our_id
+        return our_id

zerver/data_import/slack_message_conversion.py

@@ -0,0 +1,501 @@
+import contextlib
+import re
+from datetime import datetime, timezone
+from itertools import zip_longest
+from typing import Any, Literal, TypeAlias, TypedDict, cast
+
+import regex
+from django.core.exceptions import ValidationError
+from requests.utils import requote_uri
+
+from zerver.lib.timestamp import datetime_to_global_time
+from zerver.lib.types import Validator
+from zerver.lib.validator import (
+    WildValue,
+    check_dict,
+    check_int,
+    check_list,
+    check_string,
+    check_string_in,
+    check_url,
+)
+
+# stubs
+ZerverFieldsT: TypeAlias = dict[str, Any]
+SlackToZulipUserIDT: TypeAlias = dict[str, int]
+AddedChannelsT: TypeAlias = dict[str, tuple[str, int]]
+SlackFieldsT: TypeAlias = dict[str, Any]
+
+# Slack link can be in the format <http://www.foo.com|www.foo.com> and <http://foo.com/>
+LINK_REGEX = r"""
+              (<)                                                              # match '>'
+              (http:\/\/www\.|https:\/\/www\.|http:\/\/|https:\/\/|ftp:\/\/)?  # protocol and www
+                  ([a-z0-9]+([\-\.]{1}[a-z0-9]+)*)(\.)                         # domain name
+                      ([a-z]{2,63}(:[0-9]{1,5})?)                              # domain
+                  (\/[^>]*)?                                                   # path
+              (\|)?(?:\|([^>]+))?                                # char after pipe (for Slack links)
+              (>)
+              """
+
+SLACK_MAILTO_REGEX = r"""
+                      <((mailto:)?                     # match  `<mailto:`
+                      ([\w\.-]+@[\w\.-]+(\.[\w]+)+))   # match email
+                          (\|)?                        # match pipe
+                      ([\w\.-]+@[\w\.-]+(\.[\w]+)+)?>  # match email
+                      """
+
+SLACK_USERMENTION_REGEX = r"""
+                           (<@)                  # Start with '<@'
+                               ([a-zA-Z0-9]+)    # Here we have the Slack id
+                           (\|)?                 # We not always have a vertical line in mention
+                               ([a-zA-Z0-9]+)?   # If vertical line is present, this is short name
+                           (>)                   # ends with '>'
+                           """
+# Slack doesn't have mid-word message-formatting like Zulip.
+# Hence, ~stri~ke doesn't format the word in Slack, but ~~stri~~ke
+# formats the word in Zulip
+SLACK_STRIKETHROUGH_REGEX = r"""
+                             (
+                                # Capture punctuation (\p{P}), white space (\p{Zs}),
+                                # symbols (\p{S}) or newline.
+                                # Skip ~ to not reformat the same string twice
+                                # Skip @ and \
+                                # Skip closing brackets & closing quote (\p{Pf}\p{Pe})
+                                (?![~`@\\\p{Pf}\p{Pe}])
+                                [\p{P}\p{Zs}\p{S}]|^
+                             )
+                             (\~)                                  # followed by a ~
+                                 ([^~]+)                           # any character except ~
+                             (\~)                                  # followed by a ~
+                             (
+                                # Capture punctuation, white space, symbols or end of
+                                # line.
+                                # Skip ~ to not reformat the same string twice
+                                # Skip @ and \
+                                # Skip opening brackets & opening quote (\p{Pi}\p{Ps})
+                                (?![~`@\\\p{Pi}\p{Ps}])
+                                (?=[\p{P}\p{Zs}\p{S}]|$)
+                             )
+                             """
+SLACK_ITALIC_REGEX = r"""
+                      # Same as `SLACK_STRIKETHROUGH_REGEX`s. The difference
+                      # being, this skips _ instead of ~
+                      (
+                        (?![_`@\\\p{Pf}\p{Pe}])
+                        [\p{P}\p{Zs}\p{S}]|^
+                      )
+                      (\_)
+                          ([^_]+)                    # any character except _
+                      (\_)
+                      (
+                        (?![_`@\\\p{Pi}\p{Ps}])
+                        (?=[\p{P}\p{Zs}\p{S}]|$)
+                      )
+                      """
+SLACK_BOLD_REGEX = r"""
+                    # Same as `SLACK_STRIKETHROUGH_REGEX`s. The difference
+                    # being, this skips * instead of ~
+                    (
+                        (?![*`@\\\p{Pf}\p{Pe}])
+                        [\p{P}\p{Zs}\p{S}]|^
+                    )
+                    (\*)
+                        ([^*]+)                       # any character except *
+                    (\*)
+                    (
+                        (?![*`@\\\p{Pi}\p{Ps}])
+                        (?=[\p{P}\p{Zs}\p{S}]|$)
+                    )
+                    """
+
+FALLBACK_SLACK_USER_FULL_NAME = "Slack user {id}"
+
+
+def get_user_full_name(user: ZerverFieldsT) -> str:
+    if "deleted" in user and user["deleted"] is False:
+        return user["real_name"] or user["name"]
+    elif user["is_mirror_dummy"]:
+        return user["profile"].get("real_name", user["name"])
+    else:
+        return user["name"]
+
+
+def get_zulip_mention_for_slack_user(
+    slack_user_id: str | None,
+    slack_user_shortname: str | None,
+    users: list[ZerverFieldsT],
+    silent: bool = False,
+) -> str | None:
+    if slack_user_id:
+        for user in users:
+            if user["id"] == slack_user_id and (
+                slack_user_shortname is None or user["name"] == slack_user_shortname
+            ):
+                return ("@_**" if silent else "@**") + get_user_full_name(user) + "**"
+    return None
+
+
+def get_user_mentions(
+    token: str,
+    users: list[ZerverFieldsT],
+    slack_user_id_to_zulip_user_id: SlackToZulipUserIDT,
+) -> tuple[str, int | None]:
+    slack_usermention_match = re.search(SLACK_USERMENTION_REGEX, token, re.VERBOSE)
+    assert slack_usermention_match is not None
+    short_name = slack_usermention_match.group(4)
+    slack_id = slack_usermention_match.group(2)
+    zulip_mention = get_zulip_mention_for_slack_user(slack_id, short_name, users)
+    if zulip_mention is not None:
+        token = re.sub(SLACK_USERMENTION_REGEX, zulip_mention, token, flags=re.VERBOSE)
+        user_id = slack_user_id_to_zulip_user_id[slack_id]
+        return token, user_id
+    return token, None
+
+
+def convert_link_format(text: str) -> tuple[str, bool]:
+    """
+    1. Converts '<https://foo.com>' to 'https://foo.com'
+    2. Converts '<https://foo.com|foo>' to '[foo](https://foo.com)'
+    """
+    has_link = False
+    for match in re.finditer(LINK_REGEX, text, re.VERBOSE):
+        slack_url = match.group(0)
+        url_parts = slack_url[1:-1].split("|", maxsplit=1)
+        # Check if there's a pipe with text after it
+        if len(url_parts) == 2:
+            converted_url = f"[{url_parts[1]}]({url_parts[0]})"
+        else:
+            converted_url = url_parts[0]
+
+        has_link = True
+        text = text.replace(slack_url, converted_url)
+    return text, has_link
+
+
+def convert_mailto_format(text: str) -> tuple[str, bool]:
+    """
+    1. Converts '<mailto:foo@foo.com>' to 'mailto:foo@foo.com'
+    2. Converts '<mailto:foo@foo.com|foo@foo.com>' to 'mailto:foo@foo.com'
+    """
+    has_link = False
+    for match in re.finditer(SLACK_MAILTO_REGEX, text, re.VERBOSE):
+        has_link = True
+        text = text.replace(match.group(0), match.group(1))
+    return text, has_link
+
+
+# Map italic, bold and strikethrough Markdown
+def convert_markdown_syntax(text: str, pattern: str, zulip_keyword: str) -> str:
+    """
+    Returns:
+    1. For strikethrough formatting: This maps Slack's '~strike~' to Zulip's '~~strike~~'
+    2. For bold formatting: This maps Slack's '*bold*' to Zulip's '**bold**'
+    3. For italic formatting: This maps Slack's '_italic_' to Zulip's '*italic*'
+    """
+
+    def replace_slack_format(match: regex.Match[str]) -> str:
+        return match.group(1) + zulip_keyword + match.group(3) + zulip_keyword
+
+    return regex.sub(pattern, replace_slack_format, text, flags=re.VERBOSE | re.MULTILINE)
+
+
+def convert_slack_workspace_mentions(text: str) -> str:
+    # Map Slack's '<!everyone>', '<!channel>' and '<!here>'
+    # mentions to Zulip's '@**all**' wildcard mention.
+    # No regex for these as they can be present anywhere
+    # in the sentence.
+    text = text.replace("<!everyone>", "@**all**")
+    text = text.replace("<!channel>", "@**all**")
+    text = text.replace("<!here>", "@**all**")
+    return text
+
+
+def convert_slack_formatting(text: str) -> str:
+    text = convert_markdown_syntax(text, SLACK_BOLD_REGEX, "**")
+    text = convert_markdown_syntax(text, SLACK_STRIKETHROUGH_REGEX, "~~")
+    text = convert_markdown_syntax(text, SLACK_ITALIC_REGEX, "*")
+    return text
+
+
+# Markdown mapping
+def convert_to_zulip_markdown(
+    text: str,
+    users: list[ZerverFieldsT],
+    added_channels: AddedChannelsT,
+    slack_user_id_to_zulip_user_id: SlackToZulipUserIDT,
+) -> tuple[str, list[int], bool]:
+    mentioned_users_id = []
+    text = convert_slack_formatting(text)
+    text = convert_slack_workspace_mentions(text)
+
+    # Map Slack channel mention: '<#C5Z73A7RA|general>' to '#**general**'
+    for cname, ids in added_channels.items():
+        cid = ids[0]
+        text = text.replace(f"<#{cid}|{cname}>", "#**" + cname + "**")
+
+    tokens = text.split(" ")
+    for iterator in range(len(tokens)):
+        # Check user mentions and change mention format from
+        # '<@slack_id|short_name>' to '@**full_name**'
+        if re.findall(SLACK_USERMENTION_REGEX, tokens[iterator], re.VERBOSE):
+            tokens[iterator], user_id = get_user_mentions(
+                tokens[iterator], users, slack_user_id_to_zulip_user_id
+            )
+            if user_id is not None:
+                mentioned_users_id.append(user_id)
+
+    text = " ".join(tokens)
+
+    # Check and convert link format
+    text, has_link = convert_link_format(text)
+    # convert `<mailto:foo@foo.com>` to `mailto:foo@foo.com`
+    text, has_mailto_link = convert_mailto_format(text)
+
+    message_has_link = has_link or has_mailto_link
+
+    return text, mentioned_users_id, message_has_link
+
+
+class LossyConversionError(Exception):
+    pass
+
+
+def render_block(block: WildValue) -> str:
+    """
+    Raises `LossyConversionError` if it's a rich_text block type.
+    """
+    # https://api.slack.com/reference/block-kit/blocks
+    supported_types = {
+        "context",
+        "divider",
+        "header",
+        "image",
+        "section",
+    }
+    unhandled_types = {
+        # `call` is a block type we've observed in the wild in a Slack export,
+        # despite not being documented in
+        # https://docs.slack.dev/reference/block-kit/blocks/
+        # It likes maps to a request for a Slack call. If we can verify that,
+        # probably it would be worth replacing with a string indicating a Slack
+        # call occurred.
+        "call",
+        "contact_card",
+        "file",
+        "table",
+        "video",
+        # The "actions" block is used to format literal in-message clickable
+        # buttons and similar elements, which Zulip currently doesn't support.
+        # https://docs.slack.dev/reference/block-kit/blocks/actions-block
+        "actions",
+        "input",
+        "condition",
+    }
+    known_types = {
+        *supported_types,
+        *unhandled_types,
+        # rich_text is a special case. It probably composes the bulk of a message
+        # body since it's responsible for very basic text formatting--such as
+        # plain text, bold, italic, etc. Our current Slack text conversion module
+        # can't process this block since this is not formatted in Markdown.
+        "rich_text",
+    }
+    block_type = block["type"].tame(check_string_in(known_types))
+
+    if block_type in unhandled_types:
+        return ""
+    elif block_type == "rich_text":
+        raise LossyConversionError
+    elif block_type == "context" and block.get("elements"):
+        pieces = []
+        # Slack renders these pieces left-to-right, packed in as
+        # closely as possible.  We just render them above each other,
+        # for simplicity.
+        for element in block["elements"]:
+            element_type = element["type"].tame(check_string_in(["image", "plain_text", "mrkdwn"]))
+            if element_type == "image":
+                pieces.append(render_block_element(element))
+            else:
+                pieces.append(element.tame(check_text_block())["text"])
+        return "\n\n".join(piece.strip() for piece in pieces if piece.strip() != "")
+    elif block_type == "divider":
+        return "----"
+    elif block_type == "header":
+        return "## " + block["text"].tame(check_text_block(plain_text_only=True))["text"]
+    elif block_type == "image":
+        image_url = block["image_url"].tame(check_url)
+        alt_text = block["alt_text"].tame(check_string)
+        if "title" in block:
+            alt_text = block["title"].tame(check_text_block(plain_text_only=True))["text"]
+        return f"[{alt_text}]({image_url})"
+    elif block_type == "section":
+        pieces = []
+        if "text" in block:
+            pieces.append(block["text"].tame(check_text_block())["text"])
+
+        if "accessory" in block:
+            pieces.append(render_block_element(block["accessory"]))
+
+        if "fields" in block:
+            fields = block["fields"].tame(check_list(check_text_block()))
+            if len(fields) == 1:
+                # Special-case a single field to display a bit more
+                # nicely, without extraneous borders and limitations
+                # on its contents.
+                pieces.append(fields[0]["text"])
+            else:
+                # It is not possible to have newlines in a table, nor
+                # escape the pipes that make it up; replace them with
+                # whitespace.
+                field_text = [f["text"].replace("\n", " ").replace("|", " ") for f in fields]
+                # Because Slack formats this as two columns, but not
+                # necessarily a table with a bold header, we emit a
+                # blank header row first.
+                table = "| | |\n|-|-|\n"
+                # Then take the fields two-at-a-time to make the table
+                iters = [iter(field_text)] * 2
+                for left, right in zip_longest(*iters, fillvalue=""):
+                    table += f"| {left} | {right} |\n"
+                pieces.append(table)
+
+        return "\n\n".join(piece.strip() for piece in pieces if piece.strip() != "")
+
+    return ""  # nocoverage
+
+
+class TextField(TypedDict):
+    text: str
+    type: Literal["plain_text", "mrkdwn"]
+
+
+def check_text_block(plain_text_only: bool = False) -> Validator[TextField]:
+    if plain_text_only:
+        type_validator = check_string_in(["plain_text"])
+    else:
+        type_validator = check_string_in(["plain_text", "mrkdwn"])
+
+    def f(var_name: str, val: object) -> TextField:
+        block = check_dict(
+            [
+                ("type", type_validator),
+                ("text", check_string),
+            ],
+        )(var_name, val)
+
+        return cast(TextField, block)
+
+    return f
+
+
+def render_block_element(element: WildValue) -> str:
+    # https://api.slack.com/reference/block-kit/block-elements
+    # Zulip doesn't support interactive elements, so we only render images here
+    element_type = element["type"].tame(check_string)
+    if element_type == "image":
+        try:
+            image_url = element["image_url"].tame(check_url)
+            alt_text = element["alt_text"].tame(check_string)
+            return f"[{alt_text}]({image_url})"
+        except ValidationError:
+            # Just drop invalid image URLs, rather than drop the whole message.
+            return ""
+    else:
+        # Unsupported
+        return ""
+
+
+def render_attachment(attachment: WildValue) -> str:
+    # https://api.slack.com/reference/messaging/attachments
+    # Slack recommends the usage of "blocks" even within attachments; the
+    # rest of the fields we handle here are legacy fields. These fields are
+    # optional and may contain null values.
+    pieces = []
+    if attachment.get("title"):
+        title = attachment["title"].tame(check_string)
+        if attachment.get("title_link"):
+            title_link = attachment["title_link"].tame(check_url)
+            pieces.append(f"## [{title}]({title_link})")
+        else:
+            pieces.append(f"## {title}")
+    if attachment.get("pretext"):
+        pieces.append(attachment["pretext"].tame(check_string))
+    if attachment.get("text"):
+        pieces.append(attachment["text"].tame(check_string))
+    if "fields" in attachment:
+        fields = []
+        for field in attachment["fields"]:
+            if "title" in field and "value" in field and field["title"] and field["value"]:
+                title = field["title"].tame(check_string)
+                value = field["value"].tame(check_string)
+                fields.append(f"*{title}*: {value}")
+            elif field.get("title"):
+                title = field["title"].tame(check_string)
+                fields.append(f"*{title}*")
+            elif field.get("value"):
+                value = field["value"].tame(check_string)
+                fields.append(f"{value}")
+        pieces.append("\n".join(fields))
+    if attachment.get("blocks"):
+        for block in attachment["blocks"]:
+            # rich_text blocks in "attachments" elements don't have a
+            # corresponding raw value in the "text" field, so we can't
+            # fallback to the "text" field in this case like we can do
+            # with rich_text in message "blocks".
+            # TODO: We should use the raw rich_text strings instead.
+            with contextlib.suppress(LossyConversionError):  # nocoverage
+                pieces.append(render_block(block))
+    if image_url_wv := attachment.get("image_url"):
+        try:
+            image_url = image_url_wv.tame(check_url)
+        except ValidationError:  # nocoverage
+            image_url = image_url_wv.tame(check_string)
+            image_url = requote_uri(image_url)
+        pieces.append(f"[]({image_url})")
+    if attachment.get("footer"):
+        pieces.append(attachment["footer"].tame(check_string))
+    if attachment.get("ts"):
+        try:
+            time = attachment["ts"].tame(check_int)
+        except ValidationError as e:  # nocoverage
+            # In some cases Slack has the ts as a string with a float
+            # number. The reason is unknown, but we've observed it
+            # in the wild several times.
+            ts = attachment["ts"].tame(check_string)
+            try:
+                ts_float = float(ts)
+            except ValueError:
+                raise e
+
+            time = int(ts_float)
+        pieces.append(datetime_to_global_time(datetime.fromtimestamp(time, timezone.utc)))
+
+    return "\n\n".join(piece.strip() for piece in pieces if piece.strip() != "")
+
+
+def replace_links(text: str) -> str:
+    text, _ = convert_link_format(text)
+    text, _ = convert_mailto_format(text)
+    return text
+
+
+def process_slack_block_and_attachment(message: WildValue) -> str:
+    pieces: list[str] = []
+    raw_text = message["text"].tame(check_string)
+
+    if message.get("blocks"):
+        try:
+            pieces += map(render_block, message["blocks"])
+        except LossyConversionError:
+            # render_block doesn't yet handle the "rich_text" block, which is used
+            # for very basic text formatting. To avoid message content loss, process
+            # the message["text"] field instead if rich_text is used.
+            pieces.append(raw_text)
+
+    if set(pieces) in ({""}, set()):
+        pieces.append(raw_text)
+
+    # The attachments typically don't have a corresponding raw value in
+    # message["text"] and only add to it, so the output is appended unconditionally.
+    if message.get("attachments"):
+        pieces += map(render_attachment, message["attachments"])
+    return "\n".join(piece.strip() for piece in pieces if piece.strip() != "")

zerver/data_import/user_handler.py

@@ -0,0 +1,33 @@
+from typing import Any
+
+from zerver.data_import.import_util import validate_user_emails_for_import
+
+
+class UserHandler:
+    """
+    Our UserHandler class is a glorified wrapper
+    around the data that eventually goes into
+    zerver_userprofile.
+
+    The class helps us do things like map ids
+    to names for mentions.
+    """
+
+    def __init__(self) -> None:
+        self.id_to_user_map: dict[int, dict[str, Any]] = {}
+
+    def add_user(self, user: dict[str, Any]) -> None:
+        user_id = user["id"]
+        self.id_to_user_map[user_id] = user
+
+    def get_user(self, user_id: int) -> dict[str, Any]:
+        user = self.id_to_user_map[user_id]
+        return user
+
+    def get_all_users(self) -> list[dict[str, Any]]:
+        users = list(self.id_to_user_map.values())
+        return users
+
+    def validate_user_emails(self) -> None:
+        all_users = self.get_all_users()
+        validate_user_emails_for_import([user["delivery_email"] for user in all_users])

zerver/filters.py

@@ -0,0 +1,56 @@
+import re
+from typing import Any
+
+from django.conf import settings
+from django.http import HttpRequest
+from django.views.debug import SafeExceptionReporterFilter
+from typing_extensions import override
+
+
+class ZulipExceptionReporterFilter(SafeExceptionReporterFilter):
+    # Add _SALT to the standard list
+    hidden_settings = re.compile(
+        r"API|TOKEN|KEY|SECRET|PASS|SIGNATURE|HTTP_COOKIE|_SALT", flags=re.IGNORECASE
+    )
+
+    @override
+    def get_post_parameters(self, request: HttpRequest | None) -> dict[str, Any]:
+        post_data = super().get_post_parameters(request)
+
+        filtered_post = post_data.copy()
+        filtered_vars = [
+            "content",
+            "secret",
+            "password",
+            "key",
+            "api-key",
+            "subject",
+            "stream",
+            "subscriptions",
+            "to",
+            "csrfmiddlewaretoken",
+            "api_key",
+            "realm_counts",
+            "installation_counts",
+        ]
+
+        for var in filtered_vars:
+            if var in filtered_post:
+                filtered_post[var] = "**********"
+        return filtered_post
+
+    # We do not include Django settings in exception reports at all.
+    # Settings can contain secrets and other environment-specific
+    # details, and we rely on other mechanisms to inspect them when
+    # debugging. Returning an empty dict here tells ExceptionReporter
+    # to omit settings-related context entirely.
+    @override
+    def get_safe_settings(self) -> dict[str, Any]:
+        # In production, we do not include Django settings in exception reports at all.
+        # Settings can contain secrets and other environment-specific
+        # details.
+        if settings.PRODUCTION:
+            return {}
+
+        # In development, return the safe settings (secrets masked by hidden_settings regex)
+        return super().get_safe_settings()

zerver/forms.py

@@ -0,0 +1,626 @@
+import logging
+import re
+from typing import Any
+
+from django import forms
+from django.conf import settings
+from django.contrib.auth import authenticate, password_validation
+from django.contrib.auth.forms import AuthenticationForm, PasswordResetForm, SetPasswordForm
+from django.contrib.auth.tokens import PasswordResetTokenGenerator, default_token_generator
+from django.core.exceptions import ValidationError
+from django.core.validators import validate_email
+from django.http import HttpRequest
+from django.utils.translation import gettext as _
+from django.utils.translation import gettext_lazy
+from markupsafe import Markup
+from two_factor.forms import AuthenticationTokenForm as TwoFactorAuthenticationTokenForm
+from two_factor.utils import totp_digits
+from typing_extensions import override
+
+from zerver.actions.user_settings import do_change_password, do_change_user_setting
+from zerver.actions.users import do_send_password_reset_email
+from zerver.lib.captcha import CaptchaFormMixin
+from zerver.lib.email_validation import (
+    email_allowed_for_realm,
+    email_reserved_for_system_bots_error,
+    validate_is_not_disposable,
+)
+from zerver.lib.exceptions import JsonableError, RateLimitedError
+from zerver.lib.i18n import get_language_list
+from zerver.lib.name_restrictions import is_reserved_subdomain
+from zerver.lib.rate_limiter import (
+    RateLimitedObject,
+    rate_limit_request_by_ip,
+    readable_expiry_string_for_plaintext,
+    should_rate_limit,
+)
+from zerver.lib.subdomains import get_subdomain, is_root_domain_available
+from zerver.lib.users import check_full_name
+from zerver.models import PreregistrationRealm, Realm, UserProfile
+from zerver.models.realm_audit_logs import RealmAuditLog
+from zerver.models.realms import (
+    DisposableEmailError,
+    DomainNotAllowedForRealmError,
+    EmailContainsPlusError,
+    get_realm,
+)
+from zerver.models.users import get_user_by_delivery_email, is_cross_realm_bot_email
+from zproject.backends import (
+    check_password_strength,
+    email_auth_enabled,
+    email_belongs_to_ldap,
+    password_auth_enabled,
+)
+
+# We don't mark this error for translation, because it's displayed
+# only to MIT users.
+MIT_VALIDATION_ERROR = Markup(
+    "That user does not exist at MIT or is a"
+    ' <a href="https://ist.mit.edu/email-lists">mailing list</a>.'
+    " If you want to sign up an alias for Zulip,"
+    ' <a href="mailto:support@zulip.com">contact us</a>.'
+)
+
+INVALID_ACCOUNT_CREDENTIALS_ERROR = gettext_lazy("Incorrect email or password.")
+DEACTIVATED_ACCOUNT_ERROR = gettext_lazy(
+    "Your account {username} has been deactivated."
+    " Please contact your organization administrator to reactivate it."
+)
+PASSWORD_TOO_WEAK_ERROR = gettext_lazy("The password is too weak.")
+
+# Set Form.EmailField to match the default max_length on Model.EmailField,
+# can be removed when https://code.djangoproject.com/ticket/35119 is
+# completed.
+EMAIL_MAX_LENGTH = 254
+
+
+class OverridableValidationError(ValidationError):
+    pass
+
+
+def check_subdomain_available(subdomain: str, allow_reserved_subdomain: bool = False) -> None:
+    error_strings = {
+        "too short": _("Subdomain needs to have length 3 or greater."),
+        "extremal dash": _("Subdomain cannot start or end with a '-'."),
+        "bad character": _("Subdomain can only have lowercase letters, numbers, and '-'s."),
+        "unavailable": _("Subdomain is already in use. Please choose a different one."),
+        "reserved": _("Subdomain reserved. Please choose a different one."),
+    }
+
+    if subdomain == Realm.SUBDOMAIN_FOR_ROOT_DOMAIN:
+        if is_root_domain_available():
+            return
+        raise ValidationError(error_strings["unavailable"])
+    if subdomain[0] == "-" or subdomain[-1] == "-":
+        raise ValidationError(error_strings["extremal dash"])
+    if not re.match(r"^[a-z0-9-]*$", subdomain):
+        raise ValidationError(error_strings["bad character"])
+    if len(subdomain) < 3:
+        raise ValidationError(error_strings["too short"])
+    if Realm.objects.filter(string_id=subdomain).exists():
+        raise ValidationError(error_strings["unavailable"])
+    if is_reserved_subdomain(subdomain) and not allow_reserved_subdomain:
+        raise OverridableValidationError(
+            error_strings["reserved"],
+            "Pass --allow-reserved-subdomain to override",
+        )
+
+
+def email_not_system_bot(email: str) -> None:
+    if is_cross_realm_bot_email(email):
+        msg = email_reserved_for_system_bots_error(email)
+        code = msg
+        raise ValidationError(
+            msg,
+            code=code,
+            params=dict(deactivated=False),
+        )
+
+
+def email_is_not_disposable(email: str) -> None:
+    try:
+        validate_is_not_disposable(email)
+    except DisposableEmailError:
+        raise ValidationError(_("Please use your real email address."))
+
+
+class RealmDetailsForm(forms.Form):
+    realm_subdomain = forms.CharField(max_length=Realm.MAX_REALM_SUBDOMAIN_LENGTH, required=False)
+    realm_type = forms.TypedChoiceField(
+        coerce=int, choices=[(t["id"], t["name"]) for t in Realm.ORG_TYPES.values()]
+    )
+    realm_default_language = forms.ChoiceField(choices=[])
+    realm_name = forms.CharField(max_length=Realm.MAX_REALM_NAME_LENGTH)
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        # Since the superclass doesn't accept random extra kwargs, we
+        # remove it from the kwargs dict before initializing.
+        self.realm_creation = kwargs["realm_creation"]
+        del kwargs["realm_creation"]
+
+        super().__init__(*args, **kwargs)
+        self.fields["realm_default_language"] = forms.ChoiceField(
+            choices=[(lang["code"], lang["name"]) for lang in get_language_list()],
+            required=self.realm_creation,
+        )
+        self.fields["realm_type"].required = self.realm_creation
+        self.fields["realm_name"].required = self.realm_creation
+
+    def clean_realm_subdomain(self) -> str:
+        if not self.realm_creation:
+            # This field is only used if realm_creation
+            return ""
+
+        subdomain = self.cleaned_data["realm_subdomain"]
+        if "realm_in_root_domain" in self.data:
+            subdomain = Realm.SUBDOMAIN_FOR_ROOT_DOMAIN
+
+        check_subdomain_available(subdomain)
+        return subdomain
+
+
+# These are the matching field names for the options in
+# RealmAuditLog.HOW_REALM_CREATOR_FOUND_ZULIP_OPTIONS that
+# ask for more context when creating a new organization.
+HOW_FOUND_ZULIP_EXTRA_CONTEXT = {
+    "other": "how_realm_creator_found_zulip_other_text",
+    "ad": "how_realm_creator_found_zulip_where_ad",
+    "existing_user": "how_realm_creator_found_zulip_which_organization",
+    "review_site": "how_realm_creator_found_zulip_review_site",
+    "ai_chatbot": "how_realm_creator_found_zulip_which_ai_chatbot",
+}
+
+
+class HowFoundZulipFormMixin(forms.Form):
+    how_realm_creator_found_zulip = forms.ChoiceField(
+        choices=RealmAuditLog.HOW_REALM_CREATOR_FOUND_ZULIP_OPTIONS.items()
+    )
+    how_realm_creator_found_zulip_other_text = forms.CharField(max_length=100, required=False)
+    how_realm_creator_found_zulip_where_ad = forms.CharField(max_length=100, required=False)
+    how_realm_creator_found_zulip_which_organization = forms.CharField(
+        max_length=100, required=False
+    )
+    how_realm_creator_found_zulip_review_site = forms.CharField(max_length=100, required=False)
+    how_realm_creator_found_zulip_which_ai_chatbot = forms.CharField(max_length=100, required=False)
+
+
+class RegistrationForm(HowFoundZulipFormMixin, RealmDetailsForm):
+    MAX_PASSWORD_LENGTH = 100
+    full_name = forms.CharField(max_length=UserProfile.MAX_NAME_LENGTH)
+    # The required-ness of the password field gets overridden if it isn't
+    # actually required for a realm
+    password = forms.CharField(widget=forms.PasswordInput, max_length=MAX_PASSWORD_LENGTH)
+    enable_marketing_emails = forms.BooleanField(required=False)
+    email_address_visibility = forms.TypedChoiceField(
+        required=False,
+        coerce=int,
+        empty_value=None,
+        choices=list(UserProfile.EMAIL_ADDRESS_VISIBILITY_ID_TO_NAME_MAP.items()),
+    )
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        # Since the superclass doesn't except random extra kwargs, we
+        # remove it from the kwargs dict before initializing.
+        self.realm = kwargs.pop("realm", None)
+
+        super().__init__(*args, **kwargs)
+        if settings.TERMS_OF_SERVICE_VERSION is not None:
+            self.fields["terms"] = forms.BooleanField(required=True)
+        self.fields["how_realm_creator_found_zulip"].required = self.realm_creation
+
+    def clean_full_name(self) -> str:
+        try:
+            return check_full_name(
+                full_name_raw=self.cleaned_data["full_name"], user_profile=None, realm=self.realm
+            )
+        except JsonableError as e:
+            raise ValidationError(e.msg)
+
+    def clean_password(self) -> str:
+        password = self.cleaned_data["password"]
+        if self.fields["password"].required and not check_password_strength(password):
+            # The frontend code tries to stop the user from submitting the form with a weak password,
+            # but if the user bypasses that protection, this error code path will run.
+            raise ValidationError(str(PASSWORD_TOO_WEAK_ERROR))
+
+        return password
+
+
+class DemoRegistrationForm(CaptchaFormMixin, HowFoundZulipFormMixin, forms.Form):
+    terms = forms.BooleanField(required=False)
+    realm_type = forms.TypedChoiceField(
+        coerce=int, choices=[(t["id"], t["name"]) for t in Realm.ORG_TYPES.values()]
+    )
+    realm_default_language = forms.ChoiceField(choices=[])
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        self.realm_creation = True
+        super().__init__(*args, **kwargs)
+        if settings.TERMS_OF_SERVICE_VERSION is not None:
+            self.fields["terms"] = forms.BooleanField(required=True)
+
+        self.fields["realm_default_language"] = forms.ChoiceField(
+            choices=[(lang["code"], lang["name"]) for lang in get_language_list()],
+        )
+
+
+class ToSForm(forms.Form):
+    terms = forms.BooleanField(required=False)
+    enable_marketing_emails = forms.BooleanField(required=False)
+    email_address_visibility = forms.TypedChoiceField(
+        required=False,
+        coerce=int,
+        empty_value=None,
+        choices=list(UserProfile.EMAIL_ADDRESS_VISIBILITY_ID_TO_NAME_MAP.items()),
+    )
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        if settings.TERMS_OF_SERVICE_VERSION is not None:
+            self.fields["terms"] = forms.BooleanField(required=True)
+
+
+class HomepageForm(forms.Form):
+    email = forms.EmailField(max_length=EMAIL_MAX_LENGTH)
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        self.realm = kwargs.pop("realm", None)
+        self.from_multiuse_invite = kwargs.pop("from_multiuse_invite", False)
+        self.has_pending_invitation = kwargs.pop("has_pending_invitation", False)
+        self.require_password_backend = kwargs.pop("require_password_backend", False)
+        self.invited_as = kwargs.pop("invited_as", None)
+        super().__init__(*args, **kwargs)
+
+    def clean_email(self) -> str:
+        """Returns the email if and only if the user's email address is
+        allowed to join the realm they are trying to join."""
+        email = self.cleaned_data["email"]
+
+        # Otherwise, the user is trying to join a specific realm.
+        realm = self.realm
+        from_multiuse_invite = self.from_multiuse_invite
+
+        if realm is None:
+            raise ValidationError(
+                _("The organization you are trying to join using {email} does not exist.").format(
+                    email=email
+                )
+            )
+
+        if not from_multiuse_invite:
+            if realm.invite_required and not self.has_pending_invitation:
+                raise ValidationError(
+                    _(
+                        "Please request an invite for {email} from the organization administrator."
+                    ).format(email=email)
+                )
+            if self.require_password_backend and not password_auth_enabled(realm):
+                raise ValidationError(
+                    _("Can't join the organization: password authentication is not enabled.")
+                )
+
+        try:
+            email_allowed_for_realm(email, realm)
+        except DomainNotAllowedForRealmError:
+            raise ValidationError(
+                _(
+                    "Your email address, {email}, is not in one of the domains "
+                    "that are allowed to register for accounts in this organization."
+                ).format(email=email)
+            )
+        except DisposableEmailError:
+            raise ValidationError(_("Please use your real email address."))
+        except EmailContainsPlusError:
+            raise ValidationError(
+                _("Email addresses containing + are not allowed in this organization.")
+            )
+
+        if settings.BILLING_ENABLED:
+            from corporate.lib.registration import (
+                check_spare_licenses_available_for_registering_new_user,
+            )
+            from corporate.lib.stripe import LicenseLimitError
+
+            role = self.invited_as if self.invited_as is not None else UserProfile.ROLE_MEMBER
+            try:
+                check_spare_licenses_available_for_registering_new_user(realm, email, role=role)
+            except LicenseLimitError:
+                raise ValidationError(
+                    _(
+                        "New members cannot join this organization because all Zulip licenses are in use. Please contact the person who "
+                        "invited you and ask them to increase the number of licenses, then try again."
+                    )
+                )
+
+        return email
+
+
+class ImportRealmOwnerSelectionForm(forms.Form):
+    user_id = forms.IntegerField(required=False)
+
+
+class RealmCreationForm(CaptchaFormMixin, RealmDetailsForm):
+    # This form determines whether users can create a new realm.
+    email = forms.EmailField(
+        validators=[email_not_system_bot, email_is_not_disposable], max_length=EMAIL_MAX_LENGTH
+    )
+    import_from = forms.ChoiceField(
+        choices=PreregistrationRealm.IMPORT_FROM_CHOICES,
+        required=False,
+    )
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        kwargs["realm_creation"] = True
+        super().__init__(*args, **kwargs)
+
+    def clean_import_from(self) -> str:
+        # Convert "" to "none".
+        return self.cleaned_data["import_from"] or "none"
+
+
+class LoggingSetPasswordForm(SetPasswordForm[UserProfile]):
+    new_password1 = forms.CharField(
+        label=_("New password"),
+        widget=forms.PasswordInput(attrs={"autocomplete": "new-password"}),
+        strip=False,
+        help_text=password_validation.password_validators_help_text_html(),
+        max_length=RegistrationForm.MAX_PASSWORD_LENGTH,
+    )
+    new_password2 = forms.CharField(
+        label=_("New password confirmation"),
+        strip=False,
+        widget=forms.PasswordInput(attrs={"autocomplete": "new-password"}),
+        max_length=RegistrationForm.MAX_PASSWORD_LENGTH,
+    )
+    enable_marketing_emails = forms.BooleanField(required=False)
+
+    def clean_new_password1(self) -> str:
+        new_password = self.cleaned_data["new_password1"]
+        if not check_password_strength(new_password):
+            # The frontend code tries to stop the user from submitting the form with a weak password,
+            # but if the user bypasses that protection, this error code path will run.
+            raise ValidationError(str(PASSWORD_TOO_WEAK_ERROR))
+
+        return new_password
+
+    @override
+    def save(self, commit: bool = True) -> UserProfile:
+        do_change_password(self.user, self.cleaned_data["new_password1"], commit=commit)
+        enable_marketing_emails = self.cleaned_data["enable_marketing_emails"]
+        if enable_marketing_emails and not self.user.enable_marketing_emails:
+            do_change_user_setting(
+                self.user,
+                "enable_marketing_emails",
+                enable_marketing_emails,
+                acting_user=self.user,
+            )
+
+        return self.user
+
+
+class ZulipPasswordResetForm(PasswordResetForm):
+    @override
+    def save(
+        self,
+        domain_override: str | None = None,
+        subject_template_name: str = "registration/password_reset_subject.txt",
+        email_template_name: str = "registration/password_reset_email.html",
+        use_https: bool = False,
+        token_generator: PasswordResetTokenGenerator = default_token_generator,
+        from_email: str | None = None,
+        request: HttpRequest | None = None,
+        html_email_template_name: str | None = None,
+        extra_email_context: dict[str, Any] | None = None,
+    ) -> None:
+        """
+        If the email address has an account in the target realm,
+        generates a one-use only link for resetting password and sends
+        to the user.
+
+        We send a different email if an associated account does not exist in the
+        database, or an account does exist, but not in the realm.
+
+        Note: We ignore protocol and the various email template arguments (those
+        are an artifact of using Django's password reset framework).
+        """
+        email = self.cleaned_data["email"]
+        # The form is only used in zerver.views.auth.password_rest, we know that
+        # the request must not be None
+        assert request is not None
+
+        realm = get_realm(get_subdomain(request))
+
+        if not email_auth_enabled(realm):
+            logging.info(
+                "Password reset attempted for %s even though password auth is disabled.", email
+            )
+            return
+        if email_belongs_to_ldap(realm, email):
+            # TODO: Ideally, we'd provide a user-facing error here
+            # about the fact that they aren't allowed to have a
+            # password in the Zulip server and should change it in LDAP.
+            logging.info("Password reset not allowed for user in LDAP domain")
+            return
+        if realm.deactivated:
+            logging.info("Realm is deactivated")
+            return
+
+        if should_rate_limit(request):
+            try:
+                rate_limit_password_reset_form_by_email(email)
+                rate_limit_request_by_ip(request, domain="sends_email_by_ip")
+            except RateLimitedError:
+                logging.info(
+                    "Too many password reset attempts for email %s from %s",
+                    email,
+                    request.META["REMOTE_ADDR"],
+                )
+                # The view will handle the RateLimit exception and render an appropriate page
+                raise
+
+        try:
+            user = get_user_by_delivery_email(email, realm)
+        except UserProfile.DoesNotExist:
+            user = None
+
+        do_send_password_reset_email(
+            email, realm, user, token_generator=token_generator, request=request
+        )
+
+
+class RateLimitedPasswordResetByEmail(RateLimitedObject):
+    def __init__(self, email: str) -> None:
+        self.email = email
+        super().__init__()
+
+    @override
+    def key(self) -> str:
+        return f"{type(self).__name__}:{self.email}"
+
+    @override
+    def rules(self) -> list[tuple[int, int]]:
+        return settings.RATE_LIMITING_RULES["password_reset_form_by_email"]
+
+
+def rate_limit_password_reset_form_by_email(email: str) -> None:
+    ratelimited, secs_to_freedom = RateLimitedPasswordResetByEmail(email).rate_limit()
+    if ratelimited:
+        raise RateLimitedError(secs_to_freedom)
+
+
+class CreateUserForm(forms.Form):
+    full_name = forms.CharField(max_length=100)
+    email = forms.EmailField(max_length=EMAIL_MAX_LENGTH)
+
+
+class OurAuthenticationForm(AuthenticationForm):
+    logger = logging.getLogger("zulip.auth.OurAuthenticationForm")
+
+    @override
+    def clean(self) -> dict[str, Any]:
+        username = self.cleaned_data.get("username")
+        password = self.cleaned_data.get("password")
+
+        if username is not None and password:
+            assert self.request is not None
+            subdomain = get_subdomain(self.request)
+            realm = get_realm(subdomain)
+
+            return_data: dict[str, Any] = {}
+            try:
+                self.user_cache = authenticate(
+                    request=self.request,
+                    username=username,
+                    password=password,
+                    realm=realm,
+                    return_data=return_data,
+                )
+            except RateLimitedError as e:
+                assert e.secs_to_freedom is not None
+                secs_to_freedom = int(e.secs_to_freedom)
+                retry_after_string = readable_expiry_string_for_plaintext(secs_to_freedom)
+                error_message = _(
+                    "You're making too many attempts to sign in."
+                    " Try again in {retry_after_string} or contact your organization administrator"
+                    " for help."
+                )
+                raise ValidationError(error_message.format(retry_after_string=retry_after_string))
+
+            if return_data.get("inactive_realm"):
+                raise AssertionError("Programming error: inactive realm in authentication form")
+
+            if return_data.get("password_reset_needed"):
+                raise ValidationError(
+                    _(
+                        "Your password has been disabled because it is too weak. "
+                        "Reset your password to create a new one."
+                    )
+                )
+
+            if return_data.get("inactive_user") and not return_data.get("is_mirror_dummy"):
+                # We exclude mirror dummy accounts here. They should be treated as the
+                # user never having had an account, so we let them fall through to the
+                # normal invalid_login case below.
+                error_message = DEACTIVATED_ACCOUNT_ERROR.format(username=username)
+                raise ValidationError(error_message)
+
+            if return_data.get("invalid_subdomain"):
+                self.logger.info(
+                    "User attempted password login to wrong subdomain %s. Matching accounts: %s",
+                    subdomain,
+                    return_data.get("matching_user_ids_in_different_realms"),
+                )
+                # We don't want to leak information by revealing there are matching accounts
+                # on different subdomain - so we just fall through to the default error.
+                assert self.user_cache is None
+
+            if self.user_cache is None:
+                raise forms.ValidationError(
+                    INVALID_ACCOUNT_CREDENTIALS_ERROR,
+                )
+
+            self.confirm_login_allowed(self.user_cache)
+
+        return self.cleaned_data
+
+    @override
+    def add_prefix(self, field_name: str) -> str:
+        """Disable prefix, since Zulip doesn't use this Django forms feature
+        (and django-two-factor does use it), and we'd like both to be
+        happy with this form.
+        """
+        return field_name
+
+
+class AuthenticationTokenForm(TwoFactorAuthenticationTokenForm):
+    """
+    We add this form to update the widget of otp_token. The default
+    widget is an input element whose type is a number, which doesn't
+    stylistically match our theme.
+    """
+
+    otp_token = forms.IntegerField(
+        label=_("Token"), min_value=0, max_value=int("9" * totp_digits()), widget=forms.TextInput
+    )
+
+
+class MultiEmailField(forms.Field):
+    @override
+    def to_python(self, emails: str | None) -> list[str]:
+        """Normalize data to a list of strings."""
+        if not emails:
+            return []
+
+        return [email.strip() for email in emails.split(",")]
+
+    @override
+    def validate(self, emails: list[str]) -> None:
+        """Check if value consists only of valid emails."""
+        super().validate(emails)
+        for email in emails:
+            validate_email(email)
+
+
+class FindMyTeamForm(forms.Form):
+    emails = MultiEmailField(
+        help_text=_("Tip: You can enter multiple email addresses with commas between them.")
+    )
+
+    def clean_emails(self) -> list[str]:
+        emails = self.cleaned_data["emails"]
+        if len(emails) > 10:
+            raise forms.ValidationError(_("Please enter at most 10 emails."))
+
+        return emails
+
+
+class RealmRedirectForm(forms.Form):
+    subdomain = forms.CharField(max_length=Realm.MAX_REALM_SUBDOMAIN_LENGTH, required=True)
+
+    def clean_subdomain(self) -> str:
+        subdomain = self.cleaned_data["subdomain"]
+        try:
+            get_realm(subdomain)
+        except Realm.DoesNotExist:
+            raise ValidationError(_("We couldn't find that Zulip organization."))
+        return subdomain

zerver/lib/addressee.py

@@ -0,0 +1,225 @@
+from collections.abc import Iterable, Sequence
+from typing import cast
+
+from django.utils.translation import gettext as _
+
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.string_validation import check_stream_topic
+from zerver.lib.topic import (
+    maybe_rename_general_chat_to_empty_topic,
+    maybe_rename_no_topic_to_empty_topic,
+)
+from zerver.models import Realm, Stream, UserProfile
+from zerver.models.users import (
+    get_user_by_id_in_realm_including_cross_realm,
+    get_user_including_cross_realm,
+)
+
+
+def get_user_profiles(emails: Iterable[str], realm: Realm) -> list[UserProfile]:
+    user_profiles: list[UserProfile] = []
+    for email in emails:
+        try:
+            user_profile = get_user_including_cross_realm(email, realm)
+        except UserProfile.DoesNotExist:
+            raise JsonableError(_("Invalid email '{email}'").format(email=email))
+        user_profiles.append(user_profile)
+    return user_profiles
+
+
+def get_user_profiles_by_ids(user_ids: Iterable[int], realm: Realm) -> list[UserProfile]:
+    user_profiles: list[UserProfile] = []
+    for user_id in user_ids:
+        try:
+            user_profile = get_user_by_id_in_realm_including_cross_realm(user_id, realm)
+        except UserProfile.DoesNotExist:
+            raise JsonableError(_("Invalid user ID {user_id}").format(user_id=user_id))
+        user_profiles.append(user_profile)
+    return user_profiles
+
+
+class Addressee:
+    # This is really just a holder for vars that tended to be passed
+    # around in a non-type-safe way before this class was introduced.
+    #
+    # It also avoids some nonsense where you have to think about whether
+    # topic should be None or '' for a direct message, or you have to
+    # make an array of one stream.
+    #
+    # Eventually we can use this to cache Stream and UserProfile objects
+    # in memory.
+    #
+    # This should be treated as an immutable class.
+    def __init__(
+        self,
+        msg_type: str,
+        user_profiles: Sequence[UserProfile] | None = None,
+        stream: Stream | None = None,
+        stream_name: str | None = None,
+        stream_id: int | None = None,
+        topic_name: str | None = None,
+    ) -> None:
+        assert msg_type in ["stream", "private"]
+        if msg_type == "stream" and topic_name is None:
+            raise JsonableError(_("Missing topic"))
+        self._msg_type = msg_type
+        self._user_profiles = user_profiles
+        self._stream = stream
+        self._stream_name = stream_name
+        self._stream_id = stream_id
+        self._topic_name = topic_name
+
+    def is_stream(self) -> bool:
+        return self._msg_type == "stream"
+
+    def is_private(self) -> bool:
+        return self._msg_type == "private"
+
+    def user_profiles(self) -> Sequence[UserProfile]:
+        assert self.is_private()
+        assert self._user_profiles is not None
+        return self._user_profiles
+
+    def stream(self) -> Stream | None:
+        assert self.is_stream()
+        return self._stream
+
+    def stream_name(self) -> str | None:
+        assert self.is_stream()
+        return self._stream_name
+
+    def stream_id(self) -> int | None:
+        assert self.is_stream()
+        return self._stream_id
+
+    def topic_name(self) -> str:
+        assert self.is_stream()
+        assert self._topic_name is not None
+        return self._topic_name
+
+    def is_message_to_self(self, sender: UserProfile) -> bool:
+        return (
+            self.is_private()
+            and len(self.user_profiles()) == 1
+            and self.user_profiles()[0].id == sender.id
+        )
+
+    @staticmethod
+    def legacy_build(
+        sender: UserProfile,
+        recipient_type_name: str,
+        message_to: Sequence[int] | Sequence[str],
+        topic_name: str | None,
+        realm: Realm | None = None,
+    ) -> "Addressee":
+        # For legacy reason message_to used to be either a list of
+        # emails or a list of streams.  We haven't fixed all of our
+        # callers yet.
+        if realm is None:
+            realm = sender.realm
+
+        if recipient_type_name == "stream":
+            if len(message_to) > 1:
+                raise JsonableError(_("Cannot send to multiple channels"))
+
+            if message_to:
+                stream_name_or_id = message_to[0]
+            else:
+                # This is a hack to deal with the fact that we still support
+                # default streams (and the None will be converted later in the
+                # call path).
+                if sender.default_sending_stream_id:
+                    # Use the user's default stream
+                    stream_name_or_id = sender.default_sending_stream_id
+                else:
+                    raise JsonableError(_("Missing channel"))
+
+            if topic_name is None:
+                raise JsonableError(_("Missing topic"))
+
+            if isinstance(stream_name_or_id, int):
+                return Addressee.for_stream_id(stream_name_or_id, topic_name)
+
+            return Addressee.for_stream_name(stream_name_or_id, topic_name)
+        elif recipient_type_name == "private":
+            if not message_to:
+                raise JsonableError(_("Message must have recipients"))
+
+            if isinstance(message_to[0], str):
+                emails = cast(Sequence[str], message_to)
+                return Addressee.for_private(emails, realm)
+            elif isinstance(message_to[0], int):
+                user_ids = cast(Sequence[int], message_to)
+                return Addressee.for_user_ids(user_ids=user_ids, realm=realm)
+        else:
+            raise JsonableError(_("Invalid message type"))
+
+    @staticmethod
+    def for_stream(stream: Stream, topic_name: str) -> "Addressee":
+        topic_name = topic_name.strip()
+        topic_name = maybe_rename_general_chat_to_empty_topic(topic_name)
+        topic_name = maybe_rename_no_topic_to_empty_topic(topic_name)
+        check_stream_topic(topic_name)
+        return Addressee(
+            msg_type="stream",
+            stream=stream,
+            topic_name=topic_name,
+        )
+
+    @staticmethod
+    def for_stream_name(stream_name: str, topic_name: str) -> "Addressee":
+        topic_name = topic_name.strip()
+        topic_name = maybe_rename_general_chat_to_empty_topic(topic_name)
+        topic_name = maybe_rename_no_topic_to_empty_topic(topic_name)
+        check_stream_topic(topic_name)
+        return Addressee(
+            msg_type="stream",
+            stream_name=stream_name,
+            topic_name=topic_name,
+        )
+
+    @staticmethod
+    def for_stream_id(stream_id: int, topic_name: str) -> "Addressee":
+        topic_name = topic_name.strip()
+        topic_name = maybe_rename_general_chat_to_empty_topic(topic_name)
+        topic_name = maybe_rename_no_topic_to_empty_topic(topic_name)
+        check_stream_topic(topic_name)
+        return Addressee(
+            msg_type="stream",
+            stream_id=stream_id,
+            topic_name=topic_name,
+        )
+
+    @staticmethod
+    def for_private(emails: Sequence[str], realm: Realm) -> "Addressee":
+        assert len(emails) > 0
+        user_profiles = get_user_profiles(emails, realm)
+        return Addressee(
+            msg_type="private",
+            user_profiles=user_profiles,
+        )
+
+    @staticmethod
+    def for_user_ids(user_ids: Sequence[int], realm: Realm) -> "Addressee":
+        assert len(user_ids) > 0
+        user_profiles = get_user_profiles_by_ids(user_ids, realm)
+        return Addressee(
+            msg_type="private",
+            user_profiles=user_profiles,
+        )
+
+    @staticmethod
+    def for_user_profile(user_profile: UserProfile) -> "Addressee":
+        user_profiles = [user_profile]
+        return Addressee(
+            msg_type="private",
+            user_profiles=user_profiles,
+        )
+
+    @staticmethod
+    def for_user_profiles(user_profiles: Sequence[UserProfile]) -> "Addressee":
+        assert len(user_profiles) > 0
+        return Addressee(
+            msg_type="private",
+            user_profiles=user_profiles,
+        )

zerver/lib/alert_words.py

@@ -0,0 +1,84 @@
+from collections.abc import Iterable
+
+import ahocorasick
+from django.db import transaction
+from django.db.models.functions import Lower
+
+from zerver.lib.cache import (
+    cache_with_key,
+    realm_alert_words_automaton_cache_key,
+    realm_alert_words_cache_key,
+)
+from zerver.models import AlertWord, Realm, UserProfile
+from zerver.models.alert_words import flush_realm_alert_words
+
+
+@cache_with_key(lambda realm: realm_alert_words_cache_key(realm.id), timeout=3600 * 24)
+def alert_words_in_realm(realm: Realm) -> dict[int, list[str]]:
+    user_ids_and_words = AlertWord.objects.filter(realm=realm, user_profile__is_active=True).values(
+        "user_profile_id", "word"
+    )
+    user_ids_with_words: dict[int, list[str]] = {}
+    for id_and_word in user_ids_and_words:
+        user_ids_with_words.setdefault(id_and_word["user_profile_id"], [])
+        user_ids_with_words[id_and_word["user_profile_id"]].append(id_and_word["word"])
+    return user_ids_with_words
+
+
+@cache_with_key(lambda realm: realm_alert_words_automaton_cache_key(realm.id), timeout=3600 * 24)
+def get_alert_word_automaton(realm: Realm) -> ahocorasick.Automaton:
+    user_id_with_words = alert_words_in_realm(realm)
+    alert_word_automaton = ahocorasick.Automaton()
+    for user_id, alert_words in user_id_with_words.items():
+        for alert_word in alert_words:
+            alert_word_lower = alert_word.lower()
+            if alert_word_automaton.exists(alert_word_lower):
+                (_key, user_ids_for_alert_word) = alert_word_automaton.get(alert_word_lower)
+                user_ids_for_alert_word.add(user_id)
+            else:
+                alert_word_automaton.add_word(alert_word_lower, (alert_word_lower, {user_id}))
+    alert_word_automaton.make_automaton()
+    # If the kind is not AHOCORASICK after calling make_automaton, it means there is no key present
+    # and hence we cannot call items on the automaton yet. To avoid it we return None for such cases
+    # where there is no alert-words in the realm.
+    # https://pyahocorasick.readthedocs.io/en/latest/#make-automaton
+    if alert_word_automaton.kind != ahocorasick.AHOCORASICK:
+        return None
+    return alert_word_automaton
+
+
+def user_alert_words(user_profile: UserProfile) -> list[str]:
+    return list(AlertWord.objects.filter(user_profile=user_profile).values_list("word", flat=True))
+
+
+@transaction.atomic(savepoint=False)
+def add_user_alert_words(user_profile: UserProfile, new_words: Iterable[str]) -> list[str]:
+    existing_words_lower = {word.lower() for word in user_alert_words(user_profile)}
+
+    # Keeping the case, use a dictionary to get the set of
+    # case-insensitive distinct, new alert words
+    word_dict: dict[str, str] = {}
+    for word in new_words:
+        if word.lower() in existing_words_lower:
+            continue
+        word_dict[word.lower()] = word
+
+    AlertWord.objects.bulk_create(
+        AlertWord(user_profile=user_profile, word=word, realm=user_profile.realm)
+        for word in word_dict.values()
+    )
+    # Django bulk_create operations don't flush caches, so we need to do this ourselves.
+    flush_realm_alert_words(user_profile.realm_id)
+
+    return user_alert_words(user_profile)
+
+
+@transaction.atomic(savepoint=False)
+def remove_user_alert_words(user_profile: UserProfile, delete_words: Iterable[str]) -> list[str]:
+    delete_words_lower = [word.lower() for word in delete_words]
+    AlertWord.objects.annotate(word_lower=Lower("word")).filter(
+        user_profile=user_profile,
+        word_lower__in=delete_words_lower,
+    ).delete()
+    flush_realm_alert_words(user_profile.realm_id)
+    return user_alert_words(user_profile)

zerver/lib/async_utils.py

@@ -0,0 +1,23 @@
+# nocoverage
+
+import asyncio
+import sys
+
+from typing_extensions import override
+
+if sys.version_info < (3, 14):
+
+    class NoAutoCreateEventLoopPolicy(asyncio.DefaultEventLoopPolicy):
+        """
+        By default asyncio.get_event_loop() automatically creates an event
+        loop for the main thread if one isn't currently installed.  Since
+        Django intentionally uninstalls the event loop within
+        sync_to_async, that autocreation proliferates confusing extra
+        event loops that will never be run.  It is also deprecated in
+        Python 3.10.  This policy disables it so we don't rely on it by
+        accident.
+        """
+
+        @override
+        def get_event_loop(self) -> asyncio.AbstractEventLoop:
+            return asyncio.get_running_loop()

zerver/lib/attachments.py

@@ -0,0 +1,280 @@
+from datetime import timedelta
+from typing import Any
+
+from django.conf import settings
+from django.contrib.auth.models import AnonymousUser
+from django.db.models import Exists, OuterRef, QuerySet
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+
+from zerver.lib.exceptions import JsonableError, RateLimitedError
+from zerver.lib.streams import is_user_in_groups_granting_content_access
+from zerver.lib.upload import delete_message_attachment
+from zerver.lib.user_groups import get_recursive_membership_groups
+from zerver.models import (
+    ArchivedAttachment,
+    Attachment,
+    ImageAttachment,
+    Message,
+    Realm,
+    Recipient,
+    Stream,
+    Subscription,
+    UserMessage,
+    UserProfile,
+)
+
+
+def user_attachments(user_profile: UserProfile) -> list[dict[str, Any]]:
+    attachments = Attachment.objects.filter(owner=user_profile).prefetch_related("messages")
+    return [a.to_dict() for a in attachments]
+
+
+def access_attachment_by_id(
+    user_profile: UserProfile, attachment_id: int, needs_owner: bool = False
+) -> Attachment:
+    query = Attachment.objects.filter(id=attachment_id)
+    if needs_owner:
+        query = query.filter(owner=user_profile)
+
+    attachment = query.first()
+    if attachment is None:
+        raise JsonableError(_("Invalid attachment"))
+    return attachment
+
+
+def remove_attachment(user_profile: UserProfile, attachment: Attachment) -> None:
+    try:
+        delete_message_attachment(attachment.path_id)
+    except Exception:
+        raise JsonableError(
+            _("An error occurred while deleting the attachment. Please try again later.")
+        )
+    ImageAttachment.objects.filter(path_id=attachment.path_id).delete()
+    attachment.delete()
+
+
+def validate_attachment_request_for_spectator_access(
+    realm: Realm, attachment: Attachment
+) -> bool | None:
+    """Returns whether the spectator is authorized to access the
+    attachment, or None if the attachment was deleted concurrently
+    with the request.
+    """
+    if attachment.realm != realm:
+        return False
+
+    if not realm.web_public_streams_enabled():
+        return False
+
+    # Update cached is_web_public property, if necessary.
+    if attachment.is_web_public is None:
+        # Fill the cache in a single query. This is important to avoid
+        # a potential race condition between checking and setting,
+        # where the attachment could have been moved again.
+        Attachment.objects.filter(id=attachment.id, is_web_public__isnull=True).update(
+            is_web_public=Exists(
+                Message.objects.filter(
+                    # Uses index: zerver_attachment_messages_attachment_id_message_id_key
+                    realm_id=realm.id,
+                    attachment=OuterRef("id"),
+                    recipient__stream__invite_only=False,
+                    recipient__stream__is_web_public=True,
+                ),
+            ),
+        )
+
+        try:
+            attachment.refresh_from_db()
+        except Attachment.DoesNotExist:
+            return None
+
+    if not attachment.is_web_public:
+        return False
+
+    if settings.RATE_LIMITING:
+        try:
+            from zerver.lib.rate_limiter import rate_limit_spectator_attachment_access_by_file
+
+            rate_limit_spectator_attachment_access_by_file(attachment.path_id)
+        except RateLimitedError:
+            return False
+
+    return True
+
+
+def validate_attachment_request(
+    maybe_user_profile: UserProfile | AnonymousUser,
+    path_id: str,
+    realm: Realm | None = None,
+) -> tuple[bool, Attachment | None]:
+    try:
+        attachment = Attachment.objects.get(path_id=path_id)
+    except Attachment.DoesNotExist:
+        return False, None
+
+    if isinstance(maybe_user_profile, AnonymousUser):
+        assert realm is not None
+        is_authorized = validate_attachment_request_for_spectator_access(realm, attachment)
+        if is_authorized is None:
+            # The attachment was deleted concurrently with this
+            # request; report it as not existing, just as if we
+            # had observed that initially.
+            return False, None
+        return is_authorized, attachment
+
+    user_profile = maybe_user_profile
+    assert isinstance(user_profile, UserProfile)
+
+    # Update cached is_realm_public property, if necessary.
+    if attachment.is_realm_public is None:
+        # Fill the cache in a single query. This is important to avoid
+        # a potential race condition between checking and setting,
+        # where the attachment could have been moved again.
+        Attachment.objects.filter(id=attachment.id, is_realm_public__isnull=True).update(
+            is_realm_public=Exists(
+                Message.objects.filter(
+                    # Uses index: zerver_attachment_messages_attachment_id_message_id_key
+                    realm_id=user_profile.realm_id,
+                    attachment=OuterRef("id"),
+                    recipient__stream__invite_only=False,
+                ),
+            ),
+        )
+        try:
+            attachment.refresh_from_db()
+        except Attachment.DoesNotExist:
+            # The attachment was deleted concurrently with this
+            # request; report it as not existing, just as if we
+            # had observed that initially.
+            return False, None
+
+    if user_profile.id == attachment.owner_id:
+        # If you own the file, you can access it.
+        return True, attachment
+    if (
+        attachment.is_realm_public
+        and attachment.realm == user_profile.realm
+        and user_profile.can_access_public_streams()
+    ):
+        # Any user in the realm can access realm-public files
+        return True, attachment
+
+    messages = attachment.messages.all().select_related("recipient")
+
+    usermessages_channel_ids = set()
+    usermessage_rows = UserMessage.objects.filter(
+        user_profile=user_profile, message__in=messages
+    ).select_related("message", "message__recipient")
+    for um in usermessage_rows:
+        if not um.message.is_channel_message:
+            # If the attachment was sent in a direct message or group direct
+            # message then anyone who received that message can access it.
+            return True, attachment
+        else:
+            usermessages_channel_ids.add(um.message.recipient.type_id)
+
+    # These are subscriptions to a channel one of the messages was sent to
+    subscribed_channel_ids = Subscription.objects.filter(
+        user_profile=user_profile,
+        active=True,
+        recipient__type=Recipient.STREAM,
+        recipient__in=[m.recipient_id for m in messages],
+    ).values_list("recipient__type_id", flat=True)
+
+    if usermessages_channel_ids.intersection(subscribed_channel_ids):
+        # If the attachment was sent in a channel with public
+        # or protected history and the user is still subscribed
+        # to the channel then anyone who received that message
+        # can access it.
+        return True, attachment
+
+    message_channel_ids = set()
+    for message in messages:
+        if message.is_channel_message:
+            message_channel_ids.add(message.recipient.type_id)
+
+    if len(message_channel_ids) == 0:
+        # If only DMs are relevant, return early here.
+        return False, attachment
+
+    # The remaining code path is slow but should only be relevant
+    # rarely: Users trying to view an attachment that was shared with
+    # at least one channel, but the user is not subscribed to any such
+    # channel. So it's not important that the accurate check for this
+    # corner case is somewhat more expensive to check groups-based
+    # permissions; we're no longer in a hot code path.
+    message_channels = Stream.objects.filter(id__in=message_channel_ids)
+    for channel in message_channels:
+        # The user didn't receive any of the messages that included
+        # this attachment. But they might still have access to it,
+        # if it was sent to a stream they are subscribed to where
+        # history is public to subscribers.
+        if channel.id in subscribed_channel_ids and channel.is_history_public_to_subscribers():
+            return True, attachment
+
+    user_recursive_group_ids = set(
+        get_recursive_membership_groups(user_profile).values_list("id", flat=True)
+    )
+    for channel in message_channels:
+        if is_user_in_groups_granting_content_access(channel, user_recursive_group_ids):
+            if channel.is_history_public_to_subscribers():
+                return True, attachment
+            # If the user had received the message at one point of
+            # time, but they are no longer subscribed to a stream
+            # with protected history. They can still access that
+            # message and it's attachment
+            elif channel.id in usermessages_channel_ids:
+                return True, attachment
+
+    return False, attachment
+
+
+def get_old_unclaimed_attachments(
+    weeks_ago: int,
+) -> tuple[QuerySet[Attachment], QuerySet[ArchivedAttachment]]:
+    """
+    The logic in this function is fairly tricky. The essence is that
+    a file should be cleaned up if and only if it not referenced by any
+    Message, ScheduledMessage or ArchivedMessage. The way to find that out is through the
+    Attachment and ArchivedAttachment tables.
+    The queries are complicated by the fact that an uploaded file
+    may have either only an Attachment row, only an ArchivedAttachment row,
+    or both - depending on whether some, all or none of the messages
+    linking to it have been archived.
+    """
+    delta_weeks_ago = timezone_now() - timedelta(weeks=weeks_ago)
+
+    # The Attachment vs ArchivedAttachment queries are asymmetric because only
+    # Attachment has the scheduled_messages relation.
+    old_attachments = (
+        Attachment.objects.alias(
+            has_other_messages=Exists(
+                ArchivedAttachment.objects.filter(id=OuterRef("id")).exclude(messages=None)
+            )
+        )
+        .filter(
+            messages=None,
+            scheduled_messages=None,
+            create_time__lt=delta_weeks_ago,
+            has_other_messages=False,
+        )
+        .order_by("id")
+    )
+    old_archived_attachments = (
+        ArchivedAttachment.objects.alias(
+            has_other_messages=Exists(
+                Attachment.objects.filter(id=OuterRef("id")).exclude(
+                    messages=None, scheduled_messages=None
+                )
+            )
+        )
+        .filter(
+            messages=None,
+            create_time__lt=delta_weeks_ago,
+            has_other_messages=False,
+        )
+        .order_by("id")
+    )
+
+    return old_attachments, old_archived_attachments

zerver/lib/avatar.py

@@ -0,0 +1,241 @@
+import logging
+import os
+import subprocess
+from urllib.parse import urljoin
+
+from django.conf import settings
+from django.contrib.staticfiles.storage import staticfiles_storage
+
+from zerver.lib.avatar_hash import (
+    gravatar_hash,
+    user_avatar_base_path_from_ids,
+    user_avatar_content_hash,
+    user_avatar_path,
+)
+from zerver.lib.thumbnail import DEFAULT_AVATAR_SIZE, MEDIUM_AVATAR_SIZE
+from zerver.lib.upload import get_avatar_url, write_jdenticon_avatars
+from zerver.lib.url_encoding import append_url_query_string
+from zerver.models import UserProfile
+from zerver.models.users import is_cross_realm_bot_email
+
+STATIC_AVATARS_DIR = "images/static_avatars/"
+
+DEFAULT_AVATAR_FILE = "images/default-avatar.png"
+
+logger = logging.getLogger(__name__)
+
+
+def avatar_url(
+    user_profile: UserProfile, medium: bool = False, client_gravatar: bool = False
+) -> str | None:
+    return get_avatar_field(
+        user_id=user_profile.id,
+        realm_id=user_profile.realm_id,
+        email=user_profile.delivery_email,
+        avatar_source=user_profile.avatar_source,
+        avatar_version=user_profile.avatar_version,
+        medium=medium,
+        client_gravatar=client_gravatar,
+    )
+
+
+def get_system_bots_avatar_file_name(email: str) -> str:
+    system_bot_avatar_name_map = {
+        settings.WELCOME_BOT: "welcome-bot",
+        settings.NOTIFICATION_BOT: "notification-bot",
+        settings.EMAIL_GATEWAY_BOT: "emailgateway",
+    }
+    return urljoin(STATIC_AVATARS_DIR, system_bot_avatar_name_map.get(email, "unknown"))
+
+
+def get_static_avatar_url(email: str, medium: bool) -> str:
+    avatar_file_name = get_system_bots_avatar_file_name(email)
+    avatar_file_name += "-medium.png" if medium else ".png"
+
+    if settings.DEBUG:
+        # This find call may not be cheap, so we only do it in the
+        # development environment to do an assertion.
+        from django.contrib.staticfiles.finders import find
+
+        if not find(avatar_file_name):
+            raise AssertionError(f"Unknown avatar file for: {email}")
+    elif settings.STATIC_ROOT and not staticfiles_storage.exists(avatar_file_name):
+        # Fallback for the case where no avatar exists; this should
+        # never happen in practice. This logic cannot be executed
+        # while STATIC_ROOT is not defined, so the above STATIC_ROOT
+        # check is important.
+        return DEFAULT_AVATAR_FILE
+
+    return staticfiles_storage.url(avatar_file_name)
+
+
+def get_avatar_field(
+    user_id: int,
+    realm_id: int,
+    email: str,
+    avatar_source: str,
+    avatar_version: int,
+    medium: bool,
+    client_gravatar: bool,
+) -> str | None:
+    """
+    Most of the parameters to this function map to fields
+    by the same name in UserProfile (avatar_source, realm_id,
+    email, etc.).
+
+    Then there are these:
+
+        medium - This means we want a medium-sized avatar. This can
+            affect the "s" parameter for gravatar avatars, or it
+            can give us something like foo-medium.png for
+            user-uploaded avatars.
+
+        client_gravatar - If the client can compute their own
+            gravatars, this will be set to True, and we'll avoid
+            computing them on the server (mostly to save bandwidth).
+    """
+
+    # System bots have hardcoded avatars
+    if is_cross_realm_bot_email(email):
+        return get_static_avatar_url(email, medium)
+
+    """
+    If our client knows how to calculate gravatar hashes, we
+    will return None and let the client compute the gravatar
+    url.
+    """
+    if (
+        client_gravatar
+        and settings.ENABLE_GRAVATAR
+        and avatar_source == UserProfile.AVATAR_FROM_GRAVATAR
+    ):
+        return None
+
+    """
+    If we get this far, we'll compute an avatar URL based on the
+    avatar source, and then we'll add version info to try to avoid
+    stale caches.
+    """
+    if avatar_source in [UserProfile.AVATAR_FROM_USER, UserProfile.AVATAR_FROM_JDENTICON]:
+        hash_key = user_avatar_base_path_from_ids(user_id, avatar_version, realm_id)
+        return get_avatar_url(hash_key, medium=medium)
+
+    return get_gravatar_url(
+        email=email,
+        avatar_version=avatar_version,
+        realm_id=realm_id,
+        medium=medium,
+    )
+
+
+def get_gravatar_url(email: str, avatar_version: int, realm_id: int, medium: bool = False) -> str:
+    url = _get_unversioned_gravatar_url(email, medium, realm_id)
+    return append_url_query_string(url, f"version={avatar_version:d}")
+
+
+def _get_unversioned_gravatar_url(email: str, medium: bool, realm_id: int) -> str:
+    use_gravatar = settings.ENABLE_GRAVATAR
+    if realm_id in settings.GRAVATAR_REALM_OVERRIDE:
+        use_gravatar = settings.GRAVATAR_REALM_OVERRIDE[realm_id]
+
+    if use_gravatar:
+        gravitar_query_suffix = f"&s={MEDIUM_AVATAR_SIZE}" if medium else ""
+        hash_key = gravatar_hash(email)
+        return f"https://secure.gravatar.com/avatar/{hash_key}?d=identicon{gravitar_query_suffix}"
+    elif settings.DEFAULT_AVATAR_URI is not None:
+        return settings.DEFAULT_AVATAR_URI
+    else:
+        return staticfiles_storage.url("images/default-avatar.png")
+
+
+def absolute_avatar_url(
+    user_profile: UserProfile,
+    # Pass `realm_url` to avoid a DB query when `user_profile.realm` isn't
+    # already loaded but the caller has realm available from another source.
+    realm_url: str | None = None,
+) -> str:
+    """
+    Absolute URLs are used to simplify logic for applications that
+    won't be served by browsers, such as rendering GCM notifications.
+    """
+    avatar = avatar_url(user_profile)
+    # avatar_url can return None if client_gravatar=True, however here we use the default value of False
+    assert avatar is not None
+    if realm_url is None:
+        realm_url = user_profile.realm.url
+    return urljoin(realm_url, avatar)
+
+
+def is_avatar_new(ldap_avatar: bytes, user_profile: UserProfile) -> bool:
+    new_avatar_hash = user_avatar_content_hash(ldap_avatar)
+
+    if user_profile.avatar_hash and user_profile.avatar_hash == new_avatar_hash:
+        # If an avatar exists and is the same as the new avatar,
+        # then, no need to change the avatar.
+        return False
+
+    return True
+
+
+def get_avatar_for_inaccessible_user() -> str:
+    return staticfiles_storage.url("images/unknown-user-avatar.png")
+
+
+def generate_avatar_jdenticon(input: str, medium: bool) -> bytes:
+    from zerver.lib.storage import static_path
+
+    jdenticon_path = (
+        static_path("webpack-bundles/jdenticon.js")
+        if settings.PRODUCTION
+        else os.path.join(settings.DEPLOY_ROOT, "node_modules/jdenticon/bin/jdenticon.js")
+    )
+    size = str(MEDIUM_AVATAR_SIZE if medium else DEFAULT_AVATAR_SIZE)
+    command = [
+        "node",
+        jdenticon_path,
+        input,
+        "-s",
+        size,
+        "-p",
+        "0",
+        "--lightness-color",
+        "0.3,0.7",
+        "--lightness-grayscale",
+        "0.3,0.7",
+    ]
+    try:
+        stdout = subprocess.check_output(command)
+        return stdout
+    except subprocess.CalledProcessError:  # nocoverage
+        logger.exception("Jdenticon generation failed for user_id: %s", input)
+        raise
+
+
+def generate_and_upload_jdenticon_avatar(
+    user_profile: UserProfile,
+    realm_uuid: str,
+    future: bool,
+) -> None:
+    # We use a combination of user ID and realm_uuid (salt) as the key
+    # for Jdenticon generation, so that clients that prefer to use computation
+    # instead of network to provide default avatars for users can do that
+    # in the future.
+    #
+    # Using only user ID (no salt) can result in situations where a person is
+    # part of multiple zulip servers, and they find same avatar for different
+    # people in different servers.
+    #
+    # Note: The key effectively changes when user IDs are renumbered when
+    # migrating between Zulip servers,
+    jdenticon_key = f"{realm_uuid}:{user_profile.id}"
+    image_data = generate_avatar_jdenticon(jdenticon_key, medium=False)
+    image_data_medium = generate_avatar_jdenticon(jdenticon_key, medium=True)
+    file_path = user_avatar_path(user_profile, future)
+
+    write_jdenticon_avatars(
+        file_path,
+        user_profile,
+        image_data=image_data,
+        image_data_medium=image_data_medium,
+        future=future,
+    )

zerver/lib/avatar_hash.py

@@ -0,0 +1,41 @@
+import hashlib
+
+from django.conf import settings
+
+from zerver.models import UserProfile
+
+
+def gravatar_hash(email: str) -> str:
+    """Compute the Gravatar hash for an email address."""
+    # Non-ASCII characters aren't permitted by the currently active e-mail
+    # RFCs. However, the IETF has published https://tools.ietf.org/html/rfc4952,
+    # outlining internationalization of email addresses, and regardless if we
+    # typo an address or someone manages to give us a non-ASCII address, let's
+    # not error out on it.
+    return hashlib.md5(email.lower().encode()).hexdigest()
+
+
+def user_avatar_hash(uid: str, version: str) -> str:
+    # WARNING: If this method is changed, you may need to do a migration
+    # similar to zerver/migrations/0060_move_avatars_to_be_uid_based.py .
+
+    # The salt prevents unauthenticated clients from enumerating the
+    # avatars of all users.
+    user_key = uid + ":" + version + ":" + settings.AVATAR_SALT
+    return hashlib.sha256(user_key.encode()).hexdigest()[:40]
+
+
+def user_avatar_path(user_profile: UserProfile, future: bool = False) -> str:
+    # 'future' is if this is for the current avatar version, or the next one.
+    return user_avatar_base_path_from_ids(
+        user_profile.id, user_profile.avatar_version + (1 if future else 0), user_profile.realm_id
+    )
+
+
+def user_avatar_base_path_from_ids(user_profile_id: int, version: int, realm_id: int) -> str:
+    user_id_hash = user_avatar_hash(str(user_profile_id), str(version))
+    return f"{realm_id}/{user_id_hash}"
+
+
+def user_avatar_content_hash(ldap_avatar: bytes) -> str:
+    return hashlib.sha256(ldap_avatar).hexdigest()

zerver/lib/bot_config.py

@@ -0,0 +1,80 @@
+import configparser
+import importlib
+import os
+from collections import defaultdict
+
+from django.conf import settings
+from django.db.models import F, Sum
+from django.db.models.functions import Length
+
+from zerver.models import BotConfigData, UserProfile
+
+
+class ConfigError(Exception):
+    pass
+
+
+def get_bot_config(bot_profile: UserProfile) -> dict[str, str]:
+    entries = BotConfigData.objects.filter(bot_profile=bot_profile)
+    if not entries:
+        raise ConfigError("No config data available.")
+    return {entry.key: entry.value for entry in entries}
+
+
+def get_bot_configs(bot_profile_ids: list[int]) -> dict[int, dict[str, str]]:
+    if not bot_profile_ids:
+        return {}
+    entries = BotConfigData.objects.filter(bot_profile_id__in=bot_profile_ids)
+    entries_by_uid: dict[int, dict[str, str]] = defaultdict(dict)
+    for entry in entries:
+        entries_by_uid[entry.bot_profile_id].update({entry.key: entry.value})
+    return entries_by_uid
+
+
+def get_bot_config_size(bot_profile: UserProfile, key: str | None = None) -> int:
+    if key is None:
+        return (
+            BotConfigData.objects.filter(bot_profile=bot_profile)
+            .annotate(key_size=Length("key"), value_size=Length("value"))
+            .aggregate(sum=Sum(F("key_size") + F("value_size")))["sum"]
+            or 0
+        )
+    else:
+        try:
+            return len(key) + len(BotConfigData.objects.get(bot_profile=bot_profile, key=key).value)
+        except BotConfigData.DoesNotExist:
+            return 0
+
+
+def set_bot_config(bot_profile: UserProfile, key: str, value: str) -> None:
+    config_size_limit = settings.BOT_CONFIG_SIZE_LIMIT
+    old_entry_size = get_bot_config_size(bot_profile, key)
+    new_entry_size = len(key) + len(value)
+    old_config_size = get_bot_config_size(bot_profile)
+    new_config_size = old_config_size + (new_entry_size - old_entry_size)
+    if new_config_size > config_size_limit:
+        raise ConfigError(
+            f"Cannot store configuration. Request would require {new_config_size} characters. "
+            f"The current configuration size limit is {config_size_limit} characters."
+        )
+    obj, created = BotConfigData.objects.get_or_create(
+        bot_profile=bot_profile, key=key, defaults={"value": value}
+    )
+    if not created:
+        obj.value = value
+        obj.save()
+
+
+def load_bot_config_template(bot: str) -> dict[str, str]:
+    bot_module_name = f"zulip_bots.bots.{bot}"
+    bot_module = importlib.import_module(bot_module_name)
+    assert bot_module.__file__ is not None
+    bot_module_path = os.path.dirname(bot_module.__file__)
+    config_path = os.path.join(bot_module_path, f"{bot}.conf")
+    if os.path.isfile(config_path):
+        config = configparser.ConfigParser()
+        with open(config_path) as conf:
+            config.read_file(conf)
+        return dict(config.items(bot))
+    else:
+        return {}

zerver/lib/bot_lib.py

@@ -0,0 +1,166 @@
+import importlib
+import json
+from collections.abc import Callable
+from typing import Any
+
+from django.conf import settings
+from django.utils.translation import gettext as _
+from zulip_bots.lib import BotIdentity, RateLimit
+
+from zerver.actions.message_flags import do_update_message_flags
+from zerver.actions.message_send import (
+    internal_send_group_direct_message,
+    internal_send_private_message,
+    internal_send_stream_message_by_name,
+)
+from zerver.lib.bot_config import ConfigError, get_bot_config
+from zerver.lib.bot_storage import (
+    get_bot_storage,
+    is_key_in_bot_storage,
+    remove_bot_storage,
+    set_bot_storage,
+)
+from zerver.lib.integrations import EMBEDDED_BOTS
+from zerver.lib.topic import get_topic_from_message_info
+from zerver.models import UserProfile
+from zerver.models.users import get_active_user
+
+
+def get_bot_handler(service_name: str) -> Any:
+    # Check that this service is present in EMBEDDED_BOTS, add exception handling.
+    configured_service = ""
+    for embedded_bot_service in EMBEDDED_BOTS:
+        if service_name == embedded_bot_service.name:
+            configured_service = embedded_bot_service.name
+    if not configured_service:
+        return None
+    bot_module_name = f"zulip_bots.bots.{configured_service}.{configured_service}"
+    bot_module: Any = importlib.import_module(bot_module_name)
+    return bot_module.handler_class()
+
+
+class StateHandler:
+    storage_size_limit: int = settings.USER_STATE_SIZE_LIMIT
+
+    def __init__(self, user_profile: UserProfile) -> None:
+        self.user_profile = user_profile
+        self.marshal: Callable[[object], str] = json.dumps
+        self.demarshal: Callable[[str], object] = json.loads
+
+    def get(self, key: str) -> object:
+        return self.demarshal(get_bot_storage(self.user_profile, key))
+
+    def put(self, key: str, value: object) -> None:
+        set_bot_storage(self.user_profile, [(key, self.marshal(value))])
+
+    def remove(self, key: str) -> None:
+        remove_bot_storage(self.user_profile, [key])
+
+    def contains(self, key: str) -> bool:
+        return is_key_in_bot_storage(self.user_profile, key)
+
+
+class EmbeddedBotQuitError(Exception):
+    pass
+
+
+class EmbeddedBotEmptyRecipientsListError(Exception):
+    pass
+
+
+class EmbeddedBotHandler:
+    def __init__(self, user_profile: UserProfile) -> None:
+        # Only expose a subset of our UserProfile's functionality
+        self.user_profile = user_profile
+        self._rate_limit = RateLimit(20, 5)
+        self.full_name = user_profile.full_name
+        self.email = user_profile.email
+        self.storage = StateHandler(user_profile)
+        self.user_id = user_profile.id
+
+    def identity(self) -> BotIdentity:
+        return BotIdentity(self.full_name, self.email)
+
+    def react(self, message: dict[str, Any], emoji_name: str) -> dict[str, Any]:
+        return {}  # Not implemented
+
+    def send_message(self, message: dict[str, Any]) -> dict[str, Any]:
+        if not self._rate_limit.is_legal():
+            self._rate_limit.show_error_and_exit()
+
+        if message["type"] == "stream":
+            message_id = internal_send_stream_message_by_name(
+                self.user_profile.realm,
+                self.user_profile,
+                message["to"],
+                message["topic"],
+                message["content"],
+            )
+            return {"id": message_id}
+
+        assert message["type"] == "private"
+        # Ensure that it's a comma-separated list, even though the
+        # usual 'to' field could be either a List[str] or a str.
+        recipients = ",".join(message["to"]).split(",")
+
+        if len(message["to"]) == 0:
+            raise EmbeddedBotEmptyRecipientsListError(_("Message must have recipients!"))
+        elif len(message["to"]) == 1:
+            recipient_user = get_active_user(recipients[0], self.user_profile.realm)
+            message_id = internal_send_private_message(
+                self.user_profile, recipient_user, message["content"]
+            )
+        else:
+            message_id = internal_send_group_direct_message(
+                self.user_profile.realm, self.user_profile, message["content"], emails=recipients
+            )
+        return {"id": message_id}
+
+    def send_reply(
+        self, message: dict[str, Any], response: str, widget_content: str | None = None
+    ) -> dict[str, Any]:
+        if message["type"] == "private":
+            result = self.send_message(
+                dict(
+                    type="private",
+                    to=[x["email"] for x in message["display_recipient"]],
+                    content=response,
+                    sender_email=message["sender_email"],
+                )
+            )
+        else:
+            result = self.send_message(
+                dict(
+                    type="stream",
+                    to=message["display_recipient"],
+                    topic=get_topic_from_message_info(message),
+                    content=response,
+                    sender_email=message["sender_email"],
+                )
+            )
+        return {"id": result["id"]}
+
+    def update_message(self, message: dict[str, Any]) -> None:
+        pass  # Not implemented
+
+    # The bot_name argument exists only to comply with ExternalBotHandler.get_config_info().
+    def get_config_info(self, bot_name: str, optional: bool = False) -> dict[str, str]:
+        try:
+            return get_bot_config(self.user_profile)
+        except ConfigError:
+            if optional:
+                return {}
+            raise
+
+    def quit(self, message: str = "") -> None:
+        raise EmbeddedBotQuitError(message)
+
+
+def do_flag_message_triggered_bots_messages_as_processed(
+    bot_profile: UserProfile, message_ids: list[int]
+) -> None:
+    assert (
+        bot_profile.is_bot is True
+        and bot_profile.bot_type in UserProfile.MESSAGE_TRIGGERED_BOT_TYPES
+    )
+    do_update_message_flags(bot_profile, "add", "read", message_ids)

zerver/lib/bot_storage.py

@@ -0,0 +1,69 @@
+from django.conf import settings
+from django.db.models import F, Sum
+from django.db.models.functions import Length
+
+from zerver.models import BotStorageData, UserProfile
+
+
+class StateError(Exception):
+    pass
+
+
+def get_bot_storage(bot_profile: UserProfile, key: str) -> str:
+    try:
+        return BotStorageData.objects.get(bot_profile=bot_profile, key=key).value
+    except BotStorageData.DoesNotExist:
+        raise StateError("Key does not exist.")
+
+
+def get_bot_storage_size(bot_profile: UserProfile, key: str | None = None) -> int:
+    if key is None:
+        return (
+            BotStorageData.objects.filter(bot_profile=bot_profile)
+            .annotate(key_size=Length("key"), value_size=Length("value"))
+            .aggregate(sum=Sum(F("key_size") + F("value_size")))["sum"]
+            or 0
+        )
+    else:
+        try:
+            return len(key) + len(
+                BotStorageData.objects.get(bot_profile=bot_profile, key=key).value
+            )
+        except BotStorageData.DoesNotExist:
+            return 0
+
+
+def set_bot_storage(bot_profile: UserProfile, entries: list[tuple[str, str]]) -> None:
+    storage_size_limit = settings.USER_STATE_SIZE_LIMIT
+    storage_size_difference = 0
+    for key, value in entries:
+        assert isinstance(key, str), "Key type should be str."
+        assert isinstance(value, str), "Value type should be str."
+        storage_size_difference += (len(key) + len(value)) - get_bot_storage_size(bot_profile, key)
+    new_storage_size = get_bot_storage_size(bot_profile) + storage_size_difference
+    if new_storage_size > storage_size_limit:
+        raise StateError(
+            f"Request exceeds storage limit by {new_storage_size - storage_size_limit} characters. The limit is {storage_size_limit} characters."
+        )
+    else:
+        for key, value in entries:
+            BotStorageData.objects.update_or_create(
+                bot_profile=bot_profile, key=key, defaults={"value": value}
+            )
+
+
+def remove_bot_storage(bot_profile: UserProfile, keys: list[str]) -> None:
+    queryset = BotStorageData.objects.filter(bot_profile=bot_profile, key__in=keys)
+    if queryset.count() < len(keys):
+        raise StateError("Key does not exist.")
+    queryset.delete()
+
+
+def is_key_in_bot_storage(bot_profile: UserProfile, key: str) -> bool:
+    return BotStorageData.objects.filter(bot_profile=bot_profile, key=key).exists()
+
+
+def get_keys_in_bot_storage(bot_profile: UserProfile) -> list[str]:
+    return list(
+        BotStorageData.objects.filter(bot_profile=bot_profile).values_list("key", flat=True)
+    )

zerver/lib/bulk_create.py

@@ -0,0 +1,218 @@
+from collections.abc import Collection, Iterable
+from typing import Any
+
+from django.db.models import QuerySet
+from django.utils.timezone import now as timezone_now
+
+from zerver.lib.avatar import generate_and_upload_jdenticon_avatar
+from zerver.lib.create_user import create_user_profile, get_display_email_address
+from zerver.lib.initial_password import initial_password
+from zerver.lib.streams import (
+    get_default_values_for_stream_permission_group_settings,
+    render_stream_description,
+)
+from zerver.models import (
+    NamedUserGroup,
+    Realm,
+    RealmAuditLog,
+    RealmUserDefault,
+    Recipient,
+    Stream,
+    UserGroupMembership,
+    UserProfile,
+)
+from zerver.models.groups import SystemGroups
+from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.models.streams import StreamTopicsPolicyEnum
+
+
+def bulk_create_users(
+    realm: Realm,
+    users_raw: set[tuple[str, str, bool]],
+    bot_type: int | None = None,
+    bot_owner: UserProfile | None = None,
+    tos_version: str | None = None,
+    timezone: str = "",
+    is_imported_stub: bool = False,
+) -> None:
+    """
+    Creates and saves a UserProfile with the given email.
+    Has some code based off of UserManage.create_user, but doesn't .save()
+    """
+    existing_users = frozenset(
+        UserProfile.objects.filter(realm=realm).values_list("email", flat=True)
+    )
+    users = sorted(user_raw for user_raw in users_raw if user_raw[0] not in existing_users)
+
+    realm_user_default = RealmUserDefault.objects.get(realm=realm)
+    if bot_type is None:
+        email_address_visibility = realm_user_default.email_address_visibility
+    else:
+        # There is no privacy motivation for limiting access to bot email addresses,
+        # so we hardcode them to EMAIL_ADDRESS_VISIBILITY_EVERYONE.
+        email_address_visibility = UserProfile.EMAIL_ADDRESS_VISIBILITY_EVERYONE
+
+    # Now create user_profiles
+    profiles_to_create: list[UserProfile] = []
+    for email, full_name, active in users:
+        profile = create_user_profile(
+            realm,
+            email,
+            initial_password(email),
+            active,
+            bot_type,
+            full_name,
+            bot_owner,
+            False,
+            tos_version,
+            timezone,
+            is_imported_stub=is_imported_stub,
+            default_language=realm.default_language,
+            email_address_visibility=email_address_visibility,
+        )
+
+        if bot_type is None:
+            # This block simulates copy_default_settings from
+            # zerver/lib/create_user.py.
+            #
+            # We cannot use 'copy_default_settings' directly here
+            # because it calls '.save' after copying the settings, and
+            # we are bulk creating the objects here instead.
+            for settings_name in RealmUserDefault.property_types:
+                if settings_name in ["default_language", "enable_login_emails"]:
+                    continue
+                value = getattr(realm_user_default, settings_name)
+                setattr(profile, settings_name, value)
+        profiles_to_create.append(profile)
+
+    if email_address_visibility == UserProfile.EMAIL_ADDRESS_VISIBILITY_EVERYONE:
+        UserProfile.objects.bulk_create(profiles_to_create)
+    else:
+        for user_profile in profiles_to_create:
+            user_profile.email = user_profile.delivery_email
+
+        UserProfile.objects.bulk_create(profiles_to_create)
+
+        for user_profile in profiles_to_create:
+            user_profile.email = get_display_email_address(user_profile)
+        UserProfile.objects.bulk_update(profiles_to_create, ["email"])
+
+    for user_profile in profiles_to_create:
+        generate_and_upload_jdenticon_avatar(user_profile, str(realm.uuid), future=False)
+
+    RealmAuditLog.objects.bulk_create(
+        RealmAuditLog(
+            realm=realm,
+            modified_user=profile_,
+            event_type=AuditLogEventType.USER_CREATED,
+            event_time=profile_.date_joined,
+        )
+        for profile_ in profiles_to_create
+    )
+
+    full_members_system_group = NamedUserGroup.objects.get(
+        name=SystemGroups.FULL_MEMBERS, realm_for_sharding=realm, is_system_group=True
+    )
+    members_system_group = NamedUserGroup.objects.get(
+        name=SystemGroups.MEMBERS, realm_for_sharding=realm, is_system_group=True
+    )
+    group_memberships_to_create: list[UserGroupMembership] = []
+    for user_profile in profiles_to_create:
+        # All users are members since this function is only used to create bots
+        # and test and development environment users.
+        assert user_profile.role == UserProfile.ROLE_MEMBER
+        group_memberships_to_create.append(
+            UserGroupMembership(user_profile=user_profile, user_group=members_system_group)
+        )
+        if not user_profile.is_provisional_member:
+            group_memberships_to_create.append(
+                UserGroupMembership(user_profile=user_profile, user_group=full_members_system_group)
+            )
+
+    UserGroupMembership.objects.bulk_create(group_memberships_to_create)
+    now = timezone_now()
+    RealmAuditLog.objects.bulk_create(
+        RealmAuditLog(
+            realm=realm,
+            modified_user=membership.user_profile,
+            modified_user_group=membership.user_group.named_user_group,
+            event_type=AuditLogEventType.USER_GROUP_DIRECT_USER_MEMBERSHIP_ADDED,
+            event_time=now,
+            acting_user=None,
+        )
+        for membership in group_memberships_to_create
+    )
+
+
+def bulk_set_stream_recipient_fields(
+    streams: Collection[Stream] | QuerySet[Stream],
+    recipients: Iterable[Recipient] | None = None,
+) -> None:
+    if recipients is None:
+        stream_ids = [stream.id for stream in streams]
+        recipients = Recipient.objects.filter(type=Recipient.STREAM, type_id__in=stream_ids)
+
+    streams_dict = {stream.id: stream for stream in streams}
+
+    streams_to_update = set()
+    for recipient in recipients:
+        assert recipient.type == Recipient.STREAM
+        stream = streams_dict.get(recipient.type_id)
+        if stream is not None:
+            stream.recipient = recipient
+            streams_to_update.add(stream)
+    Stream.objects.bulk_update(streams_to_update, ["recipient"])
+
+
+# This is only sed in populate_db, so doesn't really need tests
+def bulk_create_streams(realm: Realm, stream_dict: dict[str, dict[str, Any]]) -> None:  # nocoverage
+    existing_streams = {
+        name.lower() for name in Stream.objects.filter(realm=realm).values_list("name", flat=True)
+    }
+    streams_to_create: list[Stream] = []
+    for name, options in stream_dict.items():
+        creator = options.get("creator", None)
+        if name.lower() not in existing_streams:
+            stream = Stream(
+                realm=realm,
+                name=name,
+                description=options["description"],
+                rendered_description=render_stream_description(options["description"], realm),
+                invite_only=options.get("invite_only", False),
+                history_public_to_subscribers=options.get("history_public_to_subscribers", True),
+                is_web_public=options.get("is_web_public", False),
+                creator=options.get("creator", None),
+                folder_id=options.get("folder_id", None),
+                topics_policy=options.get("topics_policy", StreamTopicsPolicyEnum.inherit.value),
+                **get_default_values_for_stream_permission_group_settings(realm, creator),
+            )
+            if "can_send_message_group" in options:
+                stream.can_send_message_group = options["can_send_message_group"]
+
+            streams_to_create.append(stream)
+    # Sort streams by name before creating them so that we can have a
+    # reliable ordering of `stream_id` across different python versions.
+    # This is required for test fixtures which contain `stream_id`. Prior
+    # to python 3.3 hashes were not randomized but after a security fix
+    # hash randomization was enabled in python 3.3 which made iteration
+    # of dictionaries and sets completely unpredictable. Here the order
+    # of elements while iterating `stream_dict` will be completely random
+    # for python 3.3 and later versions.
+    streams_to_create.sort(key=lambda x: x.name)
+    Stream.objects.bulk_create(streams_to_create)
+
+    recipients_to_create = [
+        Recipient(type_id=stream["id"], type=Recipient.STREAM)
+        for stream in Stream.objects.filter(realm=realm).values("id", "name")
+        if stream["name"].lower() not in existing_streams
+    ]
+    Recipient.objects.bulk_create(recipients_to_create)
+
+    bulk_set_stream_recipient_fields(streams_to_create, recipients_to_create)
+
+
+def create_users(
+    realm: Realm, name_list: Iterable[tuple[str, str]], bot_type: int | None = None
+) -> None:
+    user_set = {(email, full_name, True) for full_name, email in name_list}
+    bulk_create_users(realm, user_set, bot_type)

zerver/lib/cache.py

@@ -0,0 +1,894 @@
+# See https://zulip.readthedocs.io/en/latest/subsystems/caching.html for docs
+import hashlib
+import logging
+import os
+import re
+import secrets
+import sys
+import time
+import traceback
+from collections.abc import Callable, Iterable, Iterator, Sequence
+from functools import _lru_cache_wrapper, lru_cache, wraps
+from itertools import islice, product
+from typing import TYPE_CHECKING, Any, Generic, TypeVar
+
+from bmemcached.exceptions import MemcachedException
+from django.conf import settings
+from django.core.cache import caches
+from django.core.cache.backends.base import BaseCache
+from django.db.models import Q, QuerySet
+from typing_extensions import ParamSpec
+
+from scripts.lib.zulip_tools import DEPLOYMENTS_DIR, get_recent_deployments
+
+if TYPE_CHECKING:
+    # These modules have to be imported for type annotations but
+    # they cannot be imported at runtime due to cyclic dependency.
+    from zerver.models import Attachment, Message, MutedUser, Realm, Stream, SubMessage, UserProfile
+
+MEMCACHED_MAX_KEY_LENGTH = 250
+
+ParamT = ParamSpec("ParamT")
+ReturnT = TypeVar("ReturnT")
+
+logger = logging.getLogger()
+
+remote_cache_time_start = 0.0
+remote_cache_total_time = 0.0
+remote_cache_total_requests = 0
+
+
+def get_remote_cache_time() -> float:
+    return remote_cache_total_time
+
+
+def get_remote_cache_requests() -> int:
+    return remote_cache_total_requests
+
+
+def remote_cache_stats_start() -> None:
+    global remote_cache_time_start
+    remote_cache_time_start = time.time()
+
+
+def remote_cache_stats_finish() -> None:
+    global remote_cache_total_time, remote_cache_total_requests
+    remote_cache_total_requests += 1
+    remote_cache_total_time += time.time() - remote_cache_time_start
+
+
+def update_cached_cache_key_prefixes() -> list[str]:
+    # Clearing cache keys happens for all cache prefixes at once.
+    # Because the list of cache prefixes can only be derived from
+    # reading disk, we cache the list of cache prefixes, itself, in
+    # the cache.
+    found_prefixes: set[str] = set()
+    for deploy_dir in get_recent_deployments(None):
+        filename = os.path.join(deploy_dir, "var", "remote_cache_prefix")
+        if not os.path.exists(filename):
+            continue
+        with open(filename) as f:
+            found_prefixes.add(f.readline().removesuffix("\n"))
+
+    # We reach into the bmemcached client directly to do this set
+    # *without* compression, so that changes in our choice of
+    # bmemcached compression algorithm are always
+    # backwards-compatible.
+    caches["default"]._cache.set(  # type: ignore[attr-defined] # not in stubs
+        caches["default"].make_key("cache_key_prefixes"),
+        list(found_prefixes),
+        60 * 60 * 24,  # 24h
+        compress_level=0,
+    )
+    return list(found_prefixes)
+
+
+def get_or_create_key_prefix() -> str:
+    if settings.PUPPETEER_TESTS:
+        # This sets the prefix for the benefit of the Puppeteer tests.
+        #
+        # Having a fixed key is OK since we don't support running
+        # multiple copies of the Puppeteer tests at the same time anyway.
+        return "puppeteer_tests:"
+    elif settings.TEST_SUITE:
+        # The Python tests overwrite KEY_PREFIX on each test, but use
+        # this codepath as well, just to save running the more complex
+        # code below for reading the normal key prefix.
+        return "django_tests_unused:"
+
+    # directory `var` should exist in production
+    os.makedirs(os.path.join(settings.DEPLOY_ROOT, "var"), exist_ok=True)
+
+    filename = os.path.join(settings.DEPLOY_ROOT, "var", "remote_cache_prefix")
+    try:
+        with open(filename, "x") as f:
+            prefix = secrets.token_hex(16) + ":"
+            f.write(prefix + "\n")
+    except FileExistsError:
+        tries = 1
+        while tries < 10:
+            with open(filename) as f:
+                prefix = f.readline().removesuffix("\n")
+            if len(prefix) == 33:
+                break
+            tries += 1
+            prefix = ""
+            time.sleep(0.5)
+
+    if not prefix:
+        print("Could not read remote cache key prefix file")
+        sys.exit(1)
+
+    update_cached_cache_key_prefixes()
+    return prefix
+
+
+KEY_PREFIX: str = get_or_create_key_prefix()
+
+
+def get_all_cache_key_prefixes() -> list[str]:
+    if not settings.PRODUCTION or not os.path.exists(DEPLOYMENTS_DIR):
+        return [KEY_PREFIX]
+    return get_all_deployment_cache_key_prefixes()
+
+
+def get_all_deployment_cache_key_prefixes() -> list[str]:
+    stored_prefixes = caches["default"].get("cache_key_prefixes")
+    if stored_prefixes:
+        return stored_prefixes
+
+    return update_cached_cache_key_prefixes()
+
+
+def bounce_key_prefix_for_testing(test_name: str) -> None:
+    global KEY_PREFIX
+    KEY_PREFIX = test_name + ":" + str(os.getpid()) + ":"
+    # We are taking the hash of the KEY_PREFIX to decrease the size of the key.
+    # Memcached keys should have a length of less than 250.
+    KEY_PREFIX = hashlib.sha1(KEY_PREFIX.encode()).hexdigest() + ":"
+
+
+def get_cache_backend(cache_name: str | None) -> BaseCache:
+    if cache_name is None:
+        cache_name = "default"
+    return caches[cache_name]
+
+
+def cache_with_key(
+    keyfunc: Callable[ParamT, str],
+    cache_name: str | None = None,
+    timeout: int | None = None,
+    pickled_tupled: bool = True,
+) -> Callable[[Callable[ParamT, ReturnT]], Callable[ParamT, ReturnT]]:
+    """Decorator which applies Django caching to a function.
+
+    Decorator argument is a function which computes a cache key
+    from the original function's arguments.  You are responsible
+    for avoiding collisions with other uses of this decorator or
+    other uses of caching."""
+
+    def decorator(func: Callable[ParamT, ReturnT]) -> Callable[ParamT, ReturnT]:
+        @wraps(func)
+        def func_with_caching(*args: ParamT.args, **kwargs: ParamT.kwargs) -> ReturnT:
+            key = keyfunc(*args, **kwargs)
+
+            try:
+                val = cache_get(key, cache_name=cache_name)
+            except InvalidCacheKeyError:
+                stack_trace = traceback.format_exc()
+                log_invalid_cache_keys(stack_trace, [key])
+                return func(*args, **kwargs)
+
+            # Values are singleton tuples so that we can distinguish a
+            # result of None from a missing key.  Setting
+            # pickled_tupled=False avoids pickling the result (if it's
+            # a raw string or bytes) at the cost of losing this
+            # distinction.
+            if val is not None and pickled_tupled:
+                return val[0]
+
+            val = func(*args, **kwargs)
+            if isinstance(val, QuerySet):
+                logging.error(
+                    "cache_with_key attempted to store a full QuerySet object -- declining to cache",
+                    stack_info=True,
+                )
+            else:
+                cache_set(
+                    key, val, cache_name=cache_name, timeout=timeout, pickled_tupled=pickled_tupled
+                )
+
+            return val
+
+        return func_with_caching
+
+    return decorator
+
+
+class InvalidCacheKeyError(Exception):
+    pass
+
+
+def log_invalid_cache_keys(stack_trace: str, key: list[str]) -> None:
+    logger.warning(
+        "Invalid cache key used: %s\nStack trace: %s\n",
+        key,
+        stack_trace,
+    )
+
+
+def validate_cache_key(key: str, auto_prepend_prefix: bool = True) -> None:
+    if auto_prepend_prefix and not key.startswith(KEY_PREFIX):
+        key = KEY_PREFIX + key
+
+    # Theoretically memcached can handle non-ascii characters
+    # and only "control" characters are strictly disallowed, see:
+    # https://github.com/memcached/memcached/blob/master/doc/protocol.txt
+    # However, limiting the characters we allow in keys simiplifies things,
+    # and anyway we use a hash function when forming some keys to ensure
+    # the resulting keys fit the regex below.
+    # The regex checks "all characters between ! and ~ in the ascii table",
+    # which happens to be the set of all "nice" ascii characters.
+    if not bool(re.fullmatch(r"([!-~])+", key)):
+        raise InvalidCacheKeyError("Invalid characters in the cache key: " + key)
+    if len(key) > MEMCACHED_MAX_KEY_LENGTH:
+        raise InvalidCacheKeyError(f"Cache key too long: {key} Length: {len(key)}")
+
+
+def cache_set(
+    key: str,
+    val: Any,
+    cache_name: str | None = None,
+    timeout: int | None = None,
+    pickled_tupled: bool = True,
+) -> None:
+    final_key = KEY_PREFIX + key
+    validate_cache_key(final_key)
+
+    remote_cache_stats_start()
+    cache_backend = get_cache_backend(cache_name)
+    if pickled_tupled:
+        val = (val,)
+    try:
+        cache_backend.set(final_key, val, timeout=timeout)
+    except MemcachedException:
+        logger.exception("Error while storing to cache")
+    remote_cache_stats_finish()
+
+
+def cache_get(key: str, cache_name: str | None = None) -> Any:
+    final_key = KEY_PREFIX + key
+    validate_cache_key(final_key)
+
+    remote_cache_stats_start()
+    cache_backend = get_cache_backend(cache_name)
+    ret = cache_backend.get(final_key)
+    remote_cache_stats_finish()
+    return ret
+
+
+def cache_get_many(keys: list[str], cache_name: str | None = None) -> dict[str, Any]:
+    keys = [KEY_PREFIX + key for key in keys]
+    for key in keys:
+        validate_cache_key(key)
+    remote_cache_stats_start()
+    ret = get_cache_backend(cache_name).get_many(keys)
+    remote_cache_stats_finish()
+    return {key.removeprefix(KEY_PREFIX): value for key, value in ret.items()}
+
+
+def safe_cache_get_many(keys: list[str], cache_name: str | None = None) -> dict[str, Any]:
+    """Variant of cache_get_many that drops any keys that fail
+    validation, rather than throwing an exception visible to the
+    caller."""
+    try:
+        # Almost always the keys will all be correct, so we just try
+        # to do normal cache_get_many to avoid the overhead of
+        # validating all the keys here.
+        return cache_get_many(keys, cache_name)
+    except InvalidCacheKeyError:
+        stack_trace = traceback.format_exc()
+        good_keys, bad_keys = filter_good_and_bad_keys(keys)
+
+        log_invalid_cache_keys(stack_trace, bad_keys)
+        return cache_get_many(good_keys, cache_name)
+
+
+def cache_set_many(
+    items: dict[str, Any], cache_name: str | None = None, timeout: int | None = None
+) -> None:
+    new_items = {}
+    for key, item in items.items():
+        new_key = KEY_PREFIX + key
+        validate_cache_key(new_key)
+        new_items[new_key] = item
+    items = new_items
+    remote_cache_stats_start()
+    try:
+        get_cache_backend(cache_name).set_many(items, timeout=timeout)
+    except MemcachedException:
+        logger.exception("Error while storing to cache")
+    remote_cache_stats_finish()
+
+
+def safe_cache_set_many(
+    items: dict[str, Any], cache_name: str | None = None, timeout: int | None = None
+) -> None:
+    """Variant of cache_set_many that drops saving any keys that fail
+    validation, rather than throwing an exception visible to the
+    caller."""
+    try:
+        # Almost always the keys will all be correct, so we just try
+        # to do normal cache_set_many to avoid the overhead of
+        # validating all the keys here.
+        return cache_set_many(items, cache_name, timeout)
+    except InvalidCacheKeyError:
+        stack_trace = traceback.format_exc()
+
+        good_keys, bad_keys = filter_good_and_bad_keys(list(items.keys()))
+        log_invalid_cache_keys(stack_trace, bad_keys)
+
+        good_items = {key: items[key] for key in good_keys}
+        return cache_set_many(good_items, cache_name, timeout)
+
+
+def cache_delete(key: str, cache_name: str | None = None) -> None:
+    cache_delete_many([key], cache_name)
+
+
+def cache_delete_many(items: Iterable[str], cache_name: str | None = None) -> None:
+    remote_cache_stats_start()
+    keys = iter(e[0] + e[1] for e in product(get_all_cache_key_prefixes(), items))
+    while True:
+        batch = tuple(islice(keys, 10000))
+        if not batch:
+            break
+        for key in batch:
+            validate_cache_key(key, auto_prepend_prefix=False)
+        get_cache_backend(cache_name).delete_many(batch)
+    remote_cache_stats_finish()
+
+
+def filter_good_and_bad_keys(keys: list[str]) -> tuple[list[str], list[str]]:
+    good_keys = []
+    bad_keys = []
+    for key in keys:
+        try:
+            validate_cache_key(key)
+            good_keys.append(key)
+        except InvalidCacheKeyError:
+            bad_keys.append(key)
+
+    return good_keys, bad_keys
+
+
+# Generic_bulk_cached fetch and its helpers.  We start with declaring
+# a few type variables that help define its interface.
+
+# Type for the cache's keys; will typically be int or str.
+ObjKT = TypeVar("ObjKT")
+
+# Type for items to be fetched from the database (e.g. a Django model object)
+ItemT = TypeVar("ItemT")
+
+# Type for items to be stored in the cache (e.g. a dictionary serialization).
+# Will equal ItemT unless a cache_transformer is specified.
+CacheItemT = TypeVar("CacheItemT")
+
+# Type for compressed items for storage in the cache.  For
+# serializable objects, will be the object; if encoded, bytes.
+CompressedItemT = TypeVar("CompressedItemT")
+
+
+# Required arguments are as follows:
+# * object_ids: The list of object ids to look up
+# * cache_key_function: object_id => cache key
+# * query_function: [object_ids] => [objects from database]
+# * setter: Function to call before storing items to cache (e.g. compression)
+# * extractor: Function to call on items returned from cache
+#   (e.g. decompression).  Should be the inverse of the setter
+#   function.
+# * id_fetcher: Function mapping an object from database => object_id
+#   (in case we're using a key more complex than obj.id)
+# * cache_transformer: Function mapping an object from database =>
+#   value for cache (in case the values that we're caching are some
+#   function of the objects, not the objects themselves)
+def generic_bulk_cached_fetch(
+    cache_key_function: Callable[[ObjKT], str],
+    query_function: Callable[[list[ObjKT]], Iterable[ItemT]],
+    object_ids: Sequence[ObjKT],
+    *,
+    extractor: Callable[[CompressedItemT], CacheItemT],
+    setter: Callable[[CacheItemT], CompressedItemT],
+    id_fetcher: Callable[[ItemT], ObjKT],
+    cache_transformer: Callable[[ItemT], CacheItemT],
+    pickled_tupled: bool = True,
+) -> dict[ObjKT, CacheItemT]:
+    if len(object_ids) == 0:
+        # Nothing to fetch.
+        return {}
+
+    if pickled_tupled:
+        return generic_bulk_cached_fetch(
+            cache_key_function,
+            query_function,
+            object_ids,
+            extractor=lambda val: extractor(val[0]),
+            setter=lambda val: (setter(val),),
+            id_fetcher=id_fetcher,
+            cache_transformer=cache_transformer,
+            pickled_tupled=False,
+        )
+
+    cache_keys: dict[ObjKT, str] = {}
+    for object_id in object_ids:
+        cache_keys[object_id] = cache_key_function(object_id)
+
+    cached_objects_compressed: dict[str, CompressedItemT] = safe_cache_get_many(
+        [cache_keys[object_id] for object_id in object_ids],
+    )
+
+    cached_objects = {key: extractor(val) for key, val in cached_objects_compressed.items()}
+    needed_ids = [
+        object_id for object_id in object_ids if cache_keys[object_id] not in cached_objects
+    ]
+
+    # Only call query_function if there are some ids to fetch from the database:
+    if len(needed_ids) > 0:
+        db_objects = query_function(needed_ids)
+    else:
+        db_objects = []
+
+    items_for_remote_cache: dict[str, CompressedItemT] = {}
+    for obj in db_objects:
+        key = cache_keys[id_fetcher(obj)]
+        item = cache_transformer(obj)
+        items_for_remote_cache[key] = setter(item)
+        cached_objects[key] = item
+    if len(items_for_remote_cache) > 0:
+        safe_cache_set_many(items_for_remote_cache)
+    return {
+        object_id: cached_objects[cache_keys[object_id]]
+        for object_id in object_ids
+        if cache_keys[object_id] in cached_objects
+    }
+
+
+def bulk_cached_fetch(
+    cache_key_function: Callable[[ObjKT], str],
+    query_function: Callable[[list[ObjKT]], Iterable[ItemT]],
+    object_ids: Sequence[ObjKT],
+    *,
+    id_fetcher: Callable[[ItemT], ObjKT],
+) -> dict[ObjKT, ItemT]:
+    return generic_bulk_cached_fetch(
+        cache_key_function,
+        query_function,
+        object_ids,
+        id_fetcher=id_fetcher,
+        extractor=lambda obj: obj,
+        setter=lambda obj: obj,
+        cache_transformer=lambda obj: obj,
+    )
+
+
+def preview_url_cache_key(url: str) -> str:
+    return f"preview_url:{hashlib.sha1(url.encode()).hexdigest()}"
+
+
+def display_recipient_cache_key(recipient_id: int) -> str:
+    return f"display_recipient_dict:{recipient_id}"
+
+
+def single_user_display_recipient_cache_key(user_id: int) -> str:
+    return f"single_user_display_recipient:{user_id}"
+
+
+def user_profile_by_email_realm_id_cache_key(email: str, realm_id: int) -> str:
+    return f"user_profile:{hashlib.sha1(email.strip().encode()).hexdigest()}:{realm_id}"
+
+
+def user_profile_by_email_realm_cache_key(email: str, realm: "Realm") -> str:
+    return user_profile_by_email_realm_id_cache_key(email, realm.id)
+
+
+def user_profile_delivery_email_cache_key(delivery_email: str, realm_id: int) -> str:
+    return f"user_profile_by_delivery_email:{hashlib.sha1(delivery_email.strip().encode()).hexdigest()}:{realm_id}"
+
+
+def bot_profile_cache_key(email: str, realm_id: int) -> str:
+    return f"bot_profile:{hashlib.sha1(email.strip().encode()).hexdigest()}"
+
+
+def user_profile_by_id_cache_key(user_profile_id: int) -> str:
+    return f"user_profile_by_id:{user_profile_id}"
+
+
+def user_profile_narrow_by_id_cache_key(user_profile_id: int) -> str:
+    return f"user_profile_narrow_by_id:{user_profile_id}"
+
+
+def user_profile_by_api_key_cache_key(api_key: str) -> str:
+    return f"user_profile_by_api_key:{api_key}"
+
+
+def get_cross_realm_dicts_key() -> str:
+    emails = list(settings.CROSS_REALM_BOT_EMAILS)
+    raw_key = ",".join(sorted(emails))
+    digest = hashlib.sha1(raw_key.encode()).hexdigest()
+    return f"get_cross_realm_dicts:{digest}"
+
+
+realm_user_dict_fields: list[str] = [
+    "id",
+    "full_name",
+    "email",
+    "avatar_source",
+    "avatar_version",
+    "is_active",
+    "role",
+    "is_bot",
+    "timezone",
+    "date_joined",
+    "bot_owner_id",
+    "delivery_email",
+    "bot_type",
+    "long_term_idle",
+    "email_address_visibility",
+    "is_imported_stub",
+    "is_deleted",
+]
+
+
+def realm_user_dicts_cache_key(realm_id: int) -> str:
+    return f"realm_user_dicts:{realm_id}"
+
+
+def get_muting_users_cache_key(muted_user_id: int) -> str:
+    return f"muting_users_list:{muted_user_id}"
+
+
+def get_realm_used_upload_space_cache_key(realm_id: int) -> str:
+    return f"realm_used_upload_space:{realm_id}"
+
+
+def get_realm_seat_count_cache_key(realm_id: int) -> str:
+    return f"realm_seat_count:{realm_id}"
+
+
+def active_user_ids_cache_key(realm_id: int) -> str:
+    return f"active_user_ids:{realm_id}"
+
+
+def active_non_guest_user_ids_cache_key(realm_id: int) -> str:
+    return f"active_non_guest_user_ids:{realm_id}"
+
+
+def active_guest_user_ids_cache_key(realm_id: int) -> str:
+    return f"active_guest_user_ids:{realm_id}"
+
+
+def get_realm_system_groups_cache_key(realm_id: int) -> str:
+    return f"realm_system_groups:{realm_id}"
+
+
+bot_dict_fields: list[str] = [
+    "api_key",
+    "avatar_source",
+    "avatar_version",
+    "bot_owner_id",
+    "bot_type",
+    "default_all_public_streams",
+    "default_events_register_stream__name",
+    "default_sending_stream__name",
+    "email",
+    "full_name",
+    "id",
+    "is_active",
+    "realm_id",
+]
+
+
+def bot_dicts_in_realm_cache_key(realm_id: int) -> str:
+    return f"bot_dicts_in_realm:{realm_id}"
+
+
+def delete_user_profile_caches(user_profiles: Iterable["UserProfile"], realm_id: int) -> None:
+    # Imported here to avoid cyclic dependency.
+    from zerver.models.users import is_cross_realm_bot_email
+
+    def user_profile_key_iterator() -> Iterator[str]:
+        for user_profile in user_profiles:
+            yield user_profile_by_id_cache_key(user_profile.id)
+            yield user_profile_narrow_by_id_cache_key(user_profile.id)
+            yield user_profile_by_api_key_cache_key(user_profile.api_key)
+            yield user_profile_by_email_realm_id_cache_key(user_profile.email, realm_id)
+            yield user_profile_delivery_email_cache_key(user_profile.delivery_email, realm_id)
+            if user_profile.is_bot and is_cross_realm_bot_email(user_profile.email):
+                # Handle clearing system bots from their special cache.
+                yield bot_profile_cache_key(user_profile.email, realm_id)
+                yield get_cross_realm_dicts_key()
+
+    cache_delete_many(user_profile_key_iterator())
+
+
+def delete_display_recipient_cache(user_profiles: list["UserProfile"]) -> None:
+    from zerver.models import Subscription  # We need to import here to avoid cyclic dependency.
+
+    recipient_ids = Subscription.objects.filter(user_profile__in=user_profiles).values_list(
+        "recipient_id", flat=True
+    )
+    keys = [display_recipient_cache_key(rid) for rid in recipient_ids]
+    keys.extend(
+        [single_user_display_recipient_cache_key(user_profile.id) for user_profile in user_profiles]
+    )
+    cache_delete_many(keys)
+
+
+def changed(update_fields: Sequence[str] | None, fields: list[str]) -> bool:
+    if update_fields is None:
+        # adds/deletes should invalidate the cache
+        return True
+
+    update_fields_set = set(update_fields)
+    return any(f in update_fields_set for f in fields)
+
+
+def bulk_flush_users(
+    *,
+    user_profiles: list["UserProfile"],
+    realm: "Realm",
+    update_fields: Sequence[str] | None = None,
+) -> None:
+    delete_user_profile_caches(user_profiles, realm.id)
+
+    cache_keys_to_delete = set()
+    if changed(update_fields, realm_user_dict_fields):
+        cache_keys_to_delete.add(realm_user_dicts_cache_key(realm.id))
+
+    if changed(update_fields, ["is_active"]):
+        cache_keys_to_delete.add(active_user_ids_cache_key(realm.id))
+        cache_keys_to_delete.add(active_non_guest_user_ids_cache_key(realm.id))
+        cache_keys_to_delete.add(active_guest_user_ids_cache_key(realm.id))
+
+    if changed(update_fields, ["role"]):
+        cache_keys_to_delete.add(active_non_guest_user_ids_cache_key(realm.id))
+        cache_keys_to_delete.add(active_guest_user_ids_cache_key(realm.id))
+
+    # Invalidate our bots_in_realm info dict if any bot has
+    # changed the fields in the dict or become (in)active
+    if changed(update_fields, bot_dict_fields):
+        for user_profile in user_profiles:
+            if user_profile.is_bot:
+                cache_keys_to_delete.add(bot_dicts_in_realm_cache_key(realm.id))
+
+    cache_delete_many(list(cache_keys_to_delete))
+
+    if changed(update_fields, ["email", "full_name", "id", "is_mirror_dummy"]):
+        delete_display_recipient_cache(user_profiles)
+
+
+# Called by models/users.py to flush the user_profile cache whenever we save
+# a user_profile object
+def flush_user_profile(
+    *,
+    instance: "UserProfile",
+    update_fields: Sequence[str] | None = None,
+    **kwargs: object,
+) -> None:
+    user_profile = instance
+    bulk_flush_users(
+        user_profiles=[user_profile], realm=user_profile.realm, update_fields=update_fields
+    )
+
+
+def flush_muting_users_cache(*, instance: "MutedUser", **kwargs: object) -> None:
+    mute_object = instance
+    cache_delete(get_muting_users_cache_key(mute_object.muted_user_id))
+
+
+# Called by models/realms.py to flush various caches whenever we save
+# a Realm object.  The main tricky thing here is that Realm info is
+# generally cached indirectly through user_profile objects.
+def flush_realm(
+    *,
+    instance: "Realm",
+    update_fields: Sequence[str] | None = None,
+    from_deletion: bool = False,
+    **kwargs: object,
+) -> None:
+    realm = instance
+    users = realm.get_active_users()
+    delete_user_profile_caches(users, realm.id)
+
+    if (
+        from_deletion
+        or realm.deactivated
+        or (update_fields is not None and "string_id" in update_fields)
+    ):
+        cache_delete(realm_user_dicts_cache_key(realm.id))
+        cache_delete(active_user_ids_cache_key(realm.id))
+        cache_delete(bot_dicts_in_realm_cache_key(realm.id))
+        cache_delete(realm_alert_words_cache_key(realm.id))
+        cache_delete(realm_alert_words_automaton_cache_key(realm.id))
+        cache_delete(active_non_guest_user_ids_cache_key(realm.id))
+        cache_delete(active_guest_user_ids_cache_key(realm.id))
+        cache_delete(realm_rendered_description_cache_key(realm))
+        cache_delete(realm_text_description_cache_key(realm))
+    elif changed(update_fields, ["description"]):
+        cache_delete(realm_rendered_description_cache_key(realm))
+        cache_delete(realm_text_description_cache_key(realm))
+
+
+def realm_alert_words_cache_key(realm_id: int) -> str:
+    return f"realm_alert_words:{realm_id}"
+
+
+def realm_alert_words_automaton_cache_key(realm_id: int) -> str:
+    return f"realm_alert_words_automaton:{realm_id}"
+
+
+def realm_rendered_description_cache_key(realm: "Realm") -> str:
+    return f"realm_rendered_description:{realm.string_id}"
+
+
+def realm_text_description_cache_key(realm: "Realm") -> str:
+    return f"realm_text_description:{realm.string_id}"
+
+
+# Called by models/streams.py to flush the stream cache whenever we save a stream
+# object.
+def flush_stream(
+    *,
+    instance: "Stream",
+    update_fields: Sequence[str] | None = None,
+    **kwargs: object,
+) -> None:
+    from zerver.models import UserProfile
+
+    stream = instance
+
+    if update_fields is None or (
+        "name" in update_fields
+        and UserProfile.objects.filter(
+            Q(default_sending_stream=stream) | Q(default_events_register_stream=stream)
+        ).exists()
+    ):
+        cache_delete(bot_dicts_in_realm_cache_key(stream.realm_id))
+
+
+def flush_used_upload_space_cache(
+    *,
+    instance: "Attachment",
+    created: bool = True,
+    **kwargs: object,
+) -> None:
+    attachment = instance
+
+    if created:
+        cache_delete(get_realm_used_upload_space_cache_key(attachment.owner.realm_id))
+
+
+def to_dict_cache_key_id(message_id: int) -> str:
+    return f"message_dict:{message_id}"
+
+
+def to_dict_cache_key(message: "Message", realm_id: int | None = None) -> str:
+    return to_dict_cache_key_id(message.id)
+
+
+def open_graph_description_cache_key(content: bytes, request_url: str) -> str:
+    return f"open_graph_description_path:{hashlib.sha1(request_url.encode()).hexdigest()}"
+
+
+def zoom_server_access_token_cache_key(account_id: str) -> str:
+    return f"zoom_server_to_server_access_token:{account_id}"
+
+
+def flush_zoom_server_access_token_cache(account_id: str) -> None:
+    cache_delete(zoom_server_access_token_cache_key(account_id))
+
+
+def flush_message(*, instance: "Message", **kwargs: object) -> None:
+    message = instance
+    cache_delete(to_dict_cache_key_id(message.id))
+
+
+def flush_submessage(*, instance: "SubMessage", **kwargs: object) -> None:
+    submessage = instance
+    # submessages are not cached directly, they are part of their
+    # parent messages
+    message_id = submessage.message_id
+    cache_delete(to_dict_cache_key_id(message_id))
+
+
+class IgnoreUnhashableLruCacheWrapper(Generic[ParamT, ReturnT]):
+    def __init__(
+        self, function: Callable[ParamT, ReturnT], cached_function: "_lru_cache_wrapper[ReturnT]"
+    ) -> None:
+        self.key_prefix = KEY_PREFIX
+        self.function = function
+        self.cached_function = cached_function
+        self.cache_info = cached_function.cache_info
+        self.cache_clear = cached_function.cache_clear
+
+    def __call__(self, *args: ParamT.args, **kwargs: ParamT.kwargs) -> ReturnT:
+        if settings.DEVELOPMENT and not settings.TEST_SUITE:  # nocoverage
+            # In the development environment, we want every file
+            # change to refresh the source files from disk.
+            return self.function(*args, **kwargs)
+
+        if self.key_prefix != KEY_PREFIX:
+            # Clear cache when cache.KEY_PREFIX changes. This is used in
+            # tests.
+            self.cache_clear()
+            self.key_prefix = KEY_PREFIX
+
+        try:
+            return self.cached_function(
+                *args,
+                **kwargs,  # type: ignore[arg-type] # might be unhashable
+            )
+        except TypeError:
+            # args or kwargs contains an element which is unhashable. In
+            # this case we don't cache the result.
+            pass
+
+        # Deliberately calling this function from outside of exception
+        # handler to get a more descriptive traceback. Otherwise traceback
+        # can include the exception from cached_function as well.
+        return self.function(*args, **kwargs)
+
+
+def ignore_unhashable_lru_cache(
+    maxsize: int = 128, typed: bool = False
+) -> Callable[[Callable[ParamT, ReturnT]], IgnoreUnhashableLruCacheWrapper[ParamT, ReturnT]]:
+    """
+    This is a wrapper over lru_cache function. It adds following features on
+    top of lru_cache:
+
+        * It will not cache result of functions with unhashable arguments.
+        * It will clear cache whenever zerver.lib.cache.KEY_PREFIX changes.
+    """
+    internal_decorator = lru_cache(maxsize=maxsize, typed=typed)
+
+    def decorator(
+        user_function: Callable[ParamT, ReturnT],
+    ) -> IgnoreUnhashableLruCacheWrapper[ParamT, ReturnT]:
+        return IgnoreUnhashableLruCacheWrapper(user_function, internal_decorator(user_function))
+
+    return decorator
+
+
+def dict_to_items_tuple(user_function: Callable[..., Any]) -> Callable[..., Any]:
+    """Wrapper that converts any dict args to dict item tuples."""
+
+    def dict_to_tuple(arg: Any) -> Any:
+        if isinstance(arg, dict):
+            return tuple(sorted(arg.items()))
+        return arg
+
+    def wrapper(*args: Any, **kwargs: Any) -> Any:
+        new_args = (dict_to_tuple(arg) for arg in args)
+        return user_function(*new_args, **kwargs)
+
+    return wrapper
+
+
+def items_tuple_to_dict(user_function: Callable[..., Any]) -> Callable[..., Any]:
+    """Wrapper that converts any dict items tuple args to dicts."""
+
+    def dict_items_to_dict(arg: Any) -> Any:
+        if isinstance(arg, tuple):
+            try:
+                return dict(arg)
+            except TypeError:
+                pass
+        return arg
+
+    def wrapper(*args: Any, **kwargs: Any) -> Any:
+        new_args = (dict_items_to_dict(arg) for arg in args)
+        new_kwargs = {key: dict_items_to_dict(val) for key, val in kwargs.items()}
+        return user_function(*new_args, **new_kwargs)
+
+    return wrapper

zerver/lib/cache_helpers.py

@@ -0,0 +1,146 @@
+# See https://zulip.readthedocs.io/en/latest/subsystems/caching.html for docs
+import logging
+from collections.abc import Callable, Iterable, Iterator
+from datetime import timedelta
+from typing import Any
+
+from django.conf import settings
+from django.contrib.sessions.models import Session
+from django.db import connection
+from django.db.models import QuerySet
+from django.utils.timezone import now as timezone_now
+
+# This file needs to be different from cache.py because cache.py
+# cannot import anything from zerver.models or we'd have an import
+# loop
+from analytics.models import RealmCount
+from zerver.lib.cache import (
+    cache_set_many,
+    get_remote_cache_requests,
+    get_remote_cache_time,
+    user_profile_narrow_by_id_cache_key,
+)
+from zerver.lib.safe_session_cached_db import SessionStore
+from zerver.lib.sessions import session_engine
+from zerver.models import Client, UserProfile
+from zerver.models.clients import get_client_cache_key
+from zerver.models.users import base_get_user_narrow_queryset
+
+
+def get_narrow_users() -> Iterator[UserProfile]:
+    return (
+        base_get_user_narrow_queryset()
+        .filter(long_term_idle=False, realm__in=get_active_realm_ids())
+        .iterator()
+    )
+
+
+def user_narrow_cache_items(
+    items_for_remote_cache: dict[str, tuple[UserProfile]], user_profile: UserProfile
+) -> None:
+    items_for_remote_cache[user_profile_narrow_by_id_cache_key(user_profile.id)] = (user_profile,)
+
+
+def client_cache_items(items_for_remote_cache: dict[str, tuple[Client]], client: Client) -> None:
+    items_for_remote_cache[get_client_cache_key(client.name)] = (client,)
+
+
+def session_cache_items(
+    items_for_remote_cache: dict[str, dict[str, object]], session: Session
+) -> None:
+    if settings.SESSION_ENGINE != "zerver.lib.safe_session_cached_db":
+        # If we're not using the cached_db session engine, we there
+        # will be no store.cache_key attribute, and in any case we
+        # don't need to fill the cache, since it won't exist.
+        return
+    store = session_engine.SessionStore(session_key=session.session_key)
+    assert isinstance(store, SessionStore)
+    items_for_remote_cache[store.cache_key] = store.decode(session.session_data)
+
+
+def get_active_realm_ids() -> QuerySet[RealmCount, int]:
+    """For installations like Zulip Cloud hosting a lot of realms, it only makes
+    sense to do cache-filling work for realms that have any currently
+    active users/clients.  Otherwise, we end up with every single-user
+    trial organization that has ever been created costing us N streams
+    worth of cache work (where N is the number of default streams for
+    a new organization).
+    """
+    date = timezone_now() - timedelta(days=2)
+    return (
+        RealmCount.objects.filter(
+            end_time__gte=date,
+            property="1day_actives::day",
+            # Filtering on subgroup is important to ensure we use the good indexes.
+            subgroup=None,
+            value__gt=0,
+        )
+        .distinct("realm_id")
+        .values_list("realm_id", flat=True)
+    )
+
+
+# Format is (objects query, items filler function, timeout, batch size)
+#
+# The objects queries are put inside lambdas to prevent Django from
+# doing any setup for things we're unlikely to use (without the lambda
+# wrapper the below adds an extra 3ms or so to startup time for
+# anything importing this file).
+cache_fillers: dict[
+    str, tuple[Callable[[], Iterable[Any]], Callable[[dict[str, Any], Any], None], int, int]
+] = {
+    "user_narrow": (get_narrow_users, user_narrow_cache_items, 3600 * 24 * 7, 10000),
+    "client": (
+        lambda: Client.objects.all().iterator(),
+        client_cache_items,
+        3600 * 24 * 7,
+        10000,
+    ),
+    "session": (
+        lambda: Session.objects.all().iterator(),
+        session_cache_items,
+        3600 * 24 * 7,
+        10000,
+    ),
+}
+
+
+class SQLQueryCounter:
+    def __init__(self) -> None:
+        self.count = 0
+
+    def __call__(
+        self,
+        execute: Callable[[str, Any, bool, dict[str, Any]], Any],
+        sql: str,
+        params: Any,
+        many: bool,
+        context: dict[str, Any],
+    ) -> Any:
+        self.count += 1
+        return execute(sql, params, many, context)
+
+
+def fill_remote_cache(cache: str) -> None:
+    remote_cache_time_start = get_remote_cache_time()
+    remote_cache_requests_start = get_remote_cache_requests()
+    items_for_remote_cache: dict[str, Any] = {}
+    (objects, items_filler, timeout, batch_size) = cache_fillers[cache]
+    count = 0
+    db_query_counter = SQLQueryCounter()
+    with connection.execute_wrapper(db_query_counter):
+        for obj in objects():
+            items_filler(items_for_remote_cache, obj)
+            count += 1
+            if count % batch_size == 0:
+                cache_set_many(items_for_remote_cache, timeout=timeout)
+                items_for_remote_cache = {}
+        cache_set_many(items_for_remote_cache, timeout=timeout)
+    logging.info(
+        "Successfully populated %s cache: %d items, %d DB queries, %d memcached sets, %.2f seconds",
+        cache,
+        count,
+        db_query_counter.count,
+        get_remote_cache_requests() - remote_cache_requests_start,
+        get_remote_cache_time() - remote_cache_time_start,
+    )

zerver/lib/camo.py

@@ -0,0 +1,28 @@
+import hashlib
+import hmac
+
+from django.conf import settings
+from django.utils.crypto import constant_time_compare
+
+
+def generate_camo_url(url: str) -> str:
+    encoded_url = url.encode()
+    assert settings.CAMO_KEY is not None
+    encoded_camo_key = settings.CAMO_KEY.encode()
+    digest = hmac.new(encoded_camo_key, encoded_url, hashlib.sha1).hexdigest()
+    return f"{digest}/{encoded_url.hex()}"
+
+
+# Encodes the provided URL using the same algorithm used by the camo
+# caching https image proxy
+def get_camo_url(url: str) -> str:
+    # Only encode the URL if Camo is enabled
+    if settings.CAMO_URI == "":
+        return url
+    return f"{settings.CAMO_URI}{generate_camo_url(url)}"
+
+
+def is_camo_url_valid(digest: str, url: str) -> bool:
+    camo_url = generate_camo_url(url)
+    camo_url_digest = camo_url.split("/")[0]
+    return constant_time_compare(camo_url_digest, digest)

zerver/lib/captcha.py

@@ -0,0 +1,105 @@
+import base64
+import logging
+from typing import Any
+
+import orjson
+from altcha.v1 import verify_solution
+from django import forms
+from django.conf import settings
+from django.forms.renderers import BaseRenderer
+from django.http import HttpRequest
+from django.utils.html import format_html
+from django.utils.safestring import SafeString
+from django.utils.translation import gettext as _
+from django.utils.translation import gettext_lazy
+from typing_extensions import override
+
+
+def captcha_enabled() -> bool:
+    return bool(settings.USING_CAPTCHA and settings.ALTCHA_HMAC_KEY)
+
+
+class AltchaWidget(forms.TextInput):
+    @override
+    def render(
+        self,
+        name: str,
+        value: Any,
+        attrs: dict[str, Any] | None = None,
+        renderer: BaseRenderer | None = None,
+    ) -> SafeString:
+        return format_html(
+            (
+                "<altcha-widget"
+                '  name="captcha"'
+                '  challengeurl="/json/antispam_challenge"'
+                "  hidelogo"
+                "  hidefooter"
+                "  refetchonexpire"
+                '  style="{}"'
+                '  strings="{}"'
+                ">"
+            ),
+            "--altcha-max-width: 300px;",
+            orjson.dumps(
+                {
+                    "verified": _("Verified that you're a human user!"),
+                    "verifying": _("Verifying that you're not a bot…"),
+                }
+            ).decode(),
+        )
+
+
+class CaptchaFormMixin(forms.Form):
+    captcha = forms.CharField(
+        widget=AltchaWidget,
+        error_messages={"required": gettext_lazy("Validation failed, please try again.")},
+    )
+
+    def __init__(self, *args: Any, request: HttpRequest, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self.request = request
+        if not captcha_enabled():
+            del self.fields["captcha"]
+
+    @override
+    def clean(self) -> None:
+        super().clean()
+        # Validating the payload consumes the solved challenge, to
+        # prevent replay; defer it until the rest of the form is
+        # valid, so that a failure in some other field does not
+        # invalidate a successful solve.  The captcha field does not
+        # exist when the captcha is not enabled.
+        if "captcha" in self.fields and not self.errors:
+            try:
+                validate_captcha_payload(self.request, self.cleaned_data["captcha"])
+            except forms.ValidationError as error:
+                self.add_error("captcha", error)
+
+
+def validate_captcha_payload(request: HttpRequest, captcha_payload: str) -> None:
+    if not captcha_enabled():  # nocoverage
+        raise forms.ValidationError(_("Challenges are not enabled."))
+
+    try:
+        ok, err = verify_solution(captcha_payload, settings.ALTCHA_HMAC_KEY, check_expires=True)
+        if not ok:
+            logging.warning("Invalid altcha solution: %s", err)
+            raise forms.ValidationError(_("Validation failed, please try again."))
+    except forms.ValidationError:
+        raise
+    except Exception:
+        logging.exception("Error while validating altcha solution")
+        raise forms.ValidationError(_("Validation failed, please try again."))
+
+    captcha_data = orjson.loads(base64.b64decode(captcha_payload))
+    challenge = captcha_data["challenge"]
+    session_challenges = [e[0] for e in request.session.get("altcha_challenges", [])]
+    if challenge not in session_challenges:
+        logging.warning("Expired or replayed altcha solution")
+        raise forms.ValidationError(_("Validation failed, please try again."))
+
+    # Remove the successful solve from the session, to prevent replay
+    request.session["altcha_challenges"] = [
+        e for e in request.session.get("altcha_challenges", []) if e[0] != challenge
+    ]

zerver/lib/channel_folders.py

@@ -0,0 +1,92 @@
+from dataclasses import dataclass
+
+from django.utils.translation import gettext as _
+
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.markdown import markdown_convert
+from zerver.lib.streams import get_web_public_streams_queryset
+from zerver.lib.string_validation import check_string_is_printable
+from zerver.lib.timestamp import datetime_to_timestamp
+from zerver.models import ChannelFolder, Realm, Stream, UserProfile
+
+
+@dataclass
+class ChannelFolderData:
+    id: int
+    name: str
+    description: str
+    rendered_description: str
+    order: int
+    creator_id: int | None
+    date_created: int
+    is_archived: bool
+
+
+def check_channel_folder_name(name: str, realm: Realm) -> None:
+    if name.strip() == "":
+        raise JsonableError(_("Channel folder name can't be empty."))
+
+    invalid_character_pos = check_string_is_printable(name)
+    if invalid_character_pos is not None:
+        raise JsonableError(
+            _("Invalid character in channel folder name, at position {position}.").format(
+                position=invalid_character_pos
+            )
+        )
+
+    if ChannelFolder.objects.filter(name__iexact=name, realm=realm, is_archived=False).exists():
+        raise JsonableError(_("Channel folder name already in use"))
+
+
+def render_channel_folder_description(text: str, realm: Realm, *, acting_user: UserProfile) -> str:
+    return markdown_convert(
+        text, message_realm=realm, no_previews=True, acting_user=acting_user
+    ).rendered_content
+
+
+def get_channel_folder_data(channel_folder: ChannelFolder) -> ChannelFolderData:
+    date_created = datetime_to_timestamp(channel_folder.date_created)
+    return ChannelFolderData(
+        id=channel_folder.id,
+        name=channel_folder.name,
+        description=channel_folder.description,
+        rendered_description=channel_folder.rendered_description,
+        order=channel_folder.order,
+        date_created=date_created,
+        creator_id=channel_folder.creator_id,
+        is_archived=channel_folder.is_archived,
+    )
+
+
+def get_channel_folders_in_realm(
+    realm: Realm, include_archived: bool = False
+) -> list[ChannelFolderData]:
+    folders = ChannelFolder.objects.filter(realm=realm)
+    if not include_archived:
+        folders = folders.exclude(is_archived=True)
+
+    channel_folders = [get_channel_folder_data(channel_folder) for channel_folder in folders]
+    return sorted(channel_folders, key=lambda folder: folder.order)
+
+
+def get_channel_folder_by_id(channel_folder_id: int, realm: Realm) -> ChannelFolder:
+    try:
+        channel_folder = ChannelFolder.objects.get(id=channel_folder_id, realm=realm)
+        return channel_folder
+    except ChannelFolder.DoesNotExist:
+        raise JsonableError(_("Invalid channel folder ID"))
+
+
+def get_channel_folders_for_spectators(realm: Realm) -> list[ChannelFolderData]:
+    folder_ids_for_web_public_streams = set(
+        get_web_public_streams_queryset(realm).values_list("folder_id", flat=True)
+    )
+    folders = ChannelFolder.objects.filter(id__in=folder_ids_for_web_public_streams)
+    channel_folders = [get_channel_folder_data(channel_folder) for channel_folder in folders]
+    return sorted(channel_folders, key=lambda folder: folder.id)
+
+
+def check_channel_folder_in_use(channel_folder: ChannelFolder) -> bool:
+    if Stream.objects.filter(folder=channel_folder).exists():
+        return True
+    return False

zerver/lib/compatibility.py

@@ -0,0 +1,159 @@
+import os
+import re
+from datetime import datetime, timedelta, timezone
+
+from django.conf import settings
+from django.utils.timezone import now as timezone_now
+
+from version import DESKTOP_MINIMUM_VERSION, DESKTOP_WARNING_VERSION
+from zerver.lib.user_agent import parse_user_agent
+from zerver.models import UserProfile
+from zerver.signals import get_device_browser
+
+# LAST_SERVER_UPGRADE_TIME is the last time the server had a version deployed.
+if settings.PRODUCTION:  # nocoverage
+    timestamp = os.path.basename(os.path.abspath(settings.DEPLOY_ROOT))
+    LAST_SERVER_UPGRADE_TIME = datetime.strptime(timestamp, "%Y-%m-%d-%H-%M-%S").replace(
+        tzinfo=timezone.utc
+    )
+else:
+    LAST_SERVER_UPGRADE_TIME = timezone_now()
+
+
+def is_outdated_server(user_profile: UserProfile | None) -> bool:
+    # Release tarballs are unpacked via `tar -xf`, which means the
+    # `mtime` on files in them is preserved from when the release
+    # tarball was built.  Checking this allows us to catch cases where
+    # someone has upgraded in the last year but to a release more than
+    # a year old.
+    git_version_path = os.path.join(settings.DEPLOY_ROOT, "version.py")
+    release_build_time = datetime.fromtimestamp(os.path.getmtime(git_version_path), timezone.utc)
+
+    version_no_newer_than = min(LAST_SERVER_UPGRADE_TIME, release_build_time)
+    deadline = version_no_newer_than + timedelta(days=settings.SERVER_UPGRADE_NAG_DEADLINE_DAYS)
+
+    if user_profile is None or not user_profile.is_realm_admin:
+        # Administrators get warned at the deadline; all users 30 days later.
+        deadline += timedelta(days=30)
+
+    if timezone_now() > deadline:
+        return True
+    return False
+
+
+def pop_numerals(ver: str) -> tuple[list[int], str]:
+    match = re.search(r"^( \d+ (?: \. \d+ )* ) (.*)", ver, re.VERBOSE)
+    if match is None:
+        return [], ver
+    numerals, rest = match.groups()
+    numbers = [int(n) for n in numerals.split(".")]
+    return numbers, rest
+
+
+def version_lt(ver1: str, ver2: str) -> bool | None:
+    """
+    Compare two Zulip-style version strings.
+
+    Versions are dot-separated sequences of decimal integers,
+    followed by arbitrary trailing decoration.  Comparison is
+    lexicographic on the integer sequences, and refuses to
+    guess how any trailing decoration compares to any other,
+    to further numerals, or to nothing.
+
+    Returns:
+      True if ver1 < ver2
+      False if ver1 >= ver2
+      None if can't tell.
+    """
+    num1, rest1 = pop_numerals(ver1)
+    num2, rest2 = pop_numerals(ver2)
+    if not num1 or not num2:
+        return None
+    common_len = min(len(num1), len(num2))
+    common_num1, rest_num1 = num1[:common_len], num1[common_len:]
+    common_num2, rest_num2 = num2[:common_len], num2[common_len:]
+
+    # Leading numbers win.
+    if common_num1 != common_num2:
+        return common_num1 < common_num2
+
+    # More numbers beats end-of-string, but ??? vs trailing text.
+    # (NB at most one of rest_num1, rest_num2 is nonempty.)
+    if not rest1 and rest_num2:
+        return True
+    if rest_num1 and not rest2:
+        return False
+    if rest_num1 or rest_num2:
+        return None
+
+    # Trailing text we can only compare for equality.
+    if rest1 == rest2:
+        return False
+    return None
+
+
+def find_mobile_os(user_agent: str) -> str | None:
+    if re.search(r"\b Android \b", user_agent, re.IGNORECASE | re.VERBOSE):
+        return "android"
+    if re.search(r"\b(?: iOS | iPhone\ OS )\b", user_agent, re.IGNORECASE | re.VERBOSE):
+        return "ios"
+    return None
+
+
+def is_outdated_desktop_app(user_agent_str: str) -> tuple[bool, bool, bool]:
+    # Returns (insecure, banned, auto_update_broken)
+    user_agent = parse_user_agent(user_agent_str)
+    if user_agent["name"] == "ZulipDesktop":
+        # The deprecated QT/webkit based desktop app, last updated in ~2016.
+        return (True, True, True)
+
+    if user_agent["name"] != "ZulipElectron":
+        return (False, False, False)
+
+    if version_lt(user_agent["version"], "4.0.0"):
+        # Version 2.3.82 and older (aka <4.0.0) of the modern
+        # Electron-based Zulip desktop app with known security issues.
+        # won't auto-update; we may want a special notice to
+        # distinguish those from modern releases.
+        return (True, True, True)
+
+    if version_lt(user_agent["version"], DESKTOP_MINIMUM_VERSION):
+        # Below DESKTOP_MINIMUM_VERSION, we reject access as well.
+        return (True, True, False)
+
+    if version_lt(user_agent["version"], DESKTOP_WARNING_VERSION):
+        # Other insecure versions should just warn.
+        return (True, False, False)
+
+    return (False, False, False)
+
+
+def is_banned_browser(user_agent: str) -> tuple[bool, str | None]:
+    browser_name = get_device_browser(user_agent)
+    if browser_name == "Internet Explorer":
+        return (True, browser_name)
+    return (False, browser_name)
+
+
+def is_pronouns_field_type_supported(user_agent_str: str | None) -> bool:
+    # In order to avoid users having a bad experience with these
+    # custom profile fields disappearing after applying migration
+    # 0421_migrate_pronouns_custom_profile_fields, we provide this
+    # compatibility shim to show such custom profile fields as
+    # SHORT_TEXT to older mobile app clients.
+    #
+    # TODO/compatibility(7.0): Because this is a relatively minor
+    # detail, we can remove this compatibility hack once most users
+    # have upgraded to a sufficiently new mobile client.
+    if user_agent_str is None:
+        return True
+
+    user_agent = parse_user_agent(user_agent_str)
+    if user_agent["name"] != "ZulipMobile":
+        return True
+
+    FIRST_VERSION_TO_SUPPORT_PRONOUNS_FIELD = "27.192"
+    if version_lt(user_agent["version"], FIRST_VERSION_TO_SUPPORT_PRONOUNS_FIELD):
+        return False
+
+    return True

zerver/lib/context_managers.py

@@ -0,0 +1,50 @@
+"""
+Context managers, i.e. things you can use with the 'with' statement.
+"""
+
+import fcntl
+from collections.abc import Iterator
+from contextlib import contextmanager
+from typing import IO, Any
+
+
+@contextmanager
+def flock(lockfile: int | IO[Any], shared: bool = False) -> Iterator[None]:
+    """Lock a file object using flock(2) for the duration of a 'with' statement.
+
+    If shared is True, use a LOCK_SH lock, otherwise LOCK_EX."""
+
+    fcntl.flock(lockfile, fcntl.LOCK_SH if shared else fcntl.LOCK_EX)
+    try:
+        yield
+    finally:
+        fcntl.flock(lockfile, fcntl.LOCK_UN)
+
+
+@contextmanager
+def lockfile(filename: str, shared: bool = False) -> Iterator[None]:
+    """Lock a file using flock(2) for the duration of a 'with' statement.
+
+    If shared is True, use a LOCK_SH lock, otherwise LOCK_EX.
+
+    The file is given by name and will be created if it does not exist."""
+    with open(filename, "w") as lock, flock(lock, shared=shared):
+        yield
+
+
+@contextmanager
+def lockfile_nonblocking(filename: str) -> Iterator[bool]:  # nocoverage
+    """Lock a file using flock(2) for the duration of a 'with' statement.
+
+    Doesn't block, yields False immediately if the lock can't be acquired."""
+    with open(filename, "w") as f:
+        lock_acquired = False
+        try:
+            fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            lock_acquired = True
+            yield lock_acquired
+        except BlockingIOError:
+            yield False
+        finally:
+            if lock_acquired:
+                fcntl.flock(f, fcntl.LOCK_UN)

zerver/lib/create_user.py

@@ -0,0 +1,244 @@
+import re
+from datetime import datetime
+from email.headerregistry import Address
+
+from django.contrib.auth.models import UserManager
+from django.utils.timezone import now as timezone_now
+
+from zerver.lib.i18n import get_default_language_for_new_user
+from zerver.lib.onboarding_steps import copy_onboarding_steps
+from zerver.lib.timezone import canonicalize_timezone
+from zerver.lib.upload import copy_avatar
+from zerver.models import Realm, RealmUserDefault, Stream, UserBaseSettings, UserProfile
+from zerver.models.realms import get_fake_email_domain
+
+
+def copy_default_settings(
+    settings_source: UserProfile | RealmUserDefault, target_profile: UserProfile
+) -> None:
+    # Important note: Code run from here to configure the user's
+    # settings should not send events, as that would cause clients
+    # to throw an exception (we haven't sent the realm_user/add event
+    # yet, so that event will include the updated details of target_profile).
+    #
+    # Note that this function will do at least one save() on target_profile.
+    for settings_name in UserBaseSettings.property_types:
+        if settings_name in ["default_language", "enable_login_emails"] and isinstance(
+            settings_source, RealmUserDefault
+        ):
+            continue
+
+        if settings_name == "email_address_visibility":
+            # For email_address_visibility, the value selected in registration form
+            # is preferred over the realm-level default value and value of source
+            # profile.
+            continue
+        value = getattr(settings_source, settings_name)
+        setattr(target_profile, settings_name, value)
+
+    if isinstance(settings_source, RealmUserDefault):
+        target_profile.save()
+        return
+
+    target_profile.full_name = settings_source.full_name
+    target_profile.timezone = canonicalize_timezone(settings_source.timezone)
+    target_profile.save()
+
+    if settings_source.avatar_source == UserProfile.AVATAR_FROM_USER:
+        from zerver.actions.user_settings import do_change_avatar_fields
+
+        copy_avatar(settings_source, target_profile)
+        do_change_avatar_fields(
+            target_profile,
+            UserProfile.AVATAR_FROM_USER,
+            skip_notify=True,
+            acting_user=target_profile,
+        )
+
+    copy_onboarding_steps(settings_source, target_profile)
+
+
+def get_dummy_email_address_for_display_regex(realm: Realm) -> str:
+    """
+    Returns a regex that matches the format of dummy email addresses we
+    generate for the .email of users with limit email_address_visibility.
+
+    The reason we need a regex is that we want something that we can use both
+    for generating the dummy email addresses and recognizing them together with extraction
+    of the user ID.
+    """
+
+    # We can't directly have (\d+) in the username passed to Address, because it gets
+    # mutated by the underlying logic for escaping special characters.
+    # So we use a trick by using $ as a placeholder which will be preserved, and then
+    # replace it with (\d+) to obtain our intended regex.
+    address_template = Address(username="user$", domain=get_fake_email_domain(realm.host)).addr_spec
+    regex = re.escape(address_template).replace(r"\$", r"(\d+)", 1)
+    return regex
+
+
+def get_display_email_address(user_profile: UserProfile) -> str:
+    if not user_profile.email_address_is_realm_public():
+        # The format of the dummy email address created here needs to stay consistent
+        # with get_dummy_email_address_for_display_regex.
+        return Address(
+            username=f"user{user_profile.id}", domain=get_fake_email_domain(user_profile.realm.host)
+        ).addr_spec
+    return user_profile.delivery_email
+
+
+# create_user_profile is based on Django's User.objects.create_user,
+# except that we don't save to the database so it can used in
+# bulk_creates
+#
+# Only use this for bulk_create -- for normal usage one should use
+# create_user (below) which will also make the Subscription and
+# Recipient objects
+def create_user_profile(
+    realm: Realm,
+    email: str,
+    password: str | None,
+    active: bool,
+    bot_type: int | None,
+    full_name: str,
+    bot_owner: UserProfile | None,
+    is_mirror_dummy: bool,
+    tos_version: str | None,
+    timezone: str,
+    default_language: str,
+    is_deleted: bool = False,
+    force_date_joined: datetime | None = None,
+    is_imported_stub: bool = False,
+    *,
+    email_address_visibility: int,
+) -> UserProfile:
+    if force_date_joined is None:
+        date_joined = timezone_now()
+    else:  # nocoverage
+        date_joined = force_date_joined
+
+    email = UserManager.normalize_email(email)
+
+    user_profile = UserProfile(
+        is_staff=False,
+        is_active=active,
+        full_name=full_name,
+        last_login=date_joined,
+        date_joined=date_joined,
+        realm=realm,
+        is_bot=bool(bot_type),
+        bot_type=bot_type,
+        bot_owner=bot_owner,
+        is_mirror_dummy=is_mirror_dummy,
+        is_deleted=is_deleted,
+        tos_version=tos_version,
+        timezone=timezone,
+        default_language=default_language,
+        delivery_email=email,
+        email_address_visibility=email_address_visibility,
+        is_imported_stub=is_imported_stub,
+    )
+    if bot_type or not active:
+        password = None
+    if user_profile.email_address_is_realm_public():
+        # If emails are visible to everyone, we can set this here and save a DB query
+        user_profile.email = get_display_email_address(user_profile)
+    user_profile.set_password(password)
+    return user_profile
+
+
+def create_user(
+    email: str,
+    password: str | None,
+    realm: Realm,
+    full_name: str,
+    active: bool = True,
+    role: int | None = None,
+    bot_type: int | None = None,
+    bot_owner: UserProfile | None = None,
+    tos_version: str | None = None,
+    timezone: str = "",
+    avatar_source: str | None = None,
+    is_mirror_dummy: bool = False,
+    is_deleted: bool = False,
+    default_language: str | None = None,
+    default_sending_stream: Stream | None = None,
+    default_events_register_stream: Stream | None = None,
+    default_all_public_streams: bool | None = None,
+    source_profile: UserProfile | None = None,
+    force_date_joined: datetime | None = None,
+    enable_marketing_emails: bool | None = None,
+    email_address_visibility: int | None = None,
+) -> UserProfile:
+    realm_user_default = RealmUserDefault.objects.get(realm=realm)
+    if bot_type is None:
+        if email_address_visibility is not None:
+            user_email_address_visibility = email_address_visibility
+        else:
+            user_email_address_visibility = realm_user_default.email_address_visibility
+    else:
+        # There is no privacy motivation for limiting access to bot email addresses,
+        # so we hardcode them to EMAIL_ADDRESS_VISIBILITY_EVERYONE.
+        user_email_address_visibility = UserProfile.EMAIL_ADDRESS_VISIBILITY_EVERYONE
+
+    # Users created via the API or LDAP/SAML syncing code paths will
+    # usually not have a default_language value, and should fall back
+    # to the realm default.
+    if default_language is None:
+        default_language = get_default_language_for_new_user(realm, request=None)
+
+    user_profile = create_user_profile(
+        realm,
+        email,
+        password,
+        active,
+        bot_type,
+        full_name,
+        bot_owner,
+        is_mirror_dummy,
+        tos_version,
+        timezone,
+        default_language,
+        is_deleted,
+        force_date_joined=force_date_joined,
+        email_address_visibility=user_email_address_visibility,
+    )
+    if avatar_source is None:
+        avatar_source = realm.default_avatar_source
+    user_profile.avatar_source = avatar_source
+    user_profile.timezone = timezone
+    user_profile.default_sending_stream = default_sending_stream
+    user_profile.default_events_register_stream = default_events_register_stream
+    if role is not None:
+        user_profile.role = role
+    # Allow the ORM default to be used if not provided
+    if default_all_public_streams is not None:
+        user_profile.default_all_public_streams = default_all_public_streams
+    # If a source profile was specified, we copy settings from that
+    # user.  Note that this is positioned in a way that overrides
+    # other arguments passed in, which is correct for most defaults
+    # like time zone where the source profile likely has a better value
+    # than the guess. As we decide on details like avatars and full
+    # names for this feature, we may want to move it.
+    if source_profile is not None:
+        # copy_default_settings saves the attribute values so a secondary
+        # save is not required.
+        copy_default_settings(source_profile, user_profile)
+    elif bot_type is None:
+        copy_default_settings(realm_user_default, user_profile)
+    else:
+        # This will be executed only for bots.
+        user_profile.save()
+
+    if bot_type is None and enable_marketing_emails is not None:
+        user_profile.enable_marketing_emails = enable_marketing_emails
+        user_profile.save(update_fields=["enable_marketing_emails"])
+
+    if not user_profile.email_address_is_realm_public():
+        # With restricted access to email addresses, we can't generate
+        # the fake email addresses we use for display purposes without
+        # a User ID, which isn't generated until the .save() above.
+        user_profile.email = get_display_email_address(user_profile)
+        user_profile.save(update_fields=["email"])
+
+    return user_profile

zerver/lib/db.py

@@ -0,0 +1,54 @@
+import time
+from collections.abc import Callable, Iterable, Mapping, Sequence
+from typing import Any, TypeAlias, TypeVar
+
+from psycopg2.extensions import connection, cursor
+from psycopg2.sql import Composable
+from typing_extensions import override
+
+CursorObj = TypeVar("CursorObj", bound=cursor)
+Query: TypeAlias = str | bytes | Composable
+Params: TypeAlias = Sequence[object] | Mapping[str, object] | None
+ParamsT = TypeVar("ParamsT")
+
+
+# Similar to the tracking done in Django's CursorDebugWrapper, but done at the
+# psycopg2 cursor level so it works with SQLAlchemy.
+def wrapper_execute(
+    self: CursorObj, action: Callable[[Query, ParamsT], None], sql: Query, params: ParamsT
+) -> None:
+    start = time.time()
+    try:
+        action(sql, params)
+    finally:
+        stop = time.time()
+        duration = stop - start
+        assert isinstance(self.connection, TimeTrackingConnection)
+        self.connection.queries.append(
+            {
+                "time": f"{duration:.3f}",
+            }
+        )
+
+
+class TimeTrackingCursor(cursor):
+    """A psycopg2 cursor class that tracks the time spent executing queries."""
+
+    @override
+    def execute(self, query: Query, vars: Params = None) -> None:
+        wrapper_execute(self, super().execute, query, vars)
+
+    @override
+    def executemany(self, query: Query, vars_list: Iterable[Params]) -> None:  # nocoverage
+        wrapper_execute(self, super().executemany, query, vars_list)
+
+
+CursorT = TypeVar("CursorT", bound=cursor)
+
+
+class TimeTrackingConnection(connection):
+    """A psycopg2 connection class that uses TimeTrackingCursors."""
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        self.queries: list[dict[str, str]] = []
+        super().__init__(*args, **kwargs)

zerver/lib/db_connections.py

@@ -0,0 +1,7 @@
+from django.db import connections
+
+
+def reset_queries() -> None:
+    for conn in connections.all():
+        if conn.connection is not None:
+            conn.connection.queries = []

zerver/lib/debug.py

@@ -0,0 +1,117 @@
+import code
+import gc
+import logging
+import os
+import signal
+import socket
+import threading
+import traceback
+import tracemalloc
+from types import FrameType
+
+from django.conf import settings
+from django.utils.timezone import now as timezone_now
+
+logger = logging.getLogger("zulip.debug")
+
+
+# Interactive debugging code from
+# https://stackoverflow.com/questions/132058/showing-the-stack-trace-from-a-running-python-application
+# (that link also points to code for an interactive remote debugger
+# setup, which we might want if we move Tornado to run in a daemon
+# rather than via screen).
+def interactive_debug(sig: int, frame: FrameType | None) -> None:
+    """Interrupt running process, and provide a python prompt for
+    interactive debugging."""
+    d = {"_frame": frame}  # Allow access to frame object.
+    if frame is not None:
+        d.update(frame.f_globals)  # Unless shadowed by global
+        d.update(frame.f_locals)
+
+    message = "Signal received : entering python shell.\nTraceback:\n"
+    message += "".join(traceback.format_stack(frame))
+    i = code.InteractiveConsole(d)
+    i.interact(message)
+
+
+# SIGUSR1 => Just print the stack
+# SIGUSR2 => Print stack + open interactive debugging shell
+def interactive_debug_listen() -> None:
+    signal.signal(signal.SIGUSR1, lambda sig, stack: traceback.print_stack(stack))
+    signal.signal(signal.SIGUSR2, interactive_debug)
+
+
+def tracemalloc_dump() -> None:
+    if not tracemalloc.is_tracing():
+        logger.warning("pid %s: tracemalloc off, nothing to dump", os.getpid())
+        return
+    # Despite our name for it, `timezone_now` always deals in UTC.
+    basename = "snap.{}.{}".format(
+        os.getpid(), timezone_now().replace(tzinfo=None).isoformat("-", "seconds")
+    )
+    path = os.path.join(settings.TRACEMALLOC_DUMP_DIR, basename)
+    os.makedirs(settings.TRACEMALLOC_DUMP_DIR, exist_ok=True)
+
+    gc.collect()
+    tracemalloc.take_snapshot().dump(path)
+
+    with open(f"/proc/{os.getpid()}/stat", "rb") as f:
+        procstat = f.read().split()
+    rss_pages = int(procstat[23])
+    logger.info(
+        "tracemalloc dump: tracing %s MiB (%s MiB peak), using %s MiB; rss %s MiB; dumped %s",
+        tracemalloc.get_traced_memory()[0] // 1048576,
+        tracemalloc.get_traced_memory()[1] // 1048576,
+        tracemalloc.get_tracemalloc_memory() // 1048576,
+        rss_pages // 256,
+        basename,
+    )
+
+
+def tracemalloc_listen_sock(sock: socket.socket) -> None:
+    logger.debug("pid %s: tracemalloc_listen_sock started!", os.getpid())
+    while True:
+        sock.recv(1)
+        tracemalloc_dump()
+
+
+listener_pid: int | None = None
+
+
+def tracemalloc_listen() -> None:
+    global listener_pid
+    if listener_pid == os.getpid():
+        # Already set up -- and in this process, not just its parent.
+        return
+    logger.debug("pid %s: tracemalloc_listen working...", os.getpid())
+    listener_pid = os.getpid()
+
+    sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
+    os.makedirs(settings.TRACEMALLOC_DUMP_DIR, exist_ok=True)
+    path = os.path.join(settings.TRACEMALLOC_DUMP_DIR, f"tracemalloc.{os.getpid()}")
+    sock.bind(path)
+    thread = threading.Thread(target=lambda: tracemalloc_listen_sock(sock), daemon=True)
+    thread.start()
+    logger.debug("pid %s: tracemalloc_listen done: %s", os.getpid(), path)
+
+
+def maybe_tracemalloc_listen() -> None:
+    """If tracemalloc tracing enabled, listen for requests to dump a snapshot.
+
+    To trigger once this is listening:
+      echo | socat -u stdin unix-sendto:/var/log/zulip/tracemalloc/tracemalloc.$pid
+
+    To enable in the Zulip web server: edit /etc/zulip/uwsgi.ini ,
+    and add e.g. ` PYTHONTRACEMALLOC=5` to the `env=` line.
+    This function is called in middleware, so the process will
+    automatically start listening.
+
+    To enable in other contexts: see upstream docs
+    https://docs.python.org/3/library/tracemalloc .
+    You may also have to add a call to this function somewhere.
+
+    """
+    if os.environ.get("PYTHONTRACEMALLOC"):
+        # If the server was started with `tracemalloc` tracing on, then
+        # listen for a signal to dump `tracemalloc` snapshots.
+        tracemalloc_listen()

zerver/lib/default_streams.py

@@ -0,0 +1,19 @@
+from zerver.models import DefaultStream, Stream
+
+
+def get_slim_realm_default_streams(realm_id: int) -> set[Stream]:
+    # We really want this query to be simple and just get "thin" Stream objects
+    # in one round trip.
+    #
+    # The above is enforced by at least three tests that verify query counts,
+    # and test_query_count in test_subs.py makes sure that the query itself is
+    # not like 11000 bytes, which is what we had in a prior version that used
+    # select_related() with no arguments (and thus joined to too many tables).
+    #
+    # Please be careful about modifying this code, as it has had a history
+    # of performance problems.
+    return set(Stream.objects.filter(defaultstream__realm_id=realm_id))
+
+
+def get_default_stream_ids_for_realm(realm_id: int) -> set[int]:
+    return set(DefaultStream.objects.filter(realm_id=realm_id).values_list("stream_id", flat=True))

zerver/lib/demo_organizations.py

@@ -0,0 +1,78 @@
+import os
+from datetime import timedelta
+from functools import lru_cache
+
+import orjson
+from django.conf import settings
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+
+from zerver.actions.scheduled_messages import check_schedule_message
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.timestamp import ceiling_to_day, datetime_to_global_time
+from zerver.models.clients import get_client
+from zerver.models.realms import Realm
+from zerver.models.users import UserProfile, get_system_bot
+
+
+@lru_cache(None)
+def get_demo_organization_wordlists() -> dict[str, list[str]]:
+    path = os.path.join(settings.DEPLOY_ROOT, "zerver/lib", "demo_organization_words.json")
+    with open(path, "rb") as reader:
+        return orjson.loads(reader.read())
+
+
+def demo_organization_owner_email_exists(realm: Realm) -> bool:
+    human_owner_emails = set(realm.get_human_owner_users().values_list("delivery_email", flat=True))
+    return human_owner_emails != {""}
+
+
+def check_demo_organization_has_set_email(realm: Realm) -> None:
+    # This should be called after checking that the realm has
+    # a demo_organization_scheduled_deletion_date set.
+    assert realm.demo_organization_scheduled_deletion_date is not None
+    if not demo_organization_owner_email_exists(realm):
+        raise JsonableError(_("Configure owner account email address."))
+
+
+def get_demo_organization_deadline_days_remaining(realm: Realm) -> int:
+    assert realm.demo_organization_scheduled_deletion_date is not None
+    days_remaining = (realm.demo_organization_scheduled_deletion_date - timezone_now()).days
+    return days_remaining
+
+
+def get_scheduled_deletion_global_time(realm: Realm) -> str:
+    assert realm.demo_organization_scheduled_deletion_date is not None
+    scheduled_deletion_cronjob_timestamp = ceiling_to_day(
+        realm.demo_organization_scheduled_deletion_date
+    ) + timedelta(hours=6)
+    return datetime_to_global_time(scheduled_deletion_cronjob_timestamp)
+
+
+def schedule_demo_organization_deletion_reminder(user: UserProfile) -> None:
+    assert user.is_realm_owner
+    realm = user.realm
+    assert realm.demo_organization_scheduled_deletion_date is not None
+    deliver_at = realm.demo_organization_scheduled_deletion_date - timedelta(days=7)
+    sender = get_system_bot(settings.NOTIFICATION_BOT, realm.id)
+    client = get_client("Internal")
+
+    message_content = _("""
+As a reminder, this [demo organization]({demo_help}) will be automatically deleted on {deletion_time}, unless it's [converted into a permanent organization]({convert_demo}).
+""").format(
+        demo_help="/help/demo-organizations",
+        deletion_time=get_scheduled_deletion_global_time(realm),
+        convert_demo="/help/demo-organizations#convert-a-demo-organization-to-a-permanent-organization",
+    )
+
+    check_schedule_message(
+        sender,
+        client,
+        "private",
+        [user.id],
+        None,
+        message_content,
+        deliver_at,
+        realm,
+        skip_events=True,
+    )

zerver/lib/dev_ldap_directory.py

@@ -0,0 +1,90 @@
+import glob
+import logging
+import os
+from email.headerregistry import Address
+from typing import Any
+
+from django.conf import settings
+
+from zerver.lib.storage import static_path
+
+# See https://jackstromberg.com/2013/01/useraccountcontrol-attributeflag-values/
+# for docs on what these values mean.
+LDAP_USER_ACCOUNT_CONTROL_NORMAL = "512"
+LDAP_USER_ACCOUNT_CONTROL_DISABLED = "514"
+
+
+def generate_dev_ldap_dir(mode: str, num_users: int = 8) -> dict[str, dict[str, Any]]:
+    mode = mode.lower()
+    ldap_data = []
+    for i in range(1, num_users + 1):
+        name = f"LDAP User {i}"
+        email = f"ldapuser{i}@zulip.com"
+        phone_number = f"999999999{i}"
+        birthdate = f"19{i:02}-{i:02}-{i:02}"
+        ldap_data.append((name, email, phone_number, birthdate))
+
+    profile_images = []
+    for path in glob.glob(os.path.join(static_path("images/test-images/avatars"), "*")):
+        with open(path, "rb") as f:
+            profile_images.append(f.read())
+    ldap_dir = {}
+    for i, user_data in enumerate(ldap_data):
+        email = user_data[1].lower()
+        email_username = Address(addr_spec=email).username
+        common_data = {
+            "cn": [user_data[0]],
+            "userPassword": [email_username],
+            "phoneNumber": [user_data[2]],
+            "birthDate": [user_data[3]],
+        }
+        if mode == "a":
+            ldap_dir[f"uid={email},ou=users,dc=zulip,dc=com"] = dict(
+                uid=[email],
+                thumbnailPhoto=[profile_images[i % len(profile_images)]],
+                userAccountControl=[LDAP_USER_ACCOUNT_CONTROL_NORMAL],
+                **common_data,
+            )
+        elif mode == "b":
+            ldap_dir[f"uid={email_username},ou=users,dc=zulip,dc=com"] = dict(
+                uid=[email_username],
+                jpegPhoto=[profile_images[i % len(profile_images)]],
+                **common_data,
+            )
+        elif mode == "c":
+            ldap_dir[f"uid={email_username},ou=users,dc=zulip,dc=com"] = dict(
+                uid=[email_username], email=[email], **common_data
+            )
+
+    return ldap_dir
+
+
+def init_fakeldap(
+    directory: dict[str, dict[str, list[str]]] | None = None,
+) -> None:  # nocoverage
+    # We only use this in development.  Importing mock inside
+    # this function is an import time optimization, which
+    # avoids the expensive import of the mock module (slow
+    # because its dependency pbr uses pkgresources, which is
+    # really slow to import.)
+    from unittest import mock
+
+    from fakeldap import MockLDAP
+
+    # Silent `django_auth_ldap` logger in dev mode to avoid
+    # spammy user not found log messages.
+    ldap_auth_logger = logging.getLogger("django_auth_ldap")
+    ldap_auth_logger.setLevel(logging.CRITICAL)
+
+    fakeldap_logger = logging.getLogger("fakeldap")
+    fakeldap_logger.setLevel(logging.CRITICAL)
+
+    ldap_patcher = mock.patch("django_auth_ldap.config.ldap.initialize")
+    mock_initialize = ldap_patcher.start()
+    mock_ldap = MockLDAP()
+    mock_initialize.return_value = mock_ldap
+
+    assert settings.FAKE_LDAP_MODE is not None
+    mock_ldap.directory = directory or generate_dev_ldap_dir(
+        settings.FAKE_LDAP_MODE, settings.FAKE_LDAP_NUM_USERS
+    )

zerver/lib/devices.py

@@ -0,0 +1,70 @@
+import base64
+import binascii
+
+from django.utils.translation import gettext as _
+from typing_extensions import TypedDict
+
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.timestamp import datetime_to_timestamp
+from zerver.models.devices import Device
+from zerver.models.users import UserProfile
+
+
+class DeviceInfoDict(TypedDict):
+    push_key_id: int | None
+    push_token_id: str | None
+    pending_push_token_id: str | None
+    push_token_last_updated_timestamp: int | None
+    push_registration_error_code: str | None
+
+
+def b64encode_token_id_int(token_id_int: int) -> str:
+    token_id_bytes = token_id_int.to_bytes(8, byteorder="big", signed=True)
+    return base64.b64encode(token_id_bytes).decode()
+
+
+def b64decode_token_id_base64(token_id_base64: str) -> int:
+    try:
+        token_id_bytes = base64.b64decode(token_id_base64, validate=True)
+    except binascii.Error:
+        raise JsonableError(_("{var_name} is not Base64 encoded").format(var_name="`token_id`"))
+
+    return int.from_bytes(token_id_bytes, byteorder="big", signed=True)
+
+
+def get_devices(user_profile: UserProfile) -> dict[str, DeviceInfoDict]:
+    devices = Device.objects.filter(user=user_profile)
+
+    devices_dict: dict[str, DeviceInfoDict] = dict()
+    for device in devices:
+        push_token_id_base64 = None
+        pending_push_token_id_base64 = None
+        push_token_last_updated_timestamp = None
+
+        if device.push_token_id is not None:
+            push_token_id_base64 = b64encode_token_id_int(device.push_token_id)
+        if device.pending_push_token_id is not None:
+            pending_push_token_id_base64 = b64encode_token_id_int(device.pending_push_token_id)
+        if device.push_token_last_updated_timestamp is not None:
+            push_token_last_updated_timestamp = datetime_to_timestamp(
+                device.push_token_last_updated_timestamp
+            )
+
+        devices_dict[str(device.id)] = DeviceInfoDict(
+            push_key_id=device.push_key_id,
+            push_token_id=push_token_id_base64,
+            pending_push_token_id=pending_push_token_id_base64,
+            push_token_last_updated_timestamp=push_token_last_updated_timestamp,
+            push_registration_error_code=device.push_registration_error_code,
+        )
+
+    return devices_dict
+
+
+def check_device_id(device_id: int, user_id: int) -> Device:
+    try:
+        device = Device.objects.get(id=device_id, user_id=user_id)
+    except Device.DoesNotExist:
+        raise JsonableError(_("Invalid `device_id`"))
+
+    return device

zerver/lib/digest.py

@@ -0,0 +1,560 @@
+import functools
+import heapq
+import logging
+from collections import defaultdict
+from collections.abc import Collection, Iterator
+from dataclasses import dataclass
+from datetime import datetime, timedelta, timezone
+from typing import Any, TypeAlias
+
+from django.conf import settings
+from django.db import transaction
+from django.db.models import Exists, OuterRef, QuerySet
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+from markupsafe import Markup
+
+from confirmation.models import one_click_unsubscribe_link
+from zerver.context_processors import common_context
+from zerver.lib.email_notifications import (
+    MessageListPayload,
+    build_message_list,
+    get_channel_privacy_icon,
+    message_content_allowed_in_missedmessage_emails,
+)
+from zerver.lib.logging_util import log_to_file
+from zerver.lib.message import get_last_message_id
+from zerver.lib.queue import queue_json_publish_rollback_unsafe
+from zerver.lib.send_email import FromAddress, send_future_email
+from zerver.lib.url_encoding import stream_narrow_url
+from zerver.models import (
+    Message,
+    Realm,
+    RealmAuditLog,
+    Recipient,
+    Stream,
+    Subscription,
+    UserActivityInterval,
+    UserMessage,
+    UserProfile,
+    UserTopic,
+)
+from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.models.streams import get_active_streams
+
+logger = logging.getLogger(__name__)
+log_to_file(logger, settings.DIGEST_LOG_PATH)
+
+DIGEST_CUTOFF = 5
+MAX_HOT_TOPICS_TO_BE_INCLUDED_IN_DIGEST = 4
+
+TopicKey: TypeAlias = tuple[int, str]
+
+
+@dataclass
+class DigestTeaserData:
+    participants: list[str]
+    count: int
+    first_few_messages: MessageListPayload
+
+
+class DigestTopic:
+    def __init__(self, topic_key: TopicKey) -> None:
+        self.topic_key = topic_key
+        self.human_senders: set[str] = set()
+        self.sample_messages: list[Message] = []
+        self.num_human_messages = 0
+
+    def stream_id(self) -> int:
+        # topic_key is (stream_id, topic_name)
+        return self.topic_key[0]
+
+    def topic_name(self) -> str:
+        return self.topic_key[1]
+
+    def add_message(self, message: Message) -> None:
+        if len(self.sample_messages) < 2:
+            self.sample_messages.append(message)
+
+        if not message.sender.is_bot:
+            self.human_senders.add(message.sender.full_name)
+            self.num_human_messages += 1
+
+    def length(self) -> int:
+        return self.num_human_messages
+
+    def diversity(self) -> int:
+        return len(self.human_senders)
+
+    def teaser_data(self, user: UserProfile, stream_id_map: dict[int, Stream]) -> DigestTeaserData:
+        teaser_count = self.num_human_messages - len(self.sample_messages)
+        first_few_messages = build_message_list(
+            user=user,
+            messages=self.sample_messages,
+            stream_id_map=stream_id_map,
+        )
+        return DigestTeaserData(
+            participants=sorted(self.human_senders),
+            count=teaser_count,
+            first_few_messages=first_few_messages,
+        )
+
+
+# Digests accumulate 2 types of interesting traffic for a user:
+# 1. New streams
+# 2. Interesting stream traffic, as determined by the longest and most
+#    diversely comment upon topics.
+
+
+# Changes to this should also be reflected in
+# zerver/worker/digest_emails.py:DigestWorker.consume()
+def queue_digest_user_ids(user_ids: list[int], cutoff: datetime) -> None:
+    # Convert cutoff to epoch seconds for transit.
+    event = {"user_ids": user_ids, "cutoff": cutoff.strftime("%s")}
+    queue_json_publish_rollback_unsafe("digest_emails", event)
+
+
+def enqueue_emails(cutoff: datetime) -> None:
+    if not settings.SEND_DIGEST_EMAILS:
+        return
+
+    weekday = timezone_now().weekday()
+    for realm in Realm.objects.filter(
+        deactivated=False, digest_emails_enabled=True, digest_weekday=weekday
+    ).exclude(string_id__in=settings.SYSTEM_ONLY_REALMS):
+        _enqueue_emails_for_realm(realm, cutoff)
+
+
+def _enqueue_emails_for_realm(realm: Realm, cutoff: datetime) -> None:
+    # This should only be called directly by tests.  Use enqueue_emails
+    # to process all realms that are set up for processing on any given day.
+    twelve_hours_ago = timezone_now() - timedelta(hours=12)
+
+    target_users = (
+        UserProfile.objects.filter(
+            realm=realm,
+            is_active=True,
+            is_bot=False,
+            enable_digest_emails=True,
+        )
+        .alias(
+            recent_activity=Exists(
+                UserActivityInterval.objects.filter(user_profile_id=OuterRef("id"), end__gt=cutoff)
+            )
+        )
+        .filter(recent_activity=False)
+        .alias(
+            sent_recent_digest=Exists(
+                RealmAuditLog.objects.filter(
+                    realm_id=realm.id,
+                    event_type=AuditLogEventType.USER_DIGEST_EMAIL_CREATED,
+                    event_time__gt=twelve_hours_ago,
+                    modified_user_id=OuterRef("id"),
+                )
+            )
+        )
+        .filter(sent_recent_digest=False)
+    )
+
+    user_ids = list(target_users.order_by("id").values_list("id", flat=True))
+
+    # We process batches of 30.  We want a big enough batch
+    # to amortize work, but not so big that a single item
+    # from the queue takes too long to process.
+    chunk_size = 30
+    for i in range(0, len(user_ids), chunk_size):
+        chunk_user_ids = user_ids[i : i + chunk_size]
+        queue_digest_user_ids(chunk_user_ids, cutoff)
+        logger.info(
+            "Queuing user_ids for potential digest: %s",
+            chunk_user_ids,
+        )
+
+
+last_realm_id: int | None = None
+last_cutoff: float | None = None
+
+
+def maybe_clear_recent_topics_cache(realm_id: int, cutoff: float) -> None:
+    # As an optimization, we clear the digest caches when we switch to
+    # a new realm or cutoff value.  Since these values are part of the
+    # cache key, this is not necessary for correctness -- it merely
+    # helps reduce the memory footprint of the cache.
+    global last_realm_id, last_cutoff
+    if last_realm_id != realm_id or last_cutoff != cutoff:
+        logger.info("Flushing stream cache: %s", get_recent_topics.cache_info())
+        get_recent_topics.cache_clear()
+    last_realm_id = realm_id
+    last_cutoff = cutoff
+
+
+# We cache both by stream-id and cutoff, which ensures the per-stream
+# cache also does not contain data from old digests
+@functools.lru_cache(maxsize=5000)
+def get_recent_topics(
+    realm_id: int,
+    stream_id: int,
+    cutoff_date: datetime,
+) -> list[DigestTopic]:
+    # Gather information about topic conversations, then
+    # classify by:
+    #   * topic length
+    #   * number of senders
+
+    messages = (
+        # Uses index: zerver_message_realm_recipient_date_sent
+        Message.objects.filter(
+            realm_id=realm_id,
+            recipient__type=Recipient.STREAM,
+            recipient__type_id=stream_id,
+            date_sent__gt=cutoff_date,
+        )
+        .order_by(
+            "id",  # we will sample the first few messages
+        )
+        .select_related(
+            "recipient",  # build_message_list looks up recipient.type
+            "sender",  # we need the sender's full name
+        )
+        .defer(
+            # This construction, to only fetch the sender's full_name and is_bot,
+            # is because `.only()` doesn't work with select_related tables.
+            *{
+                f"sender__{f.name}"
+                for f in UserProfile._meta.fields
+                if f.name not in {"full_name", "is_bot"}
+            }
+        )
+    )
+
+    digest_topic_map: dict[TopicKey, DigestTopic] = {}
+    for message in messages:
+        topic_key = (stream_id, message.topic_name())
+
+        if topic_key not in digest_topic_map:
+            digest_topic_map[topic_key] = DigestTopic(topic_key)
+
+        digest_topic_map[topic_key].add_message(message)
+
+    topics = list(digest_topic_map.values())
+
+    return topics
+
+
+def get_hot_topics(
+    all_topics: list[DigestTopic],
+    stream_ids: set[int],
+) -> list[DigestTopic]:
+    topics = [topic for topic in all_topics if topic.stream_id() in stream_ids]
+
+    hot_topics = heapq.nlargest(2, topics, key=DigestTopic.diversity)
+
+    for topic in heapq.nlargest(
+        MAX_HOT_TOPICS_TO_BE_INCLUDED_IN_DIGEST, topics, key=DigestTopic.length
+    ):
+        if topic not in hot_topics:
+            hot_topics.append(topic)
+        if len(hot_topics) == MAX_HOT_TOPICS_TO_BE_INCLUDED_IN_DIGEST:
+            break
+
+    return hot_topics
+
+
+def get_recently_created_streams(realm: Realm, threshold: datetime) -> list[Stream]:
+    fields = ["id", "name", "is_web_public", "invite_only"]
+    return list(get_active_streams(realm).filter(date_created__gt=threshold).only(*fields))
+
+
+def gather_new_streams(
+    realm: Realm,
+    recently_created_streams: list[Stream],  # streams only need id and name
+    can_access_public: bool,
+) -> tuple[int, dict[str, list[Markup] | list[str]]]:
+    if can_access_public:
+        new_streams = [stream for stream in recently_created_streams if not stream.invite_only]
+    else:
+        new_streams = [stream for stream in recently_created_streams if stream.is_web_public]
+
+    channels_html = []
+    channels_plain = []
+
+    for stream in new_streams:
+        narrow_url = stream_narrow_url(realm, stream)
+        channel_link = Markup(
+            "<a href='{narrow_url}'>{channel_privacy_icon}{stream_name}</a>"
+        ).format(
+            narrow_url=narrow_url,
+            channel_privacy_icon=get_channel_privacy_icon(stream),
+            stream_name=stream.name,
+        )
+        channels_html.append(channel_link)
+        channels_plain.append(stream.name)
+
+    return len(new_streams), {"html": channels_html, "plain": channels_plain}
+
+
+def get_min_recent_message_id(realm_id: int, threshold: datetime) -> int | None:
+    # ID of the realm's earliest message at or after the threshold,
+    # or None if there are none.
+    return (
+        # Uses index: zerver_message_realm_date_sent
+        Message.objects.filter(realm_id=realm_id, date_sent__gte=threshold)
+        .order_by("date_sent")
+        .values_list("id", flat=True)
+        .first()
+    )
+
+
+def get_new_messages_count(user: UserProfile, min_recent_message_id: int | None) -> int:
+    if min_recent_message_id is None:
+        # No message was sent since the threshold.
+        return 0
+
+    # We filter on `message_id` for performance, so the planner drives
+    # from the unique (user_profile_id, message_id) index on `UserMessage`.
+    #
+    # Since we want "messages the user received since threshold",
+    # the intuition is to filter on `user_profile` and `message__date_sent`.
+    # When filtering without `message_id`, the planner drives from Message's
+    # `date_sent` index, which is not scoped to a user: it reads every message
+    # sent since threshold and joins against all UserMessage rows for the user
+    # to determine whether they received each message - which is too slow.
+    count = UserMessage.objects.filter(
+        user_profile=user,
+        message_id__gte=min_recent_message_id,
+        message__sender__is_bot=False,
+    ).count()
+    return count
+
+
+def enough_traffic(
+    hot_conversations: int, new_streams: int, new_messages: int, show_message_content: bool
+) -> bool:
+    if new_streams > 0:
+        return True
+    if not show_message_content:
+        return new_messages > 0
+    return hot_conversations > 0
+
+
+def get_user_stream_map(user_ids: list[int], cutoff_date: datetime) -> dict[int, set[int]]:
+    """Skipping streams where the user's subscription status has changed
+    when constructing digests is critical to ensure correctness for
+    streams without shared history, guest users, and long-term idle
+    users, because it means that every user has the same view of the
+    history of a given stream whose message history is being included
+    (and thus we can share a lot of work).
+
+    The downside is that newly created streams are never included in
+    the first digest email after their creation.  Should we wish to
+    change that, we will need to be very careful to avoid creating
+    bugs for any of those classes of users.
+    """
+    events = [
+        AuditLogEventType.SUBSCRIPTION_CREATED,
+        AuditLogEventType.SUBSCRIPTION_ACTIVATED,
+        AuditLogEventType.SUBSCRIPTION_DEACTIVATED,
+    ]
+    # This uses the zerver_realmauditlog_user_subscriptions_idx
+    # partial index on RealmAuditLog which is specifically for those
+    # three event types.
+    rows = (
+        Subscription.objects.filter(
+            user_profile_id__in=user_ids,
+            recipient__type=Recipient.STREAM,
+            active=True,
+            is_muted=False,
+        )
+        .alias(
+            was_modified=Exists(
+                RealmAuditLog.objects.filter(
+                    modified_stream_id=OuterRef("recipient__type_id"),
+                    modified_user_id=OuterRef("user_profile_id"),
+                    event_time__gt=cutoff_date,
+                    event_type__in=events,
+                )
+            )
+        )
+        .filter(was_modified=False)
+        .values("user_profile_id", "recipient__type_id")
+    )
+
+    # maps user_id -> {stream_id, stream_id, ...}
+    dct: dict[int, set[int]] = defaultdict(set)
+    for row in rows:
+        dct[row["user_profile_id"]].add(row["recipient__type_id"])
+
+    return dct
+
+
+def get_user_muted_topics_map(user_ids: list[int]) -> dict[int, set[tuple[int, str]]]:
+    # maps user_id -> {(stream_id, lowercased topic_name), ...};
+    # topic names are matched case-insensitively.
+    rows = UserTopic.objects.filter(
+        user_profile_id__in=user_ids,
+        visibility_policy=UserTopic.VisibilityPolicy.MUTED,
+    ).values("user_profile_id", "stream_id", "topic_name")
+
+    dct: dict[int, set[tuple[int, str]]] = defaultdict(set)
+    for row in rows:
+        dct[row["user_profile_id"]].add((row["stream_id"], row["topic_name"].lower()))
+
+    return dct
+
+
+def get_slim_stream_id_map(stream_ids: set[int]) -> dict[int, Stream]:
+    # "slim" because it only fetches the names of the stream objects,
+    # suitable for passing into build_message_list.
+    streams = Stream.objects.filter(id__in=stream_ids).only("id", "name")
+    return {stream.id: stream for stream in streams}
+
+
+def bulk_get_digest_context(
+    users: Collection[UserProfile] | QuerySet[UserProfile], cutoff: float
+) -> Iterator[tuple[UserProfile, dict[str, Any]]]:
+    # We expect a non-empty list of users all from the same realm.
+    assert users
+    realm = next(iter(users)).realm
+    for user in users:
+        assert user.realm_id == realm.id
+
+    # Convert from epoch seconds to a datetime object.
+    cutoff_date = datetime.fromtimestamp(int(cutoff), tz=timezone.utc)
+
+    maybe_clear_recent_topics_cache(realm.id, cutoff)
+
+    recently_created_streams = get_recently_created_streams(realm, cutoff_date)
+
+    user_ids = [user.id for user in users]
+    user_stream_map = get_user_stream_map(user_ids, cutoff_date)
+    stream_ids = set().union(*user_stream_map.values())
+    stream_id_map = get_slim_stream_id_map(stream_ids)
+    user_muted_topics_map = get_user_muted_topics_map(user_ids)
+
+    # ID of the realm's earliest message at or after the `cutoff`.
+    # Shared across all users, and computed lazily since it is only
+    # needed for users who have message content hidden in emails.
+    min_recent_message_id: int | None = None
+    have_min_recent_message_id = False
+
+    for user in users:
+        context = common_context(user)
+
+        # Start building email template data.
+        unsubscribe_link = one_click_unsubscribe_link(user, "digest")
+        context.update(unsubscribe_link=unsubscribe_link)
+
+        # Gather new streams.
+        new_streams_count, new_streams = gather_new_streams(
+            realm=realm,
+            recently_created_streams=recently_created_streams,
+            can_access_public=user.can_access_public_streams(),
+        )
+
+        context["new_streams_count"] = new_streams_count
+        context[
+            "message_content_disabled_by_realm"
+        ] = not realm.message_content_allowed_in_email_notifications
+        context[
+            "message_content_disabled_by_user"
+        ] = not user.message_content_in_email_notifications
+
+        if not message_content_allowed_in_missedmessage_emails(user):
+            # Count new messages when message content is hidden in email notifications.
+            if not have_min_recent_message_id:
+                min_recent_message_id = get_min_recent_message_id(realm.id, cutoff_date)
+                have_min_recent_message_id = True
+            context["new_messages_count"] = get_new_messages_count(user, min_recent_message_id)
+            context["hot_conversations"] = []
+            context["show_message_content"] = False
+        else:
+            # Otherwise, get context data for hot conversations.
+            stream_ids = user_stream_map[user.id]
+            muted_topics = user_muted_topics_map[user.id]
+            recent_topics = []
+            for stream_id in stream_ids:
+                recent_topics += [
+                    digest_topic
+                    for digest_topic in get_recent_topics(realm.id, stream_id, cutoff_date)
+                    # Topics the user has muted should not be featured
+                    # in their digest.
+                    if (digest_topic.stream_id(), digest_topic.topic_name().lower())
+                    not in muted_topics
+                ]
+            hot_topics = get_hot_topics(recent_topics, stream_ids)
+
+            context["hot_conversations"] = [
+                hot_topic.teaser_data(user, stream_id_map) for hot_topic in hot_topics
+            ]
+            context["new_channels"] = new_streams
+            context["new_messages_count"] = 0
+            context["show_message_content"] = True
+
+        yield user, context
+
+
+def get_digest_context(user: UserProfile, cutoff: float) -> dict[str, Any]:
+    for ignored, context in bulk_get_digest_context([user], cutoff):
+        return context
+    raise AssertionError("Unreachable")
+
+
+@transaction.atomic(durable=True)
+def bulk_handle_digest_email(user_ids: list[int], cutoff: float) -> None:
+    # We go directly to the database to get user objects,
+    # since inactive users are likely to not be in the cache.
+    users = (
+        UserProfile.objects.filter(id__in=user_ids, is_active=True, realm__deactivated=False)
+        .order_by("id")
+        .select_related("realm")
+    )
+    digest_users = []
+
+    for user, context in bulk_get_digest_context(users, cutoff):
+        # We don't want to send emails containing almost no information.
+        if not enough_traffic(
+            len(context["hot_conversations"]),
+            context["new_streams_count"],
+            context["new_messages_count"],
+            context["show_message_content"],
+        ):
+            continue
+
+        digest_users.append(user)
+        logger.info("Enqueuing digest email for user %s", user.id)
+
+        # Send now, as a ScheduledEmail
+        send_future_email(
+            "zerver/emails/digest",
+            user.realm,
+            to_user_ids=[user.id],
+            from_name=_("{service_name} digest").format(service_name=settings.INSTALLATION_NAME),
+            from_address=FromAddress.no_reply_placeholder,
+            context=context,
+        )
+
+    bulk_write_realm_audit_logs(digest_users)
+
+
+def bulk_write_realm_audit_logs(users: list[UserProfile]) -> None:
+    if not users:
+        return
+
+    # We write RealmAuditLog rows for auditing, and we will also
+    # use these rows during the next run to possibly exclude the
+    # users (if insufficient time has passed).
+    last_message_id = get_last_message_id()
+    now = timezone_now()
+
+    log_rows = [
+        RealmAuditLog(
+            realm_id=user.realm_id,
+            modified_user_id=user.id,
+            event_last_message_id=last_message_id,
+            event_time=now,
+            event_type=AuditLogEventType.USER_DIGEST_EMAIL_CREATED,
+        )
+        for user in users
+    ]
+
+    RealmAuditLog.objects.bulk_create(log_rows)

zerver/lib/display_recipient.py

@@ -0,0 +1,253 @@
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Optional, TypedDict
+
+from django.db.models import QuerySet
+
+from zerver.lib.cache import (
+    bulk_cached_fetch,
+    cache_with_key,
+    display_recipient_cache_key,
+    generic_bulk_cached_fetch,
+    single_user_display_recipient_cache_key,
+)
+from zerver.lib.per_request_cache import cache_for_current_request
+from zerver.lib.types import DisplayRecipientT, UserDisplayRecipient
+
+if TYPE_CHECKING:
+    from zerver.models import Recipient
+
+display_recipient_fields = [
+    "id",
+    "email",
+    "full_name",
+    "is_mirror_dummy",
+]
+
+
+class TinyStreamResult(TypedDict):
+    recipient_id: int
+    name: str
+
+
+def get_display_recipient_cache_key(
+    recipient_id: int, recipient_type: int, recipient_type_id: int | None
+) -> str:
+    return display_recipient_cache_key(recipient_id)
+
+
+# Note that the _same_ cache key is used for streams, which contain a
+# string, not a list[UserDisplayRecipient]!  This works because the
+# recipient space is distinct between the two.
+@cache_with_key(get_display_recipient_cache_key, timeout=3600 * 24 * 7)
+def get_display_recipient_remote_cache(
+    recipient_id: int, recipient_type: int, recipient_type_id: int | None
+) -> list[UserDisplayRecipient]:
+    """
+    This returns an appropriate object describing the recipient of a
+    direct message (whether individual or group).
+
+    It will be an array of dicts for each recipient.
+
+    Do not use this for streams.
+    """
+
+    from zerver.models import Recipient, UserProfile
+
+    assert recipient_type != Recipient.STREAM
+
+    # The main priority for ordering here is being deterministic.
+    # Right now, we order by ID, which matches the ordering of user
+    # names in the left sidebar.
+    user_profile_list = (
+        UserProfile.objects.filter(
+            subscription__recipient_id=recipient_id,
+        )
+        .order_by("id")
+        .values(*display_recipient_fields)
+    )
+    return list(user_profile_list)
+
+
+def user_dict_id_fetcher(user_dict: UserDisplayRecipient) -> int:
+    return user_dict["id"]
+
+
+def bulk_fetch_single_user_display_recipients(uids: list[int]) -> dict[int, UserDisplayRecipient]:
+    from zerver.models import UserProfile
+
+    return bulk_cached_fetch(
+        # Use a separate cache key to protect us from conflicts with
+        # the get_user_profile_by_id cache.
+        # (Since we fetch only several fields here)
+        cache_key_function=single_user_display_recipient_cache_key,
+        query_function=lambda ids: list(
+            UserProfile.objects.filter(id__in=ids).values(*display_recipient_fields)
+        ),
+        object_ids=uids,
+        id_fetcher=user_dict_id_fetcher,
+    )
+
+
+def bulk_fetch_stream_names(
+    recipient_tuples: set[tuple[int, int, int]],
+) -> dict[int, str]:
+    """
+    Takes set of tuples of the form (recipient_id, recipient_type, recipient_type_id)
+    Returns dict mapping recipient_id to corresponding display_recipient
+    """
+
+    from zerver.models import Stream
+
+    if len(recipient_tuples) == 0:
+        return {}
+
+    recipient_id_to_stream_id = {tup[0]: tup[2] for tup in recipient_tuples}
+    recipient_ids = [tup[0] for tup in recipient_tuples]
+
+    def get_tiny_stream_rows(
+        recipient_ids: list[int],
+    ) -> QuerySet[Stream, TinyStreamResult]:
+        stream_ids = [recipient_id_to_stream_id[recipient_id] for recipient_id in recipient_ids]
+        return Stream.objects.filter(id__in=stream_ids).values("recipient_id", "name")
+
+    def get_recipient_id(row: TinyStreamResult) -> int:
+        return row["recipient_id"]
+
+    def get_name(row: TinyStreamResult) -> str:
+        return row["name"]
+
+    # ItemT = TinyStreamResult, CacheItemT = str (name), ObjKT = int (recipient_id)
+    stream_display_recipients: dict[int, str] = generic_bulk_cached_fetch(
+        cache_key_function=display_recipient_cache_key,
+        query_function=get_tiny_stream_rows,
+        object_ids=recipient_ids,
+        id_fetcher=get_recipient_id,
+        cache_transformer=get_name,
+        setter=lambda obj: obj,
+        extractor=lambda obj: obj,
+        pickled_tupled=False,
+    )
+
+    return stream_display_recipients
+
+
+def bulk_fetch_user_display_recipients(
+    recipient_tuples: set[tuple[int, int, int]],
+    recipient_dm_user_ids: Mapping[int, set[int]] = {},
+) -> dict[int, list[UserDisplayRecipient]]:
+    """
+    Takes set of tuples of the form (recipient_id, recipient_type, recipient_type_id)
+    Returns dict mapping recipient_id to corresponding display_recipient
+
+    The participants of a DirectMessageGroup are fixed when it is created, so
+    a caller that already knows them for some recipients can pass them in via
+    dm_involved_user_ids to avoid looking them up again.
+    """
+
+    from zerver.models.recipients import bulk_get_direct_message_group_user_ids
+
+    if len(recipient_tuples) == 0:
+        return {}
+
+    get_recipient_id = lambda tup: tup[0]
+
+    direct_message_group_recipient_ids = [get_recipient_id(tup) for tup in recipient_tuples]
+
+    # Query only DirectMessageGroup participants who are not already loaded.
+    user_ids_in_direct_message_groups = bulk_get_direct_message_group_user_ids(
+        [
+            recipient_id
+            for recipient_id in direct_message_group_recipient_ids
+            if recipient_id not in recipient_dm_user_ids
+        ]
+    )
+    user_ids_in_direct_message_groups.update(recipient_dm_user_ids)
+
+    # Find all user ids whose UserProfiles we will need to fetch:
+    user_ids_to_fetch: set[int] = set()
+
+    for recipient_id in direct_message_group_recipient_ids:
+        direct_message_group_user_ids = user_ids_in_direct_message_groups[recipient_id]
+        user_ids_to_fetch |= direct_message_group_user_ids
+
+    # Fetch the needed user dictionaries.
+    user_display_recipients = bulk_fetch_single_user_display_recipients(list(user_ids_to_fetch))
+
+    result = {}
+
+    for recipient_id in direct_message_group_recipient_ids:
+        user_ids = sorted(user_ids_in_direct_message_groups[recipient_id])
+        display_recipients = [user_display_recipients[user_id] for user_id in user_ids]
+        result[recipient_id] = display_recipients
+
+    return result
+
+
+def bulk_fetch_display_recipients(
+    recipient_tuples: set[tuple[int, int, int]],
+    recipient_dm_user_ids: Mapping[int, set[int]] = {},
+) -> dict[int, DisplayRecipientT]:
+    """
+    Takes set of tuples of the form (recipient_id, recipient_type, recipient_type_id)
+    Returns dict mapping recipient_id to corresponding display_recipient
+    """
+
+    from zerver.models import Recipient
+
+    stream_recipients = {
+        recipient for recipient in recipient_tuples if recipient[1] == Recipient.STREAM
+    }
+    direct_message_recipients = recipient_tuples - stream_recipients
+
+    stream_display_recipients = bulk_fetch_stream_names(stream_recipients)
+    direct_message_display_recipients = bulk_fetch_user_display_recipients(
+        direct_message_recipients, recipient_dm_user_ids
+    )
+
+    # Glue the dicts together and return:
+    return {**stream_display_recipients, **direct_message_display_recipients}
+
+
+@cache_for_current_request
+def get_display_recipient_by_id(
+    recipient_id: int, recipient_type: int, recipient_type_id: int | None
+) -> list[UserDisplayRecipient]:
+    """
+    returns: an object describing the recipient (using a cache).
+    If the type is a stream, the type_id must be an int; a string is returned.
+    Otherwise, type_id may be None; an array of recipient dicts is returned.
+    """
+    return get_display_recipient_remote_cache(recipient_id, recipient_type, recipient_type_id)
+
+
+def get_display_recipient(recipient: "Recipient") -> list[UserDisplayRecipient]:
+    return get_display_recipient_by_id(
+        recipient.id,
+        recipient.type,
+        recipient.type_id,
+    )
+
+
+def get_recipient_ids(
+    recipient: Optional["Recipient"], user_profile_id: int
+) -> tuple[list[int], str]:
+    from zerver.models import Recipient
+
+    if recipient is None:
+        recipient_type_str = ""
+        to = []
+    elif recipient.type == Recipient.STREAM:
+        recipient_type_str = "stream"
+        to = [recipient.type_id]
+    else:
+        # We're somewhat weirdly using the display recipient cache,
+        # for the fact that it is a probably-cached lookup which knows
+        # the user-ids which are members of a group DM.
+        recipient_type_str = "private"
+        to = []
+        recipients = get_display_recipient(recipient)
+        for r in recipients:
+            assert not isinstance(r, str)  # It will only be a string for streams
+            if r["id"] != user_profile_id or len(recipients) == 1:
+                to.append(r["id"])
+    return to, recipient_type_str

zerver/lib/domains.py

@@ -0,0 +1,22 @@
+import re
+
+from django.core.exceptions import ValidationError
+from django.utils.translation import gettext as _
+
+
+def validate_domain(domain: str | None) -> None:
+    if domain is None or len(domain) == 0:
+        raise ValidationError(_("Domain can't be empty."))
+    if "." not in domain:
+        raise ValidationError(_("Domain must have at least one dot (.)"))
+    if len(domain) > 255:
+        raise ValidationError(_("Domain is too long"))
+    if domain[0] == "." or domain[-1] == ".":
+        raise ValidationError(_("Domain cannot start or end with a dot (.)"))
+    for subdomain in domain.split("."):
+        if not subdomain:
+            raise ValidationError(_("Consecutive '.' are not allowed."))
+        if subdomain[0] == "-" or subdomain[-1] == "-":
+            raise ValidationError(_("Subdomains cannot start or end with a '-'."))
+        if not re.match(r"^[a-z0-9-]*$", subdomain):
+            raise ValidationError(_("Domain can only have letters, numbers, '.' and '-'s."))

zerver/lib/drafts.py

@@ -0,0 +1,183 @@
+import time
+from collections.abc import Callable
+from dataclasses import dataclass
+from datetime import datetime
+from functools import wraps
+from typing import Annotated, Concatenate, Literal
+
+from django.core.exceptions import ValidationError
+from django.db import transaction
+from django.http import HttpRequest, HttpResponse
+from django.utils.translation import gettext as _
+from pydantic import BaseModel, ConfigDict
+from typing_extensions import ParamSpec
+
+from zerver.lib.addressee import get_user_profiles_by_ids
+from zerver.lib.exceptions import JsonableError, ResourceNotFoundError
+from zerver.lib.message import normalize_body, truncate_topic
+from zerver.lib.recipient_users import (
+    check_sender_can_access_recipients,
+    recipient_for_user_profiles,
+)
+from zerver.lib.streams import access_stream_by_id
+from zerver.lib.timestamp import timestamp_to_datetime
+from zerver.lib.typed_endpoint import RequiredStringConstraint
+from zerver.models import Draft, UserProfile
+from zerver.tornado.django_api import send_event_on_commit
+
+ParamT = ParamSpec("ParamT")
+
+
+@dataclass
+class ValidatedDraftData:
+    recipient_id: int | None
+    topic: str
+    content: str
+    last_edit_time: datetime
+
+
+class DraftData(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    type: Literal["private", "stream", ""]
+    to: list[int]
+    topic: str
+    content: Annotated[str, RequiredStringConstraint()]
+    timestamp: int | float | None = None
+
+
+def further_validated_draft_dict(
+    draft_dict: DraftData, user_profile: UserProfile
+) -> ValidatedDraftData:
+    """Take a DraftData object that was already validated by the @typed_endpoint
+    decorator then further sanitize, validate, and transform it.
+    Ultimately return this "further validated" draft dict.
+    It will have a slightly different set of keys the values
+    for which can be used to directly create a Draft object."""
+
+    content = normalize_body(draft_dict.content)
+
+    timestamp = draft_dict.timestamp
+    if timestamp is None:
+        timestamp = time.time()
+    timestamp = round(timestamp, 6)
+    if timestamp < 0:
+        # While it's not exactly an invalid timestamp, it's not something
+        # we want to allow either.
+        raise JsonableError(_("Timestamp must not be negative."))
+    last_edit_time = timestamp_to_datetime(timestamp)
+
+    topic_name = ""
+    recipient_id = None
+    to = draft_dict.to
+    if draft_dict.type == "stream":
+        topic_name = truncate_topic(draft_dict.topic)
+        if "\0" in topic_name:
+            raise JsonableError(_("Topic must not contain null bytes"))
+        if len(to) != 1:
+            raise JsonableError(_("Must specify exactly 1 channel ID for channel messages"))
+        stream, _sub = access_stream_by_id(user_profile, to[0])
+        recipient_id = stream.recipient_id
+    elif draft_dict.type == "private" and len(to) != 0:
+        to_users = get_user_profiles_by_ids(set(to), user_profile.realm)
+        # Since recipient_for_user_profiles creates the
+        # DirectMessageGroup object, and thus Subscription objects
+        # granting the sender access to the recipients, we need to
+        # check access before calling it.
+        check_sender_can_access_recipients(user_profile, to_users)
+        try:
+            recipient_id = recipient_for_user_profiles(to_users, False, None, user_profile).id
+        except ValidationError as e:  # nocoverage
+            raise JsonableError(e.messages[0])
+
+    return ValidatedDraftData(
+        recipient_id=recipient_id,
+        topic=topic_name,
+        content=content,
+        last_edit_time=last_edit_time,
+    )
+
+
+def draft_endpoint(
+    view_func: Callable[Concatenate[HttpRequest, UserProfile, ParamT], HttpResponse],
+) -> Callable[Concatenate[HttpRequest, UserProfile, ParamT], HttpResponse]:
+    @wraps(view_func)
+    def draft_view_func(
+        request: HttpRequest,
+        user_profile: UserProfile,
+        /,
+        *args: ParamT.args,
+        **kwargs: ParamT.kwargs,
+    ) -> HttpResponse:
+        if not user_profile.enable_drafts_synchronization:
+            raise JsonableError(_("User has disabled synchronizing drafts."))
+        return view_func(request, user_profile, *args, **kwargs)
+
+    return draft_view_func
+
+
+def do_create_drafts(drafts: list[DraftData], user_profile: UserProfile) -> list[Draft]:
+    """Create drafts in bulk for a given user based on the DraftData objects. Since
+    currently, the only place this method is being used (apart from tests) is from
+    the create_draft view, we assume that these are syntactically valid
+    (i.e. they satisfy the @typed_endpoint validation for DraftData)."""
+    draft_objects = []
+    for draft in drafts:
+        valid_draft_dict = further_validated_draft_dict(draft, user_profile)
+        draft_objects.append(
+            Draft(
+                user_profile=user_profile,
+                recipient_id=valid_draft_dict.recipient_id,
+                topic=valid_draft_dict.topic,
+                content=valid_draft_dict.content,
+                last_edit_time=valid_draft_dict.last_edit_time,
+            )
+        )
+
+    with transaction.atomic(durable=True):
+        created_draft_objects = Draft.objects.bulk_create(draft_objects)
+
+        event = {
+            "type": "drafts",
+            "op": "add",
+            "drafts": [draft.to_dict() for draft in created_draft_objects],
+        }
+        send_event_on_commit(user_profile.realm, event, [user_profile.id])
+
+    return created_draft_objects
+
+
+def do_edit_draft(draft_id: int, draft: DraftData, user_profile: UserProfile) -> None:
+    """Edit/update a single draft for a given user. Since the only place this method is being
+    used from (apart from tests) is the edit_draft view, we assume that the DraftData object
+    is syntactically valid (i.e. it satisfies the @typed_endpoint validation for DraftData)."""
+    try:
+        draft_object = Draft.objects.get(id=draft_id, user_profile=user_profile)
+    except Draft.DoesNotExist:
+        raise ResourceNotFoundError(_("Draft does not exist"))
+    valid_draft_dict = further_validated_draft_dict(draft, user_profile)
+    draft_object.content = valid_draft_dict.content
+    draft_object.topic = valid_draft_dict.topic
+    draft_object.recipient_id = valid_draft_dict.recipient_id
+    draft_object.last_edit_time = valid_draft_dict.last_edit_time
+
+    with transaction.atomic(durable=True):
+        draft_object.save()
+
+        event = {"type": "drafts", "op": "update", "draft": draft_object.to_dict()}
+        send_event_on_commit(user_profile.realm, event, [user_profile.id])
+
+
+@transaction.atomic(durable=True)
+def do_delete_draft(draft_id: int, user_profile: UserProfile) -> None:
+    """Delete a draft belonging to a particular user."""
+    try:
+        draft_object = Draft.objects.get(id=draft_id, user_profile=user_profile)
+    except Draft.DoesNotExist:
+        raise ResourceNotFoundError(_("Draft does not exist"))
+
+    draft_id = draft_object.id
+    draft_object.delete()
+
+    event = {"type": "drafts", "op": "remove", "draft_id": draft_id}
+    send_event_on_commit(user_profile.realm, event, [user_profile.id])

zerver/lib/email_mirror.py

@@ -0,0 +1,607 @@
+import encodings.aliases
+import logging
+import re
+import secrets
+import sys
+from email.headerregistry import Address, AddressHeader
+from email.message import EmailMessage
+from email.utils import parseaddr
+from re import Match
+
+from django.conf import settings
+from django.db.models import Q
+from django.utils.translation import gettext as _
+from typing_extensions import override
+
+from zerver.actions.message_send import (
+    check_send_message,
+    internal_send_group_direct_message,
+    internal_send_private_message,
+    internal_send_stream_message,
+)
+from zerver.lib.display_recipient import get_display_recipient
+from zerver.lib.email_mirror_helpers import (
+    ZulipEmailForwardError,
+    ZulipEmailForwardUserError,
+    decode_email_address,
+    get_email_gateway_message_string_from_address,
+)
+from zerver.lib.exceptions import JsonableError, RateLimitedError
+from zerver.lib.markdown import get_markdown_link_for_url
+from zerver.lib.markdown.from_html import convert_html_to_markdown
+from zerver.lib.message import normalize_body, truncate_content, truncate_topic
+from zerver.lib.rate_limiter import RateLimitedObject
+from zerver.lib.send_email import FromAddress
+from zerver.lib.streams import access_stream_for_send_message
+from zerver.lib.string_validation import is_character_printable
+from zerver.lib.upload import upload_message_attachment
+from zerver.models import (
+    ChannelEmailAddress,
+    Message,
+    MissedMessageEmailAddress,
+    Realm,
+    Recipient,
+    Stream,
+    UserProfile,
+)
+from zerver.models.clients import get_client
+from zerver.models.constants import MAX_TOPIC_NAME_LENGTH
+from zerver.models.streams import StreamTopicsPolicyEnum, get_stream_by_id_in_realm
+from zerver.models.users import get_system_bot
+from zproject.backends import is_user_active
+
+if sys.version_info < (3, 14):  # nocoverage
+    # https://github.com/python/cpython/issues/62824
+    encodings.aliases.aliases.update(
+        {
+            "iso_8859_8_i": "iso8859_8",
+            "iso_8859_8_e": "iso8859_8",
+        }
+    )
+
+logger = logging.getLogger(__name__)
+
+
+def redact_email_address(error_message: str) -> str:
+    if not settings.EMAIL_GATEWAY_EXTRA_PATTERN_HACK:
+        domain = Address(addr_spec=settings.EMAIL_GATEWAY_PATTERN).domain
+    else:
+        # EMAIL_GATEWAY_EXTRA_PATTERN_HACK is of the form '@example.com'
+        domain = settings.EMAIL_GATEWAY_EXTRA_PATTERN_HACK.removeprefix("@")
+
+    def redact(address_match: Match[str]) -> str:
+        email_address = address_match[0]
+        # Annotate basic info about the address before scrubbing:
+        if is_missed_message_address(email_address):
+            annotation = " <Missed message address>"
+        else:
+            try:
+                target_stream_id = decode_stream_email_address(email_address)[0].channel_id
+                annotation = f" <Address to stream id: {target_stream_id}>"
+            except ZulipEmailForwardError:
+                annotation = " <Invalid address>"
+
+        # Scrub the address from the message, to the form XXXXX@example.com:
+        return "X" * len(address_match[1]) + address_match[2] + annotation
+
+    return re.sub(rf"\b(\S*?)(@{re.escape(domain)})", redact, error_message)
+
+
+def log_error(email_message: EmailMessage, error_message: str, to: str | None) -> None:
+    recipient = to or "No recipient found"
+    error_message = "Sender: {}\nTo: {}\n{}".format(
+        email_message.get("From"), recipient, error_message
+    )
+
+    error_message = redact_email_address(error_message)
+    logger.error(error_message)
+
+
+# Temporary missed message addresses
+
+
+def generate_missed_message_token() -> str:
+    return "mm" + secrets.token_hex(16)
+
+
+def is_missed_message_address(address: str) -> bool:
+    try:
+        msg_string = get_email_gateway_message_string_from_address(address)
+    except ZulipEmailForwardError:
+        return False
+
+    return is_mm_32_format(msg_string)
+
+
+def is_mm_32_format(msg_string: str | None) -> bool:
+    """
+    Missed message strings are formatted with a little "mm" prefix
+    followed by a randomly generated 32-character string.
+    """
+    return msg_string is not None and msg_string.startswith("mm") and len(msg_string) == 34
+
+
+def get_missed_message_token_from_address(address: str) -> str:
+    msg_string = get_email_gateway_message_string_from_address(address)
+
+    if not is_mm_32_format(msg_string):
+        raise ZulipEmailForwardError("Could not parse missed message address")
+
+    return msg_string
+
+
+def get_usable_missed_message_address(address: str) -> MissedMessageEmailAddress:
+    token = get_missed_message_token_from_address(address)
+    try:
+        mm_address = MissedMessageEmailAddress.objects.select_related(
+            "user_profile",
+            "user_profile__realm",
+            "message",
+            "message__sender",
+            "message__recipient",
+        ).get(email_token=token)
+    except MissedMessageEmailAddress.DoesNotExist:
+        raise ZulipEmailForwardError("Zulip notification reply address is invalid.")
+
+    return mm_address
+
+
+def create_missed_message_address(user_profile: UserProfile, message: Message) -> str:
+    # If the email gateway isn't configured, we specify a reply
+    # address, since there's no useful way for the user to reply into
+    # Zulip.
+    if settings.EMAIL_GATEWAY_PATTERN == "":
+        return FromAddress.NOREPLY
+
+    mm_address = MissedMessageEmailAddress.objects.create(
+        message=message,
+        user_profile=user_profile,
+        email_token=generate_missed_message_token(),
+    )
+    return str(mm_address)
+
+
+def construct_zulip_body(
+    message: EmailMessage,
+    subject: str,
+    realm: Realm,
+    *,
+    sender: UserProfile,
+    show_sender: bool = False,
+    include_quotes: bool = False,
+    include_footer: bool = False,
+    prefer_text: bool = True,
+    subject_in_body: bool = False,
+) -> str:
+    body = extract_body(message, include_quotes, prefer_text)
+    # Remove null characters, since Zulip will reject
+    body = body.replace("\x00", "")
+    if not include_footer:
+        body = filter_footer(body)
+
+    if not body.endswith("\n"):
+        body += "\n"
+    if not body.rstrip():
+        body = "(No email body)"
+
+    preamble = ""
+    if show_sender:
+        from_address = str(message.get("From", ""))
+        preamble = f"**From:** {from_address}\n"
+    if subject_in_body:
+        preamble += f"**Subject:** {subject}\n"
+    if preamble != "":
+        preamble += "\n"
+
+    postamble = extract_and_upload_attachments(message, realm, sender)
+    if postamble != "":
+        postamble = "\n" + postamble
+
+    # Truncate the content ourselves, to ensure that the attachments
+    # all make it into the body-as-posted
+    body = truncate_content(
+        body,
+        settings.MAX_MESSAGE_LENGTH - len(preamble) - len(postamble),
+        "\n[message truncated]",
+    )
+    return preamble + body + postamble
+
+
+## Sending the Zulip ##
+
+
+def send_zulip(sender: UserProfile, stream: Stream, topic_name: str, content: str) -> None:
+    internal_send_stream_message(
+        sender,
+        stream,
+        truncate_topic(topic_name),
+        normalize_body(content),
+        email_gateway=True,
+    )
+
+
+def send_mm_reply_to_stream(
+    user_profile: UserProfile, stream: Stream, topic_name: str, body: str
+) -> None:
+    try:
+        check_send_message(
+            sender=user_profile,
+            client=get_client("Internal"),
+            recipient_type_name="stream",
+            message_to=[stream.id],
+            topic_name=topic_name,
+            message_content=body,
+        )
+    except JsonableError as error:
+        error_message = _(
+            "Error sending message to channel {channel_name} via message notification email reply:\n{error_message}"
+        ).format(channel_name=stream.name, error_message=error.msg)
+        internal_send_private_message(
+            get_system_bot(settings.NOTIFICATION_BOT, user_profile.realm_id),
+            user_profile,
+            error_message,
+        )
+
+
+def get_message_part_by_type(message: EmailMessage, content_type: str) -> str | None:
+    for part in message.walk():
+        if part.get_content_type() == content_type:
+            content = part.get_payload(decode=True)
+            assert isinstance(content, bytes)
+
+            charset = part.get_content_charset()
+            if charset is None:
+                # If no charset has been specified in the header, assume us-ascii,
+                # by RFC6657: https://tools.ietf.org/html/rfc6657
+                charset = "us-ascii"
+
+            try:
+                return content.decode(charset, errors="ignore")
+            except LookupError:
+                # The RFCs do not define how to handle unknown charsets,
+                # but treating as US-ASCII seems reasonable.
+                return content.decode("us-ascii", errors="ignore")
+
+    return None
+
+
+def extract_body(
+    message: EmailMessage, include_quotes: bool = False, prefer_text: bool = True
+) -> str:
+    plaintext_content = extract_plaintext_body(message, include_quotes)
+    html_content = extract_html_body(message, include_quotes)
+
+    if plaintext_content is None and html_content is None:
+        logger.warning("Content types: %s", [part.get_content_type() for part in message.walk()])
+        raise ZulipEmailForwardUserError("Unable to find plaintext or HTML message body")
+    if not plaintext_content and not html_content:
+        raise ZulipEmailForwardUserError("Email has no nonempty body sections; ignoring.")
+
+    if prefer_text:
+        if plaintext_content:
+            return plaintext_content
+        else:
+            assert html_content  # Needed for mypy. Ensured by the validating block above.
+            return html_content
+    else:
+        if html_content:
+            return html_content
+        else:
+            assert plaintext_content  # Needed for mypy. Ensured by the validating block above.
+            return plaintext_content
+
+
+talon_initialized = False
+
+
+def extract_plaintext_body(message: EmailMessage, include_quotes: bool = False) -> str | None:
+    import talon_core
+
+    global talon_initialized
+    if not talon_initialized:
+        talon_core.init()
+        talon_initialized = True
+
+    plaintext_content = get_message_part_by_type(message, "text/plain")
+    if plaintext_content is not None:
+        if include_quotes:
+            return plaintext_content
+        else:
+            return talon_core.quotations.extract_from_plain(plaintext_content)
+    else:
+        return None
+
+
+def extract_html_body(message: EmailMessage, include_quotes: bool = False) -> str | None:
+    import talon_core
+
+    global talon_initialized
+    if not talon_initialized:  # nocoverage
+        talon_core.init()
+        talon_initialized = True
+
+    html_content = get_message_part_by_type(message, "text/html")
+    if html_content is not None:
+        if include_quotes:
+            return convert_html_to_markdown(html_content)
+        else:
+            return convert_html_to_markdown(talon_core.quotations.extract_from_html(html_content))
+    else:
+        return None
+
+
+def filter_footer(text: str) -> str:
+    # Try to filter out obvious footers.
+    possible_footers = [line for line in text.split("\n") if line.strip() == "--"]
+    if len(possible_footers) != 1:
+        # Be conservative and don't try to scrub content if there
+        # isn't a trivial footer structure.
+        return text
+
+    return re.split(r"^\s*--\s*$", text, maxsplit=1, flags=re.MULTILINE)[0].strip()
+
+
+def extract_and_upload_attachments(message: EmailMessage, realm: Realm, sender: UserProfile) -> str:
+    attachment_links = []
+    for part in message.walk():
+        content_type = part.get_content_type()
+        filename = part.get_filename()
+        if filename:
+            attachment = part.get_payload(decode=True)
+            if isinstance(attachment, bytes):
+                upload_url, filename = upload_message_attachment(
+                    filename,
+                    content_type,
+                    attachment,
+                    sender,
+                    target_realm=realm,
+                )
+                formatted_link = get_markdown_link_for_url(filename, upload_url)
+                attachment_links.append(formatted_link)
+            else:
+                logger.warning(
+                    "Payload is not bytes (invalid attachment %s in message from %s).",
+                    filename,
+                    message.get("From"),
+                )
+
+    return "\n".join(attachment_links)
+
+
+def decode_stream_email_address(email: str) -> tuple[ChannelEmailAddress, dict[str, bool]]:
+    token, options = decode_email_address(email)
+
+    try:
+        channel_email_address = ChannelEmailAddress.objects.select_related(
+            "channel", "sender", "creator", "realm"
+        ).get(email_token=token)
+    except ChannelEmailAddress.DoesNotExist:
+        raise ZulipEmailForwardError("Bad stream token from email recipient " + email)
+
+    return channel_email_address, options
+
+
+def find_emailgateway_recipient(message: EmailMessage) -> str:
+    # We can't use Delivered-To; if there is a X-Gm-Original-To
+    # it is more accurate, so try to find the most-accurate
+    # recipient list in descending priority order
+    recipient_headers = [
+        "X-Gm-Original-To",
+        "Delivered-To",
+        "Envelope-To",
+        "Resent-To",
+        "Resent-CC",
+        "To",
+        "CC",
+    ]
+
+    pattern_parts = [re.escape(part) for part in settings.EMAIL_GATEWAY_PATTERN.split("%s")]
+    match_email_re = re.compile(r".*?".join(pattern_parts))
+
+    for header_name in recipient_headers:
+        for header_value in message.get_all(header_name, []):
+            if isinstance(header_value, AddressHeader):
+                emails = [addr.addr_spec for addr in header_value.addresses]
+            else:
+                emails = [parseaddr(str(header_value))[1]]
+
+            for email in emails:
+                if match_email_re.match(email):
+                    return email
+
+    raise ZulipEmailForwardError("Missing recipient in mirror email")
+
+
+def strip_from_subject(subject: str) -> str:
+    # strips RE and FWD from the subject
+    # from: https://stackoverflow.com/questions/9153629/regex-code-for-removing-fwd-re-etc-from-email-subject
+    reg = r"([\[\(] *)?\b(RE|AW|SV|FWD?) *(\[\d+\])?([-:;)\]][ :;\])-]*|$)|\]+ *$"
+    stripped = re.sub(reg, "", subject, flags=re.IGNORECASE | re.MULTILINE)
+    return stripped.strip()
+
+
+def is_forwarded(subject: str) -> bool:
+    # regex taken from strip_from_subject, we use it to detect various forms
+    # of FWD at the beginning of the subject.
+    reg = r"([\[\(] *)?\b(FWD?) *([-:;)\]][ :;\])-]*|$)|\]+ *$"
+    return bool(re.match(reg, subject, flags=re.IGNORECASE))
+
+
+def check_access_for_channel_email_address(channel_email_address: ChannelEmailAddress) -> None:
+    channel = channel_email_address.channel
+    sender = channel_email_address.sender
+    creator = channel_email_address.creator
+    realm = channel_email_address.realm
+    email_gateway_bot = get_system_bot(settings.EMAIL_GATEWAY_BOT, realm.id)
+
+    if sender.id == email_gateway_bot.id and creator is not None:
+        user_for_access_check = creator
+    else:
+        user_for_access_check = sender
+
+    # Raises JsonableError on permission denied
+    access_stream_for_send_message(user_for_access_check, channel, forwarder_user_profile=None)
+
+
+def process_stream_message(to: str, message: EmailMessage) -> None:
+    subject_header = message.get("Subject", "")
+
+    channel_email_address, options = decode_stream_email_address(to)
+    channel = channel_email_address.channel
+    sender = channel_email_address.sender
+    realm = channel_email_address.realm
+    try:
+        check_access_for_channel_email_address(channel_email_address)
+    except JsonableError as e:
+        logger.info("Failed to process email to %s (%s): %s", channel.name, realm.string_id, e)
+        return
+
+    # Don't remove quotations if message is forwarded, unless otherwise specified:
+    if "include_quotes" not in options:
+        options["include_quotes"] = is_forwarded(subject_header)
+
+    subject = strip_from_subject(subject_header)
+    # We don't want to reject email messages with disallowed characters in the Subject,
+    # so we just remove them to make it a valid Zulip topic name.
+    subject = "".join([char for char in subject if is_character_printable(char)])
+    if channel.topics_policy == StreamTopicsPolicyEnum.empty_topic_only.value:
+        options["subject_in_body"] = True
+        topic = ""
+    elif subject == "":
+        topic = _("Email with no subject")
+    else:
+        topic = subject
+
+    if len(subject) > MAX_TOPIC_NAME_LENGTH:
+        options["subject_in_body"] = True
+
+    body = construct_zulip_body(message, subject, realm, sender=sender, **options)
+    send_zulip(sender, channel, topic, body)
+    logger.info(
+        "Successfully processed email to %s (%s)",
+        channel.name,
+        realm.string_id,
+    )
+
+
+def process_missed_message(to: str, message: EmailMessage) -> None:
+    auto_submitted = message.get("Auto-Submitted", "")
+    if auto_submitted in ("auto-replied", "auto-generated"):
+        logger.info("Dropping %s message from %s", auto_submitted, message.get("From"))
+        return
+
+    mm_address = get_usable_missed_message_address(to)
+    mm_address.increment_times_used()
+
+    user_profile = mm_address.user_profile
+    topic_name = mm_address.message.topic_name()
+    recipient = mm_address.message.recipient
+
+    if not is_user_active(user_profile):
+        logger.warning("Sending user is not active. Ignoring this message notification email.")
+        return
+
+    body = construct_zulip_body(message, topic_name, user_profile.realm, sender=user_profile)
+
+    assert recipient is not None
+    if recipient.type == Recipient.STREAM:
+        stream = get_stream_by_id_in_realm(recipient.type_id, user_profile.realm)
+        send_mm_reply_to_stream(user_profile, stream, topic_name, body)
+        recipient_str = stream.name
+    elif recipient.type == Recipient.DIRECT_MESSAGE_GROUP:
+        display_recipient = get_display_recipient(recipient)
+        # Drop members who have been deactivated, matching the check in
+        # validate_recipient_user_profiles. The web app blocks composing
+        # to such a group entirely, but the email gateway can receive
+        # replies to notification emails that predate the deactivation.
+        user_ids = [user_dict["id"] for user_dict in display_recipient]
+        emails = list(
+            UserProfile.objects.filter(id__in=user_ids)
+            .filter(Q(is_active=True) | Q(is_mirror_dummy=True))
+            .values_list("email", flat=True)
+        )
+        if len(emails) <= 1:
+            # Everyone but the sender (whom we verified active above)
+            # has been deactivated; there is nobody left to receive
+            # the reply.
+            logger.info(
+                "Dropping message notification email reply from user %s to a group direct message with no active recipients",
+                user_profile.id,
+            )
+            return
+        recipient_str = ", ".join(emails)
+        internal_send_group_direct_message(user_profile.realm, user_profile, body, emails=emails)
+    else:
+        raise AssertionError("Invalid recipient type!")
+
+    logger.info(
+        "Successfully processed email from user %s to %s",
+        user_profile.id,
+        recipient_str,
+    )
+
+
+def process_message(message: EmailMessage, rcpt_to: str | None = None) -> None:
+    to: str | None = None
+
+    try:
+        if rcpt_to is not None:
+            to = rcpt_to
+        else:
+            to = find_emailgateway_recipient(message)
+
+        if is_missed_message_address(to):
+            process_missed_message(to, message)
+        else:
+            process_stream_message(to, message)
+    except ZulipEmailForwardUserError as e:
+        # TODO: notify sender of error, retry if appropriate.
+        logger.info(e.args[0])
+    except ZulipEmailForwardError as e:
+        log_error(message, e.args[0], to)
+
+
+def validate_to_address(address: str, rate_limit: bool = True) -> None:
+    if is_missed_message_address(address):
+        mm_address = get_usable_missed_message_address(address)
+        if mm_address.message.recipient.type == Recipient.STREAM:
+            # ACL's on DMs are harder to apply simply, so we
+            # just check channel messages.
+            access_stream_for_send_message(
+                mm_address.user_profile,
+                get_stream_by_id_in_realm(
+                    mm_address.message.recipient.type_id,
+                    mm_address.user_profile.realm,
+                ),
+                forwarder_user_profile=None,
+            )
+    else:
+        channel_email = decode_stream_email_address(address)[0]
+        if rate_limit:
+            # Only channel email addresses are rate-limited, since
+            # they are likely to be used as the destination for
+            # mails from automated systems.
+            rate_limit_mirror_by_realm(channel_email.realm)
+        check_access_for_channel_email_address(channel_email)
+
+
+# Email mirror rate limiter code:
+class RateLimitedRealmMirror(RateLimitedObject):
+    def __init__(self, realm: Realm) -> None:
+        self.realm = realm
+        super().__init__()
+
+    @override
+    def key(self) -> str:
+        return f"{type(self).__name__}:{self.realm.string_id}"
+
+    @override
+    def rules(self) -> list[tuple[int, int]]:
+        return settings.RATE_LIMITING_MIRROR_REALM_RULES
+
+
+def rate_limit_mirror_by_realm(recipient_realm: Realm) -> None:
+    ratelimited, secs_to_freedom = RateLimitedRealmMirror(recipient_realm).rate_limit()
+
+    if ratelimited:
+        raise RateLimitedError(secs_to_freedom)

zerver/lib/email_mirror_helpers.py

@@ -0,0 +1,122 @@
+import re
+from collections.abc import Callable
+from typing import Any
+
+from django.conf import settings
+from django.utils.text import slugify
+
+from zerver.models import ChannelEmailAddress, Stream, UserProfile
+
+
+def default_option_handler_factory(address_option: str) -> Callable[[dict[str, Any]], None]:
+    def option_setter(options_dict: dict[str, Any]) -> None:
+        options_dict[address_option.replace("-", "_")] = True
+
+    return option_setter
+
+
+optional_address_tokens = {
+    "show-sender": default_option_handler_factory("show-sender"),
+    "include-footer": default_option_handler_factory("include-footer"),
+    "include-quotes": default_option_handler_factory("include-quotes"),
+    "prefer-text": lambda options: options.update(prefer_text=True),
+    "prefer-html": lambda options: options.update(prefer_text=False),
+}
+
+
+class ZulipEmailForwardError(Exception):
+    pass
+
+
+class ZulipEmailForwardUserError(ZulipEmailForwardError):
+    pass
+
+
+def get_email_gateway_message_string_from_address(address: str) -> str:
+    if settings.EMAIL_GATEWAY_PATTERN == "":
+        raise ZulipEmailForwardError("This server is not configured for incoming email.")
+    pattern_parts = [re.escape(part) for part in settings.EMAIL_GATEWAY_PATTERN.split("%s")]
+    if settings.EMAIL_GATEWAY_EXTRA_PATTERN_HACK:
+        # Accept mails delivered to any Zulip server
+        pattern_parts[-1] = settings.EMAIL_GATEWAY_EXTRA_PATTERN_HACK
+    match_email_re = re.compile(r"(.*?)".join(pattern_parts))
+    match = match_email_re.match(address)
+
+    if not match:
+        raise ZulipEmailForwardError("Address not recognized by gateway.")
+    msg_string = match.group(1)
+
+    return msg_string
+
+
+def get_channel_email_token(stream: Stream, *, creator: UserProfile, sender: UserProfile) -> str:
+    channel_email_address, _created = ChannelEmailAddress.objects.get_or_create(
+        realm=stream.realm,
+        channel=stream,
+        creator=creator,
+        sender=sender,
+    )
+    return channel_email_address.email_token
+
+
+def encode_email_address(name: str, email_token: str, show_sender: bool = False) -> str:
+    # Some deployments may not use the email gateway
+    if settings.EMAIL_GATEWAY_PATTERN == "":
+        return ""
+
+    # Given the fact that we have almost no restrictions on stream names and
+    # that what characters are allowed in e-mail addresses is complicated and
+    # dependent on context in the address, we opt for a simple scheme:
+    # 1. Replace all substrings of non-alphanumeric characters with a single hyphen.
+    # 2. Use Django's slugify to convert the resulting name to ascii.
+    # 3. If the resulting name is shorter than the name we got in step 1,
+    # it means some letters can't be reasonably turned to ascii and have to be dropped,
+    # which would mangle the name, so we just skip the name part of the address.
+    name = re.sub(r"\W+", "-", name)
+    slug_name = slugify(name)
+    encoded_name = slug_name if len(slug_name) == len(name) else ""
+
+    # If encoded_name ends up empty, we just skip this part of the address:
+    if encoded_name:
+        encoded_token = f"{encoded_name}.{email_token}"
+    else:
+        encoded_token = email_token
+
+    if show_sender:
+        encoded_token += ".show-sender"
+
+    return settings.EMAIL_GATEWAY_PATTERN % (encoded_token,)
+
+
+def decode_email_address(email: str) -> tuple[str, dict[str, bool]]:
+    # Perform the reverse of encode_email_address. Returns a tuple of
+    # (email_token, options)
+    msg_string = get_email_gateway_message_string_from_address(email)
+
+    # Support both + and . as separators.  For background, the `+` is
+    # more aesthetically pleasing, but because Google groups silently
+    # drops the use of `+` in email addresses, which would completely
+    # break the integration, we now favor `.` as the separator between
+    # tokens in the email addresses we generate.
+    #
+    # We need to keep supporting `+` indefinitely for backwards
+    # compatibility with older versions of Zulip that offered users
+    # email addresses prioritizing using `+` for better aesthetics.
+    msg_string = msg_string.replace(".", "+")
+
+    parts = msg_string.split("+")
+    options: dict[str, bool] = {}
+    for part in parts:
+        if part in optional_address_tokens:
+            optional_address_tokens[part](options)
+
+    remaining_parts = [part for part in parts if part not in optional_address_tokens]
+
+    # There should be one or two parts left:
+    # [stream_name, email_token] or just [email_token]
+    if len(remaining_parts) == 1:
+        token = remaining_parts[0]
+    else:
+        token = remaining_parts[1]
+
+    return token, options

zerver/lib/email_mirror_server.py

@@ -0,0 +1,275 @@
+import asyncio
+import base64
+import email
+import grp
+import logging
+import os
+import pwd
+import signal
+import smtplib
+import socket
+from collections.abc import Awaitable
+from contextlib import suppress
+from ssl import SSLContext, SSLError
+from typing import Any
+from urllib.parse import SplitResult
+
+from aiosmtpd.controller import UnthreadedController
+from aiosmtpd.handlers import Message as MessageHandler
+from aiosmtpd.smtp import SMTP, Envelope, Session, TLSSetupException
+from asgiref.sync import sync_to_async
+from django.conf import settings
+from django.core.mail import EmailMultiAlternatives as DjangoEmailMultiAlternatives
+from typing_extensions import override
+
+from version import ZULIP_VERSION
+from zerver.lib.email_mirror import decode_stream_email_address, validate_to_address
+from zerver.lib.email_mirror_helpers import (
+    ZulipEmailForwardError,
+    get_email_gateway_message_string_from_address,
+)
+from zerver.lib.exceptions import JsonableError, RateLimitedError
+from zerver.lib.logging_util import log_to_file
+from zerver.lib.queue import queue_json_publish_rollback_unsafe
+
+# We add a file handler to this later, once we've dropped privileges
+logger = logging.getLogger("zerver.lib.email_mirror")
+
+
+def send_to_postmaster(msg: email.message.Message) -> None:
+    # RFC5321 says:
+    #   Any system that includes an SMTP server supporting mail relaying or
+    #   delivery MUST support the reserved mailbox "postmaster" as a case-
+    #   insensitive local name.  This postmaster address is not strictly
+    #   necessary if the server always returns 554 on connection opening (as
+    #   described in Section 3.1).  The requirement to accept mail for
+    #   postmaster implies that RCPT commands that specify a mailbox for
+    #   postmaster at any of the domains for which the SMTP server provides
+    #   mail service, as well as the special case of "RCPT TO:<Postmaster>"
+    #   (with no domain specification), MUST be supported.
+    #
+    # We forward such mail to the ZULIP_ADMINISTRATOR.
+    mail = DjangoEmailMultiAlternatives(
+        subject=f"Mail to postmaster: {msg['Subject']}",
+        from_email=settings.NOREPLY_EMAIL_ADDRESS,
+        to=[settings.ZULIP_ADMINISTRATOR],
+    )
+    mail.attach(None, msg, "message/rfc822")
+    try:
+        mail.send()
+    except smtplib.SMTPResponseException as e:
+        logger.exception(
+            "Error sending bounce email to %s with error code %s: %s",
+            mail.to,
+            e.smtp_code,
+            e.smtp_error,
+            stack_info=True,
+        )
+    except smtplib.SMTPException:
+        logger.exception("Error sending bounce email to %s", mail.to, stack_info=True)
+
+
+class ZulipMessageHandler(MessageHandler):
+    def __init__(self) -> None:
+        super().__init__(email.message.Message)
+
+    async def handle_RCPT(
+        self,
+        server: SMTP,
+        session: Session,
+        envelope: Envelope,
+        address: str,
+        rcpt_options: list[str],
+    ) -> str:
+        # Rewrite all postmaster email addresses to just "postmaster"
+        if address.lower() == "postmaster":
+            envelope.rcpt_tos.append("postmaster")
+            return "250 Continue"
+
+        with suppress(ZulipEmailForwardError):
+            if get_email_gateway_message_string_from_address(address).lower() == "postmaster":
+                envelope.rcpt_tos.append("postmaster")
+                return "250 Continue"
+
+        try:
+            # Attempt to do the ACL-check now, just to reject early.
+            # The authoritative check is done in the worker, but we
+            # wish to reject now if we can do so simply.
+            await sync_to_async(validate_to_address)(address)
+
+        except RateLimitedError:
+            recipient_realm = await sync_to_async(
+                lambda a: decode_stream_email_address(a)[0].realm
+            )(address)
+            logger.info(
+                "Rejecting a MAIL FROM: %s to realm: %s via %s - rate limited.",
+                envelope.mail_from,
+                recipient_realm.string_id,
+                str(session.peer),
+            )
+            return "550 4.7.0 Rate-limited due to too many emails on this realm."
+
+        except ZulipEmailForwardError as e:
+            return f"550 5.1.1 Bad destination mailbox address: {e}"
+
+        except JsonableError as e:
+            return f"550 5.7.1 Permission denied: {e}"
+
+        envelope.rcpt_tos.append(address)
+        return "250 Continue"
+
+    @override
+    def handle_message(self, message: email.message.Message) -> None:
+        msg_base64 = base64.b64encode(bytes(message))
+
+        logger.info(
+            "Received email to %s, from %s via %s",
+            message["X-RcptTo"],
+            message["X-MailFrom"],
+            message["X-Peer"],
+        )
+        for address in message["X-RcptTo"].split(", "):
+            if address == "postmaster":
+                send_to_postmaster(message)
+            else:
+                queue_json_publish_rollback_unsafe(
+                    "email_mirror",
+                    {
+                        "rcpt_to": address,
+                        "msg_base64": msg_base64.decode(),
+                    },
+                )
+
+    async def handle_exception(self, error: Exception) -> str:
+        if isinstance(error, TLSSetupException):  # nocoverage
+            if isinstance(error.__cause__, SSLError):
+                reason = error.__cause__.reason
+            else:
+                reason = str(error.__cause__)
+            logger.debug("Dropping invalid TLS connection: %s", reason)
+            return f"421 4.7.6 TLS error: {reason}"
+        else:
+            logger.error("SMTP session exception", exc_info=error)
+            return "500 Server error"
+
+
+def parse_listen_address(listen: str) -> tuple[str, int]:
+    host: str | None
+    port: int | None
+    if listen.isdigit():
+        host, port = None, int(listen)
+    else:
+        r = SplitResult("", listen, "", "", "")
+        try:
+            port = r.port
+        except ValueError as e:
+            # An unbracketed IPv6 address is ambiguous with
+            # address:port, and parses as a host and a non-numeric
+            # port.
+            raise RuntimeError(
+                f"{listen!r} is not a valid port, or address:port; IPv6 addresses "
+                "must be enclosed in square brackets."
+            ) from e
+        if port is None:
+            raise RuntimeError(f"{listen!r} does not have a valid port number.")
+        host = r.hostname
+    if host is None:
+        # With no address specified, bind the IPv6 wildcard address if
+        # IPv6 is available, since (with IPV6_V6ONLY disabled, as
+        # _create_server does) that accepts IPv4 connections as well.
+        host = "::" if socket.has_dualstack_ipv6() else "0.0.0.0"  # noqa: S104
+    return host, port
+
+
+class PermissionDroppingUnthreadedController(UnthreadedController):  # nocoverage
+    @override
+    def __init__(
+        self,
+        user: str | None = None,
+        group: str | None = None,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__(**kwargs)
+
+        # These may remain None in development, when elevated
+        # privileges are not needed because a non-low port is chosen.
+        self.user_id: int | None = None
+        self.group_id: int | None = None
+        if user is not None:
+            self.user_id = pwd.getpwnam(user).pw_uid
+        if group is not None:
+            self.group_id = grp.getgrnam(group).gr_gid
+
+    @override
+    def _create_server(self) -> Awaitable[asyncio.AbstractServer]:
+        # Make the listen socket, then drop privileges before starting
+        # the server
+        family, type_, proto, _, sockaddr = socket.getaddrinfo(
+            self.hostname,
+            self.port,
+            type=socket.SOCK_STREAM,
+            flags=socket.AI_PASSIVE,
+        )[0]
+        server_socket = socket.socket(family, type_, proto)
+        server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        if family == socket.AF_INET6:
+            # Also accept IPv4 connections when binding the IPv6
+            # wildcard address; this has no effect when binding a
+            # specific address.
+            server_socket.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0)
+        server_socket.bind(sockaddr)
+        # Log the address which was actually bound; if the requested
+        # address was a hostname with more than one address, only the
+        # first one that the resolver returned is used.
+        bound_host, bound_port = server_socket.getsockname()[:2]
+        if family == socket.AF_INET6:
+            bound_host = f"[{bound_host}]"
+        logger.info("Listening on %s:%d", bound_host, bound_port)
+        if os.geteuid() == 0:
+            assert self.user_id is not None
+            assert self.group_id is not None
+            # We may have a logfile owned by root, from before we
+            # fixed it to be owned by zulip; chown it if it exists, so
+            # we don't fail below.
+            if os.path.exists(settings.EMAIL_MIRROR_LOG_PATH):
+                os.chown(settings.EMAIL_MIRROR_LOG_PATH, self.user_id, self.group_id)
+
+            logger.info("Dropping privileges to uid %d / gid %d", self.user_id, self.group_id)
+            os.setgid(self.group_id)
+            os.setuid(self.user_id)
+
+        log_to_file(logger, settings.EMAIL_MIRROR_LOG_PATH)
+
+        server = self.loop.create_server(
+            self._factory_invoker,
+            sock=server_socket,
+            ssl=self.ssl_context,
+        )
+
+        return server
+
+
+def run_smtp_server(
+    user: str | None, group: str | None, host: str, port: int, tls_context: SSLContext | None
+) -> None:  # nocoverage
+    logger.info("Starting SMTP server on %s:%d", f"[{host}]" if ":" in host else host, port)
+    server = PermissionDroppingUnthreadedController(
+        user=user,
+        group=group,
+        hostname=host,
+        port=port,
+        handler=ZulipMessageHandler(),
+        tls_context=tls_context,
+        ident=f"Zulip Server {ZULIP_VERSION}",
+    )
+
+    server.loop.add_signal_handler(signal.SIGINT, server.loop.stop)
+
+    server.begin()
+    with suppress(KeyboardInterrupt):
+        # The KeyboardInterrupt will exit the loop, but there's no
+        # reason to throw a stacktrace rather than conduct an ordered
+        # exit.
+        server.loop.run_forever()
+
+    server.end()

zerver/lib/email_validation.py

@@ -0,0 +1,232 @@
+from collections.abc import Callable
+from email.errors import HeaderParseError
+from email.headerregistry import Address
+
+import idna
+from django.core import validators
+from django.core.exceptions import ValidationError
+from django.utils.translation import gettext as _
+
+from zerver.lib.name_restrictions import is_disposable_domain
+
+# TODO: Move DisposableEmailError, etc. into here.
+from zerver.models import Realm, RealmDomain
+from zerver.models.realms import (
+    DisposableEmailError,
+    DomainNotAllowedForRealmError,
+    EmailContainsPlusError,
+)
+from zerver.models.users import get_users_by_delivery_email, is_cross_realm_bot_email
+
+
+def validate_is_not_disposable(email: str) -> None:
+    try:
+        domain = Address(addr_spec=email).domain
+    except (HeaderParseError, ValueError):
+        raise DisposableEmailError
+
+    if is_disposable_domain(domain):
+        raise DisposableEmailError
+
+
+def get_realm_email_validator(realm: Realm) -> Callable[[str], None]:
+    if not realm.emails_restricted_to_domains:
+        # Should we also do '+' check for non-restricted realms?
+        if realm.disallow_disposable_email_addresses:
+            return validate_is_not_disposable
+
+        # allow any email through
+        return lambda email: None
+
+    """
+    RESTRICTIVE REALMS:
+
+    Some realms only allow emails within a set
+    of domains that are configured in RealmDomain.
+
+    We get the set of domains up front so that
+    folks can validate multiple emails without
+    multiple round trips to the database.
+    """
+
+    query = RealmDomain.objects.filter(realm=realm)
+    rows = list(query.values("allow_subdomains", "domain"))
+
+    allowed_domains = {r["domain"] for r in rows}
+
+    allowed_subdomains = {r["domain"] for r in rows if r["allow_subdomains"]}
+
+    def validate(email: str) -> None:
+        """
+        We don't have to do a "disposable" check for restricted
+        domains, since the realm is already giving us
+        a small whitelist.
+        """
+
+        address = Address(addr_spec=email)
+        if "+" in address.username:
+            raise EmailContainsPlusError
+
+        domain = address.domain.lower()
+
+        if domain in allowed_domains:
+            return
+
+        while len(domain) > 0:
+            _subdomain, _sep, domain = domain.partition(".")
+            if domain in allowed_subdomains:
+                return
+
+        raise DomainNotAllowedForRealmError
+
+    return validate
+
+
+# Is a user with the given email address allowed to be in the given realm?
+# (This function does not check whether the user has been invited to the realm.
+# So for invite-only realms, this is the test for whether a user can be invited,
+# not whether the user can sign up currently.)
+def email_allowed_for_realm(email: str, realm: Realm) -> None:
+    """
+    Avoid calling this in a loop!
+    Instead, call get_realm_email_validator()
+    outside of the loop.
+    """
+    get_realm_email_validator(realm)(email)
+
+
+def validate_email_is_valid(
+    email: str,
+    validate_email_allowed_in_realm: Callable[[str], None],
+) -> str | None:
+    try:
+        validators.validate_email(email)
+    except ValidationError:
+        return _("Invalid address.")
+
+    # Django's EmailValidator is looser than IDNA 2008 about what it
+    # accepts in a domain, and also accepts RFC 5321 address-literal
+    # domains (user@[192.168.0.1]) which we don't want to deliver to.
+    # Require the domain to IDNA-encode cleanly, which is what SMTP
+    # delivery will need to do.
+    try:
+        domain = Address(addr_spec=email).domain
+        idna.encode(domain, uts46=True)
+    except (HeaderParseError, ValueError, idna.IDNAError):
+        return _("Invalid address.")
+
+    try:
+        validate_email_allowed_in_realm(email)
+    except DomainNotAllowedForRealmError:
+        return _("Outside your domain.")
+    except DisposableEmailError:
+        return _("Please use your real email address.")
+    except EmailContainsPlusError:
+        return _("Email addresses containing + are not allowed.")
+
+    return None
+
+
+def email_reserved_for_system_bots_error(email: str) -> str:
+    return f"{email} is reserved for system bots"
+
+
+def get_existing_user_errors(
+    target_realm: Realm,
+    emails: set[str],
+    *,
+    allow_inactive_mirror_dummies: bool,
+    verbose: bool = False,
+) -> dict[str, tuple[str, bool]]:
+    """
+    We use this function even for a list of one emails.
+
+    It checks "new" emails to make sure that they don't
+    already exist.  There's a bit of fiddly logic related
+    to cross-realm bots and mirror dummies too.
+    """
+
+    errors: dict[str, tuple[str, bool]] = {}
+
+    users = get_users_by_delivery_email(emails, target_realm).only(
+        "delivery_email",
+        "is_active",
+        "is_mirror_dummy",
+    )
+
+    """
+    A note on casing: We will preserve the casing used by
+    the user for email in most of this code.  The only
+    exception is when we do existence checks against
+    the `user_dict` dictionary.  (We don't allow two
+    users in the same realm to have the same effective
+    delivery email.)
+    """
+    user_dict = {user.delivery_email.lower(): user for user in users}
+
+    def process_email(email: str) -> None:
+        if is_cross_realm_bot_email(email):
+            if verbose:
+                msg = email_reserved_for_system_bots_error(email)
+            else:
+                msg = _("Reserved for system bots.")
+            deactivated = False
+            errors[email] = (msg, deactivated)
+            return
+
+        existing_user_profile = user_dict.get(email.lower())
+
+        if existing_user_profile is None:
+            # HAPPY PATH!  Most people invite users that don't exist yet.
+            return
+
+        if existing_user_profile.is_mirror_dummy and allow_inactive_mirror_dummies:
+            if existing_user_profile.is_active:
+                raise AssertionError("Mirror dummy user is already active!")
+            return
+
+        """
+        Email has already been taken by a "normal" user.
+        """
+        deactivated = not existing_user_profile.is_active
+
+        if existing_user_profile.is_active:
+            if verbose:
+                msg = _("{email} already has an account").format(email=email)
+            else:
+                msg = _("Already has an account.")
+        else:
+            msg = _("Account has been deactivated.")
+
+        errors[email] = (msg, deactivated)
+
+    for email in emails:
+        process_email(email)
+
+    return errors
+
+
+def validate_email_not_already_in_realm(
+    target_realm: Realm, email: str, *, allow_inactive_mirror_dummies: bool, verbose: bool = True
+) -> None:
+    """
+    NOTE:
+        Only use this to validate that a single email
+        is not already used in the realm.
+
+        We should start using bulk_check_new_emails()
+        for any endpoint that takes multiple emails,
+        such as the "invite" interface.
+    """
+    error_dict = get_existing_user_errors(
+        target_realm,
+        {email},
+        allow_inactive_mirror_dummies=allow_inactive_mirror_dummies,
+        verbose=verbose,
+    )
+
+    # Loop through errors, the only key should be our email.
+    for key, error_info in error_dict.items():
+        assert key == email
+        msg, _deactivated = error_info
+        raise ValidationError(msg)

zerver/lib/emoji.py

@@ -0,0 +1,163 @@
+import hashlib
+import os
+import re
+from dataclasses import dataclass
+
+import orjson
+from django.conf import settings
+from django.contrib.staticfiles.storage import staticfiles_storage
+from django.utils.translation import gettext as _
+
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.mime_types import guess_extension
+from zerver.lib.storage import static_path
+from zerver.models import Reaction, Realm, RealmEmoji, UserProfile
+from zerver.models.realm_emoji import (
+    get_all_custom_emoji_for_realm,
+    get_name_keyed_dict_for_active_realm_emoji,
+)
+
+emoji_codes_path = static_path("generated/emoji/emoji_codes.json")
+if not os.path.exists(emoji_codes_path):  # nocoverage
+    # During the collectstatic step of build-release-tarball,
+    # prod-static/serve/generated/emoji won't exist yet.
+    emoji_codes_path = os.path.join(
+        os.path.dirname(__file__),
+        "../../static/generated/emoji/emoji_codes.json",
+    )
+
+with open(emoji_codes_path, "rb") as fp:
+    emoji_codes = orjson.loads(fp.read())
+
+name_to_codepoint = emoji_codes["name_to_codepoint"]
+codepoint_to_name = emoji_codes["codepoint_to_name"]
+EMOTICON_CONVERSIONS = emoji_codes["emoticon_conversions"]
+
+possible_emoticons = EMOTICON_CONVERSIONS.keys()
+possible_emoticon_regexes = (re.escape(emoticon) for emoticon in possible_emoticons)
+terminal_symbols = r",.;?!()\[\] \"'\n\t"  # from composebox_typeahead.ts
+EMOTICON_RE = (
+    rf"(?<![^{terminal_symbols}])(?P<emoticon>("
+    + r")|(".join(possible_emoticon_regexes)
+    + rf"))(?![^{terminal_symbols}])"
+)
+
+
+def data_url() -> str:
+    # This bakes a hash into the URL, which looks something like
+    # static/webpack-bundles/files/64.0cdafdf0b6596657a9be.png
+    # This is how Django deals with serving static files in a cacheable way.
+    # See PR #22275 for details.
+    return staticfiles_storage.url("generated/emoji/emoji_api.json")
+
+
+# Translates emoticons to their colon syntax, e.g. `:smiley:`.
+def translate_emoticons(text: str) -> str:
+    translated = text
+
+    for emoticon in EMOTICON_CONVERSIONS:
+        translated = re.sub(re.escape(emoticon), EMOTICON_CONVERSIONS[emoticon], translated)
+
+    return translated
+
+
+@dataclass
+class EmojiData:
+    emoji_code: str
+    reaction_type: str
+
+
+def get_emoji_data(realm_id: int, emoji_name: str) -> EmojiData:
+    # Even if emoji_name is either in name_to_codepoint or named "zulip",
+    # we still need to call get_realm_active_emoji.
+    realm_emoji_dict = get_name_keyed_dict_for_active_realm_emoji(realm_id)
+    realm_emoji = realm_emoji_dict.get(emoji_name)
+
+    if realm_emoji is not None:
+        emoji_code = str(realm_emoji["id"])
+        return EmojiData(emoji_code=emoji_code, reaction_type=Reaction.REALM_EMOJI)
+
+    if emoji_name == "zulip":
+        return EmojiData(emoji_code=emoji_name, reaction_type=Reaction.ZULIP_EXTRA_EMOJI)
+
+    if emoji_name in name_to_codepoint:
+        emoji_code = name_to_codepoint[emoji_name]
+        return EmojiData(emoji_code=emoji_code, reaction_type=Reaction.UNICODE_EMOJI)
+
+    raise JsonableError(_("Emoji '{emoji_name}' does not exist").format(emoji_name=emoji_name))
+
+
+def check_emoji_request(realm: Realm, emoji_name: str, emoji_code: str, emoji_type: str) -> None:
+    # For a given realm and emoji type, checks whether an emoji
+    # code is valid for new reactions, or not.
+    if emoji_type == "realm_emoji":
+        # We cache emoji, so this generally avoids a round trip,
+        # but it does require deserializing a bigger data structure
+        # than we need.
+        realm_emojis = get_all_custom_emoji_for_realm(realm.id)
+        realm_emoji = realm_emojis.get(emoji_code)
+        if realm_emoji is None:
+            raise JsonableError(_("Invalid custom emoji."))
+        if realm_emoji["name"] != emoji_name:
+            raise JsonableError(_("Invalid custom emoji name."))
+        if realm_emoji["deactivated"]:
+            raise JsonableError(_("This custom emoji has been deactivated."))
+    elif emoji_type == "zulip_extra_emoji":
+        if emoji_code != "zulip":
+            raise JsonableError(_("Invalid emoji code."))
+        if emoji_name != emoji_code:
+            raise JsonableError(_("Invalid emoji name."))
+    elif emoji_type == "unicode_emoji":
+        if emoji_code not in codepoint_to_name:
+            raise JsonableError(_("Invalid emoji code."))
+        if name_to_codepoint.get(emoji_name) != emoji_code:
+            raise JsonableError(_("Invalid emoji name."))
+    else:
+        # The above are the only valid emoji types
+        raise JsonableError(_("Invalid emoji type."))
+
+
+def check_remove_custom_emoji(user_profile: UserProfile, emoji_name: str) -> None:
+    # normal users can remove emoji they themselves added
+    if user_profile.is_realm_admin:
+        return
+
+    emoji = RealmEmoji.objects.filter(
+        realm=user_profile.realm, name=emoji_name, deactivated=False
+    ).first()
+    current_user_is_author = emoji is not None and emoji.author.id == user_profile.id
+    if not current_user_is_author:
+        raise JsonableError(_("Must be an organization administrator or emoji author"))
+
+
+def check_valid_emoji_name(emoji_name: str) -> None:
+    if emoji_name:
+        if re.match(r"^[0-9a-z\-_]+(?<![\-_])$", emoji_name):
+            return
+        if re.match(r"^[0-9a-z\-_]+$", emoji_name):
+            raise JsonableError(_("Emoji names must end with either a letter or digit."))
+        raise JsonableError(
+            _(
+                "Emoji names must contain only lowercase English letters, digits, spaces, dashes, and underscores.",
+            )
+        )
+    raise JsonableError(_("Emoji name is missing"))
+
+
+def get_emoji_file_name(content_type: str, emoji_id: int) -> str:
+    image_ext = guess_extension(content_type, strict=False)
+    # The only callsite of this pre-limits the content_type to a
+    # reasonable set that we know have extensions.
+    assert image_ext is not None
+
+    # We salt this with a server-side secret so that it is not
+    # enumerable by clients, and will not collide on the server.  New
+    # realm imports may pass a synthetic emoji_id, which is fine as
+    # long as it starts at 1, and as such later emoji cannot collide
+    # unless there is a legit hash collision.
+    #
+    # We truncate the hash at 8 characters, as this is enough entropy
+    # to make collisions vanishingly unlikely.  In the event of a
+    # collusion, the id will advance and a manual retry will succeed.
+    hash_key = settings.AVATAR_SALT.encode() + b":" + str(emoji_id).encode()
+    return "".join((hashlib.sha256(hash_key).hexdigest()[0:8], image_ext))

zerver/lib/emoji_utils.py

@@ -0,0 +1,25 @@
+# This file doesn't import from django so that we can use it in `build_emoji`
+
+
+def unqualify_emoji(emoji: str) -> str:
+    # Starting from version 4.0.0, `emoji_datasource` package has started to
+    # add an emoji presentation variation selector for certain emojis which
+    # have defined variation sequences. The emoji presentation selector
+    # "qualifies" an emoji, and an "unqualified" version of an emoji does
+    # not have an emoji presentation selector.
+    #
+    # Since in informal environments(like texting and chat), it is more
+    # appropriate for an emoji to have a colorful display so until emoji
+    # characters have a text presentation selector, it should have a
+    # colorful display. Hence we can continue using emoji characters
+    # without appending emoji presentation selector.
+    # (http://unicode.org/reports/tr51/index.html#Presentation_Style)
+    return emoji.replace("\ufe0f", "")
+
+
+def emoji_to_hex_codepoint(emoji: str) -> str:
+    return "-".join(f"{ord(c):04x}" for c in emoji)
+
+
+def hex_codepoint_to_emoji(hex: str) -> str:
+    return "".join(chr(int(h, 16)) for h in hex.split("-"))

zerver/lib/event_schema.py

@@ -0,0 +1,766 @@
+# This module is a collection of testing helpers for validating the
+# schema of "events" sent by Zulip's server-to-client push system.
+#
+# By policy, every event generated by Zulip's API should be validated
+# by a test in test_events.py with a schema checker here.
+#
+# See https://zulip.readthedocs.io/en/latest/subsystems/events-system.html
+import inspect
+from collections.abc import Callable
+from enum import Enum
+from pprint import PrettyPrinter
+from typing import cast
+
+from pydantic import BaseModel
+
+from zerver.lib.event_types import (
+    AlertWordsEvent,
+    AllowMessageEditingData,
+    AttachmentAddEvent,
+    AttachmentRemoveEvent,
+    AttachmentUpdateEvent,
+    AuthenticationData,
+    BaseEvent,
+    BotServicesEmbedded,
+    BotServicesOutgoing,
+    ChannelFolderAddEvent,
+    ChannelFolderReorderEvent,
+    ChannelFolderUpdateEvent,
+    CustomProfileFieldsEvent,
+    DefaultStreamGroupsEvent,
+    DefaultStreamsEvent,
+    DeleteMessageEvent,
+    DeviceAddEvent,
+    DeviceRemoveEvent,
+    DeviceUpdateEvent,
+    DirectMessageEvent,
+    DraftsAddEvent,
+    DraftsRemoveEvent,
+    DraftsUpdateEvent,
+    GroupSettingUpdateData,
+    HasWebexTokenEvent,
+    HasZoomTokenEvent,
+    HeartbeatEvent,
+    IconData,
+    InvitesChangedEvent,
+    LegacyPresenceEvent,
+    LogoData,
+    MessageContentEditLimitSecondsData,
+    MessageEvent,
+    ModernPresenceEvent,
+    MutedTopicsEvent,
+    MutedUsersEvent,
+    NavigationViewAddEvent,
+    NavigationViewRemoveEvent,
+    NavigationViewUpdateEvent,
+    NightLogoData,
+    OnboardingStepsEvent,
+    PersonAvatarFields,
+    PersonBotOwnerId,
+    PersonCustomProfileField,
+    PersonDateJoined,
+    PersonDeliveryEmail,
+    PersonEmail,
+    PersonFullName,
+    PersonIsActive,
+    PersonIsImportedStub,
+    PersonRole,
+    PersonTimezone,
+    PlanTypeData,
+    ReactionAddEvent,
+    ReactionRemoveEvent,
+    RealmBotAddEvent,
+    RealmBotDeleteEvent,
+    RealmBotUpdateEvent,
+    RealmDeactivatedEvent,
+    RealmDescriptionData,
+    RealmDomainsAddEvent,
+    RealmDomainsChangeEvent,
+    RealmDomainsRemoveEvent,
+    RealmEmojiAddEvent,
+    RealmEmojiUpdateEvent,
+    RealmEmojiUpdateOneEvent,
+    RealmExportConsentEvent,
+    RealmExportEvent,
+    RealmLinkifiersEvent,
+    RealmPlaygroundsEvent,
+    RealmTopicsPolicyData,
+    RealmUpdateDictEvent,
+    RealmUpdateEvent,
+    RealmUserAddEvent,
+    RealmUserRemoveEvent,
+    RealmUserSettingsDefaultsUpdateEvent,
+    RealmUserUpdateEvent,
+    RemindersAddEvent,
+    RemindersRemoveEvent,
+    RestartEvent,
+    SavedSnippetsAddEvent,
+    SavedSnippetsRemoveEvent,
+    SavedSnippetsUpdateEvent,
+    ScheduledMessagesAddEvent,
+    ScheduledMessagesRemoveEvent,
+    ScheduledMessagesUpdateEvent,
+    StreamCreateEvent,
+    StreamDeleteEvent,
+    StreamUpdateEvent,
+    SubmessageEvent,
+    SubscriptionAddEvent,
+    SubscriptionPeerAddEvent,
+    SubscriptionPeerRemoveEvent,
+    SubscriptionRemoveEvent,
+    SubscriptionUpdateEvent,
+    TypingEditMessageStartEvent,
+    TypingEditMessageStopEvent,
+    TypingStartEvent,
+    TypingStopEvent,
+    UpdateMessageEvent,
+    UpdateMessageFlagsAddEvent,
+    UpdateMessageFlagsRemoveEvent,
+    UserGroupAddEvent,
+    UserGroupAddMembersEvent,
+    UserGroupAddSubgroupsEvent,
+    UserGroupRemoveEvent,
+    UserGroupRemoveMembersEvent,
+    UserGroupRemoveSubgroupsEvent,
+    UserGroupUpdateEvent,
+    UserSettingsUpdateEvent,
+    UserStatusEvent,
+    UserTopicEvent,
+    WebReloadClientEvent,
+)
+from zerver.lib.topic import ORIG_TOPIC, TOPIC_NAME
+from zerver.lib.types import UserGroupMembersDict
+from zerver.models import Realm, RealmUserDefault, Stream, UserProfile
+from zerver.models.streams import StreamTopicsPolicyEnum
+
+
+def validate_with_model(data: dict[str, object], model: type[BaseModel]) -> None:
+    allowed_fields = set(model.model_fields.keys())
+    if not set(data.keys()).issubset(allowed_fields):  # nocoverage
+        raise ValueError(f"Extra fields not allowed: {set(data.keys()) - allowed_fields}")
+
+    model.model_validate(data, strict=True)
+
+
+def make_checker(base_model: type[BaseEvent]) -> Callable[[str, dict[str, object]], None]:
+    def f(label: str, event: dict[str, object]) -> None:
+        assert isinstance(event["id"], int)
+        event = dict(event)
+        del event["id"]
+        try:
+            validate_with_model(event, base_model)
+        except Exception:  # nocoverage
+            print(f"""
+FAILURE:
+
+The event below fails the check to make sure it has the
+correct "shape" of data:
+
+    {label}
+
+Often this is a symptom that the following type definition
+is either broken or needs to be updated due to other
+changes that you have made:
+
+    {base_model}
+
+A traceback should follow to help you debug this problem.
+
+Here is the event:
+""")
+
+            PrettyPrinter(indent=4).pprint(event)
+            raise
+
+    return f
+
+
+check_alert_words = make_checker(AlertWordsEvent)
+check_attachment_add = make_checker(AttachmentAddEvent)
+check_attachment_remove = make_checker(AttachmentRemoveEvent)
+check_attachment_update = make_checker(AttachmentUpdateEvent)
+check_channel_folder_add = make_checker(ChannelFolderAddEvent)
+check_channel_folder_reorder = make_checker(ChannelFolderReorderEvent)
+check_custom_profile_fields = make_checker(CustomProfileFieldsEvent)
+check_default_stream_groups = make_checker(DefaultStreamGroupsEvent)
+check_default_streams = make_checker(DefaultStreamsEvent)
+check_device_add = make_checker(DeviceAddEvent)
+check_device_remove = make_checker(DeviceRemoveEvent)
+check_device_update = make_checker(DeviceUpdateEvent)
+check_direct_message = make_checker(DirectMessageEvent)
+check_draft_add = make_checker(DraftsAddEvent)
+check_draft_remove = make_checker(DraftsRemoveEvent)
+check_draft_update = make_checker(DraftsUpdateEvent)
+check_heartbeat = make_checker(HeartbeatEvent)
+check_invites_changed = make_checker(InvitesChangedEvent)
+check_message = make_checker(MessageEvent)
+check_muted_users = make_checker(MutedUsersEvent)
+check_navigation_view_add = make_checker(NavigationViewAddEvent)
+check_navigation_view_remove = make_checker(NavigationViewRemoveEvent)
+check_navigation_view_update = make_checker(NavigationViewUpdateEvent)
+check_onboarding_steps = make_checker(OnboardingStepsEvent)
+check_reaction_add = make_checker(ReactionAddEvent)
+check_reaction_remove = make_checker(ReactionRemoveEvent)
+check_realm_bot_delete = make_checker(RealmBotDeleteEvent)
+check_realm_deactivated = make_checker(RealmDeactivatedEvent)
+check_realm_domains_add = make_checker(RealmDomainsAddEvent)
+check_realm_domains_change = make_checker(RealmDomainsChangeEvent)
+check_realm_domains_remove = make_checker(RealmDomainsRemoveEvent)
+check_realm_emoji_add = make_checker(RealmEmojiAddEvent)
+check_realm_emoji_update_one = make_checker(RealmEmojiUpdateOneEvent)
+check_realm_export_consent = make_checker(RealmExportConsentEvent)
+check_realm_linkifiers = make_checker(RealmLinkifiersEvent)
+check_realm_playgrounds = make_checker(RealmPlaygroundsEvent)
+check_realm_user_add = make_checker(RealmUserAddEvent)
+check_realm_user_remove = make_checker(RealmUserRemoveEvent)
+check_reminder_add = make_checker(RemindersAddEvent)
+check_reminder_remove = make_checker(RemindersRemoveEvent)
+check_restart = make_checker(RestartEvent)
+check_saved_snippets_add = make_checker(SavedSnippetsAddEvent)
+check_saved_snippets_remove = make_checker(SavedSnippetsRemoveEvent)
+check_saved_snippets_update = make_checker(SavedSnippetsUpdateEvent)
+check_scheduled_message_add = make_checker(ScheduledMessagesAddEvent)
+check_scheduled_message_remove = make_checker(ScheduledMessagesRemoveEvent)
+check_scheduled_message_update = make_checker(ScheduledMessagesUpdateEvent)
+check_stream_create = make_checker(StreamCreateEvent)
+check_stream_delete = make_checker(StreamDeleteEvent)
+check_submessage = make_checker(SubmessageEvent)
+check_subscription_add = make_checker(SubscriptionAddEvent)
+check_subscription_peer_add = make_checker(SubscriptionPeerAddEvent)
+check_subscription_peer_remove = make_checker(SubscriptionPeerRemoveEvent)
+check_subscription_remove = make_checker(SubscriptionRemoveEvent)
+check_typing_start = make_checker(TypingStartEvent)
+check_typing_stop = make_checker(TypingStopEvent)
+check_typing_edit_message_start = make_checker(TypingEditMessageStartEvent)
+check_typing_edit_message_stop = make_checker(TypingEditMessageStopEvent)
+check_update_message_flags_add = make_checker(UpdateMessageFlagsAddEvent)
+check_update_message_flags_remove = make_checker(UpdateMessageFlagsRemoveEvent)
+check_user_group_add = make_checker(UserGroupAddEvent)
+check_user_group_add_members = make_checker(UserGroupAddMembersEvent)
+check_user_group_add_subgroups = make_checker(UserGroupAddSubgroupsEvent)
+check_user_group_remove = make_checker(UserGroupRemoveEvent)
+check_user_group_remove_members = make_checker(UserGroupRemoveMembersEvent)
+check_user_group_remove_subgroups = make_checker(UserGroupRemoveSubgroupsEvent)
+check_user_topic = make_checker(UserTopicEvent)
+check_web_reload_client_event = make_checker(WebReloadClientEvent)
+
+
+# Now for the slightly more tricky bits.  All the following functions
+# get wrapped with more stringent checkers.  Some of the wrappers are
+# reasonably sane functions that just check the data, not the shape of
+# the data.
+#
+# But there are some ugly wrappers that work around the fact that
+# a) our events are not very consistent and/or b) our types aren't
+# robust.
+#
+# TODO: work through the bottom of this file to try to find ways to
+#       simplify our types or make them more robust
+
+_check_channel_folder_update = make_checker(ChannelFolderUpdateEvent)
+_check_delete_message = make_checker(DeleteMessageEvent)
+_check_has_zoom_token = make_checker(HasZoomTokenEvent)
+_check_has_webex_token = make_checker(HasWebexTokenEvent)
+_check_legacy_presence = make_checker(LegacyPresenceEvent)
+_check_modern_presence = make_checker(ModernPresenceEvent)
+_check_muted_topics = make_checker(MutedTopicsEvent)
+_check_realm_bot_add = make_checker(RealmBotAddEvent)
+_check_realm_bot_update = make_checker(RealmBotUpdateEvent)
+_check_realm_default_update = make_checker(RealmUserSettingsDefaultsUpdateEvent)
+_check_realm_emoji_update = make_checker(RealmEmojiUpdateEvent)
+_check_realm_export = make_checker(RealmExportEvent)
+_check_realm_update = make_checker(RealmUpdateEvent)
+_check_realm_update_dict = make_checker(RealmUpdateDictEvent)
+_check_realm_user_update = make_checker(RealmUserUpdateEvent)
+_check_stream_update = make_checker(StreamUpdateEvent)
+_check_subscription_update = make_checker(SubscriptionUpdateEvent)
+_check_update_message = make_checker(UpdateMessageEvent)
+_check_user_group_update = make_checker(UserGroupUpdateEvent)
+_check_user_settings_update = make_checker(UserSettingsUpdateEvent)
+_check_user_status = make_checker(UserStatusEvent)
+
+
+PERSON_TYPES: dict[str, type[BaseModel]] = dict(
+    avatar_fields=PersonAvatarFields,
+    bot_owner_id=PersonBotOwnerId,
+    custom_profile_field=PersonCustomProfileField,
+    date_joined=PersonDateJoined,
+    delivery_email=PersonDeliveryEmail,
+    email=PersonEmail,
+    full_name=PersonFullName,
+    role=PersonRole,
+    timezone=PersonTimezone,
+    is_active=PersonIsActive,
+    is_imported_stub=PersonIsImportedStub,
+)
+
+
+def check_channel_folder_update(var_name: str, event: dict[str, object], fields: set[str]) -> None:
+    _check_channel_folder_update(var_name, event)
+
+    assert isinstance(event["data"], dict)
+    assert set(event["data"].keys()) == fields
+
+
+def check_delete_message(
+    var_name: str,
+    event: dict[str, object],
+    message_type: str,
+    num_message_ids: int,
+    is_legacy: bool,
+) -> None:
+    _check_delete_message(var_name, event)
+
+    keys = {"id", "type", "message_type"}
+
+    assert event["message_type"] == message_type
+
+    if message_type == "stream":
+        keys |= {"stream_id", "topic"}
+    elif message_type == "private":
+        pass
+    else:
+        raise AssertionError("unexpected message_type")
+
+    if is_legacy:
+        assert num_message_ids == 1
+        keys.add("message_id")
+    else:
+        assert isinstance(event["message_ids"], list)
+        assert num_message_ids == len(event["message_ids"])
+        keys.add("message_ids")
+
+    assert set(event.keys()) == keys
+
+
+def check_has_zoom_token(
+    var_name: str,
+    event: dict[str, object],
+    value: bool,
+) -> None:
+    _check_has_zoom_token(var_name, event)
+    assert event["value"] == value
+
+
+def check_has_webex_token(
+    var_name: str,
+    event: dict[str, object],
+    value: bool,
+) -> None:
+    _check_has_webex_token(var_name, event)
+    assert event["value"] == value
+
+
+def check_muted_topics(
+    var_name: str,
+    event: dict[str, object],
+) -> None:
+    _check_muted_topics(var_name, event)
+    muted_topics = event["muted_topics"]
+    assert isinstance(muted_topics, list)
+    for muted_topic in muted_topics:
+        muted_topic_tuple = tuple(muted_topic)
+        assert list(map(type, muted_topic_tuple)) == [str, str, int]
+
+
+def check_legacy_presence(
+    var_name: str,
+    event: dict[str, object],
+    has_email: bool,
+    presence_key: str,
+    status: str,
+) -> None:
+    _check_legacy_presence(var_name, event)
+
+    assert ("email" in event) == has_email
+
+    assert isinstance(event["presence"], dict)
+
+    # Our tests only have one presence value.
+    [(event_presence_key, event_presence_value)] = event["presence"].items()
+    assert event_presence_key == presence_key
+    assert event_presence_value["status"] == status
+
+
+def check_modern_presence(var_name: str, event: dict[str, object], user_id: int) -> None:
+    _check_modern_presence(var_name, event)
+
+    assert isinstance(event["presences"], dict)
+
+    [(event_presences_key, _event_presences_value)] = event["presences"].items()
+    assert event_presences_key == str(user_id)
+
+
+def check_realm_bot_add(
+    var_name: str,
+    event: dict[str, object],
+    bot_type: int,
+) -> None:
+    _check_realm_bot_add(var_name, event)
+
+    assert isinstance(event["bot"], dict)
+
+    services = event["bot"]["services"]
+
+    if bot_type == UserProfile.DEFAULT_BOT:
+        assert services == []
+    elif bot_type == UserProfile.OUTGOING_WEBHOOK_BOT:
+        assert len(services) == 1
+        validate_with_model(services[0], BotServicesOutgoing)
+    elif bot_type == UserProfile.EMBEDDED_BOT:
+        assert len(services) == 1
+        validate_with_model(services[0], BotServicesEmbedded)
+    else:
+        raise AssertionError(f"Unknown bot_type: {bot_type}")
+
+
+def check_realm_bot_update(
+    # Check schema plus the field.
+    var_name: str,
+    event: dict[str, object],
+    field: str,
+) -> None:
+    # Check the overall schema first.
+    _check_realm_bot_update(var_name, event)
+
+    assert isinstance(event["bot"], dict)
+    assert {"user_id", field} == set(event["bot"].keys())
+
+
+def check_realm_emoji_update(var_name: str, event: dict[str, object]) -> None:
+    """
+    The way we send realm emojis is kinda clumsy--we
+    send a dict mapping the emoji id to a sub_dict with
+    the fields (including the id).  Ideally we can streamline
+    this and just send a list of dicts.  The clients can make
+    a Map as needed.
+    """
+    _check_realm_emoji_update(var_name, event)
+
+    assert isinstance(event["realm_emoji"], dict)
+    for k, v in event["realm_emoji"].items():
+        assert v["id"] == k
+
+
+def check_realm_export(
+    var_name: str,
+    event: dict[str, object],
+    has_export_url: bool,
+    has_deleted_timestamp: bool,
+    has_failed_timestamp: bool,
+) -> None:
+    # Check the overall event first, knowing it has some
+    # optional types.
+    _check_realm_export(var_name, event)
+
+    # It's possible to have multiple data exports, but the events tests do not
+    # exercise that case, so we do strict validation for a single export here.
+    assert isinstance(event["exports"], list)
+    assert len(event["exports"]) == 1
+    export = event["exports"][0]
+
+    # Now verify which fields have non-None values.
+    assert has_export_url == (export["export_url"] is not None)
+    assert has_deleted_timestamp == (export["deleted_timestamp"] is not None)
+    assert has_failed_timestamp == (export["failed_timestamp"] is not None)
+
+
+def check_realm_update(
+    var_name: str,
+    event: dict[str, object],
+    prop: str,
+) -> None:
+    """
+    Realm updates have these two fields:
+
+        property
+        value
+
+    We check not only the basic schema, but also that
+    the value people actually matches the type from
+    Realm.property_types that we have configured
+    for the property.
+
+    For certain properties, there are extra fields.
+    For example, when property is "description", there's also
+    "rendered_description".
+    """
+    _check_realm_update(var_name, event)
+
+    assert prop == event["property"]
+    value = event["value"]
+
+    if prop in [
+        "moderation_request_channel_id",
+        "new_stream_announcements_stream_id",
+        "signup_announcements_stream_id",
+        "zulip_update_announcements_stream_id",
+        "org_type",
+    ]:
+        assert isinstance(value, int)
+        return
+
+    property_type = Realm.property_types[prop]
+    if inspect.isclass(property_type) and issubclass(property_type, Enum):
+        assert isinstance(value, str)
+        property_type[value]
+    else:
+        assert isinstance(value, property_type)
+
+
+def check_realm_default_update(
+    var_name: str,
+    event: dict[str, object],
+    prop: str,
+) -> None:
+    _check_realm_default_update(var_name, event)
+
+    assert prop == event["property"]
+    assert prop != "default_language"
+    assert prop in RealmUserDefault.property_types
+
+    prop_type = RealmUserDefault.property_types[prop]
+    value = event["value"]
+    if inspect.isclass(prop_type) and issubclass(prop_type, Enum):
+        assert isinstance(value, str)
+        prop_type[value]
+    else:
+        assert isinstance(value, prop_type)
+
+
+def check_realm_update_dict(
+    # handle union types
+    var_name: str,
+    event: dict[str, object],
+) -> None:
+    _check_realm_update_dict(var_name, event)
+
+    if event["property"] == "default":
+        assert isinstance(event["data"], dict)
+
+        if "allow_message_editing" in event["data"]:
+            sub_type: type[BaseModel] = AllowMessageEditingData
+        elif "message_content_edit_limit_seconds" in event["data"]:
+            sub_type = MessageContentEditLimitSecondsData
+        elif "authentication_methods" in event["data"]:
+            sub_type = AuthenticationData
+        elif any(
+            setting_name in event["data"] for setting_name in Realm.REALM_PERMISSION_GROUP_SETTINGS
+        ):
+            sub_type = GroupSettingUpdateData
+        elif "plan_type" in event["data"]:
+            sub_type = PlanTypeData
+        elif "topics_policy" in event["data"]:
+            sub_type = RealmTopicsPolicyData
+        elif "description" in event["data"]:
+            sub_type = RealmDescriptionData
+        else:
+            raise AssertionError("unhandled fields in data")
+
+    elif event["property"] == "icon":
+        sub_type = IconData
+    elif event["property"] == "logo":
+        sub_type = LogoData
+    elif event["property"] == "night_logo":
+        sub_type = NightLogoData
+    else:
+        raise AssertionError("unhandled property: {event['property']}")
+
+    validate_with_model(cast(dict[str, object], event["data"]), sub_type)
+
+
+def check_realm_user_update(
+    # person_flavor tells us which extra fields we need
+    var_name: str,
+    event: dict[str, object],
+    person_flavor: str,
+) -> None:
+    _check_realm_user_update(var_name, event)
+
+    sub_type = PERSON_TYPES[person_flavor]
+    validate_with_model(cast(dict[str, object], event["person"]), sub_type)
+
+
+def check_stream_update(
+    var_name: str,
+    event: dict[str, object],
+) -> None:
+    _check_stream_update(var_name, event)
+    prop = event["property"]
+    value = event["value"]
+
+    extra_keys = set(event.keys()) - {
+        "id",
+        "type",
+        "op",
+        "property",
+        "value",
+        "name",
+        "stream_id",
+        "first_message_id",
+        "is_archived",
+        "folder_id",
+    }
+
+    if prop == "description":
+        assert extra_keys == {"rendered_description"}
+        assert isinstance(value, str)
+    elif prop == "invite_only":
+        assert extra_keys == {"history_public_to_subscribers", "is_web_public"}
+        assert isinstance(value, bool)
+    elif prop == "message_retention_days":
+        assert extra_keys == set()
+        if value is not None:
+            assert isinstance(value, int)
+    elif prop == "name":
+        assert extra_keys == set()
+        assert isinstance(value, str)
+    elif prop == "stream_post_policy":
+        assert extra_keys == set()
+        assert value in Stream.STREAM_POST_POLICY_TYPES
+    elif prop in Stream.stream_permission_group_settings:
+        assert extra_keys == set()
+        assert isinstance(value, int | dict)
+        # We cannot validate a TypedDict using isinstance, thus
+        # requiring this check.
+        if isinstance(value, dict):
+            expected_keys = set(inspect.get_annotations(UserGroupMembersDict).keys())
+            keys = set(value.keys())
+            assert expected_keys == keys
+    elif prop == "first_message_id":
+        assert extra_keys == set()
+        assert isinstance(value, int)
+    elif prop == "topics_policy":
+        assert extra_keys == set()
+        assert value in [e.name for e in StreamTopicsPolicyEnum]
+    elif prop == "is_recently_active":
+        assert extra_keys == set()
+        assert isinstance(value, bool)
+    elif prop == "is_announcement_only":
+        assert extra_keys == set()
+        assert isinstance(value, bool)
+    elif prop == "is_archived":
+        assert extra_keys == set()
+        assert isinstance(value, bool)
+    elif prop == "folder_id":
+        assert extra_keys == set()
+        assert value is None or isinstance(value, int)
+    elif prop == "default_push_notifications":
+        assert extra_keys == set()
+        assert isinstance(value, bool)
+    else:
+        raise AssertionError(f"Unknown property: {prop}")
+
+
+def check_subscription_update(
+    var_name: str, event: dict[str, object], property: str, value: bool
+) -> None:
+    _check_subscription_update(var_name, event)
+    assert event["property"] == property
+    assert event["value"] == value
+
+
+def check_user_settings_update(
+    var_name: str,
+    event: dict[str, object],
+) -> None:
+    _check_user_settings_update(var_name, event)
+    setting_name = event["property"]
+    value = event["value"]
+
+    assert isinstance(setting_name, str)
+    if setting_name == "timezone":
+        assert isinstance(value, str)
+    else:
+        setting_type = UserProfile.property_types[setting_name]
+        if inspect.isclass(setting_type) and issubclass(setting_type, Enum):
+            assert isinstance(value, str)
+            setting_type[value]
+        else:
+            assert isinstance(value, setting_type)
+
+    if setting_name == "default_language":
+        assert "language_name" in event
+    else:
+        assert "language_name" not in event
+
+
+def check_update_message(
+    var_name: str,
+    event: dict[str, object],
+    is_stream_message: bool,
+    has_content: bool,
+    has_topic: bool,
+    has_new_stream_id: bool,
+    is_embedded_update_only: bool,
+) -> None:
+    # Always check the basic schema first.
+    _check_update_message(var_name, event)
+
+    actual_keys = set(event.keys())
+    expected_keys = {
+        "id",
+        "type",
+        "user_id",
+        "edit_timestamp",
+        "message_id",
+        "flags",
+        "message_ids",
+        "rendering_only",
+    }
+
+    if is_stream_message:
+        expected_keys |= {
+            "stream_id",
+            "stream_name",
+        }
+
+    if has_content:
+        expected_keys |= {
+            "is_me_message",
+            "orig_content",
+            "orig_rendered_content",
+            "content",
+            "rendered_content",
+        }
+
+    if has_topic:
+        expected_keys |= {
+            "topic_links",
+            ORIG_TOPIC,
+            TOPIC_NAME,
+            "propagate_mode",
+        }
+
+    if has_new_stream_id:
+        expected_keys |= {
+            "new_stream_id",
+            ORIG_TOPIC,
+            "propagate_mode",
+        }
+
+    if is_embedded_update_only:
+        expected_keys |= {
+            "content",
+            "rendered_content",
+        }
+        assert event["user_id"] is None
+    else:
+        assert isinstance(event["user_id"], int)
+
+    assert event["rendering_only"] == is_embedded_update_only
+    assert expected_keys == actual_keys
+
+
+def check_user_group_update(var_name: str, event: dict[str, object], fields: set[str]) -> None:
+    _check_user_group_update(var_name, event)
+
+    assert isinstance(event["data"], dict)
+
+    assert set(event["data"].keys()) == fields
+
+
+def check_user_status(var_name: str, event: dict[str, object], fields: set[str]) -> None:
+    _check_user_status(var_name, event)
+
+    assert set(event.keys()) == {"id", "type", "user_id"} | fields