snapshot of https://github.com/zulip/zulip.git @ 48c69911c7dd4e03b20552eb3bd77ad8151c1357 (2026-09-15)

zerver/actions/alert_words.py

@@ -0,0 +1,24 @@
+from collections.abc import Iterable, Sequence
+
+from django.db import transaction
+
+from zerver.lib.alert_words import add_user_alert_words, remove_user_alert_words
+from zerver.models import UserProfile
+from zerver.tornado.django_api import send_event_on_commit
+
+
+def notify_alert_words(user_profile: UserProfile, words: Sequence[str]) -> None:
+    event = dict(type="alert_words", alert_words=words)
+    send_event_on_commit(user_profile.realm, event, [user_profile.id])
+
+
+@transaction.atomic(durable=True)
+def do_add_alert_words(user_profile: UserProfile, alert_words: Iterable[str]) -> None:
+    words = add_user_alert_words(user_profile, alert_words)
+    notify_alert_words(user_profile, words)
+
+
+@transaction.atomic(durable=True)
+def do_remove_alert_words(user_profile: UserProfile, alert_words: Iterable[str]) -> None:
+    words = remove_user_alert_words(user_profile, alert_words)
+    notify_alert_words(user_profile, words)

zerver/actions/bots.py

@@ -0,0 +1,188 @@
+from django.db import transaction
+from django.utils.timezone import now as timezone_now
+
+from zerver.actions.create_user import created_bot_event
+from zerver.models import RealmAuditLog, Stream, UserProfile
+from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.models.users import active_user_ids, bot_owner_user_ids
+from zerver.tornado.django_api import send_event_on_commit
+
+
+def send_bot_owner_update_events(
+    user_profile: UserProfile, bot_owner: UserProfile, previous_owner: UserProfile | None
+) -> None:
+    # Since `bot_owner_id` is included in the user profile dict we need
+    # to update the users dict with the new bot owner id
+    event = dict(
+        type="realm_user",
+        op="update",
+        person=dict(
+            user_id=user_profile.id,
+            bot_owner_id=bot_owner.id,
+        ),
+    )
+    send_event_on_commit(user_profile.realm, event, active_user_ids(user_profile.realm_id))
+
+    # Delete the bot from previous owner's bot data.
+    if previous_owner and not previous_owner.is_realm_admin:
+        delete_event = dict(
+            type="realm_bot",
+            op="delete",
+            bot=dict(
+                user_id=user_profile.id,
+            ),
+        )
+        previous_owner_id = previous_owner.id
+        send_event_on_commit(
+            user_profile.realm,
+            delete_event,
+            {previous_owner_id},
+        )
+
+    # Notify the new owner that the bot has been added.
+    if not bot_owner.is_realm_admin:
+        add_event = created_bot_event(user_profile)
+        send_event_on_commit(user_profile.realm, add_event, {bot_owner.id})
+
+
+@transaction.atomic(durable=True)
+def do_change_bot_owner(
+    user_profile: UserProfile, bot_owner: UserProfile, acting_user: UserProfile | None
+) -> None:
+    previous_owner = user_profile.bot_owner
+    user_profile.bot_owner = bot_owner
+    user_profile.save()  # Can't use update_fields because of how the foreign key works.
+    event_time = timezone_now()
+    RealmAuditLog.objects.create(
+        realm=user_profile.realm,
+        acting_user=acting_user,
+        modified_user=user_profile,
+        event_type=AuditLogEventType.USER_BOT_OWNER_CHANGED,
+        event_time=event_time,
+        extra_data={
+            RealmAuditLog.OLD_VALUE: None if previous_owner is None else previous_owner.id,
+            RealmAuditLog.NEW_VALUE: bot_owner.id,
+        },
+    )
+
+    send_bot_owner_update_events(user_profile, bot_owner, previous_owner)
+
+
+@transaction.atomic(durable=True)
+def do_change_default_sending_stream(
+    user_profile: UserProfile, stream: Stream | None, *, acting_user: UserProfile | None
+) -> None:
+    old_value = user_profile.default_sending_stream_id
+    user_profile.default_sending_stream = stream
+    user_profile.save(update_fields=["default_sending_stream"])
+
+    event_time = timezone_now()
+    RealmAuditLog.objects.create(
+        realm=user_profile.realm,
+        event_type=AuditLogEventType.USER_DEFAULT_SENDING_STREAM_CHANGED,
+        event_time=event_time,
+        modified_user=user_profile,
+        acting_user=acting_user,
+        extra_data={
+            RealmAuditLog.OLD_VALUE: old_value,
+            RealmAuditLog.NEW_VALUE: None if stream is None else stream.id,
+        },
+    )
+
+    if user_profile.is_bot:
+        if stream:
+            stream_name: str | None = stream.name
+        else:
+            stream_name = None
+        event = dict(
+            type="realm_bot",
+            op="update",
+            bot=dict(
+                user_id=user_profile.id,
+                default_sending_stream=stream_name,
+            ),
+        )
+        send_event_on_commit(
+            user_profile.realm,
+            event,
+            bot_owner_user_ids(user_profile),
+        )
+
+
+@transaction.atomic(durable=True)
+def do_change_default_events_register_stream(
+    user_profile: UserProfile, stream: Stream | None, *, acting_user: UserProfile | None
+) -> None:
+    old_value = user_profile.default_events_register_stream_id
+    user_profile.default_events_register_stream = stream
+    user_profile.save(update_fields=["default_events_register_stream"])
+
+    event_time = timezone_now()
+    RealmAuditLog.objects.create(
+        realm=user_profile.realm,
+        event_type=AuditLogEventType.USER_DEFAULT_REGISTER_STREAM_CHANGED,
+        event_time=event_time,
+        modified_user=user_profile,
+        acting_user=acting_user,
+        extra_data={
+            RealmAuditLog.OLD_VALUE: old_value,
+            RealmAuditLog.NEW_VALUE: None if stream is None else stream.id,
+        },
+    )
+
+    if user_profile.is_bot:
+        if stream:
+            stream_name: str | None = stream.name
+        else:
+            stream_name = None
+
+        event = dict(
+            type="realm_bot",
+            op="update",
+            bot=dict(
+                user_id=user_profile.id,
+                default_events_register_stream=stream_name,
+            ),
+        )
+        send_event_on_commit(
+            user_profile.realm,
+            event,
+            bot_owner_user_ids(user_profile),
+        )
+
+
+@transaction.atomic(durable=True)
+def do_change_default_all_public_streams(
+    user_profile: UserProfile, value: bool, *, acting_user: UserProfile | None
+) -> None:
+    old_value = user_profile.default_all_public_streams
+    user_profile.default_all_public_streams = value
+    user_profile.save(update_fields=["default_all_public_streams"])
+
+    event_time = timezone_now()
+    RealmAuditLog.objects.create(
+        realm=user_profile.realm,
+        event_type=AuditLogEventType.USER_DEFAULT_ALL_PUBLIC_STREAMS_CHANGED,
+        event_time=event_time,
+        modified_user=user_profile,
+        acting_user=acting_user,
+        extra_data={
+            RealmAuditLog.OLD_VALUE: old_value,
+            RealmAuditLog.NEW_VALUE: value,
+        },
+    )
+
+    if user_profile.is_bot:
+        event = dict(
+            type="realm_bot",
+            op="update",
+            bot=dict(
+                user_id=user_profile.id,
+                default_all_public_streams=user_profile.default_all_public_streams,
+            ),
+        )
+        send_event_on_commit(
+            user_profile.realm,
+            event,
+            bot_owner_user_ids(user_profile),
+        )

zerver/actions/channel_folders.py

@@ -0,0 +1,159 @@
+from dataclasses import asdict
+
+from django.db import transaction
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+
+from zerver.lib.channel_folders import get_channel_folder_data, render_channel_folder_description
+from zerver.lib.exceptions import JsonableError
+from zerver.models import ChannelFolder, Realm, RealmAuditLog, UserProfile
+from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.models.users import active_user_ids
+from zerver.tornado.django_api import send_event_on_commit
+
+
+@transaction.atomic(durable=True)
+def check_add_channel_folder(
+    realm: Realm, name: str, description: str, *, acting_user: UserProfile
+) -> ChannelFolder:
+    rendered_description = render_channel_folder_description(
+        description, realm, acting_user=acting_user
+    )
+    channel_folder = ChannelFolder.objects.create(
+        realm=realm,
+        name=name,
+        description=description,
+        rendered_description=rendered_description,
+        creator_id=acting_user.id,
+    )
+    channel_folder.order = channel_folder.id
+    channel_folder.save(update_fields=["order"])
+
+    creation_time = timezone_now()
+    RealmAuditLog.objects.create(
+        realm=realm,
+        acting_user=acting_user,
+        event_type=AuditLogEventType.CHANNEL_FOLDER_CREATED,
+        event_time=creation_time,
+        modified_channel_folder=channel_folder,
+    )
+
+    event = dict(
+        type="channel_folder",
+        op="add",
+        channel_folder=asdict(get_channel_folder_data(channel_folder)),
+    )
+    send_event_on_commit(realm, event, active_user_ids(realm.id))
+
+    return channel_folder
+
+
+@transaction.atomic(durable=True)
+def try_reorder_realm_channel_folders(realm: Realm, order: list[int]) -> None:
+    order_mapping = {_[1]: _[0] for _ in enumerate(order)}
+    channel_folders = ChannelFolder.objects.filter(realm=realm)
+    for channel_folder in channel_folders:
+        if channel_folder.id not in order_mapping:
+            raise JsonableError(_("Invalid order mapping."))
+    for channel_folder in channel_folders:
+        channel_folder.order = order_mapping[channel_folder.id]
+        channel_folder.save(update_fields=["order"])
+
+    event = dict(
+        type="channel_folder",
+        op="reorder",
+        order=order,
+    )
+    send_event_on_commit(realm, event, active_user_ids(realm.id))
+
+
+def do_send_channel_folder_update_event(
+    channel_folder: ChannelFolder, data: dict[str, str | bool]
+) -> None:
+    realm = channel_folder.realm
+    event = dict(type="channel_folder", op="update", channel_folder_id=channel_folder.id, data=data)
+    send_event_on_commit(realm, event, active_user_ids(realm.id))
+
+
+@transaction.atomic(durable=True)
+def do_change_channel_folder_name(
+    channel_folder: ChannelFolder, name: str, *, acting_user: UserProfile
+) -> None:
+    old_value = channel_folder.name
+    channel_folder.name = name
+    channel_folder.save(update_fields=["name"])
+
+    RealmAuditLog.objects.create(
+        realm=acting_user.realm,
+        acting_user=acting_user,
+        event_type=AuditLogEventType.CHANNEL_FOLDER_NAME_CHANGED,
+        event_time=timezone_now(),
+        modified_channel_folder=channel_folder,
+        extra_data={
+            RealmAuditLog.OLD_VALUE: old_value,
+            RealmAuditLog.NEW_VALUE: name,
+        },
+    )
+
+    do_send_channel_folder_update_event(channel_folder, dict(name=name))
+
+
+@transaction.atomic(durable=True)
+def do_change_channel_folder_description(
+    channel_folder: ChannelFolder, description: str, *, acting_user: UserProfile
+) -> None:
+    old_value = channel_folder.description
+    rendered_description = render_channel_folder_description(
+        description, acting_user.realm, acting_user=acting_user
+    )
+    channel_folder.description = description
+    channel_folder.rendered_description = rendered_description
+    channel_folder.save(update_fields=["description", "rendered_description"])
+
+    RealmAuditLog.objects.create(
+        realm=acting_user.realm,
+        acting_user=acting_user,
+        event_type=AuditLogEventType.CHANNEL_FOLDER_DESCRIPTION_CHANGED,
+        event_time=timezone_now(),
+        modified_channel_folder=channel_folder,
+        extra_data={
+            RealmAuditLog.OLD_VALUE: old_value,
+            RealmAuditLog.NEW_VALUE: description,
+        },
+    )
+
+    do_send_channel_folder_update_event(
+        channel_folder, dict(description=description, rendered_description=rendered_description)
+    )
+
+
+@transaction.atomic(durable=True)
+def do_archive_channel_folder(channel_folder: ChannelFolder, *, acting_user: UserProfile) -> None:
+    channel_folder.is_archived = True
+    channel_folder.save(update_fields=["is_archived"])
+
+    RealmAuditLog.objects.create(
+        realm=acting_user.realm,
+        acting_user=acting_user,
+        event_type=AuditLogEventType.CHANNEL_FOLDER_ARCHIVED,
+        event_time=timezone_now(),
+        modified_channel_folder=channel_folder,
+    )
+
+    do_send_channel_folder_update_event(channel_folder, dict(is_archived=True))
+
+
+@transaction.atomic(durable=True)
+def do_unarchive_channel_folder(channel_folder: ChannelFolder, *, acting_user: UserProfile) -> None:
+    channel_folder.is_archived = False
+    channel_folder.save(update_fields=["is_archived"])
+
+    RealmAuditLog.objects.create(
+        realm=acting_user.realm,
+        acting_user=acting_user,
+        event_type=AuditLogEventType.CHANNEL_FOLDER_UNARCHIVED,
+        event_time=timezone_now(),
+        modified_channel_folder=channel_folder,
+    )
+
+    do_send_channel_folder_update_event(channel_folder, dict(is_archived=False))

zerver/actions/create_realm.py

@@ -0,0 +1,404 @@
+import logging
+from datetime import datetime, timedelta
+from typing import Any
+
+from django.conf import settings
+from django.db import transaction
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+from django.utils.translation import override as override_language
+
+from confirmation import settings as confirmation_settings
+from zerver.actions.realm_settings import (
+    do_add_deactivated_redirect,
+    do_change_realm_plan_type,
+    do_deactivate_realm,
+)
+from zerver.lib.bulk_create import create_users
+from zerver.lib.push_notifications import sends_notifications_directly
+from zerver.lib.realm_description import render_realm_description
+from zerver.lib.remote_server import maybe_enqueue_audit_log_upload
+from zerver.lib.server_initialization import create_internal_realm, server_initialized
+from zerver.lib.sessions import delete_realm_user_sessions
+from zerver.lib.streams import ensure_stream
+from zerver.lib.user_groups import (
+    create_system_user_groups_for_realm,
+    get_role_based_system_groups_dict,
+)
+from zerver.lib.zulip_update_announcements import get_latest_zulip_update_announcements_level
+from zerver.models import (
+    DefaultStream,
+    PreregistrationRealm,
+    Realm,
+    RealmAuditLog,
+    RealmAuthenticationMethod,
+    RealmUserDefault,
+    UserProfile,
+)
+from zerver.models.groups import SystemGroups
+from zerver.models.presence import PresenceSequence
+from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.models.scheduled_jobs import ScheduledMessage
+from zerver.models.users import get_system_bot
+from zproject.backends import all_default_backend_names
+
+DEFAULT_EMAIL_ADDRESS_VISIBILITY_FOR_REALM = RealmUserDefault.EMAIL_ADDRESS_VISIBILITY_ADMINS
+
+
+def do_change_realm_subdomain(
+    realm: Realm,
+    new_subdomain: str,
+    *,
+    acting_user: UserProfile | None,
+    add_deactivated_redirect: bool = True,
+) -> None:
+    """Changing a realm's subdomain is a highly disruptive operation,
+    because all existing clients will need to be updated to point to
+    the new URL.  Further, requests to fetch data from existing event
+    queues will fail with an authentication error when this change
+    happens (because the old subdomain is no longer associated with
+    the realm), making it hard for us to provide a graceful update
+    experience for clients.
+    """
+    old_subdomain = realm.subdomain
+    old_url = realm.url
+    was_demo_organization = realm.demo_organization_scheduled_deletion_date is not None
+    # If the realm had been a demo organization scheduled for
+    # deleting, clear that state.
+    realm.demo_organization_scheduled_deletion_date = None
+    realm.string_id = new_subdomain
+    with transaction.atomic(durable=True):
+        realm.save(update_fields=["string_id", "demo_organization_scheduled_deletion_date"])
+        RealmAuditLog.objects.create(
+            realm=realm,
+            event_type=AuditLogEventType.REALM_SUBDOMAIN_CHANGED,
+            event_time=timezone_now(),
+            acting_user=acting_user,
+            # Old RealmAuditLog entries for this used "old_subdomain" and
+            # "new_subdomain" keys to store this data.
+            extra_data={
+                RealmAuditLog.OLD_VALUE: old_subdomain,
+                RealmAuditLog.NEW_VALUE: new_subdomain,
+                "was_demo_organization": was_demo_organization,
+            },
+        )
+
+        # If a realm if being renamed multiple times, we should find all the placeholder
+        # realms and reset their deactivated_redirect field to point to the new realm url
+        placeholder_realms = Realm.objects.filter(deactivated_redirect=old_url, deactivated=True)
+        for placeholder_realm in placeholder_realms:
+            do_add_deactivated_redirect(placeholder_realm, realm.url)
+
+        # If a realm was a demo organization delete any scheduled messages about the
+        # automatic deletion of the demo organization.
+        if was_demo_organization:
+            sender = get_system_bot(settings.NOTIFICATION_BOT, realm.id)
+            scheduled_notification_bot_messages = ScheduledMessage.objects.filter(
+                sender=sender,
+                delivery_type=ScheduledMessage.SEND_LATER,
+                realm=realm,
+                delivered=False,
+                failed=False,
+            )
+            for scheduled_message in scheduled_notification_bot_messages:
+                scheduled_message.delete()
+
+    # The below block isn't executed in a transaction with the earlier code due to
+    # the functions called below being complex and potentially sending events,
+    # which we don't want to do in atomic blocks.
+    # When we change a realm's subdomain the realm with old subdomain is basically
+    # deactivated. We are creating a deactivated realm using old subdomain and setting
+    # it's deactivated redirect to new_subdomain so that we can tell the users that
+    # the realm has been moved to a new subdomain.
+    if add_deactivated_redirect:
+        placeholder_realm = do_create_realm(old_subdomain, realm.name)
+        do_deactivate_realm(
+            placeholder_realm,
+            acting_user=None,
+            deactivation_reason="subdomain_change",
+            email_owners=False,
+        )
+        do_add_deactivated_redirect(placeholder_realm, realm.url)
+
+    # Sessions can't be deleted inside a transaction.
+    delete_realm_user_sessions(realm)
+
+
+@transaction.atomic(savepoint=False)
+def set_default_for_realm_permission_group_settings(
+    realm: Realm, group_settings_defaults_for_org_types: dict[str, dict[int, str]] | None = None
+) -> None:
+    system_groups_dict = get_role_based_system_groups_dict(realm)
+
+    for setting_name, permission_configuration in Realm.REALM_PERMISSION_GROUP_SETTINGS.items():
+        group_name = permission_configuration.default_group_name
+
+        # Below code updates the group_name if the setting default depends on org_type.
+        if (
+            group_settings_defaults_for_org_types is not None
+            and setting_name in group_settings_defaults_for_org_types
+        ):
+            setting_org_type_defaults = group_settings_defaults_for_org_types[setting_name]
+            if realm.org_type in setting_org_type_defaults:
+                group_name = setting_org_type_defaults[realm.org_type]
+
+        setattr(realm, setting_name, system_groups_dict[group_name].usergroup_ptr)
+
+    realm.save(update_fields=list(Realm.REALM_PERMISSION_GROUP_SETTINGS.keys()))
+
+
+def setup_realm_internal_bots(realm: Realm) -> None:
+    """Create this realm's internal bots.
+
+    This function is idempotent; it does nothing for a bot that
+    already exists.
+    """
+    internal_bots = [
+        (bot["name"], bot["email_template"] % (settings.INTERNAL_BOT_DOMAIN,))
+        for bot in settings.REALM_INTERNAL_BOTS
+    ]
+    create_users(realm, internal_bots, bot_type=UserProfile.DEFAULT_BOT)
+    bots = UserProfile.objects.filter(
+        realm=realm,
+        email__in=[bot_info[1] for bot_info in internal_bots],
+        bot_owner__isnull=True,
+    )
+    for bot in bots:
+        bot.bot_owner = bot
+        bot.save()
+
+
+def get_email_address_visibility_default(org_type: int | None = None) -> int:
+    # For the majority of organization types, the default email address visibility
+    # setting for new users should initially be admins only.
+    realm_default_email_address_visibility = DEFAULT_EMAIL_ADDRESS_VISIBILITY_FOR_REALM
+    if org_type in (
+        Realm.ORG_TYPES["education_nonprofit"]["id"],
+        Realm.ORG_TYPES["education"]["id"],
+    ):
+        # Email address of users should be initially visible to moderators and admins.
+        realm_default_email_address_visibility = (
+            RealmUserDefault.EMAIL_ADDRESS_VISIBILITY_MODERATORS
+        )
+    elif org_type == Realm.ORG_TYPES["business"]["id"]:
+        # Email address of users can be visible to all users for business organizations.
+        realm_default_email_address_visibility = RealmUserDefault.EMAIL_ADDRESS_VISIBILITY_EVERYONE
+
+    return realm_default_email_address_visibility
+
+
+def do_create_realm(
+    string_id: str,
+    name: str,
+    *,
+    emails_restricted_to_domains: bool | None = None,
+    description: str | None = None,
+    invite_required: bool | None = None,
+    plan_type: int | None = None,
+    org_type: int | None = None,
+    default_language: str | None = None,
+    date_created: datetime | None = None,
+    is_demo_organization: bool = False,
+    enable_read_receipts: bool | None = None,
+    enable_spectator_access: bool | None = None,
+    prereg_realm: PreregistrationRealm | None = None,
+    how_realm_creator_found_zulip: str | None = None,
+    how_realm_creator_found_zulip_extra_context: str | None = None,
+) -> Realm:
+    if string_id in [settings.SOCIAL_AUTH_SUBDOMAIN, settings.SELF_HOSTING_MANAGEMENT_SUBDOMAIN]:
+        raise AssertionError(
+            "Creating a realm on SOCIAL_AUTH_SUBDOMAIN or SELF_HOSTING_MANAGEMENT_SUBDOMAIN is not allowed!"
+        )
+    if Realm.objects.filter(string_id=string_id).exists():
+        raise AssertionError(f"Realm {string_id} already exists!")
+    if not server_initialized():
+        logging.info("Server not yet initialized. Creating the internal realm first.")
+        create_internal_realm()
+
+    kwargs: dict[str, Any] = {}
+    if emails_restricted_to_domains is not None:
+        kwargs["emails_restricted_to_domains"] = emails_restricted_to_domains
+    if description is not None:
+        kwargs["description"] = description
+        rendered_description, version = render_realm_description(description, realm=None)
+        kwargs["rendered_description"] = rendered_description
+        kwargs["rendered_description_version"] = version
+    if invite_required is not None:
+        kwargs["invite_required"] = invite_required
+    if plan_type is not None:
+        kwargs["plan_type"] = plan_type
+    if org_type is not None:
+        kwargs["org_type"] = org_type
+    if default_language is not None:
+        kwargs["default_language"] = default_language
+    if enable_spectator_access is not None:
+        if enable_spectator_access:
+            # Realms with LIMITED plan cannot have spectators enabled.
+            assert plan_type != Realm.PLAN_TYPE_LIMITED
+            assert plan_type is not None or not settings.BILLING_ENABLED
+        kwargs["enable_spectator_access"] = enable_spectator_access
+
+    if date_created is not None:
+        # The date_created parameter is intended only for use by test
+        # suites that want to backdate the date of a realm's creation.
+        assert not settings.PRODUCTION
+        kwargs["date_created"] = date_created
+
+    if is_demo_organization:
+        # To enable creating demo organizations, a deadline of the number
+        # of days after creation that a demo organization should be deleted
+        # needs to be set on the server.
+        assert settings.DEMO_ORG_DEADLINE_DAYS is not None
+
+    # Generally, closed organizations like companies want read
+    # receipts, whereas it's unclear what an open organization's
+    # preferences will be. We enable the setting by default only for
+    # closed organizations.
+    if enable_read_receipts is not None:
+        kwargs["enable_read_receipts"] = enable_read_receipts
+    else:
+        # Hacky: The default of invited_required is True, so we need
+        # to check for None too.
+        kwargs["enable_read_receipts"] = (
+            invite_required is None or invite_required is True or emails_restricted_to_domains
+        )
+    # Initialize this property correctly in the case that no network activity
+    # is required to do so correctly.
+    kwargs["push_notifications_enabled"] = sends_notifications_directly()
+
+    with transaction.atomic(durable=True):
+        realm = Realm(string_id=string_id, name=name, **kwargs)
+        if is_demo_organization:
+            assert settings.DEMO_ORG_DEADLINE_DAYS is not None
+            realm.demo_organization_scheduled_deletion_date = realm.date_created + timedelta(
+                days=settings.DEMO_ORG_DEADLINE_DAYS
+            )
+
+        # For now a dummy value of -1 is given to groups fields which
+        # is changed later before the transaction is committed.
+        for setting_name in Realm.REALM_PERMISSION_GROUP_SETTINGS:
+            setattr(realm, setting_name + "_id", -1)
+
+        realm.save()
+
+        RealmAuditLog.objects.create(
+            # acting_user will be set as the initial realm owner inside
+            # do_create_user(..., realm_creation=True).
+            acting_user=None,
+            realm=realm,
+            event_type=AuditLogEventType.REALM_CREATED,
+            event_time=realm.date_created,
+            extra_data={
+                "how_realm_creator_found_zulip": how_realm_creator_found_zulip,
+                "how_realm_creator_found_zulip_extra_context": how_realm_creator_found_zulip_extra_context,
+            },
+        )
+
+        RealmUserDefault.objects.create(
+            realm=realm,
+            email_address_visibility=get_email_address_visibility_default(realm.org_type),
+        )
+
+        create_system_user_groups_for_realm(realm)
+
+        group_settings_defaults_for_org_types = {
+            "can_add_subscribers_group": {
+                Realm.ORG_TYPES["education_nonprofit"]["id"]: SystemGroups.MODERATORS,
+                Realm.ORG_TYPES["education"]["id"]: SystemGroups.MODERATORS,
+            },
+            "can_create_public_channel_group": {
+                Realm.ORG_TYPES["education_nonprofit"]["id"]: SystemGroups.ADMINISTRATORS,
+                Realm.ORG_TYPES["education"]["id"]: SystemGroups.ADMINISTRATORS,
+            },
+            "can_create_groups": {
+                Realm.ORG_TYPES["education_nonprofit"]["id"]: SystemGroups.MODERATORS,
+                Realm.ORG_TYPES["education"]["id"]: SystemGroups.MODERATORS,
+            },
+            "can_invite_users_group": {
+                Realm.ORG_TYPES["education_nonprofit"]["id"]: SystemGroups.ADMINISTRATORS,
+                Realm.ORG_TYPES["education"]["id"]: SystemGroups.ADMINISTRATORS,
+            },
+            "can_move_messages_between_channels_group": {
+                Realm.ORG_TYPES["education_nonprofit"]["id"]: SystemGroups.MODERATORS,
+                Realm.ORG_TYPES["education"]["id"]: SystemGroups.MODERATORS,
+            },
+        }
+        set_default_for_realm_permission_group_settings(
+            realm, group_settings_defaults_for_org_types
+        )
+
+        RealmAuthenticationMethod.objects.bulk_create(
+            [
+                RealmAuthenticationMethod(name=backend_name, realm=realm)
+                for backend_name in all_default_backend_names()
+            ]
+        )
+
+        PresenceSequence.objects.create(realm=realm, last_update_id=0)
+
+        maybe_enqueue_audit_log_upload(realm)
+
+    # Create channels once Realm object has been saved
+    with override_language(realm.default_language):
+        zulip_discussion_channel = ensure_stream(
+            realm,
+            str(Realm.ZULIP_DISCUSSION_CHANNEL_NAME),
+            stream_description=_("Questions and discussion about using Zulip."),
+            acting_user=None,
+        )
+        zulip_sandbox_channel = ensure_stream(
+            realm,
+            str(Realm.ZULIP_SANDBOX_CHANNEL_NAME),
+            stream_description=_("Experiment with Zulip here. :test_tube:"),
+            acting_user=None,
+        )
+        new_stream_announcements_stream = ensure_stream(
+            realm,
+            str(Realm.DEFAULT_NOTIFICATION_STREAM_NAME),
+            stream_description=_("For team-wide conversations"),
+            acting_user=None,
+        )
+
+    # By default, 'New stream' & 'Zulip update' announcements are sent to the same stream.
+    realm.new_stream_announcements_stream = new_stream_announcements_stream
+    realm.zulip_update_announcements_stream = new_stream_announcements_stream
+
+    # With the current initial streams situation, the public channels are
+    # 'zulip_discussion_channel', 'zulip_sandbox_channel', 'new_stream_announcements_stream'.
+    public_channels = [
+        DefaultStream(stream=zulip_discussion_channel, realm=realm),
+        DefaultStream(stream=zulip_sandbox_channel, realm=realm),
+        DefaultStream(stream=new_stream_announcements_stream, realm=realm),
+    ]
+    DefaultStream.objects.bulk_create(public_channels)
+
+    # New realm is initialized with the latest zulip update announcements
+    # level as it shouldn't receive a bunch of old updates.
+    realm.zulip_update_announcements_level = get_latest_zulip_update_announcements_level()
+
+    realm.save(
+        update_fields=[
+            "new_stream_announcements_stream",
+            "zulip_update_announcements_stream",
+            "zulip_update_announcements_level",
+        ]
+    )
+
+    if plan_type is None and settings.BILLING_ENABLED:
+        # We use acting_user=None for setting the initial plan type.
+        do_change_realm_plan_type(realm, Realm.PLAN_TYPE_LIMITED, acting_user=None)
+
+    if prereg_realm is not None:
+        prereg_realm.status = confirmation_settings.STATUS_USED
+        prereg_realm.created_realm = realm
+        prereg_realm.save(update_fields=["status", "created_realm"])
+
+    if settings.CORPORATE_ENABLED:
+        # Send a notification to the admin realm when a new organization registers.
+        from corporate.lib.stripe import RealmBillingSession
+
+        billing_session = RealmBillingSession(user=None, realm=realm)
+        billing_session.send_realm_created_internal_admin_message(is_demo_organization)
+
+    setup_realm_internal_bots(realm)
+    return realm

zerver/actions/create_user.py

@@ -0,0 +1,839 @@
+from collections import defaultdict
+from collections.abc import Iterable, Sequence
+from contextlib import suppress
+from typing import Any
+
+from django.conf import settings
+from django.db import IntegrityError, transaction
+from django.db.models import F
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+from django.utils.translation import override as override_language
+
+from confirmation import settings as confirmation_settings
+from zerver.actions.message_send import (
+    internal_send_group_direct_message,
+    internal_send_private_message,
+    internal_send_stream_message,
+)
+from zerver.actions.streams import bulk_add_subscriptions, send_peer_subscriber_events
+from zerver.actions.user_groups import (
+    bulk_add_members_to_user_groups,
+    do_send_user_group_members_update_event,
+)
+from zerver.actions.users import (
+    change_user_is_active,
+    get_service_dicts_for_bot,
+    send_update_events_for_anonymous_group_settings,
+)
+from zerver.lib.avatar import generate_and_upload_jdenticon_avatar
+from zerver.lib.create_user import create_user
+from zerver.lib.default_streams import get_slim_realm_default_streams
+from zerver.lib.email_notifications import enqueue_welcome_emails, send_account_registered_email
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.invites import notify_invites_changed
+from zerver.lib.mention import silent_mention_syntax_for_user
+from zerver.lib.remote_server import maybe_enqueue_audit_log_upload
+from zerver.lib.send_email import clear_scheduled_invitation_emails
+from zerver.lib.streams import can_access_stream_history
+from zerver.lib.subscription_info import bulk_get_subscriber_peer_info
+from zerver.lib.user_counts import realm_user_count, realm_user_count_by_role
+from zerver.lib.user_groups import get_system_user_group_for_user
+from zerver.lib.users import (
+    can_access_delivery_email,
+    format_user_row,
+    get_data_for_inaccessible_user,
+    get_user_ids_who_can_access_user,
+    user_access_restricted_in_realm,
+    user_profile_to_user_row,
+)
+from zerver.lib.utils import assert_is_not_none
+from zerver.models import (
+    DefaultStreamGroup,
+    Message,
+    NamedUserGroup,
+    OnboardingStep,
+    OnboardingUserMessage,
+    PreregistrationRealm,
+    PreregistrationUser,
+    Realm,
+    RealmAuditLog,
+    Recipient,
+    Stream,
+    Subscription,
+    UserGroupMembership,
+    UserMessage,
+    UserProfile,
+)
+from zerver.models.groups import SystemGroups
+from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.models.streams import StreamTopicsPolicyEnum
+from zerver.models.users import ExternalAuthID, active_user_ids, bot_owner_user_ids, get_system_bot
+from zerver.tornado.django_api import send_event_on_commit
+
+MAX_NUM_RECENT_MESSAGES = 1000
+MAX_NUM_RECENT_UNREAD_MESSAGES = 20
+
+
+def send_message_to_signup_notification_stream(
+    sender: UserProfile, realm: Realm, message: str
+) -> None:
+    signup_announcements_stream = realm.signup_announcements_stream
+    if signup_announcements_stream is None:
+        return
+
+    with override_language(realm.default_language):
+        topic_name = _("signups")
+        if (
+            signup_announcements_stream.topics_policy
+            == StreamTopicsPolicyEnum.empty_topic_only.value
+        ):
+            topic_name = ""
+
+    internal_send_stream_message(sender, signup_announcements_stream, topic_name, message)
+
+
+def send_group_direct_message_to_admins(sender: UserProfile, realm: Realm, content: str) -> None:
+    administrators = list(realm.get_human_admin_users())
+    internal_send_group_direct_message(
+        realm,
+        sender,
+        content,
+        recipient_users=administrators,
+    )
+
+
+def notify_new_user(user_profile: UserProfile) -> None:
+    user_count = realm_user_count(user_profile.realm)
+    sender_email = settings.NOTIFICATION_BOT
+    sender = get_system_bot(sender_email, user_profile.realm_id)
+
+    is_first_user = user_count == 1
+    if not is_first_user:
+        with override_language(user_profile.realm.default_language):
+            message = _("{user} joined this organization.").format(
+                user=silent_mention_syntax_for_user(user_profile), user_count=user_count
+            )
+            send_message_to_signup_notification_stream(sender, user_profile.realm, message)
+
+        if settings.BILLING_ENABLED:
+            from corporate.lib.registration import generate_licenses_low_warning_message_if_required
+
+            licenses_low_warning_message = generate_licenses_low_warning_message_if_required(
+                user_profile.realm
+            )
+            if licenses_low_warning_message is not None:
+                message += "\n"
+                message += licenses_low_warning_message
+                send_group_direct_message_to_admins(sender, user_profile.realm, message)
+
+
+def set_up_streams_and_groups_for_new_human_user(
+    *,
+    user_profile: UserProfile,
+    prereg_user: PreregistrationUser | None = None,
+    default_stream_groups: Sequence[DefaultStreamGroup] = [],
+    add_initial_stream_subscriptions: bool = True,
+    realm_creation: bool = False,
+) -> None:
+    realm = user_profile.realm
+
+    if prereg_user is not None:
+        streams: list[Stream] = list(prereg_user.streams.all())
+        user_groups: list[NamedUserGroup] = list(prereg_user.groups.all())
+        acting_user: UserProfile | None = prereg_user.referred_by
+
+        # A PregistrationUser should not be used for another UserProfile
+        assert prereg_user.created_user is None, "PregistrationUser should not be reused"
+    else:
+        streams = []
+        user_groups = []
+        acting_user = None
+
+    if add_initial_stream_subscriptions:
+        # If prereg_user.include_realm_default_subscriptions is true, we
+        # add the default streams for the realm to the list of streams.
+        # Note that we are fine with "slim" Stream objects for calling
+        # bulk_add_subscriptions and add_new_user_history, which we verify
+        # in StreamSetupTest tests that check query counts.
+        if prereg_user is None or prereg_user.include_realm_default_subscriptions:
+            default_streams = get_slim_realm_default_streams(realm.id)
+            streams = list(set(streams) | default_streams)
+
+        for default_stream_group in default_stream_groups:
+            default_stream_group_streams = default_stream_group.streams.all()
+            for stream in default_stream_group_streams:
+                if stream not in streams:
+                    streams.append(stream)
+    else:
+        streams = []
+
+    bulk_add_subscriptions(
+        realm,
+        streams,
+        [user_profile],
+        from_user_creation=True,
+        acting_user=acting_user,
+    )
+
+    bulk_add_members_to_user_groups(
+        user_groups,
+        [user_profile.id],
+        acting_user=acting_user,
+    )
+
+    add_new_user_history(user_profile, streams, realm_creation=realm_creation)
+
+
+def add_new_user_history(
+    user_profile: UserProfile,
+    streams: Iterable[Stream],
+    *,
+    realm_creation: bool = False,
+) -> None:
+    """
+    Give the user some messages in their feed, so that they can learn
+    how to use the home view in a realistic way.
+
+    For realms having older onboarding messages, mark the very
+    most recent messages as unread. Otherwise, ONLY mark the
+    messages tracked in 'OnboardingUserMessage' as unread.
+    """
+
+    realm = user_profile.realm
+    # Find recipient ids for the user's streams, limiting to just
+    # those where we can access the streams' full history.
+    #
+    # TODO: This will do database queries in a loop if many private
+    # streams are involved.
+    recipient_ids = [
+        assert_is_not_none(stream.recipient_id)
+        for stream in streams
+        if can_access_stream_history(user_profile, stream)
+    ]
+
+    # Start by finding recent messages matching those recipients.
+    recent_message_ids = set(
+        Message.objects.filter(
+            # Uses index: zerver_message_realm_recipient_id
+            realm_id=realm.id,
+            recipient_id__in=recipient_ids,
+        )
+        .order_by("-id")
+        .values_list("id", flat=True)[0:MAX_NUM_RECENT_MESSAGES]
+    )
+
+    tracked_onboarding_message_ids = set()
+    message_id_to_onboarding_user_message = {}
+    onboarding_user_messages_queryset = OnboardingUserMessage.objects.filter(realm_id=realm.id)
+    for onboarding_user_message in onboarding_user_messages_queryset:
+        tracked_onboarding_message_ids.add(onboarding_user_message.message_id)
+        message_id_to_onboarding_user_message[onboarding_user_message.message_id] = (
+            onboarding_user_message
+        )
+    tracked_onboarding_messages_exist = len(tracked_onboarding_message_ids) > 0
+
+    message_history_ids = recent_message_ids.union(tracked_onboarding_message_ids)
+
+    if len(message_history_ids) > 0:
+        # Handle the race condition where a message arrives between
+        # bulk_add_subscriptions above and the recent message query just above
+        already_used_ids = set(
+            UserMessage.objects.filter(
+                message_id__in=recent_message_ids, user_profile=user_profile
+            ).values_list("message_id", flat=True)
+        )
+
+        # Exclude the already-used ids and sort them.
+        backfill_message_ids = sorted(message_history_ids - already_used_ids)
+
+        # Find which message ids we should mark as read.
+        # (We don't want too many unread messages.)
+        older_message_ids = set()
+        if not tracked_onboarding_messages_exist:
+            older_message_ids = set(backfill_message_ids[:-MAX_NUM_RECENT_UNREAD_MESSAGES])
+
+        # Create UserMessage rows for the backfill.
+        ums_to_create = []
+        for message_id in backfill_message_ids:
+            um = UserMessage(user_profile=user_profile, message_id=message_id)
+            # Only onboarding messages are available for realm creator.
+            # They are not marked as historical.
+            if not realm_creation:
+                um.flags = UserMessage.flags.historical
+            if tracked_onboarding_messages_exist:
+                if message_id not in tracked_onboarding_message_ids:
+                    um.flags |= UserMessage.flags.read
+                elif message_id_to_onboarding_user_message[message_id].flags.starred.is_set:
+                    um.flags |= UserMessage.flags.starred
+            elif message_id in older_message_ids:
+                um.flags |= UserMessage.flags.read
+            ums_to_create.append(um)
+
+        UserMessage.objects.bulk_create(ums_to_create)
+
+
+# Does the processing for a new user account:
+# * Subscribes to default/invitation streams
+# * Adds to initial user groups
+# * Fills in some recent historical messages
+# * Notifies other users in realm and Zulip about the signup
+# * Deactivates PreregistrationUser objects
+# * Mark 'visibility_policy_banner' as read
+def process_new_human_user(
+    user_profile: UserProfile,
+    prereg_user: PreregistrationUser | None = None,
+    default_stream_groups: Sequence[DefaultStreamGroup] = [],
+    realm_creation: bool = False,
+    add_initial_stream_subscriptions: bool = True,
+) -> None:
+    # subscribe to default/invitation streams, add to groups and
+    # fill in some recent historical messages
+    set_up_streams_and_groups_for_new_human_user(
+        user_profile=user_profile,
+        prereg_user=prereg_user,
+        default_stream_groups=default_stream_groups,
+        add_initial_stream_subscriptions=add_initial_stream_subscriptions,
+        realm_creation=realm_creation,
+    )
+
+    realm = user_profile.realm
+
+    if (
+        prereg_user is not None
+        and prereg_user.referred_by is not None
+        and prereg_user.referred_by.is_active
+        and prereg_user.notify_referrer_on_join
+    ):
+        # This is a cross-realm direct message.
+        with override_language(prereg_user.referred_by.default_language):
+            internal_send_private_message(
+                get_system_bot(settings.NOTIFICATION_BOT, prereg_user.referred_by.realm_id),
+                prereg_user.referred_by,
+                _("{user} accepted your invitation to join Zulip!").format(
+                    user=silent_mention_syntax_for_user(user_profile)
+                ),
+            )
+
+    # For the sake of tracking the history of UserProfiles,
+    # we want to tie the newly created user to the PreregistrationUser
+    # it was created from.
+    if prereg_user is not None:
+        prereg_user.status = confirmation_settings.STATUS_USED
+        prereg_user.created_user = user_profile
+        prereg_user.save(update_fields=["status", "created_user"])
+
+    # Mark any other PreregistrationUsers in the realm that are STATUS_USED as
+    # inactive so we can keep track of the PreregistrationUser we
+    # actually used for analytics.
+    if prereg_user is not None:
+        PreregistrationUser.objects.filter(
+            email__iexact=user_profile.delivery_email, realm=user_profile.realm
+        ).exclude(id=prereg_user.id).update(status=confirmation_settings.STATUS_REVOKED)
+    else:
+        PreregistrationUser.objects.filter(
+            email__iexact=user_profile.delivery_email, realm=user_profile.realm
+        ).update(status=confirmation_settings.STATUS_REVOKED)
+
+    if prereg_user is not None and prereg_user.referred_by is not None:
+        notify_invites_changed(user_profile.realm, changed_invite_referrer=prereg_user.referred_by)
+
+    notify_new_user(user_profile)
+    # Clear any scheduled invitation emails to prevent them
+    # from being sent after the user is created.
+    clear_scheduled_invitation_emails(user_profile.delivery_email)
+    if realm.send_welcome_emails:
+        enqueue_welcome_emails(user_profile, realm_creation)
+
+    # Schedule an initial email with the user's
+    # new account details and log-in information.
+    send_account_registered_email(user_profile, realm_creation)
+
+    # We have an import loop here; it's intentional, because we want
+    # to keep all the onboarding code in zerver/lib/onboarding.py.
+    from zerver.lib.onboarding import send_initial_direct_messages_to_user
+
+    welcome_message_custom_text = realm.welcome_message_custom_text
+    if prereg_user is not None and prereg_user.welcome_message_custom_text is not None:
+        welcome_message_custom_text = prereg_user.welcome_message_custom_text
+    initial_direct_message_ids = send_initial_direct_messages_to_user(
+        user_profile,
+        realm_creation=realm_creation,
+        welcome_message_custom_text=welcome_message_custom_text,
+    )
+    message_id_list = [initial_direct_message_ids.welcome_bot_intro_message_id]
+    if initial_direct_message_ids.welcome_bot_custom_message_id is not None:
+        message_id_list.append(initial_direct_message_ids.welcome_bot_custom_message_id)
+
+    UserMessage.objects.filter(user_profile=user_profile, message_id__in=message_id_list).update(
+        flags=F("flags").bitor(UserMessage.flags.starred)
+    )
+
+    # The 'visibility_policy_banner' is only displayed to existing users.
+    # Mark it as read for a new user.
+    #
+    # If the new user opted to import settings from an existing account, and
+    # 'visibility_policy_banner' is already marked as read for the existing account,
+    # 'copy_onboarding_steps' function already did the needed copying.
+    # Simply ignore the IntegrityError in that case.
+    #
+    # The extremely brief nature of this subtransaction makes a savepoint safe.
+    # See https://postgres.ai/blog/20210831-postgresql-subtransactions-considered-harmful
+    # for context on risks around savepoints.
+    with suppress(IntegrityError), transaction.atomic(savepoint=True):
+        OnboardingStep.objects.create(user=user_profile, onboarding_step="visibility_policy_banner")
+
+
+def notify_created_user(user_profile: UserProfile, notify_user_ids: list[int]) -> None:
+    user_row = user_profile_to_user_row(user_profile)
+
+    format_user_row_kwargs: dict[str, Any] = {
+        "realm_id": user_profile.realm_id,
+        "row": user_row,
+        # Since we don't know what the client
+        # supports at this point in the code, we
+        # just assume client_gravatar and
+        # user_avatar_url_field_optional = False :(
+        "client_gravatar": False,
+        "user_avatar_url_field_optional": False,
+        # We assume there's no custom profile
+        # field data for a new user; initial
+        # values are expected to be added in a
+        # later event.
+        "custom_profile_field_data": {},
+    }
+
+    user_ids_without_access_to_created_user: list[int] = []
+    users_with_access_to_created_users: list[UserProfile] = []
+
+    if notify_user_ids:
+        # This is currently used to send creation event when a guest
+        # gains access to a user, so we depend on the caller to make
+        # sure that only accessible users receive the user data.
+        users_with_access_to_created_users = list(
+            user_profile.realm.get_active_users().filter(id__in=notify_user_ids)
+        )
+    else:
+        active_realm_users = list(user_profile.realm.get_active_users())
+
+        if user_access_restricted_in_realm(user_profile):
+            for user in active_realm_users:
+                if user.is_guest:
+                    # This logic assumes that can_access_all_users_group
+                    # setting can only be set to EVERYONE or MEMBERS.
+                    user_ids_without_access_to_created_user.append(user.id)
+                else:
+                    users_with_access_to_created_users.append(user)
+        else:
+            users_with_access_to_created_users = active_realm_users
+
+    user_ids_with_real_email_access = []
+    user_ids_without_real_email_access = []
+
+    person_for_real_email_access_users = None
+    person_for_without_real_email_access_users = None
+    for recipient_user in users_with_access_to_created_users:
+        if can_access_delivery_email(
+            recipient_user, user_profile.id, user_row["email_address_visibility"]
+        ):
+            user_ids_with_real_email_access.append(recipient_user.id)
+            if person_for_real_email_access_users is None:
+                # This caller assumes that "format_user_row" only depends on
+                # specific value of "acting_user" among users in a realm in
+                # email_address_visibility.
+                person_for_real_email_access_users = format_user_row(
+                    **format_user_row_kwargs,
+                    acting_user=recipient_user,
+                )
+        else:
+            user_ids_without_real_email_access.append(recipient_user.id)
+            if person_for_without_real_email_access_users is None:
+                person_for_without_real_email_access_users = format_user_row(
+                    **format_user_row_kwargs,
+                    acting_user=recipient_user,
+                )
+
+    if user_ids_with_real_email_access:
+        assert person_for_real_email_access_users is not None
+        event: dict[str, Any] = dict(
+            type="realm_user", op="add", person=person_for_real_email_access_users
+        )
+        send_event_on_commit(user_profile.realm, event, user_ids_with_real_email_access)
+
+    if user_ids_without_real_email_access:
+        assert person_for_without_real_email_access_users is not None
+        event = dict(type="realm_user", op="add", person=person_for_without_real_email_access_users)
+        send_event_on_commit(user_profile.realm, event, user_ids_without_real_email_access)
+
+    if user_ids_without_access_to_created_user:
+        event = dict(
+            type="realm_user",
+            op="add",
+            person=get_data_for_inaccessible_user(user_profile.realm, user_profile.id),
+            inaccessible_user=True,
+        )
+        send_event_on_commit(user_profile.realm, event, user_ids_without_access_to_created_user)
+
+
+def created_bot_event(user_profile: UserProfile) -> dict[str, Any]:
+    def stream_name(stream: Stream | None) -> str | None:
+        if not stream:
+            return None
+        return stream.name
+
+    default_sending_stream_name = stream_name(user_profile.default_sending_stream)
+    default_events_register_stream_name = stream_name(user_profile.default_events_register_stream)
+
+    bot = dict(
+        user_id=user_profile.id,
+        default_sending_stream=default_sending_stream_name,
+        default_events_register_stream=default_events_register_stream_name,
+        default_all_public_streams=user_profile.default_all_public_streams,
+        services=get_service_dicts_for_bot(user_profile.id),
+    )
+
+    return dict(type="realm_bot", op="add", bot=bot)
+
+
+def notify_created_bot(user_profile: UserProfile) -> None:
+    event = created_bot_event(user_profile)
+    send_event_on_commit(user_profile.realm, event, bot_owner_user_ids(user_profile))
+
+
+@transaction.atomic(savepoint=False)
+def do_create_user(
+    email: str,
+    password: str | None,
+    realm: Realm,
+    full_name: str,
+    bot_type: int | None = None,
+    role: int | None = None,
+    bot_owner: UserProfile | None = None,
+    tos_version: str | None = None,
+    timezone: str = "",
+    avatar_source: str | None = None,
+    default_language: str | None = None,
+    default_sending_stream: Stream | None = None,
+    default_events_register_stream: Stream | None = None,
+    default_all_public_streams: bool | None = None,
+    prereg_user: PreregistrationUser | None = None,
+    prereg_realm: PreregistrationRealm | None = None,
+    default_stream_groups: Sequence[DefaultStreamGroup] = [],
+    source_profile: UserProfile | None = None,
+    realm_creation: bool = False,
+    *,
+    acting_user: UserProfile | None,
+    enable_marketing_emails: bool = True,
+    email_address_visibility: int | None = None,
+    add_initial_stream_subscriptions: bool = True,
+    external_auth_id_dict: dict[str, str] | None = None,
+) -> UserProfile:
+    if settings.BILLING_ENABLED:
+        from corporate.lib.stripe import RealmBillingSession
+
+    user_profile = create_user(
+        email=email,
+        password=password,
+        realm=realm,
+        full_name=full_name,
+        role=role,
+        bot_type=bot_type,
+        bot_owner=bot_owner,
+        tos_version=tos_version,
+        timezone=timezone,
+        avatar_source=avatar_source,
+        default_language=default_language,
+        default_sending_stream=default_sending_stream,
+        default_events_register_stream=default_events_register_stream,
+        default_all_public_streams=default_all_public_streams,
+        source_profile=source_profile,
+        enable_marketing_emails=enable_marketing_emails,
+        email_address_visibility=email_address_visibility,
+    )
+
+    if user_profile.avatar_source == UserProfile.AVATAR_FROM_JDENTICON:
+        generate_and_upload_jdenticon_avatar(user_profile, str(realm.uuid), future=False)
+
+    event_time = user_profile.date_joined
+    if not acting_user:
+        acting_user = user_profile
+    if realm_creation:
+        # If this user just created a realm, make sure they are
+        # properly tagged as the creator of the realm.
+        realm_creation_audit_log = (
+            RealmAuditLog.objects.filter(event_type=AuditLogEventType.REALM_CREATED, realm=realm)
+            .order_by("id")
+            .last()
+        )
+        assert realm_creation_audit_log is not None
+        realm_creation_audit_log.acting_user = user_profile
+        realm_creation_audit_log.save(update_fields=["acting_user"])
+
+    system_user_group = get_system_user_group_for_user(user_profile)
+    UserGroupMembership.objects.create(user_profile=user_profile, user_group=system_user_group)
+    is_new_full_member = (
+        user_profile.role == UserProfile.ROLE_MEMBER and not user_profile.is_provisional_member
+    )
+    if is_new_full_member:
+        full_members_system_group = NamedUserGroup.objects.get(
+            name=SystemGroups.FULL_MEMBERS,
+            realm_for_sharding=user_profile.realm,
+            is_system_group=True,
+        )
+        UserGroupMembership.objects.create(
+            user_profile=user_profile, user_group=full_members_system_group
+        )
+
+    # These are recorded only once the system group memberships above
+    # have been created, because realm_user_count_by_role counts the
+    # workplace users from memberships of realm.workplace_users_group,
+    # for cases where we cannot compute it directly from role counts.
+    RealmAuditLog.objects.create(
+        realm=user_profile.realm,
+        acting_user=acting_user,
+        modified_user=user_profile,
+        event_type=AuditLogEventType.USER_CREATED,
+        event_time=event_time,
+        extra_data={
+            RealmAuditLog.ROLE_COUNT: realm_user_count_by_role(user_profile.realm),
+        },
+    )
+    RealmAuditLog.objects.create(
+        realm=user_profile.realm,
+        modified_user=user_profile,
+        modified_user_group=system_user_group,
+        event_type=AuditLogEventType.USER_GROUP_DIRECT_USER_MEMBERSHIP_ADDED,
+        event_time=event_time,
+        acting_user=acting_user,
+    )
+    if is_new_full_member:
+        RealmAuditLog.objects.create(
+            realm=user_profile.realm,
+            modified_user=user_profile,
+            modified_user_group=full_members_system_group,
+            event_type=AuditLogEventType.USER_GROUP_DIRECT_USER_MEMBERSHIP_ADDED,
+            event_time=event_time,
+            acting_user=acting_user,
+        )
+    maybe_enqueue_audit_log_upload(user_profile.realm)
+
+    if settings.BILLING_ENABLED:
+        billing_session = RealmBillingSession(user=user_profile, realm=user_profile.realm)
+        billing_session.update_license_ledger_if_needed(event_time)
+
+    # Note that for bots, the caller will send an additional event
+    # with bot-specific info like services.
+    notify_created_user(user_profile, [])
+
+    do_send_user_group_members_update_event("add_members", system_user_group, [user_profile.id])
+    if is_new_full_member:
+        do_send_user_group_members_update_event(
+            "add_members", full_members_system_group, [user_profile.id]
+        )
+
+    if external_auth_id_dict:
+        for (
+            external_auth_method_name,
+            external_auth_id,
+        ) in external_auth_id_dict.items():
+            ExternalAuthID.objects.create(
+                user=user_profile,
+                realm=user_profile.realm,
+                external_auth_method_name=external_auth_method_name,
+                external_auth_id=external_auth_id,
+            )
+
+    if prereg_realm is not None:
+        prereg_realm.created_user = user_profile
+        prereg_realm.save(update_fields=["created_user"])
+
+    if realm_creation:
+        from zerver.lib.onboarding import send_initial_realm_messages
+
+        with override_language(realm.default_language):
+            send_initial_realm_messages(realm)
+
+    if bot_type is None:
+        process_new_human_user(
+            user_profile,
+            prereg_user=prereg_user,
+            default_stream_groups=default_stream_groups,
+            realm_creation=realm_creation,
+            add_initial_stream_subscriptions=add_initial_stream_subscriptions,
+        )
+
+    return user_profile
+
+
+def do_activate_mirror_dummy_user(
+    user_profile: UserProfile, *, acting_user: UserProfile | None
+) -> None:
+    """Called to have a user "take over" a "mirror dummy" user
+    (i.e. is_mirror_dummy=True) account when they sign up with the
+    same email address.
+
+    Essentially, the result should be as though we had created the
+    UserProfile just now with do_create_user, except that the mirror
+    dummy user may appear as the recipient or sender of messages from
+    before their account was fully created.
+
+    TODO: This function likely has bugs resulting from this being a
+    parallel code path to do_create_user; e.g. it likely does not
+    handle preferences or default streams properly.
+    """
+    assert user_profile.is_mirror_dummy
+
+    if settings.BILLING_ENABLED:
+        from corporate.lib.stripe import RealmBillingSession
+
+    with transaction.atomic(savepoint=False):
+        change_user_is_active(user_profile, True)
+        user_profile.is_mirror_dummy = False
+        user_profile.set_unusable_password()
+        user_profile.date_joined = timezone_now()
+        user_profile.tos_version = settings.TERMS_OF_SERVICE_VERSION
+        user_profile.save(
+            update_fields=["date_joined", "password", "is_mirror_dummy", "tos_version"]
+        )
+
+        event_time = user_profile.date_joined
+        RealmAuditLog.objects.create(
+            realm=user_profile.realm,
+            modified_user=user_profile,
+            acting_user=acting_user,
+            event_type=AuditLogEventType.USER_ACTIVATED,
+            event_time=event_time,
+            extra_data={
+                RealmAuditLog.ROLE_COUNT: realm_user_count_by_role(user_profile.realm),
+            },
+        )
+        maybe_enqueue_audit_log_upload(user_profile.realm)
+        if settings.BILLING_ENABLED:
+            billing_session = RealmBillingSession(user=user_profile, realm=user_profile.realm)
+            billing_session.update_license_ledger_if_needed(event_time)
+
+    notify_created_user(user_profile, [])
+
+
+@transaction.atomic(savepoint=False)
+def do_reactivate_user(user_profile: UserProfile, *, acting_user: UserProfile | None) -> None:
+    """Reactivate a user that had previously been deactivated"""
+    if user_profile.is_deleted:
+        raise JsonableError(_("Cannot reactivate a deleted user"))
+
+    if user_profile.is_mirror_dummy:
+        raise JsonableError(
+            _("Cannot activate a placeholder account; ask the user to sign up, instead.")
+        )
+
+    change_user_is_active(user_profile, True)
+
+    event_time = timezone_now()
+    RealmAuditLog.objects.create(
+        realm=user_profile.realm,
+        modified_user=user_profile,
+        acting_user=acting_user,
+        event_type=AuditLogEventType.USER_REACTIVATED,
+        event_time=event_time,
+        extra_data={
+            RealmAuditLog.ROLE_COUNT: realm_user_count_by_role(user_profile.realm),
+        },
+    )
+    maybe_enqueue_audit_log_upload(user_profile.realm)
+
+    bot_owner_changed = False
+    if (
+        user_profile.is_bot
+        and user_profile.bot_owner is not None
+        and not user_profile.bot_owner.is_active
+        and acting_user is not None
+    ):
+        previous_owner = user_profile.bot_owner
+        user_profile.bot_owner = acting_user
+        user_profile.save()  # Can't use update_fields because of how the foreign key works.
+        RealmAuditLog.objects.create(
+            realm=user_profile.realm,
+            acting_user=acting_user,
+            modified_user=user_profile,
+            event_type=AuditLogEventType.USER_BOT_OWNER_CHANGED,
+            event_time=event_time,
+            extra_data={
+                RealmAuditLog.OLD_VALUE: previous_owner.id,
+                RealmAuditLog.NEW_VALUE: acting_user.id,
+            },
+        )
+        bot_owner_changed = True
+
+    if settings.BILLING_ENABLED:
+        from corporate.lib.stripe import RealmBillingSession
+
+        billing_session = RealmBillingSession(user=user_profile, realm=user_profile.realm)
+        billing_session.update_license_ledger_if_needed(event_time)
+
+    event = dict(
+        type="realm_user", op="update", person=dict(user_id=user_profile.id, is_active=True)
+    )
+    send_event_on_commit(user_profile.realm, event, get_user_ids_who_can_access_user(user_profile))
+
+    if not user_profile.is_bot:
+        realm_export_consent_event = dict(
+            type="realm_export_consent",
+            user_id=user_profile.id,
+            consented=user_profile.allow_private_data_export,
+        )
+        send_event_on_commit(
+            user_profile.realm,
+            realm_export_consent_event,
+            list(user_profile.realm.get_human_admin_users().values_list("id", flat=True)),
+        )
+
+    if user_profile.is_bot and bot_owner_changed:
+        from zerver.actions.bots import send_bot_owner_update_events
+
+        assert acting_user is not None
+        send_bot_owner_update_events(user_profile, acting_user, previous_owner)
+
+    subscribed_recipient_ids = Subscription.objects.filter(
+        user_profile_id=user_profile.id, active=True, recipient__type=Recipient.STREAM
+    ).values_list("recipient__type_id", flat=True)
+    subscribed_streams = Stream.objects.filter(id__in=subscribed_recipient_ids, deactivated=False)
+    subscriber_peer_info = bulk_get_subscriber_peer_info(
+        realm=user_profile.realm,
+        streams=subscribed_streams,
+    )
+
+    altered_user_dict: dict[int, set[int]] = defaultdict(set)
+    for stream in subscribed_streams:
+        altered_user_dict[stream.id] = {user_profile.id}
+
+    stream_dict = {stream.id: stream for stream in subscribed_streams}
+
+    send_peer_subscriber_events(
+        op="peer_add",
+        realm=user_profile.realm,
+        altered_user_dict=altered_user_dict,
+        stream_dict=stream_dict,
+        subscriber_peer_info=subscriber_peer_info,
+    )
+
+    member_user_groups = user_profile.direct_groups.select_related("named_user_group").order_by(
+        "id"
+    )
+    named_user_groups = []
+    setting_user_groups = []
+    for group in member_user_groups:
+        if hasattr(group, "named_user_group"):
+            named_user_groups.append(group)
+        else:
+            setting_user_groups.append(group)
+
+    for user_group in named_user_groups:
+        do_send_user_group_members_update_event(
+            "add_members", user_group.named_user_group, [user_profile.id]
+        )
+
+    if setting_user_groups:
+        notify_user_ids = active_user_ids(user_profile.realm_id)
+        send_update_events_for_anonymous_group_settings(
+            setting_user_groups, user_profile.realm, list(notify_user_ids)
+        )

zerver/actions/custom_profile_fields.py

@@ -0,0 +1,323 @@
+from collections.abc import Iterable
+
+import orjson
+from django.db import transaction
+from django.utils.translation import gettext as _
+
+from zerver.actions.message_send import send_user_profile_update_notification
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.external_accounts import DEFAULT_EXTERNAL_ACCOUNTS
+from zerver.lib.mention import silent_mention_syntax_for_user
+from zerver.lib.streams import render_stream_description
+from zerver.lib.types import ProfileDataElementUpdateDict, ProfileFieldData, UserProfileChangeDict
+from zerver.lib.users import get_user_ids_who_can_access_user
+from zerver.models import CustomProfileField, CustomProfileFieldValue, Realm, UserProfile
+from zerver.models.custom_profile_fields import custom_profile_fields_for_realm
+from zerver.models.users import active_user_ids
+from zerver.tornado.django_api import send_event_on_commit
+
+
+def notify_realm_custom_profile_fields(realm: Realm) -> None:
+    fields = custom_profile_fields_for_realm(realm.id)
+    event = dict(type="custom_profile_fields", fields=[f.as_dict() for f in fields])
+    send_event_on_commit(realm, event, active_user_ids(realm.id))
+
+
+@transaction.atomic(durable=True)
+def try_add_realm_default_custom_profile_field(
+    realm: Realm,
+    field_subtype: str,
+    display_in_profile_summary: bool = False,
+    required: bool = False,
+    editable_by_user: bool = True,
+    use_for_user_matching: bool = False,
+) -> CustomProfileField:
+    field_data = DEFAULT_EXTERNAL_ACCOUNTS[field_subtype]
+    custom_profile_field = CustomProfileField(
+        realm=realm,
+        name=str(field_data.name),
+        field_type=CustomProfileField.EXTERNAL_ACCOUNT,
+        hint=field_data.hint,
+        field_data=orjson.dumps(dict(subtype=field_subtype)).decode(),
+        display_in_profile_summary=display_in_profile_summary,
+        required=required,
+        editable_by_user=editable_by_user,
+        use_for_user_matching=use_for_user_matching,
+    )
+    custom_profile_field.save()
+    custom_profile_field.order = custom_profile_field.id
+    custom_profile_field.save(update_fields=["order"])
+    notify_realm_custom_profile_fields(realm)
+    return custom_profile_field
+
+
+@transaction.atomic(durable=True)
+def try_add_realm_custom_profile_field(
+    realm: Realm,
+    name: str,
+    field_type: int,
+    hint: str = "",
+    field_data: ProfileFieldData | None = None,
+    display_in_profile_summary: bool = False,
+    required: bool = False,
+    editable_by_user: bool = True,
+    use_for_user_matching: bool = False,
+) -> CustomProfileField:
+    custom_profile_field = CustomProfileField(
+        realm=realm,
+        name=name,
+        field_type=field_type,
+        display_in_profile_summary=display_in_profile_summary,
+        required=required,
+        editable_by_user=editable_by_user,
+        use_for_user_matching=use_for_user_matching,
+    )
+    custom_profile_field.hint = hint
+    if custom_profile_field.field_type in (
+        CustomProfileField.DROPDOWN,
+        CustomProfileField.EXTERNAL_ACCOUNT,
+    ):
+        custom_profile_field.field_data = orjson.dumps(field_data or {}).decode()
+
+    custom_profile_field.save()
+    custom_profile_field.order = custom_profile_field.id
+    custom_profile_field.save(update_fields=["order"])
+    notify_realm_custom_profile_fields(realm)
+    return custom_profile_field
+
+
+@transaction.atomic(durable=True)
+def do_remove_realm_custom_profile_field(realm: Realm, field: CustomProfileField) -> None:
+    """
+    Deleting a field will also delete the user profile data
+    associated with it in CustomProfileFieldValue model.
+    """
+    field.delete()
+    notify_realm_custom_profile_fields(realm)
+
+
+def do_remove_realm_custom_profile_fields(realm: Realm) -> None:
+    CustomProfileField.objects.filter(realm=realm).delete()
+
+
+def remove_custom_profile_field_value_if_required(
+    field: CustomProfileField, field_data: ProfileFieldData
+) -> None:
+    old_values = set(orjson.loads(field.field_data).keys())
+    new_values = set(field_data.keys())
+    removed_values = old_values - new_values
+
+    if not removed_values:
+        return
+
+    values_to_delete = CustomProfileFieldValue.objects.filter(
+        field=field, value__in=removed_values
+    ).select_related("user_profile")
+
+    updated_users = [field_value.user_profile for field_value in values_to_delete]
+
+    values_to_delete.delete()
+
+    for user_profile in updated_users:
+        notify_user_update_custom_profile_data(
+            user_profile,
+            {
+                "id": field.id,
+                "value": None,
+                "rendered_value": None,
+                "type": field.field_type,
+            },
+        )
+
+
+@transaction.atomic(durable=True)
+def try_update_realm_custom_profile_field(
+    realm: Realm,
+    field: CustomProfileField,
+    name: str | None = None,
+    hint: str | None = None,
+    field_data: ProfileFieldData | None = None,
+    display_in_profile_summary: bool | None = None,
+    required: bool | None = None,
+    editable_by_user: bool | None = None,
+    use_for_user_matching: bool | None = None,
+) -> None:
+    if name is not None:
+        field.name = name
+    if hint is not None:
+        field.hint = hint
+    if required is not None:
+        field.required = required
+    if editable_by_user is not None:
+        field.editable_by_user = editable_by_user
+    if display_in_profile_summary is not None:
+        field.display_in_profile_summary = display_in_profile_summary
+    if use_for_user_matching is not None:
+        field.use_for_user_matching = use_for_user_matching
+
+    if field.field_type in (
+        CustomProfileField.DROPDOWN,
+        CustomProfileField.EXTERNAL_ACCOUNT,
+    ):
+        # If field_data is None, field_data is unchanged and there is no need for
+        # comparing field_data values.
+        if field_data is not None and field.field_type == CustomProfileField.DROPDOWN:
+            remove_custom_profile_field_value_if_required(field, field_data)
+
+        # If field.field_data is the default empty string, we will set field_data
+        # to an empty dict.
+        if field_data is not None or field.field_data == "":
+            field.field_data = orjson.dumps(field_data or {}).decode()
+    field.save()
+    notify_realm_custom_profile_fields(realm)
+
+
+@transaction.atomic(durable=True)
+def try_reorder_realm_custom_profile_fields(realm: Realm, order: Iterable[int]) -> None:
+    order_mapping = {_[1]: _[0] for _ in enumerate(order)}
+    custom_profile_fields = CustomProfileField.objects.filter(realm=realm)
+    for custom_profile_field in custom_profile_fields:
+        if custom_profile_field.id not in order_mapping:
+            raise JsonableError(_("Invalid order mapping."))
+    for custom_profile_field in custom_profile_fields:
+        custom_profile_field.order = order_mapping[custom_profile_field.id]
+        custom_profile_field.save(update_fields=["order"])
+    notify_realm_custom_profile_fields(realm)
+
+
+def notify_user_update_custom_profile_data(
+    user_profile: UserProfile, field: dict[str, int | str | list[int] | None]
+) -> None:
+    data = dict(id=field["id"], value=field["value"])
+
+    if field["rendered_value"]:
+        data["rendered_value"] = field["rendered_value"]
+    payload = dict(user_id=user_profile.id, custom_profile_field=data)
+    event = dict(type="realm_user", op="update", person=payload)
+    send_event_on_commit(user_profile.realm, event, get_user_ids_who_can_access_user(user_profile))
+
+
+@transaction.atomic(savepoint=False)
+def do_update_user_custom_profile_data_if_changed(
+    user_profile: UserProfile,
+    data: list[ProfileDataElementUpdateDict],
+    acting_user: UserProfile | None,
+    notify: bool,
+) -> None:
+    changes: list[UserProfileChangeDict] = []
+
+    for custom_profile_field in data:
+        field_value, created = CustomProfileFieldValue.objects.get_or_create(
+            user_profile=user_profile, field_id=custom_profile_field["id"]
+        )
+
+        # field_value.value is a TextField() so we need to have field["value"]
+        # in string form to correctly make comparisons and assignments.
+        if isinstance(custom_profile_field["value"], str):
+            custom_profile_field_value_string = custom_profile_field["value"]
+        else:
+            custom_profile_field_value_string = orjson.dumps(custom_profile_field["value"]).decode()
+
+        if not created and field_value.value == custom_profile_field_value_string:
+            # If the field value isn't actually being changed to a different one,
+            # we have nothing to do here for this field.
+            continue
+
+        old_value = get_custom_profile_field_display_value(field_value)
+
+        field_value.value = custom_profile_field_value_string
+        if field_value.field.is_renderable():
+            field_value.rendered_value = render_stream_description(
+                custom_profile_field_value_string, user_profile.realm
+            )
+            field_value.save(update_fields=["value", "rendered_value"])
+        else:
+            field_value.save(update_fields=["value"])
+        notify_user_update_custom_profile_data(
+            user_profile,
+            {
+                "id": field_value.field_id,
+                "value": field_value.value,
+                "rendered_value": field_value.rendered_value,
+                "type": field_value.field.field_type,
+            },
+        )
+
+        new_value = get_custom_profile_field_display_value(field_value)
+
+        changes.append(
+            UserProfileChangeDict(
+                field_name=field_value.field.name,
+                old_value=old_value,
+                new_value=new_value,
+            )
+        )
+
+    if changes and notify:
+        send_user_profile_update_notification(
+            user_profile=user_profile, acting_user=acting_user, changes=changes
+        )
+
+
+def get_custom_profile_field_display_value(field_value: CustomProfileFieldValue) -> str:
+    """Convert custom profile field value to human-readable string based on field type."""
+    if not field_value.value:
+        return ""
+    type = field_value.field.field_type
+
+    if type == CustomProfileField.DROPDOWN:
+        field_data_dict = orjson.loads(field_value.field.field_data)
+        value_key = field_value.value
+        return field_data_dict[value_key]["text"]
+
+    if type == CustomProfileField.USER:
+        user_ids = orjson.loads(field_value.value)
+        users = UserProfile.objects.filter(id__in=user_ids).only("id", "full_name")
+        users_mentions = [silent_mention_syntax_for_user(user) for user in users]
+        return (", ").join(users_mentions)
+
+    return field_value.value
+
+
+@transaction.atomic(savepoint=False)
+def check_remove_custom_profile_field_value(
+    user_profile: UserProfile, field_id: int, acting_user: UserProfile, notify: bool
+) -> None:
+    try:
+        custom_profile_field = CustomProfileField.objects.get(realm=user_profile.realm, id=field_id)
+        if not acting_user.is_realm_admin and not custom_profile_field.editable_by_user:
+            raise JsonableError(
+                _(
+                    "You are not allowed to change this field. Contact an administrator to update it."
+                )
+            )
+
+        field_value = CustomProfileFieldValue.objects.get(
+            field=custom_profile_field, user_profile=user_profile
+        )
+        old_value = get_custom_profile_field_display_value(field_value)
+        field_value.delete()
+        notify_user_update_custom_profile_data(
+            user_profile,
+            {
+                "id": field_id,
+                "value": None,
+                "rendered_value": None,
+                "type": custom_profile_field.field_type,
+            },
+        )
+        if notify:
+            changes: list[UserProfileChangeDict] = [
+                UserProfileChangeDict(
+                    field_name=custom_profile_field.name,
+                    old_value=old_value,
+                    new_value="",
+                )
+            ]
+            send_user_profile_update_notification(
+                user_profile=user_profile, acting_user=acting_user, changes=changes
+            )
+    except CustomProfileField.DoesNotExist:
+        raise JsonableError(_("Field id {id} not found.").format(id=field_id))
+    except CustomProfileFieldValue.DoesNotExist:
+        pass

zerver/actions/data_import.py

@@ -0,0 +1,235 @@
+import glob
+import logging
+import os
+import shutil
+import tempfile
+from datetime import timedelta
+from typing import Any
+
+from django.conf import settings
+from django.db import IntegrityError
+from django.utils.timezone import now as timezone_now
+
+from confirmation import settings as confirmation_settings
+from zerver.actions.create_realm import get_email_address_visibility_default
+from zerver.actions.create_user import do_reactivate_user
+from zerver.actions.realm_settings import do_delete_realm
+from zerver.actions.users import do_change_user_role
+from zerver.context_processors import is_realm_import_enabled
+from zerver.data_import.slack import do_convert_zipfile
+from zerver.lib.exceptions import SlackImportInvalidFileError
+from zerver.lib.import_realm import do_import_realm
+from zerver.lib.upload import save_attachment_contents
+from zerver.models.prereg_users import PreregistrationRealm
+from zerver.models.realms import Realm, get_realm
+from zerver.models.users import RealmUserDefault, UserProfile, get_user_by_delivery_email
+
+logger = logging.getLogger("zulip.registration")
+
+
+def is_string_id_unique_violation(e: BaseException) -> bool:
+    """True when e is the IntegrityError from violating the unique constraint
+    on Realm.string_id -- i.e. the subdomain is already taken by another
+    realm. Other IntegrityErrors (e.g. genuine bugs elsewhere in the import)
+    return False, so they get the normal failure handling rather than being
+    mistaken for a taken subdomain."""
+    if not isinstance(e, IntegrityError):
+        return False
+    # Django wraps the underlying psycopg2 error as __cause__; its
+    # diagnostics carry the Postgres constraint name.
+    diag = getattr(e.__cause__, "diag", None)
+    return getattr(diag, "constraint_name", None) == "zerver_realm_string_id_key"
+
+
+def import_slack_data(event: dict[str, Any]) -> None:
+    # This is only possible if data imports were enqueued before the
+    # setting was turned off.
+    assert is_realm_import_enabled()
+
+    preregistration_realm = PreregistrationRealm.objects.get(id=event["preregistration_realm_id"])
+    string_id = preregistration_realm.string_id
+
+    if preregistration_realm.created_realm is not None:
+        # A prior delivery of this event already finished the import. The
+        # worker can die after a successful import but before the event is
+        # acked, which redelivers the event; there is nothing left to do.
+        logger.info("(%s) Slack import already completed; skipping redelivery", string_id)
+        return
+
+    try:
+        existing_realm: Realm | None = get_realm(string_id)
+    except Realm.DoesNotExist:
+        existing_realm = None
+    if existing_realm is not None:
+        if preregistration_realm.data_import_metadata.get("import_created_realm_id") == (
+            existing_realm.id
+        ):
+            # A prior attempt of *this* import created the realm and then the
+            # worker died before finishing, and the event was redelivered.
+            # The realm is our own half-imported orphan, so delete it and
+            # re-import below, rather than treating it as a foreign conflict.
+            logger.warning(
+                "(%s) Cleaning up our own orphaned realm from a prior import attempt", string_id
+            )
+            orphan = Realm.objects.get(id=existing_realm.id)
+            assert orphan.date_created >= timezone_now() - timedelta(hours=24)
+            do_delete_realm(orphan)
+        else:
+            # The subdomain is taken by an earlier or concurrent import of the
+            # same subdomain that won the race to create the realm (string_id
+            # is unique). Importing would fail on that unique constraint, and
+            # we must not touch the other import's realm, so abort before the
+            # expensive conversion work and surface the conflict to the user.
+            logger.error("(%s) Aborting Slack import: subdomain is already in use", string_id)
+            preregistration_realm.data_import_metadata["is_import_work_queued"] = False
+            preregistration_realm.data_import_metadata["subdomain_unavailable"] = True
+            preregistration_realm.save(update_fields=["data_import_metadata"])
+            return
+
+    output_dir = tempfile.mkdtemp(
+        prefix=f"import-{preregistration_realm.id}-converted-",
+        dir=settings.IMPORT_TMPFILE_DIRECTORY,
+    )
+    logger.info(
+        "(%s) Starting Slack import from %s / %s",
+        string_id,
+        event["filename"],
+        event["slack_access_token"],
+    )
+
+    try:
+        with tempfile.NamedTemporaryFile(
+            prefix=f"import-{preregistration_realm.id}-slack-",
+            suffix=".zip",
+            dir=settings.IMPORT_TMPFILE_DIRECTORY,
+        ) as fh:
+            save_attachment_contents(event["filename"], fh)
+            fh.flush()
+            logger.info(
+                "(%s) Wrote %d bytes of %s to %s", string_id, fh.tell(), event["filename"], fh.name
+            )
+            do_convert_zipfile(
+                fh.name,
+                output_dir,
+                event["slack_access_token"],
+                convert_slack_threads=True,
+            )
+            attachment_size = sum(
+                os.path.getsize(os.path.join(dirpath, filename))
+                for dirpath, dirnames, filenames in os.walk(f"{output_dir}/uploads/")
+                for filename in filenames
+            )
+            message_count = len(glob.glob(f"{output_dir}/messages-*.json")) * 1000
+            logger.info(
+                "(%s) Completed conversion into %s (%d MB attachments, <= %d messages)",
+                string_id,
+                output_dir,
+                attachment_size / 1024 / 1024,
+                message_count,
+            )
+
+            def record_created_realm(new_realm: Realm) -> None:
+                # Stamp the registration with the realm it owns, atomically
+                # with the realm's creation, so a redelivered import (after a
+                # crash) can recognize and clean up its own half-imported realm
+                # instead of mistaking it for a foreign conflict.
+                preregistration_realm.data_import_metadata["import_created_realm_id"] = new_realm.id
+                preregistration_realm.save(update_fields=["data_import_metadata"])
+
+            realm = do_import_realm(output_dir, string_id, on_realm_created=record_created_realm)
+            logger.info("(%s) Completed import, performing post-import steps", string_id)
+            realm.org_type = preregistration_realm.org_type
+            realm.default_language = preregistration_realm.default_language
+            realm.save()
+
+            realm_user_default = RealmUserDefault.objects.get(realm=realm)
+            realm_user_default.email_address_visibility = (
+                preregistration_realm.data_import_metadata.get(
+                    "email_address_visibility",
+                    get_email_address_visibility_default(realm.org_type),
+                )
+            )
+            realm_user_default.save(update_fields=["email_address_visibility"])
+
+            # Set email address visibility for all users in the realm.
+            UserProfile.objects.filter(realm=realm, is_bot=False).update(
+                email_address_visibility=realm_user_default.email_address_visibility
+            )
+
+            # Try finding the user who imported this realm and make them owner.
+            try:
+                importing_user: UserProfile | None = get_user_by_delivery_email(
+                    preregistration_realm.email, realm
+                )
+            except UserProfile.DoesNotExist:
+                importing_user = None
+
+            if importing_user is None or importing_user.is_bot or importing_user.is_mirror_dummy:
+                # The email address that the importing user validated
+                # with Zulip either does not appear in the data export,
+                # or maps to an account that cannot own the
+                # organization (a bot or a placeholder/mirror-dummy
+                # account). Prompt them to pick which account is theirs.
+                preregistration_realm.data_import_metadata["need_select_realm_owner"] = True
+            else:
+                if not importing_user.is_active:
+                    # The importer's account was deactivated in the
+                    # export; reactivate it so they can own the realm.
+                    do_reactivate_user(importing_user, acting_user=None)
+                if importing_user.role != UserProfile.ROLE_REALM_OWNER:
+                    do_change_user_role(
+                        importing_user,
+                        UserProfile.ROLE_REALM_OWNER,
+                        acting_user=importing_user,
+                        notify=False,
+                    )
+                preregistration_realm.status = confirmation_settings.STATUS_USED
+
+            preregistration_realm.created_realm = realm
+            preregistration_realm.data_import_metadata["is_import_work_queued"] = False
+            preregistration_realm.save()
+            logger.info("(%s) All done!", string_id)
+    except Exception as e:
+        logger.exception("Error during Slack import")
+        try:
+            preregistration_realm.created_realm = None
+            preregistration_realm.data_import_metadata["is_import_work_queued"] = False
+            if type(e) is SlackImportInvalidFileError:
+                # Store the error to be displayed to the user.
+                preregistration_realm.data_import_metadata["invalid_file_error_message"] = str(e)
+
+            # do_import_realm creates the realm with this string_id. If a
+            # concurrent or earlier import of the same subdomain won the
+            # race to create it, our realm insert violates the unique
+            # constraint on Realm.string_id and do_import_realm raises that
+            # IntegrityError. The existing realm belongs to the other
+            # import, so we must not clean it up: deleting it would destroy
+            # their data and can deadlock against their in-progress work on
+            # zerver_userprofile. We record the conflict for the status poll
+            # to surface instead.
+            subdomain_taken = is_string_id_unique_violation(e)
+            if subdomain_taken:
+                preregistration_realm.data_import_metadata["subdomain_unavailable"] = True
+            preregistration_realm.save()
+
+            # For any other failure -- including IntegrityErrors from
+            # genuine bugs on some other constraint -- delete the realm
+            # this import created, if it got that far. A failure before
+            # do_import_realm creates the realm (e.g. an invalid-file error
+            # during conversion) leaves no such realm, so the get() below
+            # raises Realm.DoesNotExist, which we swallow. Otherwise the
+            # realm with this string_id is the (possibly half-imported) one
+            # we created. The age assertion is a last line of defense
+            # against a bug ever pointing this deletion at an established
+            # realm; a Slack import's realm is created at conversion time,
+            # so it is always far younger than a day by the time we get
+            # here.
+            if not subdomain_taken:
+                realm = Realm.objects.get(string_id=string_id)
+                assert realm.date_created >= timezone_now() - timedelta(hours=24)
+                do_delete_realm(realm)
+        except Realm.DoesNotExist:
+            pass
+        raise
+    finally:
+        shutil.rmtree(output_dir)

zerver/actions/default_streams.py

@@ -0,0 +1,196 @@
+from collections.abc import Iterable
+from typing import Any
+
+from django.db import transaction
+from django.utils.translation import gettext as _
+
+from zerver.lib.default_streams import get_default_stream_ids_for_realm
+from zerver.lib.exceptions import JsonableError
+from zerver.models import DefaultStream, DefaultStreamGroup, Realm, Stream
+from zerver.models.streams import get_default_stream_groups
+from zerver.models.users import active_non_guest_user_ids
+from zerver.tornado.django_api import send_event_on_commit
+
+
+def check_default_stream_group_name(group_name: str) -> None:
+    if group_name.strip() == "":
+        raise JsonableError(
+            _("Invalid default channel group name '{group_name}'").format(group_name=group_name)
+        )
+    if len(group_name) > DefaultStreamGroup.MAX_NAME_LENGTH:
+        raise JsonableError(
+            _("Default channel group name too long (limit: {max_length} characters)").format(
+                max_length=DefaultStreamGroup.MAX_NAME_LENGTH,
+            )
+        )
+    for i in group_name:
+        if ord(i) == 0:
+            raise JsonableError(
+                _(
+                    "Default channel group name '{group_name}' contains NULL (0x00) characters."
+                ).format(
+                    group_name=group_name,
+                )
+            )
+
+
+def lookup_default_stream_groups(
+    default_stream_group_names: list[str], realm: Realm
+) -> list[DefaultStreamGroup]:
+    groups_by_name = {
+        group.name: group
+        for group in DefaultStreamGroup.objects.filter(
+            realm=realm, name__in=default_stream_group_names
+        )
+    }
+    default_stream_groups = []
+    for group_name in default_stream_group_names:
+        if group_name not in groups_by_name:
+            raise JsonableError(
+                _("Invalid default channel group {group_name}").format(group_name=group_name)
+            )
+        default_stream_groups.append(groups_by_name[group_name])
+    return default_stream_groups
+
+
+def notify_default_streams(realm: Realm) -> None:
+    event = dict(
+        type="default_streams",
+        default_streams=list(get_default_stream_ids_for_realm(realm.id)),
+    )
+    send_event_on_commit(realm, event, active_non_guest_user_ids(realm.id))
+
+
+def notify_default_stream_groups(realm: Realm) -> None:
+    event = dict(
+        type="default_stream_groups",
+        default_stream_groups=default_stream_groups_to_dicts_sorted(
+            get_default_stream_groups(realm)
+        ),
+    )
+    send_event_on_commit(realm, event, active_non_guest_user_ids(realm.id))
+
+
+def do_add_default_stream(stream: Stream) -> None:
+    realm_id = stream.realm_id
+    stream_id = stream.id
+    if not DefaultStream.objects.filter(realm_id=realm_id, stream_id=stream_id).exists():
+        DefaultStream.objects.create(realm_id=realm_id, stream_id=stream_id)
+        notify_default_streams(stream.realm)
+
+
+@transaction.atomic(savepoint=False)
+def do_remove_default_stream(stream: Stream) -> None:
+    realm_id = stream.realm_id
+    stream_id = stream.id
+    DefaultStream.objects.filter(realm_id=realm_id, stream_id=stream_id).delete()
+    notify_default_streams(stream.realm)
+
+
+def do_create_default_stream_group(
+    realm: Realm, group_name: str, description: str, streams: list[Stream]
+) -> None:
+    default_stream_ids = get_default_stream_ids_for_realm(realm.id)
+    for stream in streams:
+        if stream.id in default_stream_ids:
+            raise JsonableError(
+                _(
+                    "'{channel_name}' is a default channel and cannot be added to '{group_name}'",
+                ).format(channel_name=stream.name, group_name=group_name)
+            )
+
+    check_default_stream_group_name(group_name)
+    (group, created) = DefaultStreamGroup.objects.get_or_create(
+        name=group_name, realm=realm, description=description
+    )
+    if not created:
+        raise JsonableError(
+            _(
+                "Default channel group '{group_name}' already exists",
+            ).format(group_name=group_name)
+        )
+
+    group.streams.set(streams)
+    notify_default_stream_groups(realm)
+
+
+def do_add_streams_to_default_stream_group(
+    realm: Realm, group: DefaultStreamGroup, streams: list[Stream]
+) -> None:
+    default_stream_ids = get_default_stream_ids_for_realm(realm.id)
+    for stream in streams:
+        if stream.id in default_stream_ids:
+            raise JsonableError(
+                _(
+                    "'{channel_name}' is a default channel and cannot be added to '{group_name}'",
+                ).format(channel_name=stream.name, group_name=group.name)
+            )
+        if stream in group.streams.all():
+            raise JsonableError(
+                _(
+                    "Channel '{channel_name}' is already present in default channel group '{group_name}'",
+                ).format(channel_name=stream.name, group_name=group.name)
+            )
+        group.streams.add(stream)
+
+    group.save()
+    notify_default_stream_groups(realm)
+
+
+def do_remove_streams_from_default_stream_group(
+    realm: Realm, group: DefaultStreamGroup, streams: list[Stream]
+) -> None:
+    group_stream_ids = {stream.id for stream in group.streams.all()}
+    for stream in streams:
+        if stream.id not in group_stream_ids:
+            raise JsonableError(
+                _(
+                    "Channel '{channel_name}' is not present in default channel group '{group_name}'",
+                ).format(channel_name=stream.name, group_name=group.name)
+            )
+
+    delete_stream_ids = {stream.id for stream in streams}
+
+    group.streams.remove(*delete_stream_ids)
+    notify_default_stream_groups(realm)
+
+
+def do_change_default_stream_group_name(
+    realm: Realm, group: DefaultStreamGroup, new_group_name: str
+) -> None:
+    if group.name == new_group_name:
+        raise JsonableError(
+            _("This default channel group is already named '{group_name}'").format(
+                group_name=new_group_name
+            )
+        )
+
+    if DefaultStreamGroup.objects.filter(name=new_group_name, realm=realm).exists():
+        raise JsonableError(
+            _("Default channel group '{group_name}' already exists").format(
+                group_name=new_group_name
+            )
+        )
+
+    group.name = new_group_name
+    group.save()
+    notify_default_stream_groups(realm)
+
+
+def do_change_default_stream_group_description(
+    realm: Realm, group: DefaultStreamGroup, new_description: str
+) -> None:
+    group.description = new_description
+    group.save()
+    notify_default_stream_groups(realm)
+
+
+def do_remove_default_stream_group(realm: Realm, group: DefaultStreamGroup) -> None:
+    group.delete()
+    notify_default_stream_groups(realm)
+
+
+def default_stream_groups_to_dicts_sorted(
+    groups: Iterable[DefaultStreamGroup],
+) -> list[dict[str, Any]]:
+    return sorted((group.to_dict() for group in groups), key=lambda elt: elt["name"])

zerver/actions/devices.py

@@ -0,0 +1,27 @@
+from zerver.lib.devices import check_device_id
+from zerver.models.devices import Device
+from zerver.models.users import UserProfile
+from zerver.tornado.django_api import send_event_on_commit
+
+
+def do_register_device(user_profile: UserProfile) -> int:
+    device = Device.objects.create(user=user_profile)
+    event = dict(
+        type="device",
+        op="add",
+        device_id=device.id,
+    )
+    send_event_on_commit(user_profile.realm, event, [user_profile.id])
+    return device.id
+
+
+def do_remove_device(user_profile: UserProfile, device_id: int) -> None:
+    device = check_device_id(device_id, user_profile.id)
+    device.delete()
+
+    event = dict(
+        type="device",
+        op="remove",
+        device_id=device_id,
+    )
+    send_event_on_commit(user_profile.realm, event, [user_profile.id])

zerver/actions/invites.py

@@ -0,0 +1,580 @@
+import logging
+from collections.abc import Collection, Sequence
+from dataclasses import asdict, dataclass
+from datetime import datetime, timedelta
+from email.utils import format_datetime as email_format_datetime
+from typing import Any
+
+from django.conf import settings
+from django.contrib.contenttypes.models import ContentType
+from django.db import transaction
+from django.db.models import Q, QuerySet, Sum
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+from zxcvbn import zxcvbn
+
+from analytics.lib.counts import COUNT_STATS, do_increment_logging_stat
+from analytics.models import RealmCount
+from confirmation import settings as confirmation_settings
+from confirmation.models import (
+    Confirmation,
+    confirmation_url_for,
+    create_confirmation_link,
+    create_confirmation_object,
+)
+from zerver.context_processors import common_context
+from zerver.lib.email_validation import (
+    get_existing_user_errors,
+    get_realm_email_validator,
+    validate_email_is_valid,
+)
+from zerver.lib.exceptions import InvitationError
+from zerver.lib.invites import notify_invites_changed
+from zerver.lib.queue import queue_event_on_commit
+from zerver.lib.send_email import FromAddress, clear_scheduled_invitation_emails, send_future_email
+from zerver.lib.timestamp import datetime_to_timestamp
+from zerver.lib.types import Invitee
+from zerver.lib.utils import assert_is_not_none
+from zerver.models import (
+    Message,
+    MultiuseInvite,
+    NamedUserGroup,
+    PreregistrationUser,
+    Realm,
+    RealmAuditLog,
+    Stream,
+    UserProfile,
+)
+from zerver.models.prereg_users import filter_to_valid_prereg_users
+from zerver.models.realm_audit_logs import AuditLogEventType
+
+
+def estimate_recent_invites(realms: Collection[Realm] | QuerySet[Realm], *, days: int) -> int:
+    """An upper bound on the number of invites sent in the last `days` days"""
+    recent_invites = RealmCount.objects.filter(
+        realm__in=realms,
+        property="invites_sent::day",
+        # It's important to filter on this even though the count
+        # doesn't use subgroup, so that we use the index.
+        subgroup=None,
+        end_time__gte=timezone_now() - timedelta(days=days),
+    ).aggregate(Sum("value"))["value__sum"]
+    if recent_invites is None:
+        return 0
+    return recent_invites
+
+
+def too_many_recent_realm_invites(realm: Realm, num_invitees: int) -> bool:
+    # Basic check that we're blow the realm-set limit
+    recent_invites = estimate_recent_invites([realm], days=1)
+    if num_invitees + recent_invites > realm.max_invites:
+        return True
+
+    if realm.plan_type != Realm.PLAN_TYPE_LIMITED:
+        return False
+    if realm.max_invites != settings.INVITES_DEFAULT_REALM_DAILY_MAX:
+        return False
+
+    # If they're a non-paid plan with default invitation limits,
+    # we further limit how many invitations can be sent in a day
+    # as a function of how many current users they have. The
+    # allowed ratio has some heuristics to lock down likely-spammy
+    # realms.  This ratio likely only matters for the first
+    # handful of invites; if those users accept, then the realm is
+    # unlikely to hit these limits.  If a real realm hits them,
+    # the resulting message suggests that they contact support if
+    # they have a real use case.
+    warning_flags = []
+    if zxcvbn(realm.string_id)["score"] == 4:
+        # Very high entropy realm names are suspicious
+        warning_flags.append("random-realm-name")
+
+    if not realm.description:
+        warning_flags.append("no-realm-description")
+
+    if realm.icon_source == Realm.ICON_FROM_GRAVATAR:
+        warning_flags.append("no-realm-icon")
+
+    if realm.date_created >= timezone_now() - timedelta(hours=1):
+        warning_flags.append("realm-created-in-last-hour")
+
+    current_user_count = UserProfile.objects.filter(
+        realm=realm, is_bot=False, is_active=True
+    ).count()
+    if current_user_count == 1:
+        warning_flags.append("only-one-user")
+
+    estimated_sent = RealmCount.objects.filter(
+        realm=realm, property="messages_sent:message_type:day"
+    ).aggregate(messages=Sum("value"))
+    if (
+        not estimated_sent["messages"]
+        # Only after we've done the rough-estimate check, take the
+        # time to do the exact check:
+        and not Message.objects.filter(
+            # Uses index: zerver_message_realm_sender_recipient (prefix)
+            realm=realm,
+            sender__is_bot=False,
+        ).exists()
+    ):
+        warning_flags.append("no-messages-sent")
+
+    if len(warning_flags) == 6:
+        permitted_ratio = 2
+    elif len(warning_flags) >= 3:
+        permitted_ratio = 3
+    else:
+        permitted_ratio = 5
+
+    ratio = (num_invitees + recent_invites) / current_user_count
+    logging.log(
+        logging.WARNING if ratio > permitted_ratio else logging.INFO,
+        "%s (!: %s) inviting %d more, have %d recent, but only %d current users.  Ratio %.1f, %d allowed",
+        realm.string_id,
+        ",".join(warning_flags),
+        num_invitees,
+        recent_invites,
+        current_user_count,
+        ratio,
+        permitted_ratio,
+    )
+    return ratio > permitted_ratio
+
+
+def check_invite_limit(realm: Realm, num_invitees: int) -> None:
+    """Discourage using invitation emails as a vector for carrying spam."""
+    msg = _(
+        "To protect users, Zulip limits the number of invitations you can send in one day. Because you have reached the limit, no invitations were sent."
+    )
+    if not settings.OPEN_REALM_CREATION:
+        return
+
+    if too_many_recent_realm_invites(realm, num_invitees):
+        raise InvitationError(
+            msg,
+            [],
+            sent_invitations=False,
+            daily_limit_reached=True,
+        )
+
+    default_max = settings.INVITES_DEFAULT_REALM_DAILY_MAX
+    newrealm_age = timedelta(days=settings.INVITES_NEW_REALM_DAYS)
+    if realm.date_created <= timezone_now() - newrealm_age:
+        # If this isn't a "newly-created" realm, we're done. The
+        # remaining code applies an aggregate limit across all
+        # "new" realms, to address sudden bursts of spam realms.
+        return
+
+    if realm.max_invites > default_max:
+        # If a user is on a realm where we've bumped up
+        # max_invites, then we exempt them from invite limits.
+        return
+
+    new_realms = Realm.objects.filter(
+        date_created__gte=timezone_now() - newrealm_age,
+        _max_invites__lte=default_max,
+    ).all()
+
+    for days, count in settings.INVITES_NEW_REALM_LIMIT_DAYS:
+        recent_invites = estimate_recent_invites(new_realms, days=days)
+        if num_invitees + recent_invites > count:
+            raise InvitationError(
+                msg,
+                [],
+                sent_invitations=False,
+                daily_limit_reached=True,
+            )
+
+
+@transaction.atomic(durable=True)
+def do_invite_users(
+    user_profile: UserProfile,
+    invitees: Collection[Invitee],
+    streams: Collection[Stream],
+    notify_referrer_on_join: bool = True,
+    user_groups: Collection[NamedUserGroup] = [],
+    *,
+    invite_expires_in_minutes: int | None,
+    include_realm_default_subscriptions: bool,
+    invite_as: int = PreregistrationUser.INVITE_AS["MEMBER"],
+    welcome_message_custom_text: str | None = None,
+) -> list[tuple[str, str, bool]]:
+    num_invites = len(invitees)
+
+    # Lock the realm, since we need to not race with other invitations
+    realm = Realm.objects.select_for_update(no_key=True).get(id=user_profile.realm_id)
+    check_invite_limit(realm, num_invites)
+
+    if settings.BILLING_ENABLED:
+        from corporate.lib.registration import check_spare_licenses_available_for_inviting_new_users
+
+        if invite_as == PreregistrationUser.INVITE_AS["GUEST_USER"]:
+            check_spare_licenses_available_for_inviting_new_users(
+                realm, extra_guests_count=num_invites
+            )
+        else:
+            check_spare_licenses_available_for_inviting_new_users(
+                realm, extra_non_guests_count=num_invites
+            )
+
+    if not realm.invite_required:
+        # Inhibit joining an open realm to send spam invitations.
+        min_age = timedelta(days=settings.INVITES_MIN_USER_AGE_DAYS)
+        if user_profile.date_joined > timezone_now() - min_age and not user_profile.is_realm_admin:
+            raise InvitationError(
+                _(
+                    "Your account is too new to send invites for this organization. "
+                    "Ask an organization admin, or a more experienced user."
+                ),
+                [],
+                sent_invitations=False,
+            )
+
+    good_invitees: set[Invitee] = set()
+    errors: list[tuple[str, str, bool]] = []
+    validate_email_allowed_in_realm = get_realm_email_validator(realm)
+    for invitee in invitees:
+        email_error = validate_email_is_valid(
+            invitee.email,
+            validate_email_allowed_in_realm,
+        )
+
+        if email_error:
+            errors.append((invitee.email, email_error, False))
+        else:
+            good_invitees.add(invitee)
+
+    """
+    good_invitees are invitee names and emails that look ok so far,
+    but we still need to make sure they're not
+    gonna conflict with existing users
+    """
+    error_dict = get_existing_user_errors(
+        realm, {invitee.email for invitee in good_invitees}, allow_inactive_mirror_dummies=True
+    )
+
+    skipped: list[tuple[str, str, bool]] = []
+    for email in error_dict:
+        msg, deactivated = error_dict[email]
+        skipped.append((email, msg, deactivated))
+        good_invitees = {item for item in good_invitees if item.email != email}
+
+    validated_invitees = list(good_invitees)
+
+    if errors:
+        raise InvitationError(
+            _("Some emails did not validate, so we didn't send any invitations."),
+            errors + skipped,
+            sent_invitations=False,
+        )
+
+    if skipped and len(skipped) == len(invitees):
+        # All e-mails were skipped, so we didn't actually invite anyone.
+        raise InvitationError(
+            _("We weren't able to invite anyone."), skipped, sent_invitations=False
+        )
+
+    # Now that we are past all the possible errors, we actually create
+    # the PreregistrationUser objects and trigger the email invitations.
+    for validated_invitee in validated_invitees:
+        # The logged in user is the referrer.
+        prereg_user = PreregistrationUser(
+            email=validated_invitee.email,
+            # We include a name if specified in the invitation request, but leave full_name_validated=False,
+            # so the receiving user can spell their name as they like after accepting the invitation.
+            full_name=validated_invitee.full_name,
+            referred_by=user_profile,
+            invited_as=invite_as,
+            realm=realm,
+            include_realm_default_subscriptions=include_realm_default_subscriptions,
+            notify_referrer_on_join=notify_referrer_on_join,
+            welcome_message_custom_text=welcome_message_custom_text,
+        )
+        prereg_user.save()
+        stream_ids = [stream.id for stream in streams]
+        prereg_user.streams.set(stream_ids)
+        groups_ids = [user_group.id for user_group in user_groups]
+        prereg_user.groups.set(groups_ids)
+
+        confirmation = create_confirmation_object(
+            prereg_user, Confirmation.INVITATION, validity_in_minutes=invite_expires_in_minutes
+        )
+        do_send_user_invite_email(
+            prereg_user,
+            confirmation=confirmation,
+            invite_expires_in_minutes=invite_expires_in_minutes,
+        )
+
+    notify_invites_changed(realm, changed_invite_referrer=user_profile)
+
+    return skipped
+
+
+def get_invitation_expiry_date(confirmation_obj: Confirmation) -> int | None:
+    expiry_date = confirmation_obj.expiry_date
+    if expiry_date is None:
+        return expiry_date
+    return datetime_to_timestamp(expiry_date)
+
+
+@dataclass
+class PreregistrationInviteData:
+    email: str
+    invited_by_user_id: int
+    invited: int
+    expiry_date: int | None
+    id: int
+    invited_as: int
+    is_multiuse: bool
+    notify_referrer_on_join: bool
+
+
+@dataclass
+class MultiuseInviteData:
+    invited_by_user_id: int
+    invited: int
+    expiry_date: int | None
+    id: int
+    link_url: str
+    invited_as: int
+    is_multiuse: bool
+
+
+def do_get_invites_controlled_by_user(user_profile: UserProfile) -> list[dict[str, Any]]:
+    """
+    Returns a list of dicts representing invitations that can be controlled by user_profile.
+    This isn't necessarily the same as all the invitations generated by the user, as administrators
+    can control also invitations that they did not themselves create.
+    """
+    if user_profile.is_realm_admin:
+        prereg_users = filter_to_valid_prereg_users(
+            PreregistrationUser.objects.filter(realm=user_profile.realm, referred_by__isnull=False)
+        )
+    else:
+        prereg_users = filter_to_valid_prereg_users(
+            PreregistrationUser.objects.filter(referred_by=user_profile)
+        )
+
+    invites: list[PreregistrationInviteData | MultiuseInviteData] = []
+
+    for invitee in prereg_users:
+        assert invitee.referred_by is not None
+        invites.append(
+            PreregistrationInviteData(
+                email=invitee.email,
+                invited_by_user_id=invitee.referred_by.id,
+                invited=datetime_to_timestamp(invitee.invited_at),
+                expiry_date=get_invitation_expiry_date(invitee.confirmation.get()),
+                id=invitee.id,
+                invited_as=invitee.invited_as,
+                is_multiuse=False,
+                notify_referrer_on_join=invitee.notify_referrer_on_join,
+            )
+        )
+
+    if user_profile.is_realm_admin:
+        multiuse_invite_ids = (
+            MultiuseInvite.objects.filter(realm=user_profile.realm)
+            .exclude(status=confirmation_settings.STATUS_REVOKED)
+            .values_list("id", flat=True)
+        )
+        multiuse_confirmation_objs = Confirmation.objects.filter(
+            realm=user_profile.realm,
+            type=Confirmation.MULTIUSE_INVITE,
+            content_type=ContentType.objects.get_for_model(MultiuseInvite),
+            object_id__in=multiuse_invite_ids,
+        ).filter(Q(expiry_date__gte=timezone_now()) | Q(expiry_date=None))
+    else:
+        multiuse_invite_ids = (
+            MultiuseInvite.objects.filter(referred_by=user_profile)
+            .exclude(status=confirmation_settings.STATUS_REVOKED)
+            .values_list("id", flat=True)
+        )
+        multiuse_confirmation_objs = Confirmation.objects.filter(
+            type=Confirmation.MULTIUSE_INVITE,
+            content_type=ContentType.objects.get_for_model(MultiuseInvite),
+            object_id__in=multiuse_invite_ids,
+        ).filter(Q(expiry_date__gte=timezone_now()) | Q(expiry_date=None))
+
+    for confirmation_obj in multiuse_confirmation_objs:
+        invite = confirmation_obj.content_object
+        assert invite is not None
+
+        assert invite.status != confirmation_settings.STATUS_REVOKED
+        invites.append(
+            MultiuseInviteData(
+                invited_by_user_id=invite.referred_by.id,
+                invited=datetime_to_timestamp(confirmation_obj.date_sent),
+                expiry_date=get_invitation_expiry_date(confirmation_obj),
+                id=invite.id,
+                link_url=confirmation_url_for(confirmation_obj),
+                invited_as=invite.invited_as,
+                is_multiuse=True,
+            )
+        )
+    return [asdict(invite) for invite in invites]
+
+
+@transaction.atomic(durable=True)
+def do_create_multiuse_invite_link(
+    referred_by: UserProfile,
+    invited_as: int,
+    invite_expires_in_minutes: int | None,
+    include_realm_default_subscriptions: bool,
+    streams: Sequence[Stream] = [],
+    user_groups: Sequence[NamedUserGroup] = [],
+    welcome_message_custom_text: str | None = None,
+) -> str:
+    realm = referred_by.realm
+    invite = MultiuseInvite.objects.create(
+        realm=realm,
+        referred_by=referred_by,
+        include_realm_default_subscriptions=include_realm_default_subscriptions,
+        welcome_message_custom_text=welcome_message_custom_text,
+    )
+    if streams:
+        invite.streams.set(streams)
+    if user_groups:
+        invite.groups.set(user_groups)
+    invite.invited_as = invited_as
+    invite.save()
+    notify_invites_changed(referred_by.realm, changed_invite_referrer=referred_by)
+    return create_confirmation_link(
+        invite, Confirmation.MULTIUSE_INVITE, validity_in_minutes=invite_expires_in_minutes
+    )
+
+
+@transaction.atomic(durable=True)
+def do_revoke_user_invite(
+    prereg_user: PreregistrationUser, *, acting_user: UserProfile | None = None
+) -> None:
+    email = prereg_user.email
+    realm = prereg_user.realm
+    assert realm is not None
+
+    prereg_user.status = confirmation_settings.STATUS_REVOKED
+    prereg_user.save(update_fields=["status"])
+    clear_scheduled_invitation_emails(email)
+
+    confirmation = Confirmation.objects.get(
+        type=Confirmation.INVITATION,
+        content_type=ContentType.objects.get_for_model(PreregistrationUser),
+        object_id=prereg_user.id,
+    )
+    # Defensive assert.
+    assert confirmation.content_object == prereg_user
+
+    RealmAuditLog.objects.create(
+        realm=realm,
+        event_type=AuditLogEventType.INVITATION_REVOKED,
+        event_time=timezone_now(),
+        acting_user=acting_user,
+        extra_data={
+            "confirmation_id": confirmation.id,
+            "invitation_type": Confirmation.INVITATION,
+            "invitation_object_id": prereg_user.id,
+        },
+    )
+
+    notify_invites_changed(realm, changed_invite_referrer=prereg_user.referred_by)
+
+
+@transaction.atomic(durable=True)
+def do_revoke_multi_use_invite(
+    multiuse_invite: MultiuseInvite, *, acting_user: UserProfile | None = None
+) -> None:
+    realm = multiuse_invite.referred_by.realm
+
+    multiuse_invite.status = confirmation_settings.STATUS_REVOKED
+    multiuse_invite.save(update_fields=["status"])
+
+    confirmation = Confirmation.objects.get(
+        type=Confirmation.MULTIUSE_INVITE,
+        content_type=ContentType.objects.get_for_model(MultiuseInvite),
+        object_id=multiuse_invite.id,
+    )
+    # Defensive assert.
+    assert confirmation.content_object == multiuse_invite
+
+    RealmAuditLog.objects.create(
+        realm=realm,
+        event_type=AuditLogEventType.INVITATION_REVOKED,
+        event_time=timezone_now(),
+        acting_user=acting_user,
+        extra_data={
+            "confirmation_id": confirmation.id,
+            "invitation_type": Confirmation.MULTIUSE_INVITE,
+            "invitation_object_id": multiuse_invite.id,
+        },
+    )
+
+    notify_invites_changed(realm, changed_invite_referrer=multiuse_invite.referred_by)
+
+
+@transaction.atomic(savepoint=False)
+def do_send_user_invite_email(
+    prereg_user: PreregistrationUser,
+    *,
+    confirmation: Confirmation | None = None,
+    event_time: datetime | None = None,
+    invite_expires_in_minutes: float | None = None,
+) -> None:
+    # Take a lock on the realm, so we can check for invitation limits without races
+    realm_id = assert_is_not_none(prereg_user.realm_id)
+    realm = Realm.objects.select_for_update(no_key=True).get(id=realm_id)
+    check_invite_limit(realm, 1)
+    referrer = assert_is_not_none(prereg_user.referred_by)
+
+    if event_time is None:
+        event_time = prereg_user.invited_at
+    do_increment_logging_stat(realm, COUNT_STATS["invites_sent::day"], None, event_time)
+
+    if confirmation is None:
+        confirmation = prereg_user.confirmation.get()
+
+    event = {
+        "template_prefix": "zerver/emails/invitation",
+        "to_emails": [prereg_user.email],
+        "from_address": FromAddress.tokenized_no_reply_address(),
+        "language": realm.default_language,
+        "context": {
+            "referrer_full_name": referrer.full_name,
+            "referrer_email": referrer.delivery_email,
+            "activate_url": confirmation_url_for(confirmation),
+            "referrer_realm_name": realm.name,
+            "corporate_enabled": settings.CORPORATE_ENABLED,
+        },
+        "remove_suppressed_destination": True,
+        "realm_id": realm.id,
+        "date": email_format_datetime(event_time),
+    }
+    queue_event_on_commit("email_senders", event)
+
+    clear_scheduled_invitation_emails(prereg_user.email)
+    if invite_expires_in_minutes is None and confirmation.expiry_date is not None:
+        # Pull the remaining time from the confirmation object
+        invite_expires_in_minutes = (confirmation.expiry_date - event_time).total_seconds() / 60
+
+    if invite_expires_in_minutes is None or invite_expires_in_minutes < 4 * 24 * 60:
+        # We do not queue reminder email for never expiring
+        # invitations. This is probably a low importance bug; it
+        # would likely be more natural to send a reminder after 7
+        # days.
+        return
+
+    context = common_context(referrer)
+    context.update(
+        activate_url=confirmation_url_for(confirmation),
+        referrer_name=referrer.full_name,
+        referrer_email=referrer.delivery_email,
+        referrer_realm_name=realm.name,
+    )
+    send_future_email(
+        "zerver/emails/invitation_reminder",
+        realm,
+        to_emails=[prereg_user.email],
+        from_address=FromAddress.tokenized_no_reply_placeholder,
+        language=realm.default_language,
+        context=context,
+        delay=timedelta(minutes=invite_expires_in_minutes - (2 * 24 * 60)),
+    )

zerver/actions/message_delete.py

@@ -0,0 +1,128 @@
+from collections.abc import Iterable
+from typing import TypedDict
+
+from django.db.models import Q
+from pydantic import BaseModel
+
+from zerver.lib import retention
+from zerver.lib.retention import move_messages_to_archive
+from zerver.lib.streams import get_public_streams_queryset
+from zerver.models import Message, Realm, Stream, UserProfile
+from zerver.models.recipients import Recipient
+from zerver.tornado.django_api import send_event_on_commit
+
+
+class DeleteMessagesEvent(TypedDict, total=False):
+    type: str
+    message_ids: list[int]
+    message_type: str
+    topic: str
+    stream_id: int
+
+
+class DeactivateUserActions(BaseModel):
+    delete_profile: bool = False
+    delete_public_channel_messages: bool = False
+    delete_private_channel_messages: bool = False
+    delete_direct_messages: bool = False
+
+
+def check_update_first_message_id(
+    realm: Realm, stream: Stream, message_ids: list[int], users_to_notify: Iterable[int]
+) -> None:
+    # This will not update the `first_message_id` of streams where the
+    # first message was deleted prior to the implementation of this function.
+    assert stream.recipient_id is not None
+    if stream.first_message_id not in message_ids:
+        return
+    current_first_message_id = (
+        Message.objects.filter(realm_id=realm.id, recipient_id=stream.recipient_id)
+        .values_list("id", flat=True)
+        .order_by("id")
+        .first()
+    )
+
+    stream.first_message_id = current_first_message_id
+    stream.save(update_fields=["first_message_id"])
+
+    stream_event = dict(
+        type="stream",
+        op="update",
+        property="first_message_id",
+        value=stream.first_message_id,
+        stream_id=stream.id,
+        name=stream.name,
+    )
+    send_event_on_commit(realm, stream_event, users_to_notify)
+
+
+def do_delete_messages(
+    realm: Realm, messages: Iterable[Message], *, acting_user: UserProfile | None
+) -> None:
+    message_ids = [message.id for message in messages]
+    move_messages_to_archive(
+        message_ids,
+        realm=realm,
+        chunk_size=retention.STREAM_MESSAGE_BATCH_SIZE,
+        acting_user=acting_user,
+    )
+
+
+def do_delete_messages_by_sender(user: UserProfile, skip_notify: bool = False) -> None:
+    message_ids = list(
+        # Uses index: zerver_message_realm_sender_recipient (prefix)
+        Message.objects.filter(realm_id=user.realm_id, sender=user)
+        .values_list("id", flat=True)
+        .order_by("id")
+    )
+    move_messages_to_archive(
+        message_ids,
+        user.realm,
+        chunk_size=retention.STREAM_MESSAGE_BATCH_SIZE,
+        skip_notify=skip_notify,
+    )
+
+
+def delete_deactivated_user_messages(
+    realm: Realm,
+    user_profile: UserProfile,
+    deactivate_user_actions: DeactivateUserActions,
+    acting_user: UserProfile | None,
+) -> None:
+    delete_public_channel_messages = deactivate_user_actions.delete_public_channel_messages
+    delete_private_channel_messages = deactivate_user_actions.delete_private_channel_messages
+    delete_direct_messages = deactivate_user_actions.delete_direct_messages
+
+    if not (
+        delete_public_channel_messages or delete_private_channel_messages or delete_direct_messages
+    ):
+        return
+
+    message_filter_query = Q()
+    message_exclude_query = Q()
+
+    if delete_direct_messages:
+        message_filter_query |= Q(recipient__type=Recipient.DIRECT_MESSAGE_GROUP)
+
+    public_channel_ids = get_public_streams_queryset(realm).values_list("id", flat=True)
+    if delete_public_channel_messages and delete_private_channel_messages:
+        message_filter_query |= Q(recipient__type=Recipient.STREAM)
+    elif delete_public_channel_messages:
+        message_filter_query |= Q(
+            recipient__type=Recipient.STREAM,
+            recipient__type_id__in=list(public_channel_ids),
+        )
+    elif delete_private_channel_messages:
+        message_filter_query |= Q(recipient__type=Recipient.STREAM)
+        message_exclude_query |= Q(
+            recipient__type=Recipient.STREAM,
+            recipient__type_id__in=list(public_channel_ids),
+        )
+
+    messages = list(
+        Message.objects.filter(sender=user_profile, realm_id=user_profile.realm_id)
+        .filter(message_filter_query)
+        .exclude(message_exclude_query)
+        .select_related("recipient")
+    )
+    do_delete_messages(user_profile.realm, messages, acting_user=acting_user)

zerver/actions/message_flags.py

@@ -0,0 +1,461 @@
+import time
+from collections import defaultdict
+from dataclasses import asdict, dataclass, field
+
+from django.db import transaction
+from django.db.models import F
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+
+from analytics.lib.counts import COUNT_STATS, do_increment_logging_stat
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.message import (
+    bulk_access_messages,
+    format_unread_message_details,
+    get_raw_unread_data,
+)
+from zerver.lib.queue import mobile_notifications_queue_name, queue_event_on_commit
+from zerver.lib.stream_subscription import get_subscribed_stream_recipient_ids_for_user
+from zerver.lib.topic import filter_by_topic_name_via_message
+from zerver.lib.user_message import DEFAULT_HISTORICAL_FLAGS, create_historical_user_messages
+from zerver.models import Device, Message, PushDeviceToken, Recipient, UserMessage, UserProfile
+from zerver.tornado.django_api import send_event_on_commit, send_event_rollback_unsafe
+
+
+@dataclass
+class ReadMessagesEvent:
+    messages: list[int]
+    all: bool
+    type: str = field(default="update_message_flags", init=False)
+    op: str = field(default="add", init=False)
+    operation: str = field(default="add", init=False)
+    flag: str = field(default="read", init=False)
+
+
+def do_mark_all_as_read(user_profile: UserProfile, *, timeout: float | None = None) -> int | None:
+    start_time = time.monotonic()
+
+    # First, we clear mobile push notifications.  This is safer in the
+    # event that the below logic times out and we're killed.
+    all_push_message_ids = list(
+        UserMessage.objects.filter(
+            user_profile=user_profile,
+        )
+        .extra(  # noqa: S610
+            where=[UserMessage.where_active_push_notification()],
+        )
+        .values_list("message_id", flat=True)[0:10000]
+    )
+    do_clear_mobile_push_notifications_for_ids([user_profile.id], all_push_message_ids)
+
+    batch_size = 2000
+    count = 0
+    while True:
+        if timeout is not None and time.monotonic() >= start_time + timeout:
+            return None
+
+        with transaction.atomic(durable=True):
+            query = (
+                UserMessage.select_for_update_query()
+                .filter(user_profile=user_profile)
+                .extra(where=[UserMessage.where_unread()])[:batch_size]  # noqa: S610
+            )
+            # This updated_count is the same as the number of UserMessage
+            # rows selected, because due to the FOR UPDATE lock, we're guaranteed
+            # that all the selected rows will indeed be updated.
+            # UPDATE queries don't support LIMIT, so we have to use a subquery
+            # to do batching.
+            updated_count = UserMessage.objects.filter(id__in=query).update(
+                flags=F("flags").bitor(UserMessage.flags.read),
+            )
+
+            event_time = timezone_now()
+            do_increment_logging_stat(
+                user_profile,
+                COUNT_STATS["messages_read::hour"],
+                None,
+                event_time,
+                increment=updated_count,
+            )
+            do_increment_logging_stat(
+                user_profile,
+                COUNT_STATS["messages_read_interactions::hour"],
+                None,
+                event_time,
+                increment=min(1, updated_count),
+            )
+
+            count += updated_count
+            if updated_count < batch_size:
+                break
+
+    event = asdict(
+        ReadMessagesEvent(
+            messages=[],  # we don't send messages, since the client reloads anyway
+            all=True,
+        )
+    )
+    send_event_rollback_unsafe(user_profile.realm, event, [user_profile.id])
+
+    return count
+
+
+@transaction.atomic(durable=True)
+def do_mark_stream_messages_as_read(
+    user_profile: UserProfile, stream_recipient_id: int, topic_name: str | None = None
+) -> int:
+    query = (
+        UserMessage.select_for_update_query()
+        .filter(
+            user_profile=user_profile,
+            message__recipient_id=stream_recipient_id,
+        )
+        .extra(  # noqa: S610
+            where=[UserMessage.where_unread()],
+        )
+    )
+
+    if topic_name:
+        query = filter_by_topic_name_via_message(
+            query=query,
+            topic_name=topic_name,
+        )
+
+    message_ids = list(query.values_list("message_id", flat=True))
+
+    if len(message_ids) == 0:
+        return 0
+
+    count = query.update(
+        flags=F("flags").bitor(UserMessage.flags.read),
+    )
+
+    event = asdict(
+        ReadMessagesEvent(
+            messages=message_ids,
+            all=False,
+        )
+    )
+    event_time = timezone_now()
+
+    send_event_on_commit(user_profile.realm, event, [user_profile.id])
+    do_clear_mobile_push_notifications_for_ids([user_profile.id], message_ids)
+
+    do_increment_logging_stat(
+        user_profile, COUNT_STATS["messages_read::hour"], None, event_time, increment=count
+    )
+    do_increment_logging_stat(
+        user_profile,
+        COUNT_STATS["messages_read_interactions::hour"],
+        None,
+        event_time,
+        increment=min(1, count),
+    )
+    return count
+
+
+@transaction.atomic(savepoint=False)
+def do_mark_muted_user_messages_as_read(
+    user_profile: UserProfile,
+    muted_user: UserProfile,
+) -> int:
+    query = (
+        UserMessage.select_for_update_query()
+        .filter(
+            user_profile=user_profile,
+            message__sender=muted_user,
+            # Filtering by message.realm_id is redundant, but required to use index:
+            # zerver_message_realm_sender_recipient
+            message__realm_id=user_profile.realm_id,
+        )
+        .extra(where=[UserMessage.where_unread()])  # noqa: S610
+    )
+    message_ids = list(query.values_list("message_id", flat=True))
+
+    if len(message_ids) == 0:
+        return 0
+
+    count = query.update(
+        flags=F("flags").bitor(UserMessage.flags.read),
+    )
+
+    event = asdict(
+        ReadMessagesEvent(
+            messages=message_ids,
+            all=False,
+        )
+    )
+    event_time = timezone_now()
+
+    send_event_on_commit(user_profile.realm, event, [user_profile.id])
+    do_clear_mobile_push_notifications_for_ids([user_profile.id], message_ids)
+
+    do_increment_logging_stat(
+        user_profile, COUNT_STATS["messages_read::hour"], None, event_time, increment=count
+    )
+    do_increment_logging_stat(
+        user_profile,
+        COUNT_STATS["messages_read_interactions::hour"],
+        None,
+        event_time,
+        increment=min(1, count),
+    )
+    return count
+
+
+def do_update_mobile_push_notification(
+    message: Message,
+    prior_mention_user_ids: set[int],
+    mentions_user_ids: set[int],
+    stream_push_user_ids: set[int],
+) -> None:
+    # Called during the message edit code path to remove mobile push
+    # notifications for users who are no longer mentioned following
+    # the edit.  See #15428 for details.
+    #
+    # A perfect implementation would also support updating the message
+    # in a sent notification if a message was edited to mention a
+    # group rather than a user (or vice versa), though it is likely
+    # not worth the effort to do such a change.
+    if not message.is_channel_message:
+        return
+
+    remove_notify_users = prior_mention_user_ids - mentions_user_ids - stream_push_user_ids
+    do_clear_mobile_push_notifications_for_ids(list(remove_notify_users), [message.id])
+
+
+def do_clear_mobile_push_notifications_for_ids(
+    user_profile_ids: list[int] | None, message_ids: list[int]
+) -> None:
+    if len(message_ids) == 0:
+        return
+
+    if user_profile_ids is not None:
+        # This block gets executed in the following cases:
+        # * Message(s) marked as read by a user
+        # * A message edited to remove mention(s)
+        if len(user_profile_ids) == 0:
+            return
+
+        # This supports clearing notifications for several users only for
+        # the message-edit use case where we'll have a single message_id.
+        assert len(user_profile_ids) == 1 or len(message_ids) == 1
+
+        notifications_to_update = (
+            UserMessage.objects.filter(
+                message_id__in=message_ids,
+                user_profile_id__in=user_profile_ids,
+            )
+            .extra(  # noqa: S610
+                where=[UserMessage.where_active_push_notification()],
+            )
+            .values_list("id", "user_profile_id", "message_id")
+        )
+    else:
+        # This block handles clearing notifications when message(s) get deleted.
+        notifications_to_update = (
+            # Uses index: zerver_usermessage_message_active_mobile_push_notification_idx
+            UserMessage.objects.filter(
+                message_id__in=message_ids,
+            )
+            .extra(  # noqa: S610
+                where=[UserMessage.where_active_push_notification()],
+            )
+            .values_list("id", "user_profile_id", "message_id")
+        )
+
+    messages_by_user = defaultdict(list)
+    for unused, user_id, message_id in notifications_to_update:
+        messages_by_user[user_id].append(message_id)
+
+    clear_notifications_user_ids = set(messages_by_user.keys())
+    push_device_registered_user_ids = set(
+        PushDeviceToken.objects.filter(user_id__in=clear_notifications_user_ids)
+        .values_list("user_id", flat=True)
+        .union(
+            # Uses index "zerver_device_user_push_token_id_idx".
+            Device.objects.filter(
+                user_id__in=clear_notifications_user_ids, push_token_id__isnull=False
+            ).values_list("user_id", flat=True)
+        )
+    )
+    push_device_not_registered_user_ids = (
+        clear_notifications_user_ids - push_device_registered_user_ids
+    )
+
+    usermessages_to_update_ids = []
+    for um_id, user_id, unused in notifications_to_update:
+        if user_id in push_device_not_registered_user_ids:
+            usermessages_to_update_ids.append(um_id)
+
+    if usermessages_to_update_ids:
+        BATCH_SIZE = 1000
+        for i in range(0, len(usermessages_to_update_ids), BATCH_SIZE):
+            with transaction.atomic(savepoint=False):
+                UserMessage.select_for_update_query().filter(
+                    id__in=usermessages_to_update_ids[i : i + BATCH_SIZE]
+                ).update(
+                    flags=F("flags").bitand(~UserMessage.flags.active_mobile_push_notification)
+                )
+
+    for user_profile_id, event_message_ids in messages_by_user.items():
+        if user_profile_id in push_device_not_registered_user_ids:
+            continue
+
+        notice = {
+            "type": "remove",
+            "user_profile_id": user_profile_id,
+            "message_ids": event_message_ids,
+        }
+        queue_event_on_commit(mobile_notifications_queue_name(user_profile_id), notice)
+
+
+def do_update_message_flags(
+    user_profile: UserProfile, operation: str, flag: str, messages: list[int]
+) -> tuple[int, list[int]]:
+    valid_flags = [item for item in UserMessage.flags if item not in UserMessage.NON_API_FLAGS]
+    if flag not in valid_flags:
+        raise JsonableError(_("Invalid flag: '{flag}'").format(flag=flag))
+    if flag in UserMessage.NON_EDITABLE_FLAGS:
+        raise JsonableError(_("Flag not editable: '{flag}'").format(flag=flag))
+    if operation not in ("add", "remove"):
+        raise JsonableError(
+            _("Invalid message flag operation: '{operation}'").format(operation=operation)
+        )
+    is_adding = operation == "add"
+    flagattr = getattr(UserMessage.flags, flag)
+    flag_target = flagattr if is_adding else 0
+
+    ignored_because_not_subscribed_channels = []
+    with transaction.atomic(durable=True):
+        if flag == "read" and not is_adding:
+            # We have an invariant that all stream messages marked as
+            # unread must be in streams the user is subscribed to.
+            #
+            # When marking as unread, we enforce this invariant by
+            # ignoring any messages in streams the user is not
+            # currently subscribed to.
+            subscribed_recipient_ids = get_subscribed_stream_recipient_ids_for_user(user_profile)
+
+            messages_in_unsubscribed_streams = set(
+                # Uses index: zerver_message_pkey
+                Message.objects.select_related("recipient")
+                .filter(id__in=messages, recipient__type=Recipient.STREAM)
+                .exclude(recipient_id__in=subscribed_recipient_ids)
+                .values_list("id", "recipient__type_id")
+            )
+
+            message_ids_in_unsubscribed_streams = {
+                message[0] for message in messages_in_unsubscribed_streams
+            }
+
+            ignored_because_not_subscribed_channels = list(
+                {message[1] for message in messages_in_unsubscribed_streams}
+            )
+
+            messages = [
+                message_id
+                for message_id in messages
+                if message_id not in message_ids_in_unsubscribed_streams
+            ]
+
+        ums = {
+            um.message_id: um
+            for um in UserMessage.select_for_update_query().filter(
+                user_profile=user_profile, message_id__in=messages
+            )
+        }
+
+        # Filter out rows that already have the desired flag.  We do
+        # this here, rather than in the original database query,
+        # because not all flags have database indexes and we want to
+        # bound the cost of this operation.
+        messages = [
+            message_id
+            for message_id in messages
+            if (int(ums[message_id].flags) if message_id in ums else DEFAULT_HISTORICAL_FLAGS)
+            & flagattr
+            != flag_target
+        ]
+        count = len(messages)
+
+        if DEFAULT_HISTORICAL_FLAGS & flagattr != flag_target:
+            # When marking messages as read, creating "historical"
+            # UserMessage rows would be a waste of storage, because
+            # `flags.read | flags.historical` is exactly the flags we
+            # simulate when processing a message for which a user has
+            # access but no UserMessage row.
+            #
+            # Users can mutate flags for messages that don't have a
+            # UserMessage yet.  Validate that the user is even allowed
+            # to access these message_ids; if so, we will create
+            # "historical" UserMessage rows for the messages in question.
+            #
+            # See create_historical_user_messages for a more detailed
+            # explanation.
+            historical_message_ids = set(messages) - set(ums.keys())
+            historical_messages = bulk_access_messages(
+                user_profile,
+                list(
+                    # Uses index: zerver_message_pkey
+                    Message.objects.filter(id__in=historical_message_ids).prefetch_related(
+                        "recipient"
+                    )
+                ),
+                is_modifying_message=False,
+            )
+            if len(historical_messages) != len(historical_message_ids):
+                raise JsonableError(_("Invalid message(s)"))
+
+            create_historical_user_messages(
+                user_id=user_profile.id,
+                message_ids=list(historical_message_ids),
+                flagattr=flagattr,
+                flag_target=flag_target,
+            )
+
+        to_update = UserMessage.objects.filter(
+            user_profile=user_profile, message_id__in=set(messages) & set(ums.keys())
+        )
+        if is_adding:
+            to_update.update(flags=F("flags").bitor(flagattr))
+        else:
+            to_update.update(flags=F("flags").bitand(~flagattr))
+
+        event = {
+            "type": "update_message_flags",
+            "op": operation,
+            "operation": operation,
+            "flag": flag,
+            "messages": messages,
+            "all": False,
+        }
+
+        if flag == "read" and not is_adding:
+            # When removing the read flag (i.e. marking messages as
+            # unread), extend the event with an additional object with
+            # details on the messages required to update the client's
+            # `unread_msgs` data structure.
+            raw_unread_data = get_raw_unread_data(user_profile, messages)
+            event["message_details"] = format_unread_message_details(
+                user_profile.id, raw_unread_data
+            )
+
+        send_event_on_commit(user_profile.realm, event, [user_profile.id])
+
+        if flag == "read" and is_adding:
+            event_time = timezone_now()
+            do_clear_mobile_push_notifications_for_ids([user_profile.id], messages)
+
+            do_increment_logging_stat(
+                user_profile, COUNT_STATS["messages_read::hour"], None, event_time, increment=count
+            )
+            do_increment_logging_stat(
+                user_profile,
+                COUNT_STATS["messages_read_interactions::hour"],
+                None,
+                event_time,
+                increment=min(1, count),
+            )
+
+    return (count, ignored_because_not_subscribed_channels)

zerver/actions/message_summary.py

@@ -0,0 +1,215 @@
+import time
+from typing import Any, Literal
+
+import orjson
+from django.conf import settings
+from django.utils.timezone import now as timezone_now
+from openai import OpenAI
+from openai.types.chat import ChatCompletionMessageParam
+
+from analytics.lib.counts import COUNT_STATS, do_increment_logging_stat
+from zerver.lib.markdown import markdown_convert
+from zerver.lib.message import messages_for_ids
+from zerver.lib.narrow import (
+    LARGER_THAN_MAX_MESSAGE_ID,
+    AnchorInfo,
+    NarrowParameter,
+    clean_narrow_for_message_fetch,
+    fetch_messages,
+)
+from zerver.models import UserProfile
+from zerver.models.realms import MessageEditHistoryVisibilityPolicyEnum
+
+# Maximum number of messages that can be summarized in a single request.
+MAX_MESSAGES_SUMMARIZED = 100
+
+ai_time_start = 0.0
+ai_total_time = 0.0
+ai_total_requests = 0
+
+
+def get_ai_time() -> float:
+    return ai_total_time
+
+
+def ai_stats_start() -> None:
+    global ai_time_start
+    ai_time_start = time.time()
+
+
+def get_ai_requests() -> int:
+    return ai_total_requests
+
+
+def ai_stats_finish() -> None:
+    global ai_total_time, ai_total_requests
+    ai_total_requests += 1
+    ai_total_time += time.time() - ai_time_start
+
+
+def format_zulip_messages_for_model(zulip_messages: list[dict[str, Any]]) -> str:
+    # Format the Zulip messages for processing by the model.
+    #
+    # - We don't need to encode the recipient, since that's the same for
+    #   every message in the conversation.
+    # - We use full names to reference senders, since we want the
+    #   model to refer to users by name. We may want to experiment
+    #   with using silent-mention syntax for users if we move to
+    #   Markdown-rendering what the model returns.
+    # - We don't include timestamps, since experiments with current models
+    #   suggest they do not make relevant use of them.
+    # - We haven't figured out a useful way to include reaction metadata (either
+    #   the emoji themselves or just the counter).
+    # - Polls/TODO widgets are currently sent to the model as empty messages,
+    #   since this logic doesn't inspect SubMessage objects.
+    zulip_messages_list = [
+        {"sender": message["sender_full_name"], "content": message["content"]}
+        for message in zulip_messages
+    ]
+    return orjson.dumps(zulip_messages_list).decode()
+
+
+def make_message(
+    content: str, role: Literal["user", "system"] = "user"
+) -> ChatCompletionMessageParam:
+    if role == "system":
+        return {"content": content, "role": "system"}
+    return {"content": content, "role": "user"}
+
+
+def get_max_summary_length(conversation_length: int) -> int:
+    # Longer summaries work better for longer conversation.
+    # TODO: Test more with message content length.
+    return min(6, 4 + int((conversation_length - 10) / 10))
+
+
+def do_summarize_narrow(
+    user_profile: UserProfile,
+    narrow: list[NarrowParameter] | None,
+) -> str | None:
+    model = settings.TOPIC_SUMMARIZATION_MODEL
+    if model is None:  # nocoverage
+        return None
+
+    # TODO: This implementation does not attempt to make use of
+    # caching previous summaries of the same conversation or rolling
+    # summaries. Doing so correctly will require careful work around
+    # invalidation of caches when messages are edited, moved, or sent.
+    narrow = clean_narrow_for_message_fetch(narrow, user_profile.realm, user_profile)
+    query_info = fetch_messages(
+        narrow=narrow,
+        user_profile=user_profile,
+        realm=user_profile.realm,
+        is_web_public_query=False,
+        anchor_info=AnchorInfo(type="message_id", value=LARGER_THAN_MAX_MESSAGE_ID),
+        include_anchor=True,
+        num_before=MAX_MESSAGES_SUMMARIZED,
+        num_after=0,
+    )
+
+    if len(query_info.rows) == 0:  # nocoverage
+        return None
+
+    result_message_ids: list[int] = []
+    user_message_flags: dict[int, list[str]] = {}
+    for row in query_info.rows:
+        message_id = row[0]
+        result_message_ids.append(message_id)
+        # We skip populating flags, since they would be ignored below anyway.
+        user_message_flags[message_id] = []
+
+    message_list = messages_for_ids(
+        message_ids=result_message_ids,
+        user_message_flags=user_message_flags,
+        search_fields={},
+        # We currently prefer the plain-text content of messages to
+        apply_markdown=False,
+        # Avoid wasting resources computing gravatars.
+        client_gravatar=True,
+        allow_empty_topic_name=False,
+        # Avoid fetching edit history, which won't be passed to the model.
+        message_edit_history_visibility_policy=MessageEditHistoryVisibilityPolicyEnum.none.value,
+        user_profile=user_profile,
+        realm=user_profile.realm,
+    )
+
+    # IDEA: We could consider translating input and output text to
+    # English to improve results when using a summarization model that
+    # is primarily trained on English.
+    conversation_length = len(message_list)
+    max_summary_length = get_max_summary_length(conversation_length)
+    intro = "The following is a chat conversation in the Zulip team chat app."
+    topic: str | None = None
+    channel: str | None = None
+    if narrow and len(narrow) == 2:
+        for term in narrow:
+            assert not term.negated
+            if term.operator == "channel":
+                channel = term.operand
+            if term.operator == "topic":
+                topic = term.operand
+    if channel:
+        intro += f" channel: {channel}"
+    if topic:
+        intro += f", topic: {topic}"
+
+    formatted_conversation = format_zulip_messages_for_model(message_list)
+    prompt = (
+        f"Succinctly summarize this conversation based only on the information provided, "
+        f"in up to {max_summary_length} sentences, for someone who is familiar with the context. "
+        f"Mention key conclusions and actions, if any. Refer to specific people as appropriate. "
+        f"Don't use an intro phrase. You can use Zulip's CommonMark based formatting."
+    )
+    messages = [
+        make_message(intro, "system"),
+        make_message(formatted_conversation),
+        make_message(prompt),
+    ]
+
+    # Stats for database queries are tracked separately.
+    ai_stats_start()
+
+    # TODO when implementing user plans:
+    # - Before querying the model, check whether we've enough tokens left using
+    # an estimated token count.
+    # - Then increase the `LoggingCountStat` using the estimated token count.
+    # (These first two steps should be a short database transaction that
+    # locks the `LoggingCountStat` row).
+    # - Then query the model.
+    # - Then adjust the `LoggingCountStat` by `(actual - estimated)`,
+    # being careful to avoid doing this to the next day if the query
+    # happened milliseconds before midnight; changing the
+    # `LoggingCountStat` we added the estimate to.
+    # That way, you can't easily get extra tokens by sending
+    # 25 requests all at once when you're just below the limit.
+
+    client = OpenAI(
+        api_key=settings.TOPIC_SUMMARIZATION_API_KEY,
+        base_url=settings.TOPIC_SUMMARIZATION_API_BASE,
+    )
+    response = client.chat.completions.create(
+        model=model,
+        messages=messages,
+        **settings.TOPIC_SUMMARIZATION_PARAMETERS,
+    )
+    assert response.usage is not None
+    input_tokens = response.usage.prompt_tokens
+    output_tokens = response.usage.completion_tokens
+
+    # Divide by 1 billion to get actual cost in USD.
+    credits_used = (output_tokens * settings.OUTPUT_COST_PER_GIGATOKEN) + (
+        input_tokens * settings.INPUT_COST_PER_GIGATOKEN
+    )
+    ai_stats_finish()
+
+    do_increment_logging_stat(
+        user_profile, COUNT_STATS["ai_credit_usage::day"], None, timezone_now(), credits_used
+    )
+
+    summary = response.choices[0].message.content
+    assert summary is not None
+    # TODO: This may want to fetch `MentionData`, in order to be able
+    # to process channel or user mentions that might be in the
+    # content. Requires a prompt that supports it.
+    rendered_summary = markdown_convert(summary, message_realm=user_profile.realm).rendered_content
+    return rendered_summary

zerver/actions/muted_users.py

@@ -0,0 +1,51 @@
+from datetime import datetime
+
+from django.db import transaction
+from django.utils.timezone import now as timezone_now
+
+from zerver.actions.message_flags import do_mark_muted_user_messages_as_read
+from zerver.lib.muted_users import add_user_mute, get_user_mutes
+from zerver.models import MutedUser, RealmAuditLog, UserProfile
+from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.tornado.django_api import send_event_on_commit
+
+
+@transaction.atomic(durable=True)
+def do_mute_user(
+    user_profile: UserProfile,
+    muted_user: UserProfile,
+    date_muted: datetime | None = None,
+) -> None:
+    if date_muted is None:
+        date_muted = timezone_now()
+    add_user_mute(user_profile, muted_user, date_muted)
+    do_mark_muted_user_messages_as_read(user_profile, muted_user)
+    event = dict(type="muted_users", muted_users=get_user_mutes(user_profile))
+    send_event_on_commit(user_profile.realm, event, [user_profile.id])
+
+    RealmAuditLog.objects.create(
+        realm=user_profile.realm,
+        acting_user=user_profile,
+        modified_user=user_profile,
+        event_type=AuditLogEventType.USER_MUTED,
+        event_time=date_muted,
+        extra_data={"muted_user_id": muted_user.id},
+    )
+
+
+@transaction.atomic(durable=True)
+def do_unmute_user(mute_object: MutedUser) -> None:
+    user_profile = mute_object.user_profile
+    muted_user = mute_object.muted_user
+    mute_object.delete()
+    event = dict(type="muted_users", muted_users=get_user_mutes(user_profile))
+    send_event_on_commit(user_profile.realm, event, [user_profile.id])
+
+    RealmAuditLog.objects.create(
+        realm=user_profile.realm,
+        acting_user=user_profile,
+        modified_user=user_profile,
+        event_type=AuditLogEventType.USER_UNMUTED,
+        event_time=timezone_now(),
+        extra_data={"unmuted_user_id": muted_user.id},
+    )

zerver/actions/navigation_views.py

@@ -0,0 +1,121 @@
+from django.db import transaction
+from django.utils.timezone import now as timezone_now
+
+from zerver.lib.navigation_views import get_navigation_view_dict
+from zerver.models import NavigationView, RealmAuditLog, UserProfile
+from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.tornado.django_api import send_event_on_commit
+
+
+@transaction.atomic(durable=True)
+def do_add_navigation_view(
+    user: UserProfile,
+    fragment: str,
+    is_pinned: bool,
+    name: str | None = None,
+) -> NavigationView:
+    navigation_view = NavigationView.objects.create(
+        user=user,
+        fragment=fragment,
+        is_pinned=is_pinned,
+        name=name,
+    )
+
+    RealmAuditLog.objects.create(
+        realm=user.realm,
+        acting_user=user,
+        modified_user=user,
+        event_type=AuditLogEventType.NAVIGATION_VIEW_CREATED,
+        event_time=timezone_now(),
+        extra_data={"fragment": fragment},
+    )
+
+    event = {
+        "type": "navigation_view",
+        "op": "add",
+        "navigation_view": get_navigation_view_dict(navigation_view),
+    }
+    send_event_on_commit(user.realm, event, [user.id])
+    return navigation_view
+
+
+@transaction.atomic(durable=True)
+def do_update_navigation_view(
+    user: UserProfile,
+    navigation_view: NavigationView,
+    is_pinned: bool | None,
+    name: str | None = None,
+) -> None:
+    update_dict: dict[str, str | bool] = {}
+    audit_logs_extra_data: list[dict[str, str | bool | None]] = []
+    if name is not None:
+        old_name = navigation_view.name
+        navigation_view.name = name
+        update_dict["name"] = name
+        audit_logs_extra_data.append(
+            {
+                "fragment": navigation_view.fragment,
+                RealmAuditLog.OLD_VALUE: old_name,
+                RealmAuditLog.NEW_VALUE: name,
+                "property": "name",
+            }
+        )
+
+    if is_pinned is not None:
+        old_is_pinned_value = navigation_view.is_pinned
+        navigation_view.is_pinned = is_pinned
+        update_dict["is_pinned"] = is_pinned
+        audit_logs_extra_data.append(
+            {
+                "fragment": navigation_view.fragment,
+                RealmAuditLog.OLD_VALUE: old_is_pinned_value,
+                RealmAuditLog.NEW_VALUE: is_pinned,
+                "property": "is_pinned",
+            }
+        )
+
+    navigation_view.save(update_fields=["name", "is_pinned"])
+
+    now = timezone_now()
+    for audit_log_extra_data in audit_logs_extra_data:
+        RealmAuditLog.objects.create(
+            realm=user.realm,
+            acting_user=user,
+            modified_user=user,
+            event_type=AuditLogEventType.NAVIGATION_VIEW_UPDATED,
+            event_time=now,
+            extra_data=audit_log_extra_data,
+        )
+
+    event = {
+        "type": "navigation_view",
+        "op": "update",
+        "fragment": navigation_view.fragment,
+        "data": update_dict,
+    }
+    send_event_on_commit(user.realm, event, [user.id])
+
+
+@transaction.atomic(durable=True)
+def do_remove_navigation_view(
+    user: UserProfile,
+    navigation_view: NavigationView,
+) -> None:
+    fragment = navigation_view.fragment
+    navigation_view.delete()
+
+    RealmAuditLog.objects.create(
+        realm=user.realm,
+        acting_user=user,
+        modified_user=user,
+        event_type=AuditLogEventType.NAVIGATION_VIEW_DELETED,
+        event_time=timezone_now(),
+        extra_data={"fragment": fragment},
+    )
+
+    event = {
+        "type": "navigation_view",
+        "op": "remove",
+        "fragment": navigation_view.fragment,
+    }
+    send_event_on_commit(user.realm, event, [user.id])

zerver/actions/onboarding_steps.py

@@ -0,0 +1,17 @@
+from dataclasses import asdict
+
+from django.db import transaction
+
+from zerver.lib.onboarding_steps import get_next_onboarding_steps
+from zerver.models import OnboardingStep, UserProfile
+from zerver.tornado.django_api import send_event_on_commit
+
+
+@transaction.atomic(durable=True)
+def do_mark_onboarding_step_as_read(user: UserProfile, onboarding_step: str) -> None:
+    OnboardingStep.objects.get_or_create(user=user, onboarding_step=onboarding_step)
+    event = dict(
+        type="onboarding_steps",
+        onboarding_steps=[asdict(step) for step in get_next_onboarding_steps(user)],
+    )
+    send_event_on_commit(user.realm, event, [user.id])

zerver/actions/presence.py

@@ -0,0 +1,300 @@
+import logging
+import time
+from datetime import datetime, timedelta
+
+from django.conf import settings
+from django.db import connection, transaction
+from psycopg2 import sql
+
+from zerver.actions.user_activity import update_user_activity_interval
+from zerver.lib.presence import (
+    format_legacy_presence_dict,
+    get_modern_user_presence_info,
+    user_presence_datetime_with_date_joined_default,
+)
+from zerver.lib.users import get_user_ids_who_can_access_user
+from zerver.models import Client, UserPresence, UserProfile
+from zerver.models.clients import get_client
+from zerver.models.users import active_user_ids
+from zerver.tornado.django_api import send_event_rollback_unsafe
+
+logger = logging.getLogger(__name__)
+
+
+def send_presence_changed(
+    user_profile: UserProfile, presence: UserPresence, *, force_send_update: bool = False
+) -> None:
+    # Most presence data is sent to clients in the main presence
+    # endpoint in response to the user's own presence; this results
+    # data that is 1-2 minutes stale for who is online.  The flaw with
+    # this plan is when a user comes back online and then immediately
+    # sends a message, recipients may still see that user as offline!
+    # We solve that by sending an immediate presence update clients.
+    #
+    # The API documentation explains this interaction in more detail.
+    if settings.CAN_ACCESS_ALL_USERS_GROUP_LIMITS_PRESENCE:
+        user_ids = get_user_ids_who_can_access_user(user_profile)
+    else:
+        user_ids = active_user_ids(user_profile.realm_id)
+
+    if (
+        len(user_ids) > settings.USER_LIMIT_FOR_SENDING_PRESENCE_UPDATE_EVENTS
+        and not force_send_update
+    ):
+        # These immediate presence generate quadratic work for Tornado
+        # (linear number of users in each event and the frequency of
+        # users coming online grows linearly with userbase too).  In
+        # organizations with thousands of users, this can overload
+        # Tornado, especially if much of the realm comes online at the
+        # same time.
+        #
+        # The utility of these live-presence updates goes down as
+        # organizations get bigger (since one is much less likely to
+        # be paying attention to the sidebar); so beyond a limit, we
+        # stop sending them at all.
+        return
+
+    last_active_time = user_presence_datetime_with_date_joined_default(
+        presence.last_active_time, user_profile.date_joined
+    )
+    last_connected_time = user_presence_datetime_with_date_joined_default(
+        presence.last_connected_time, user_profile.date_joined
+    )
+
+    # The mobile app handles these events so we need to use the old format.
+    # The format of the event should also account for the slim_presence
+    # API parameter when this becomes possible in the future.
+    legacy_presence_dict = format_legacy_presence_dict(last_active_time, last_connected_time)
+    modern_presence_dict = get_modern_user_presence_info(last_active_time, last_connected_time)
+    event = dict(
+        type="presence",
+        email=user_profile.email,
+        user_id=user_profile.id,
+        server_timestamp=time.time(),
+        legacy_presence={legacy_presence_dict["client"]: legacy_presence_dict},
+        modern_presence=modern_presence_dict,
+    )
+    send_event_rollback_unsafe(user_profile.realm, event, user_ids)
+
+
+def consolidate_client(client: Client) -> Client:
+    # The web app reports a client as 'website'
+    # The desktop app reports a client as ZulipDesktop
+    # due to it setting a custom user agent. We want both
+    # to count as web users
+
+    # Alias ZulipDesktop to website
+    if client.name == "ZulipDesktop":
+        return get_client("website")
+    else:
+        return client
+
+
+# This function takes a very hot lock on the PresenceSequence row for the user's realm.
+# Since all presence updates in the realm all compete for this lock, we need to be
+# maximally efficient and only hold it as briefly as possible.
+# For that reason, we need durable=True to ensure we're not running inside a larger
+# transaction, which may stay alive longer than we'd like, holding the lock.
+@transaction.atomic(durable=True)
+def do_update_user_presence(
+    user_profile: UserProfile,
+    client: Client,
+    log_time: datetime,
+    status: int,
+    *,
+    force_send_update: bool = False,
+) -> None:
+    # This function requires some careful handling around setting the
+    # last_update_id field when updatng UserPresence objects. See the
+    # PresenceSequence model and the comments throughout the code for more details.
+
+    client = consolidate_client(client)
+
+    # If the user doesn't have a UserPresence row yet, we create one with
+    # sensible defaults. If we're getting a presence update, clearly the user
+    # at least connected, so last_connected_time should be set. last_active_time
+    # will depend on whether the status sent is idle or active.
+    defaults = dict(
+        last_active_time=None,
+        last_connected_time=log_time,
+        realm_id=user_profile.realm_id,
+    )
+    if status == UserPresence.LEGACY_STATUS_ACTIVE_INT:
+        defaults["last_active_time"] = log_time
+
+    try:
+        presence = UserPresence.objects.select_for_update(no_key=True).get(
+            user_profile=user_profile
+        )
+        creating = False
+    except UserPresence.DoesNotExist:
+        # We're not ready to write until we know the next last_update_id value.
+        # We don't want to hold the lock on PresenceSequence for too long,
+        # so we defer that until the last moment.
+        # Create the presence object in-memory only for now.
+        presence = UserPresence(**defaults, user_profile=user_profile)
+        creating = True
+
+    # We initialize these values as a large delta so that if the user
+    # was never active, we always treat the user as newly online.
+    time_since_last_active_for_comparison = timedelta(days=1)
+    time_since_last_connected_for_comparison = timedelta(days=1)
+    if presence.last_active_time is not None:
+        time_since_last_active_for_comparison = log_time - presence.last_active_time
+    if presence.last_connected_time is not None:
+        time_since_last_connected_for_comparison = log_time - presence.last_connected_time
+
+    assert (3 * settings.PRESENCE_PING_INTERVAL_SECS + 20) <= settings.OFFLINE_THRESHOLD_SECS
+    now_online = time_since_last_active_for_comparison > timedelta(
+        # Here, we decide whether the user is newly online, and we need to consider
+        # sending an immediate presence update via the events system that this user is now online,
+        # rather than waiting for other clients to poll the presence update.
+        # Sending these presence update events adds load to the system, so we only want to do this
+        # if the user has missed a couple regular presence check-ins
+        # (so their state is at least 2 * PRESENCE_PING_INTERVAL_SECS + 10 old),
+        # and also is under the risk of being shown by clients as offline before the next regular presence check-in
+        # (so at least `settings.OFFLINE_THRESHOLD_SECS - settings.PRESENCE_PING_INTERVAL_SECS - 10`).
+        # These two values happen to be the same in the default configuration.
+        seconds=settings.OFFLINE_THRESHOLD_SECS - settings.PRESENCE_PING_INTERVAL_SECS - 10
+    )
+    became_online = status == UserPresence.LEGACY_STATUS_ACTIVE_INT and now_online
+
+    update_fields = []
+
+    # This check is to prevent updating `last_connected_time` several
+    # times per minute with multiple connected browser windows.
+    # We also need to be careful not to wrongly "update" the timestamp if we actually already
+    # have newer presence than the reported log_time.
+    if not creating and time_since_last_connected_for_comparison > timedelta(
+        seconds=settings.PRESENCE_UPDATE_MIN_FREQ_SECONDS
+    ):
+        presence.last_connected_time = log_time
+        update_fields.append("last_connected_time")
+    if (
+        not creating
+        and status == UserPresence.LEGACY_STATUS_ACTIVE_INT
+        and time_since_last_active_for_comparison
+        > timedelta(seconds=settings.PRESENCE_UPDATE_MIN_FREQ_SECONDS)
+    ):
+        presence.last_active_time = log_time
+        update_fields.append("last_active_time")
+        if presence.last_connected_time is None or log_time > presence.last_connected_time:
+            # Update last_connected_time as well to ensure
+            # last_connected_time >= last_active_time.
+            presence.last_connected_time = log_time
+            update_fields.append("last_connected_time")
+
+    # WARNING: Delicate, performance-sensitive block.
+
+    # It's time to determine last_update_id and update the presence object in the database.
+    # This briefly takes the crucial lock on the PresenceSequence row for the user's realm.
+    # We're doing this in a single SQL query to avoid any unnecessary overhead, in particular
+    # database round-trips.
+    # We're also intentionally doing this at the very end of the function, at the last step
+    # before the transaction commits. This ensures the lock is held for the shortest
+    # time possible.
+    # Note: The lock isn't acquired explicitly via something like SELECT FOR UPDATE,
+    # but rather we rely on the UPDATE statement taking an implicit row lock.
+
+    # Equivalent Python code:
+    # if creating or len(update_fields) > 0:
+    #     presence_sequence = PresenceSequence.objects.select_for_update(no_key=True).get(realm_id=user_profile.realm_id)
+    #     new_last_update_id = presence_sequence.last_update_id + 1
+    #     presence_sequence.last_update_id = new_last_update_id
+    #     if creating:
+    #         presence.last_update_id = new_last_update_id
+    #         presence.save()
+    #     elif len(update_fields) > 0:
+    #         presence.last_update_id = new_last_update_id
+    #         presence.save(update_fields=[*update_fields, "last_update_id"])
+    #     presence_sequence.save(update_fields=["last_update_id"])
+    # But let's do it in a single, direct SQL query instead.
+
+    if creating or len(update_fields) > 0:
+        query = sql.SQL("""
+            WITH new_last_update_id AS (
+                UPDATE zerver_presencesequence
+                SET last_update_id = last_update_id + 1
+                WHERE realm_id = {realm_id}
+                RETURNING last_update_id
+            )
+        """).format(realm_id=sql.Literal(user_profile.realm_id))
+
+        if creating:
+            # There's a small possibility of a race where a different process may have
+            # already created a row for this user. Given the extremely close timing
+            # of these events, there's no clear reason to prefer one over the other,
+            # so we choose the simplest option of DO NOTHING here and let the other
+            # concurrent transaction win.
+            # This also allows us to avoid sending a spurious presence update event,
+            # by checking if the row was actually created.
+            query += sql.SQL("""
+                INSERT INTO zerver_userpresence (user_profile_id, last_active_time, last_connected_time, realm_id, last_update_id)
+                VALUES ({user_profile_id}, {last_active_time}, {last_connected_time}, {realm_id}, (SELECT last_update_id FROM new_last_update_id))
+                ON CONFLICT (user_profile_id) DO NOTHING
+                """).format(
+                user_profile_id=sql.Literal(user_profile.id),
+                last_active_time=sql.Literal(presence.last_active_time),
+                last_connected_time=sql.Literal(presence.last_connected_time),
+                realm_id=sql.Literal(user_profile.realm_id),
+            )
+        else:
+            assert len(update_fields) > 0
+            update_fields_segment = sql.SQL(", ").join(
+                sql.SQL("{field} = {value}  ").format(
+                    field=sql.Identifier(field), value=sql.Literal(getattr(presence, field))
+                )
+                for field in update_fields
+            )
+            query += sql.SQL("""
+                UPDATE zerver_userpresence
+                SET {update_fields_segment}, last_update_id = (SELECT last_update_id FROM new_last_update_id)
+                WHERE id = {presence_id}
+            """).format(
+                update_fields_segment=update_fields_segment, presence_id=sql.Literal(presence.id)
+            )
+
+        with connection.cursor() as cursor:
+            cursor.execute(query)
+            if creating:
+                # Check if the row was actually created or if we
+                # hit the ON CONFLICT DO NOTHING case.
+                actually_created = cursor.rowcount > 0
+
+    if creating and not actually_created:
+        # If we ended up doing nothing due to something else creating the row
+        # in the meantime, then we shouldn't send an event here.
+        logger.info("UserPresence row already created for %s, returning.", user_profile.id)
+        return
+
+    if force_send_update or (
+        not user_profile.realm.presence_disabled and (creating or became_online)
+    ):
+        # We do the transaction.on_commit here, rather than inside
+        # send_presence_changed, to help keep presence transactions
+        # brief; the active_user_ids call there is more expensive than
+        # this whole function.
+        transaction.on_commit(
+            lambda: send_presence_changed(
+                user_profile, presence, force_send_update=force_send_update
+            )
+        )
+
+
+def update_user_presence(
+    user_profile: UserProfile,
+    client: Client,
+    log_time: datetime,
+    status: int,
+    new_user_input: bool,
+) -> None:
+    logger.debug(
+        "Processing presence update for user %s, client %s, status %s",
+        user_profile.id,
+        client,
+        status,
+    )
+    if user_profile.presence_enabled:
+        do_update_user_presence(user_profile, client, log_time, status)
+    if new_user_input:
+        update_user_activity_interval(user_profile, log_time)

zerver/actions/push_notifications.py

@@ -0,0 +1,94 @@
+from django.utils.timezone import now as timezone_now
+
+from zerver.lib.timestamp import datetime_to_timestamp
+from zerver.models import UserProfile
+from zerver.models.devices import Device
+from zerver.tornado.django_api import send_event_on_commit
+
+
+def do_register_push_device(
+    user_profile: UserProfile,
+    device: Device,
+    *,
+    token_kind: str,
+    push_key_bytes: bytes,
+    push_key_id: int,
+    token_id_int: int,
+    token_id_base64: str,
+) -> None:
+    registered_at = timezone_now()
+    device.push_key = push_key_bytes
+    device.push_key_id = push_key_id
+    device.pending_push_token_id = token_id_int
+    device.push_token_kind = token_kind
+    device.push_token_last_updated_timestamp = registered_at
+    device.push_registration_error_code = None
+    device.save(
+        update_fields=[
+            "push_key",
+            "push_key_id",
+            "pending_push_token_id",
+            "push_token_kind",
+            "push_token_last_updated_timestamp",
+            "push_registration_error_code",
+        ]
+    )
+
+    event = dict(
+        type="device",
+        op="update",
+        device_id=device.id,
+        push_key_id=device.push_key_id,
+        pending_push_token_id=token_id_base64,
+        push_token_last_updated_timestamp=datetime_to_timestamp(registered_at),
+        push_registration_error_code=None,
+    )
+    send_event_on_commit(user_profile.realm, event, [user_profile.id])
+
+
+def do_rotate_push_key(
+    user_profile: UserProfile, device: Device, push_key_bytes: bytes, push_key_id: int
+) -> None:
+    if (
+        device.push_key_id == push_key_id
+        and device.push_key is not None
+        and bytes(device.push_key) == push_key_bytes
+    ):
+        return
+
+    device.push_key = push_key_bytes
+    device.push_key_id = push_key_id
+    device.save(update_fields=["push_key", "push_key_id"])
+
+    event = dict(
+        type="device",
+        op="update",
+        device_id=device.id,
+        push_key_id=device.push_key_id,
+    )
+    send_event_on_commit(user_profile.realm, event, [user_profile.id])
+
+
+def do_rotate_token(
+    user_profile: UserProfile, device: Device, token_id_int: int, token_id_base64: str
+) -> None:
+    token_updated_at = timezone_now()
+    device.pending_push_token_id = token_id_int
+    device.push_token_last_updated_timestamp = token_updated_at
+    device.push_registration_error_code = None
+    device.save(
+        update_fields=[
+            "pending_push_token_id",
+            "push_token_last_updated_timestamp",
+            "push_registration_error_code",
+        ]
+    )
+    event = dict(
+        type="device",
+        op="update",
+        device_id=device.id,
+        pending_push_token_id=token_id_base64,
+        push_token_last_updated_timestamp=datetime_to_timestamp(token_updated_at),
+        push_registration_error_code=None,
+    )
+    send_event_on_commit(user_profile.realm, event, [user_profile.id])

zerver/actions/reactions.py

@@ -0,0 +1,190 @@
+from typing import Literal
+
+from typing_extensions import TypedDict
+
+from zerver.actions.user_topics import do_set_user_topic_visibility_policy
+from zerver.lib.emoji import check_emoji_request, get_emoji_data
+from zerver.lib.exceptions import ReactionExistsError
+from zerver.lib.message import (
+    access_message_and_usermessage,
+    event_recipient_ids_for_action_on_messages,
+    set_visibility_policy_possible,
+    should_change_visibility_policy,
+    visibility_policy_for_participation,
+)
+from zerver.lib.message_cache import update_message_cache
+from zerver.lib.streams import access_stream_by_id
+from zerver.lib.user_message import create_historical_user_messages
+from zerver.models import Message, Reaction, UserProfile
+from zerver.tornado.django_api import send_event_on_commit
+
+
+class ReactionEvent(TypedDict):
+    type: Literal["reaction"]
+    op: Literal["add", "remove"]
+    user_id: int
+    message_id: int
+    emoji_name: str
+    emoji_code: str
+    reaction_type: str
+
+
+def notify_reaction_update(
+    user_profile: UserProfile,
+    message: Message,
+    reaction: Reaction,
+    op: Literal["add", "remove"],
+) -> None:
+    event: ReactionEvent = {
+        "type": "reaction",
+        "op": op,
+        "user_id": user_profile.id,
+        "message_id": message.id,
+        "emoji_name": reaction.emoji_name,
+        "emoji_code": reaction.emoji_code,
+        "reaction_type": reaction.reaction_type,
+    }
+
+    # Update the cached message since new reaction is added.
+    update_message_cache([message])
+
+    user_ids = event_recipient_ids_for_action_on_messages([message.id], message.is_channel_message)
+    send_event_on_commit(user_profile.realm, event, list(user_ids))
+
+
+def do_add_reaction(
+    user_profile: UserProfile,
+    message: Message,
+    emoji_name: str,
+    emoji_code: str,
+    reaction_type: str,
+) -> None:
+    """Should be called while holding a SELECT FOR UPDATE lock
+    (e.g. via access_message(..., lock_message=True)) on the
+    Message row, to prevent race conditions.
+    """
+
+    reaction = Reaction(
+        user_profile=user_profile,
+        message=message,
+        emoji_name=emoji_name,
+        emoji_code=emoji_code,
+        reaction_type=reaction_type,
+    )
+
+    reaction.save()
+
+    # Determine and set the visibility_policy depending on 'automatically_follow_topics_policy'
+    # and 'automatically_unmute_topics_in_muted_streams_policy'.
+    if set_visibility_policy_possible(
+        user_profile, message
+    ) and UserProfile.AUTOMATICALLY_CHANGE_VISIBILITY_POLICY_ON_PARTICIPATION in [
+        user_profile.automatically_follow_topics_policy,
+        user_profile.automatically_unmute_topics_in_muted_streams_policy,
+    ]:
+        stream_id = message.recipient.type_id
+        (stream, sub) = access_stream_by_id(user_profile, stream_id)
+        assert stream is not None
+        if sub:
+            new_visibility_policy = visibility_policy_for_participation(user_profile, sub.is_muted)
+            if new_visibility_policy and should_change_visibility_policy(
+                new_visibility_policy,
+                user_profile,
+                stream_id,
+                topic_name=message.topic_name(),
+            ):
+                do_set_user_topic_visibility_policy(
+                    user_profile=user_profile,
+                    stream=stream,
+                    topic_name=message.topic_name(),
+                    visibility_policy=new_visibility_policy,
+                )
+
+    notify_reaction_update(user_profile, message, reaction, "add")
+
+
+def check_add_reaction(
+    user_profile: UserProfile,
+    message_id: int,
+    emoji_name: str,
+    emoji_code: str | None,
+    reaction_type: str | None,
+) -> None:
+    message, user_message = access_message_and_usermessage(
+        user_profile, message_id, lock_message=True, is_modifying_message=True
+    )
+
+    if emoji_code is None or reaction_type is None:
+        emoji_data = get_emoji_data(message.realm_id, emoji_name)
+
+        if emoji_code is None:
+            # The emoji_code argument is only required for rare corner
+            # cases discussed in the long block comment below.  For simple
+            # API clients, we allow specifying just the name, and just
+            # look up the code using the current name->code mapping.
+            emoji_code = emoji_data.emoji_code
+
+        if reaction_type is None:
+            reaction_type = emoji_data.reaction_type
+
+    if Reaction.objects.filter(
+        user_profile=user_profile,
+        message=message,
+        emoji_code=emoji_code,
+        reaction_type=reaction_type,
+    ).exists():
+        raise ReactionExistsError
+
+    query = Reaction.objects.filter(
+        message=message, emoji_code=emoji_code, reaction_type=reaction_type
+    )
+    if query.exists():
+        # If another user has already reacted to this message with
+        # same emoji code, we treat the new reaction as a vote for the
+        # existing reaction.  So the emoji name used by that earlier
+        # reaction takes precedence over whatever was passed in this
+        # request.  This is necessary to avoid a message having 2
+        # "different" emoji reactions with the same emoji code (and
+        # thus same image) on the same message, which looks ugly.
+        #
+        # In this "voting for an existing reaction" case, we shouldn't
+        # check whether the emoji code and emoji name match, since
+        # it's possible that the (emoji_type, emoji_name, emoji_code)
+        # triple for this existing reaction may not pass validation
+        # now (e.g. because it is for a realm emoji that has been
+        # since deactivated).  We still want to allow users to add a
+        # vote any old reaction they see in the UI even if that is a
+        # deactivated custom emoji, so we just use the emoji name from
+        # the existing reaction with no further validation.
+        reaction = query.first()
+        assert reaction is not None
+        emoji_name = reaction.emoji_name
+    else:
+        # Otherwise, use the name provided in this request, but verify
+        # it is valid in the user's realm (e.g. not a deactivated
+        # realm emoji).
+        check_emoji_request(user_profile.realm, emoji_name, emoji_code, reaction_type)
+
+    if user_message is None:
+        # See called function for more context.
+        create_historical_user_messages(user_id=user_profile.id, message_ids=[message.id])
+
+    do_add_reaction(user_profile, message, emoji_name, emoji_code, reaction_type)
+
+
+def do_remove_reaction(
+    user_profile: UserProfile, message: Message, emoji_code: str, reaction_type: str
+) -> None:
+    """Should be called while holding a SELECT FOR UPDATE lock
+    (e.g. via access_message(..., lock_message=True)) on the
+    Message row, to prevent race conditions.
+    """
+    reaction = Reaction.objects.filter(
+        user_profile=user_profile,
+        message=message,
+        emoji_code=emoji_code,
+        reaction_type=reaction_type,
+    ).get()
+    reaction.delete()
+
+    notify_reaction_update(user_profile, message, reaction, "remove")

zerver/actions/realm_domains.py

@@ -0,0 +1,104 @@
+from django.db import transaction
+from django.utils.timezone import now as timezone_now
+
+from zerver.actions.realm_settings import do_set_realm_property
+from zerver.models import Realm, RealmAuditLog, RealmDomain, UserProfile
+from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.models.realms import RealmDomainDict, get_realm_domains
+from zerver.models.users import active_user_ids
+from zerver.tornado.django_api import send_event_on_commit
+
+
+@transaction.atomic(durable=True)
+def do_add_realm_domain(
+    realm: Realm, domain: str, allow_subdomains: bool, *, acting_user: UserProfile | None
+) -> RealmDomain:
+    realm_domain = RealmDomain.objects.create(
+        realm=realm, domain=domain, allow_subdomains=allow_subdomains
+    )
+    added_domain = RealmDomainDict(domain=domain, allow_subdomains=allow_subdomains)
+
+    RealmAuditLog.objects.create(
+        realm=realm,
+        acting_user=acting_user,
+        event_type=AuditLogEventType.REALM_DOMAIN_ADDED,
+        event_time=timezone_now(),
+        extra_data={
+            "realm_domains": get_realm_domains(realm),
+            "added_domain": added_domain,
+        },
+    )
+
+    event = dict(
+        type="realm_domains",
+        op="add",
+        realm_domain=RealmDomainDict(
+            domain=realm_domain.domain, allow_subdomains=realm_domain.allow_subdomains
+        ),
+    )
+    send_event_on_commit(realm, event, active_user_ids(realm.id))
+
+    return realm_domain
+
+
+@transaction.atomic(durable=True)
+def do_change_realm_domain(
+    realm_domain: RealmDomain, allow_subdomains: bool, *, acting_user: UserProfile | None
+) -> None:
+    realm_domain.allow_subdomains = allow_subdomains
+    realm_domain.save(update_fields=["allow_subdomains"])
+    changed_domain = RealmDomainDict(
+        domain=realm_domain.domain,
+        allow_subdomains=realm_domain.allow_subdomains,
+    )
+
+    RealmAuditLog.objects.create(
+        realm=realm_domain.realm,
+        acting_user=acting_user,
+        event_type=AuditLogEventType.REALM_DOMAIN_CHANGED,
+        event_time=timezone_now(),
+        extra_data={
+            "realm_domains": get_realm_domains(realm_domain.realm),
+            "changed_domain": changed_domain,
+        },
+    )
+
+    event = dict(
+        type="realm_domains",
+        op="change",
+        realm_domain=dict(
+            domain=realm_domain.domain, allow_subdomains=realm_domain.allow_subdomains
+        ),
+    )
+    send_event_on_commit(realm_domain.realm, event, active_user_ids(realm_domain.realm_id))
+
+
+@transaction.atomic(durable=True)
+def do_remove_realm_domain(realm_domain: RealmDomain, *, acting_user: UserProfile | None) -> None:
+    realm = realm_domain.realm
+    domain = realm_domain.domain
+    realm_domain.delete()
+    removed_domain = RealmDomainDict(
+        domain=realm_domain.domain,
+        allow_subdomains=realm_domain.allow_subdomains,
+    )
+
+    RealmAuditLog.objects.create(
+        realm=realm,
+        acting_user=acting_user,
+        event_type=AuditLogEventType.REALM_DOMAIN_REMOVED,
+        event_time=timezone_now(),
+        extra_data={
+            "realm_domains": get_realm_domains(realm),
+            "removed_domain": removed_domain,
+        },
+    )
+
+    if not RealmDomain.objects.filter(realm=realm).exists() and realm.emails_restricted_to_domains:
+        # If this was the last realm domain, we mark the realm as no
+        # longer restricted to domain, because the feature doesn't do
+        # anything if there are no domains, and this is probably less
+        # confusing than the alternative.
+        do_set_realm_property(realm, "emails_restricted_to_domains", False, acting_user=acting_user)
+    event = dict(type="realm_domains", op="remove", domain=domain)
+    send_event_on_commit(realm, event, active_user_ids(realm.id))

zerver/actions/realm_emoji.py

@@ -0,0 +1,97 @@
+from typing import IO
+
+from django.core.exceptions import ValidationError
+from django.db import transaction
+from django.db.utils import IntegrityError
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+
+from zerver.lib.emoji import get_emoji_file_name
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.mime_types import INLINE_MIME_TYPES, bare_content_type
+from zerver.lib.thumbnail import THUMBNAIL_ACCEPT_IMAGE_TYPES, BadImageError
+from zerver.lib.upload import upload_emoji_image
+from zerver.models import Realm, RealmAuditLog, RealmEmoji, UserProfile
+from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.models.realm_emoji import emoji_as_emojiinfo, get_all_custom_emoji_for_realm
+from zerver.models.users import active_user_ids
+from zerver.tornado.django_api import send_event_on_commit
+
+
+@transaction.atomic(durable=True)
+def check_add_realm_emoji(
+    realm: Realm, name: str, author: UserProfile, image_file: IO[bytes], content_type: str
+) -> RealmEmoji:
+    content_type = bare_content_type(content_type)
+    try:
+        realm_emoji = RealmEmoji(realm=realm, name=name, author=author)
+        realm_emoji.full_clean()
+        realm_emoji.save()
+    except (IntegrityError, ValidationError):
+        # This exists to handle races; upload_emoji checked prior to
+        # calling, and also hand-validated the name, but we're now in
+        # a transaction.  The error string should match the one in
+        # upload_emoji.
+        raise JsonableError(_("A custom emoji with this name already exists."))
+
+    # This mirrors the check in upload_emoji_image because we want to
+    # have some reasonable guarantees on the content-type before
+    # deriving the file extension from it.
+    if content_type not in THUMBNAIL_ACCEPT_IMAGE_TYPES or content_type not in INLINE_MIME_TYPES:
+        raise BadImageError(_("Invalid image format"))
+
+    emoji_file_name = get_emoji_file_name(content_type, realm_emoji.id)
+    is_animated = upload_emoji_image(image_file, emoji_file_name, author, content_type)
+
+    realm_emoji.file_name = emoji_file_name
+    realm_emoji.is_animated = is_animated
+    realm_emoji.save(update_fields=["file_name", "is_animated"])
+
+    new_emoji = emoji_as_emojiinfo(realm_emoji)
+    RealmAuditLog.objects.create(
+        realm=realm,
+        acting_user=author,
+        event_type=AuditLogEventType.REALM_EMOJI_ADDED,
+        event_time=timezone_now(),
+        extra_data={
+            "added_emoji": new_emoji,
+        },
+    )
+    # TODO/compatibility: Remove realm_emoji_dict once we can require that clients support
+    # the individual_emoji_changes client capability.
+    realm_emoji_dict = get_all_custom_emoji_for_realm(realm.id)
+    event = dict(type="realm_emoji", op="add", emoji=new_emoji, realm_emoji=realm_emoji_dict)
+    send_event_on_commit(realm, event, active_user_ids(realm.id))
+    return realm_emoji
+
+
+@transaction.atomic(durable=True)
+def do_remove_realm_emoji(realm: Realm, name: str, *, acting_user: UserProfile | None) -> None:
+    realm_emoji = RealmEmoji.objects.get(realm=realm, name=name, deactivated=False)
+    realm_emoji.deactivated = True
+    realm_emoji.save(update_fields=["deactivated"])
+
+    deactivated_emoji = emoji_as_emojiinfo(realm_emoji)
+    RealmAuditLog.objects.create(
+        realm=realm,
+        acting_user=acting_user,
+        event_type=AuditLogEventType.REALM_EMOJI_REMOVED,
+        event_time=timezone_now(),
+        extra_data={
+            "deactivated_emoji": deactivated_emoji,
+        },
+    )
+
+    # TODO/compatibility: Remove realm_emoji_dict once we can require that clients support
+    # the individual_emoji_changes client capability.
+    realm_emoji_dict = get_all_custom_emoji_for_realm(realm.id)
+    event = dict(
+        type="realm_emoji",
+        op="update_one",
+        emoji_id=str(realm_emoji.id),
+        data=dict(
+            deactivated=True,
+        ),
+        realm_emoji=realm_emoji_dict,
+    )
+    send_event_on_commit(realm, event, active_user_ids(realm.id))

zerver/actions/realm_export.py

@@ -0,0 +1,34 @@
+from django.db import transaction
+from django.utils.timezone import now as timezone_now
+
+from zerver.lib.export import get_realm_exports_serialized
+from zerver.lib.upload import delete_export_tarball
+from zerver.models import Realm, RealmAuditLog, RealmExport, UserProfile
+from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.tornado.django_api import send_event_on_commit
+
+
+def notify_realm_export(realm: Realm) -> None:
+    event = dict(type="realm_export", exports=get_realm_exports_serialized(realm))
+    send_event_on_commit(realm, event, realm.get_human_admin_users().values_list("id", flat=True))
+
+
+@transaction.atomic(durable=True)
+def do_delete_realm_export(export_row: RealmExport, acting_user: UserProfile) -> None:
+    export_path = export_row.export_path
+    assert export_path is not None
+
+    delete_export_tarball(export_path)
+
+    export_row.status = RealmExport.DELETED
+    export_row.date_deleted = timezone_now()
+    export_row.save(update_fields=["status", "date_deleted"])
+    notify_realm_export(export_row.realm)
+
+    RealmAuditLog.objects.create(
+        acting_user=acting_user,
+        realm=export_row.realm,
+        event_type=AuditLogEventType.REALM_EXPORT_DELETED,
+        event_time=export_row.date_deleted,
+        extra_data={"realm_export_id": export_row.id},
+    )

zerver/actions/realm_icon.py

@@ -0,0 +1,38 @@
+from django.db import transaction
+from django.utils.timezone import now as timezone_now
+
+from zerver.lib.realm_icon import realm_icon_url
+from zerver.models import Realm, RealmAuditLog, UserProfile
+from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.models.users import active_user_ids
+from zerver.tornado.django_api import send_event_on_commit
+
+
+@transaction.atomic(durable=True)
+def do_change_icon_source(
+    realm: Realm, icon_source: str, *, acting_user: UserProfile | None
+) -> None:
+    realm.icon_source = icon_source
+    realm.icon_version += 1
+    realm.save(update_fields=["icon_source", "icon_version"])
+
+    event_time = timezone_now()
+    RealmAuditLog.objects.create(
+        realm=realm,
+        event_type=AuditLogEventType.REALM_ICON_SOURCE_CHANGED,
+        extra_data={"icon_source": icon_source, "icon_version": realm.icon_version},
+        event_time=event_time,
+        acting_user=acting_user,
+    )
+
+    event = dict(
+        type="realm",
+        op="update_dict",
+        property="icon",
+        data=dict(icon_source=realm.icon_source, icon_url=realm_icon_url(realm)),
+    )
+    send_event_on_commit(
+        realm,
+        event,
+        active_user_ids(realm.id),
+    )

zerver/actions/realm_linkifiers.py

@@ -0,0 +1,230 @@
+from django.db import transaction
+from django.db.models import Max
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.types import LinkifierDict
+from zerver.models import Realm, RealmAuditLog, RealmFilter, UserProfile
+from zerver.models.linkifiers import flush_linkifiers, linkifiers_for_realm, normalize_optional_text
+from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.models.users import active_user_ids
+from zerver.tornado.django_api import send_event_on_commit
+
+
+def notify_linkifiers(realm: Realm, realm_linkifiers: list[LinkifierDict]) -> None:
+    event: dict[str, object] = dict(type="realm_linkifiers", realm_linkifiers=realm_linkifiers)
+    send_event_on_commit(realm, event, active_user_ids(realm.id))
+
+
+# NOTE: Regexes must be simple enough that they can be easily translated to JavaScript
+# RegExp syntax. In addition to JS-compatible syntax, the following features are available:
+#   * Named groups will be converted to numbered groups automatically
+#   * Inline-regex flags will be stripped, and where possible translated to RegExp-wide flags
+@transaction.atomic(durable=True)
+def do_add_linkifier(
+    realm: Realm,
+    pattern: str,
+    url_template: str,
+    example_input: str | None = None,
+    reverse_template: str | None = None,
+    alternative_url_templates: list[str] | None = None,
+    *,
+    acting_user: UserProfile | None,
+) -> int:
+    pattern = pattern.strip()
+    url_template = url_template.strip()
+    example_input = normalize_optional_text(example_input)
+    reverse_template = normalize_optional_text(reverse_template)
+    if alternative_url_templates is None:
+        alternative_url_templates = []
+    # This makes sure that the new linkifier is always ordered the last modulo
+    # the rare race condition.
+    max_order = RealmFilter.objects.aggregate(Max("order"))["order__max"]
+    if max_order is None:
+        linkifier = RealmFilter(
+            realm=realm,
+            pattern=pattern,
+            url_template=url_template,
+            example_input=example_input,
+            reverse_template=reverse_template,
+            alternative_url_templates=alternative_url_templates,
+        )
+    else:
+        linkifier = RealmFilter(
+            realm=realm,
+            pattern=pattern,
+            url_template=url_template,
+            example_input=example_input,
+            reverse_template=reverse_template,
+            alternative_url_templates=alternative_url_templates,
+            order=max_order + 1,
+        )
+    linkifier.full_clean()
+    linkifier.save()
+
+    realm_linkifiers = linkifiers_for_realm(realm.id)
+    RealmAuditLog.objects.create(
+        realm=realm,
+        acting_user=acting_user,
+        event_type=AuditLogEventType.REALM_LINKIFIER_ADDED,
+        event_time=timezone_now(),
+        extra_data={
+            "realm_linkifiers": realm_linkifiers,
+            "added_linkifier": LinkifierDict(
+                pattern=pattern,
+                url_template=url_template,
+                id=linkifier.id,
+                example_input=example_input,
+                reverse_template=reverse_template,
+                alternative_url_templates=linkifier.alternative_url_templates,
+            ),
+        },
+    )
+    notify_linkifiers(realm, realm_linkifiers)
+
+    return linkifier.id
+
+
+@transaction.atomic(durable=True)
+def do_remove_linkifier(
+    realm: Realm,
+    pattern: str | None = None,
+    id: int | None = None,
+    *,
+    acting_user: UserProfile | None = None,
+) -> None:
+    if pattern is not None:
+        realm_linkifier = RealmFilter.objects.get(realm=realm, pattern=pattern)
+    else:
+        assert id is not None
+        realm_linkifier = RealmFilter.objects.get(realm=realm, id=id)
+
+    pattern = realm_linkifier.pattern
+    url_template = realm_linkifier.url_template
+    realm_linkifier.delete()
+
+    realm_linkifiers = linkifiers_for_realm(realm.id)
+    RealmAuditLog.objects.create(
+        realm=realm,
+        acting_user=acting_user,
+        event_type=AuditLogEventType.REALM_LINKIFIER_REMOVED,
+        event_time=timezone_now(),
+        extra_data={
+            "realm_linkifiers": realm_linkifiers,
+            "removed_linkifier": {
+                "pattern": pattern,
+                "url_template": url_template,
+                "example_input": realm_linkifier.example_input,
+                "reverse_template": realm_linkifier.reverse_template,
+                "alternative_url_templates": realm_linkifier.alternative_url_templates,
+            },
+        },
+    )
+    notify_linkifiers(realm, realm_linkifiers)
+
+
+@transaction.atomic(durable=True)
+def do_update_linkifier(
+    realm: Realm,
+    id: int,
+    pattern: str,
+    url_template: str,
+    example_input: str | None = None,
+    reverse_template: str | None = None,
+    alternative_url_templates: list[str] | None = None,
+    *,
+    acting_user: UserProfile | None,
+) -> None:
+    pattern = pattern.strip()
+    url_template = url_template.strip()
+    linkifier = RealmFilter.objects.get(realm=realm, id=id)
+    linkifier.pattern = pattern
+    linkifier.url_template = url_template
+    linkifier.example_input = normalize_optional_text(example_input)
+    linkifier.reverse_template = normalize_optional_text(reverse_template)
+    if alternative_url_templates is not None:
+        linkifier.alternative_url_templates = alternative_url_templates
+
+    # Validation of fields is done by full_clean.
+    linkifier.full_clean()
+    update_fields = ["pattern", "url_template"]
+    # None means unchanged. Empty string means null, normalize_optional_text
+    # takes care of converting empty string to None.
+    if example_input is not None:
+        update_fields.append("example_input")
+    if reverse_template is not None:
+        update_fields.append("reverse_template")
+    if alternative_url_templates is not None:
+        update_fields.append("alternative_url_templates")
+    linkifier.save(update_fields=update_fields)
+
+    realm_linkifiers = linkifiers_for_realm(realm.id)
+    RealmAuditLog.objects.create(
+        realm=realm,
+        acting_user=acting_user,
+        event_type=AuditLogEventType.REALM_LINKIFIER_CHANGED,
+        event_time=timezone_now(),
+        extra_data={
+            "realm_linkifiers": realm_linkifiers,
+            "changed_linkifier": LinkifierDict(
+                pattern=pattern,
+                url_template=url_template,
+                id=linkifier.id,
+                example_input=linkifier.example_input,
+                reverse_template=linkifier.reverse_template,
+                alternative_url_templates=linkifier.alternative_url_templates,
+            ),
+        },
+    )
+
+    notify_linkifiers(realm, realm_linkifiers)
+
+
+@transaction.atomic(durable=True)
+def check_reorder_linkifiers(
+    realm: Realm, ordered_linkifier_ids: list[int], *, acting_user: UserProfile | None
+) -> None:
+    """ordered_linkifier_ids should contain ids of all existing linkifiers.
+    In the rare situation when any of the linkifier gets deleted that more ids
+    are passed, the checks below are sufficient to detect inconsistencies most of
+    the time."""
+    # Repeated IDs in the user request would collapse into the same key when
+    # constructing the set.
+    linkifier_id_set = set(ordered_linkifier_ids)
+    if len(linkifier_id_set) < len(ordered_linkifier_ids):
+        raise JsonableError(_("The ordered list must not contain duplicated linkifiers"))
+
+    linkifiers = RealmFilter.objects.filter(realm=realm)
+    if {linkifier.id for linkifier in linkifiers} != linkifier_id_set:
+        raise JsonableError(
+            _("The ordered list must enumerate all existing linkifiers exactly once")
+        )
+
+    # After the validation, we are sure that there is nothing to do. Return
+    # early to avoid flushing the cache and populating the audit logs.
+    if len(linkifiers) == 0:
+        return
+
+    id_to_new_order = {
+        linkifier_id: order for order, linkifier_id in enumerate(ordered_linkifier_ids)
+    }
+
+    for linkifier in linkifiers:
+        assert linkifier.id in id_to_new_order
+        linkifier.order = id_to_new_order[linkifier.id]
+    RealmFilter.objects.bulk_update(linkifiers, fields=["order"])
+    flush_linkifiers(instance=linkifiers[0])
+
+    # This roundtrip re-fetches the linkifiers sorted in the new order.
+    realm_linkifiers = linkifiers_for_realm(realm.id)
+    RealmAuditLog.objects.create(
+        realm=realm,
+        acting_user=acting_user,
+        event_type=AuditLogEventType.REALM_LINKIFIERS_REORDERED,
+        event_time=timezone_now(),
+        extra_data={
+            "realm_linkifiers": realm_linkifiers,
+        },
+    )
+    notify_linkifiers(realm, realm_linkifiers)

zerver/actions/realm_logo.py

@@ -0,0 +1,38 @@
+from django.db import transaction
+from django.utils.timezone import now as timezone_now
+
+from zerver.lib.realm_logo import get_realm_logo_data
+from zerver.models import Realm, RealmAuditLog, UserProfile
+from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.models.users import active_user_ids
+from zerver.tornado.django_api import send_event_on_commit
+
+
+@transaction.atomic(durable=True)
+def do_change_logo_source(
+    realm: Realm, logo_source: str, night: bool, *, acting_user: UserProfile | None
+) -> None:
+    if not night:
+        realm.logo_source = logo_source
+        realm.logo_version += 1
+        realm.save(update_fields=["logo_source", "logo_version"])
+
+    else:
+        realm.night_logo_source = logo_source
+        realm.night_logo_version += 1
+        realm.save(update_fields=["night_logo_source", "night_logo_version"])
+
+    RealmAuditLog.objects.create(
+        event_type=AuditLogEventType.REALM_LOGO_CHANGED,
+        realm=realm,
+        event_time=timezone_now(),
+        acting_user=acting_user,
+    )
+
+    event = dict(
+        type="realm",
+        op="update_dict",
+        property="night_logo" if night else "logo",
+        data=get_realm_logo_data(realm, night),
+    )
+    send_event_on_commit(realm, event, active_user_ids(realm.id))

zerver/actions/realm_playgrounds.py

@@ -0,0 +1,87 @@
+from django.core.exceptions import ValidationError
+from django.db import transaction
+from django.utils.timezone import now as timezone_now
+
+from zerver.lib.exceptions import ValidationFailureError
+from zerver.lib.types import RealmPlaygroundDict
+from zerver.models import Realm, RealmAuditLog, RealmPlayground, UserProfile
+from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.models.realm_playgrounds import get_realm_playgrounds
+from zerver.models.users import active_user_ids
+from zerver.tornado.django_api import send_event_on_commit
+
+
+def notify_realm_playgrounds(realm: Realm, realm_playgrounds: list[RealmPlaygroundDict]) -> None:
+    event = dict(type="realm_playgrounds", realm_playgrounds=realm_playgrounds)
+    send_event_on_commit(realm, event, active_user_ids(realm.id))
+
+
+@transaction.atomic(durable=True)
+def check_add_realm_playground(
+    realm: Realm,
+    *,
+    acting_user: UserProfile | None,
+    name: str,
+    pygments_language: str,
+    url_template: str,
+) -> int:
+    realm_playground = RealmPlayground(
+        realm=realm,
+        name=name,
+        pygments_language=pygments_language,
+        url_template=url_template,
+    )
+    # The additional validations using url_template_validation
+    # check_pygments_language, etc are included in full_clean.
+    # Because we want to avoid raising ValidationError from this check_*
+    # function, we do error handling here to turn it into a JsonableError.
+    try:
+        realm_playground.full_clean()
+    except ValidationError as e:
+        raise ValidationFailureError(e)
+    realm_playground.save()
+    realm_playgrounds = get_realm_playgrounds(realm)
+    RealmAuditLog.objects.create(
+        realm=realm,
+        acting_user=acting_user,
+        event_type=AuditLogEventType.REALM_PLAYGROUND_ADDED,
+        event_time=timezone_now(),
+        extra_data={
+            "realm_playgrounds": realm_playgrounds,
+            "added_playground": RealmPlaygroundDict(
+                id=realm_playground.id,
+                name=realm_playground.name,
+                pygments_language=realm_playground.pygments_language,
+                url_template=realm_playground.url_template,
+            ),
+        },
+    )
+    notify_realm_playgrounds(realm, realm_playgrounds)
+    return realm_playground.id
+
+
+@transaction.atomic(durable=True)
+def do_remove_realm_playground(
+    realm: Realm, realm_playground: RealmPlayground, *, acting_user: UserProfile | None
+) -> None:
+    removed_playground = {
+        "name": realm_playground.name,
+        "pygments_language": realm_playground.pygments_language,
+        "url_template": realm_playground.url_template,
+    }
+
+    realm_playground.delete()
+    realm_playgrounds = get_realm_playgrounds(realm)
+
+    RealmAuditLog.objects.create(
+        realm=realm,
+        acting_user=acting_user,
+        event_type=AuditLogEventType.REALM_PLAYGROUND_REMOVED,
+        event_time=timezone_now(),
+        extra_data={
+            "realm_playgrounds": realm_playgrounds,
+            "removed_playground": removed_playground,
+        },
+    )
+
+    notify_realm_playgrounds(realm, realm_playgrounds)

zerver/actions/reminders.py

@@ -0,0 +1,49 @@
+import datetime
+
+from django.db import transaction
+
+from zerver.actions.message_send import check_message
+from zerver.actions.scheduled_messages import do_schedule_messages
+from zerver.lib.addressee import Addressee
+from zerver.lib.message import access_message
+from zerver.lib.reminders import get_reminder_formatted_content, notify_remove_reminder
+from zerver.models import Client, ScheduledMessage, UserProfile
+
+
+def schedule_reminder_for_message(
+    current_user: UserProfile,
+    client: Client,
+    message_id: int,
+    deliver_at: datetime.datetime,
+    note: str,
+) -> int:
+    message = access_message(current_user, message_id, is_modifying_message=False)
+    # Even though reminder will be sent from NOTIFICATION_BOT, we still
+    # set current_user as the sender here to help us make the permission checks easier.
+    addressee = Addressee.for_user_profile(current_user)
+    # This can raise an exception in the unlikely event that the current user cannot DM themself.
+    send_request = check_message(
+        current_user,
+        client,
+        addressee,
+        get_reminder_formatted_content(message, current_user, note),
+        current_user.realm,
+    )
+    send_request.deliver_at = deliver_at
+    send_request.reminder_target_message_id = message_id
+    send_request.reminder_note = note
+
+    return do_schedule_messages(
+        [send_request],
+        current_user,
+        read_by_sender=False,
+        delivery_type=ScheduledMessage.REMIND,
+    )[0]
+
+
+@transaction.atomic(durable=True)
+def do_delete_reminder(user_profile: UserProfile, reminder: ScheduledMessage) -> None:
+    assert reminder.delivery_type == ScheduledMessage.REMIND
+    reminder_id = reminder.id
+    reminder.delete()
+    notify_remove_reminder(user_profile, reminder_id)

zerver/actions/saved_snippets.py

@@ -0,0 +1,91 @@
+from typing import Any
+
+from django.db import transaction
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+
+from zerver.lib.exceptions import ResourceNotFoundError
+from zerver.models import RealmAuditLog, SavedSnippet, UserProfile
+from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.tornado.django_api import send_event_on_commit
+
+
+@transaction.atomic(durable=True)
+def do_create_saved_snippet(
+    title: str,
+    content: str,
+    user_profile: UserProfile,
+) -> SavedSnippet:
+    saved_snippet = SavedSnippet.objects.create(
+        realm=user_profile.realm,
+        user_profile=user_profile,
+        title=title,
+        content=content,
+    )
+
+    RealmAuditLog.objects.create(
+        realm=user_profile.realm,
+        acting_user=user_profile,
+        modified_user=user_profile,
+        event_type=AuditLogEventType.SAVED_SNIPPET_CREATED,
+        event_time=timezone_now(),
+        extra_data={"saved_snippet_id": saved_snippet.id},
+    )
+
+    event = {
+        "type": "saved_snippets",
+        "op": "add",
+        "saved_snippet": saved_snippet.to_api_dict(),
+    }
+    send_event_on_commit(user_profile.realm, event, [user_profile.id])
+
+    return saved_snippet
+
+
+def do_edit_saved_snippet(
+    saved_snippet_id: int,
+    title: str | None,
+    content: str | None,
+    user_profile: UserProfile,
+) -> SavedSnippet:
+    try:
+        saved_snippet = SavedSnippet.objects.get(id=saved_snippet_id, user_profile=user_profile)
+    except SavedSnippet.DoesNotExist:
+        raise ResourceNotFoundError(_("Saved snippet does not exist."))
+
+    if title is not None:
+        saved_snippet.title = title
+    if content is not None:
+        saved_snippet.content = content
+
+    with transaction.atomic(durable=True):
+        saved_snippet.save()
+
+        event = {
+            "type": "saved_snippets",
+            "op": "update",
+            "saved_snippet": saved_snippet.to_api_dict(),
+        }
+        send_event_on_commit(user_profile.realm, event, [user_profile.id])
+
+    return saved_snippet
+
+
+def do_get_saved_snippets(user_profile: UserProfile) -> list[dict[str, Any]]:
+    saved_snippets = SavedSnippet.objects.filter(user_profile=user_profile)
+
+    return [saved_snippet.to_api_dict() for saved_snippet in saved_snippets]
+
+
+def do_delete_saved_snippet(
+    saved_snippet_id: int,
+    user_profile: UserProfile,
+) -> None:
+    try:
+        saved_snippet = SavedSnippet.objects.get(id=saved_snippet_id, user_profile=user_profile)
+    except SavedSnippet.DoesNotExist:
+        raise ResourceNotFoundError(_("Saved snippet does not exist."))
+    saved_snippet.delete()
+
+    event = {"type": "saved_snippets", "op": "remove", "saved_snippet_id": saved_snippet_id}
+    send_event_on_commit(user_profile.realm, event, [user_profile.id])

zerver/actions/scheduled_messages.py

@@ -0,0 +1,500 @@
+import logging
+from collections.abc import Sequence
+from datetime import datetime, timedelta
+
+from django.conf import settings
+from django.db import transaction
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+from django.utils.translation import override as override_language
+
+from zerver.actions.message_send import (
+    check_message,
+    do_send_messages,
+    internal_send_private_message,
+)
+from zerver.actions.uploads import check_attachment_reference_change, do_claim_attachments
+from zerver.lib.addressee import Addressee
+from zerver.lib.display_recipient import get_recipient_ids
+from zerver.lib.exceptions import (
+    DeliveryTimeNotInFutureError,
+    JsonableError,
+    RealmDeactivatedError,
+    UserDeactivatedError,
+)
+from zerver.lib.message import SendMessageRequest, access_message, truncate_topic
+from zerver.lib.recipient_parsing import extract_direct_message_recipient_ids, extract_stream_id
+from zerver.lib.reminders import get_reminder_formatted_content, notify_remove_reminder
+from zerver.lib.scheduled_messages import access_scheduled_message
+from zerver.lib.string_validation import check_stream_topic
+from zerver.lib.timestamp import datetime_to_global_time
+from zerver.models import Client, Realm, ScheduledMessage, Subscription, UserProfile
+from zerver.models.users import get_system_bot, is_cross_realm_bot_email
+from zerver.tornado.django_api import send_event_on_commit
+
+SCHEDULED_MESSAGE_LATE_CUTOFF_MINUTES = 10
+
+
+def check_schedule_message(
+    sender: UserProfile,
+    client: Client,
+    recipient_type_name: str,
+    message_to: list[int],
+    topic_name: str | None,
+    message_content: str,
+    deliver_at: datetime,
+    realm: Realm | None = None,
+    *,
+    read_by_sender: bool | None = None,
+    skip_events: bool = False,
+) -> int:
+    addressee = Addressee.legacy_build(sender, recipient_type_name, message_to, topic_name, realm)
+    send_request = check_message(
+        sender,
+        client,
+        addressee,
+        message_content,
+        realm=realm,
+    )
+    send_request.deliver_at = deliver_at
+
+    if read_by_sender is None:
+        # Legacy default: a scheduled message you sent from a non-API client is
+        # automatically marked as read for yourself, unless it was sent to
+        # yourself only.
+        read_by_sender = client.default_read_by_sender() and not addressee.is_message_to_self(
+            sender
+        )
+
+    return do_schedule_messages(
+        [send_request],
+        sender,
+        read_by_sender=read_by_sender,
+        skip_events=skip_events,
+        delivery_type=ScheduledMessage.SEND_LATER,
+    )[0]
+
+
+def notify_new_scheduled_message(
+    user_profile: UserProfile, scheduled_messages: list[ScheduledMessage]
+) -> None:
+    event = {
+        "type": "scheduled_messages",
+        "op": "add",
+        "scheduled_messages": [
+            scheduled_message.to_dict() for scheduled_message in scheduled_messages
+        ],
+    }
+    send_event_on_commit(user_profile.realm, event, [user_profile.id])
+
+
+def notify_new_reminder(user_profile: UserProfile, reminders: list[ScheduledMessage]) -> None:
+    event = {
+        "type": "reminders",
+        "op": "add",
+        "reminders": [reminder.to_reminder_dict() for reminder in reminders],
+    }
+    send_event_on_commit(user_profile.realm, event, [user_profile.id])
+
+
+def do_schedule_messages(
+    send_message_requests: Sequence[SendMessageRequest],
+    sender: UserProfile,
+    *,
+    read_by_sender: bool = False,
+    skip_events: bool = False,
+    delivery_type: int,
+) -> list[int]:
+    scheduled_messages: list[tuple[ScheduledMessage, SendMessageRequest]] = []
+
+    for send_request in send_message_requests:
+        scheduled_message = ScheduledMessage()
+        scheduled_message.sender = send_request.message.sender
+        scheduled_message.recipient = send_request.message.recipient
+        topic_name = send_request.message.topic_name()
+        scheduled_message.set_topic_name(topic_name=topic_name)
+        scheduled_message.content = send_request.message.content
+        scheduled_message.rendered_content = send_request.rendering_result.rendered_content
+        scheduled_message.sending_client = send_request.message.sending_client
+        scheduled_message.stream = send_request.stream
+        scheduled_message.realm = send_request.realm
+        assert send_request.deliver_at is not None
+        scheduled_message.scheduled_timestamp = send_request.deliver_at
+        scheduled_message.read_by_sender = read_by_sender
+        scheduled_message.delivery_type = delivery_type
+
+        if delivery_type == ScheduledMessage.REMIND:
+            scheduled_message.reminder_target_message_id = send_request.reminder_target_message_id
+            scheduled_message.reminder_note = send_request.reminder_note
+
+        scheduled_messages.append((scheduled_message, send_request))
+
+    with transaction.atomic(durable=True):
+        scheduled_message_objects = [
+            scheduled_message for scheduled_message, ignored in scheduled_messages
+        ]
+        ScheduledMessage.objects.bulk_create(scheduled_message_objects)
+        for scheduled_message, send_request in scheduled_messages:
+            if do_claim_attachments(
+                scheduled_message, send_request.rendering_result.potential_attachment_path_ids
+            ):
+                scheduled_message.has_attachment = True
+                scheduled_message.save(update_fields=["has_attachment"])
+
+        if not skip_events:
+            if delivery_type == ScheduledMessage.REMIND:
+                notify_new_reminder(sender, scheduled_message_objects)
+            else:
+                notify_new_scheduled_message(sender, scheduled_message_objects)
+    return [scheduled_message.id for scheduled_message, ignored in scheduled_messages]
+
+
+def notify_update_scheduled_message(
+    user_profile: UserProfile, scheduled_message: ScheduledMessage
+) -> None:
+    event = {
+        "type": "scheduled_messages",
+        "op": "update",
+        "scheduled_message": scheduled_message.to_dict(),
+    }
+    send_event_on_commit(user_profile.realm, event, [user_profile.id])
+
+
+@transaction.atomic(durable=True)
+def edit_scheduled_message(
+    sender: UserProfile,
+    client: Client,
+    scheduled_message_id: int,
+    recipient_type_name: str | None,
+    message_to: int | list[int] | None,
+    topic_name: str | None,
+    message_content: str | None,
+    deliver_at: datetime | None,
+    realm: Realm,
+) -> None:
+    scheduled_message_object = access_scheduled_message(
+        sender, scheduled_message_id, lock_message=True
+    )
+
+    # We locked the row above, so a concurrent delivery has already
+    # committed and this guard reliably observes its delivered=True.
+    if scheduled_message_object.delivered is True:
+        raise JsonableError(_("Scheduled message was already sent"))
+
+    # If the server failed to send the scheduled message, a new scheduled
+    # delivery timestamp (`deliver_at`) is required.
+    if scheduled_message_object.failed and deliver_at is None:
+        raise DeliveryTimeNotInFutureError
+
+    # Get existing scheduled message's recipient IDs and recipient_type_name.
+    existing_recipient, existing_recipient_type_name = get_recipient_ids(
+        scheduled_message_object.recipient, sender.id
+    )
+
+    send_request: SendMessageRequest | None = None
+    # If any recipient information or message content has been updated,
+    # we check the message again.
+    if recipient_type_name is not None or message_to is not None or message_content is not None:
+        # Update message type if changed.
+        if recipient_type_name is not None:
+            updated_recipient_type_name = recipient_type_name
+        else:
+            updated_recipient_type_name = existing_recipient_type_name
+
+        # Update message recipient if changed.
+        if message_to is not None:
+            if updated_recipient_type_name == "stream":
+                stream_id = extract_stream_id(message_to)
+                updated_recipient = [stream_id]
+            else:
+                updated_recipient = extract_direct_message_recipient_ids(message_to)
+        else:
+            updated_recipient = existing_recipient
+
+        # Update topic name if changed.
+        if topic_name is not None:
+            updated_topic_name = topic_name
+        else:
+            # This will be ignored in Addressee.legacy_build if type
+            # is being changed from stream to direct.
+            updated_topic_name = scheduled_message_object.topic_name()
+
+        # Update message content if changed.
+        if message_content is not None:
+            updated_content = message_content
+        else:
+            updated_content = scheduled_message_object.content
+
+        # Check message again.
+        addressee = Addressee.legacy_build(
+            sender, updated_recipient_type_name, updated_recipient, updated_topic_name
+        )
+        send_request = check_message(
+            sender,
+            client,
+            addressee,
+            updated_content,
+            realm=realm,
+        )
+
+    if recipient_type_name is not None or message_to is not None:
+        assert send_request is not None
+        # User has updated the scheduled message's recipient.
+        scheduled_message_object.recipient = send_request.message.recipient
+        scheduled_message_object.stream = send_request.stream
+        # Update the topic based on the new recipient information.
+        new_topic_name = send_request.message.topic_name()
+        scheduled_message_object.set_topic_name(topic_name=new_topic_name)
+    elif topic_name is not None and existing_recipient_type_name == "stream":
+        # User has updated the scheduled message's topic, but not
+        # the existing recipient information. We ignore topics sent
+        # for scheduled direct messages.
+        check_stream_topic(topic_name)
+        new_topic_name = truncate_topic(topic_name)
+        scheduled_message_object.set_topic_name(topic_name=new_topic_name)
+
+    if message_content is not None:
+        assert send_request is not None
+        # User has updated the scheduled messages's content.
+        scheduled_message_object.content = send_request.message.content
+        scheduled_message_object.rendered_content = send_request.rendering_result.rendered_content
+        attachment_reference_change = check_attachment_reference_change(
+            scheduled_message_object, send_request.rendering_result
+        )
+        scheduled_message_object.has_attachment = attachment_reference_change.did_attachment_change
+
+    if deliver_at is not None:
+        # User has updated the scheduled message's send timestamp.
+        scheduled_message_object.scheduled_timestamp = deliver_at
+
+    # Update for most recent Client information.
+    scheduled_message_object.sending_client = client
+
+    # If the user is editing a scheduled message that the server tried
+    # and failed to send, we need to update the `failed` boolean field
+    # as well as the associated `failure_message` field.
+    if scheduled_message_object.failed:
+        scheduled_message_object.failed = False
+        scheduled_message_object.failure_message = None
+
+    scheduled_message_object.save()
+
+    notify_update_scheduled_message(sender, scheduled_message_object)
+
+
+def notify_remove_scheduled_message(user_profile: UserProfile, scheduled_message_id: int) -> None:
+    event = {
+        "type": "scheduled_messages",
+        "op": "remove",
+        "scheduled_message_id": scheduled_message_id,
+    }
+    send_event_on_commit(user_profile.realm, event, [user_profile.id])
+
+
+@transaction.atomic(durable=True)
+def delete_scheduled_message(user_profile: UserProfile, scheduled_message_id: int) -> None:
+    scheduled_message_object = access_scheduled_message(user_profile, scheduled_message_id)
+    scheduled_message_id = scheduled_message_object.id
+    scheduled_message_object.delete()
+    notify_remove_scheduled_message(user_profile, scheduled_message_id)
+
+
+def send_reminder(scheduled_message: ScheduledMessage) -> None:
+    message_id = scheduled_message.reminder_target_message_id
+    assert message_id is not None
+    current_user = scheduled_message.sender
+    try:
+        message = access_message(current_user, message_id, is_modifying_message=False)
+        content = get_reminder_formatted_content(
+            message, current_user, scheduled_message.reminder_note
+        )
+    except JsonableError:
+        # If we no longer have access to the message, we send the reminder with the
+        # last known message position and content.
+        content = scheduled_message.content
+    # Reminder messages are always sent from the notification bot. We use acting_user
+    # to have appropriate permissions for the messages.
+    message_id = internal_send_private_message(
+        get_system_bot(settings.NOTIFICATION_BOT, scheduled_message.realm.id),
+        current_user,
+        content,
+        acting_user=current_user,
+    )
+    if message_id is None:
+        # internal_send_private_message swallows JsonableError from
+        # check_message and returns None; surface that as a failure so
+        # the worker marks the row failed rather than delivered with a
+        # NULL delivered_message_id.
+        raise JsonableError(_("Reminder could not be sent."))
+    scheduled_message.delivered_message_id = message_id
+    scheduled_message.delivered = True
+    scheduled_message.save(update_fields=["delivered", "delivered_message_id"])
+    notify_remove_reminder(current_user, scheduled_message.id)
+
+
+def send_scheduled_message(scheduled_message: ScheduledMessage) -> None:
+    assert not scheduled_message.delivered
+    assert not scheduled_message.failed
+
+    # Repeat the checks from validate_account_and_subdomain, in case
+    # the state changed since the message was scheduled.
+    if scheduled_message.realm.deactivated:
+        raise RealmDeactivatedError
+
+    if not scheduled_message.sender.is_active:
+        raise UserDeactivatedError
+
+    # Reminders go to the sender's own DMs from Notification Bot, so
+    # the late-cutoff concern (stale messages surprising other
+    # recipients) doesn't apply. Dispatch before the cutoff check so
+    # reminders still fire if the worker is backed up.
+    if scheduled_message.delivery_type == ScheduledMessage.REMIND:
+        send_reminder(scheduled_message)
+        return
+
+    # Limit how late we're willing to send a scheduled message.
+    latest_send_time = scheduled_message.scheduled_timestamp + timedelta(
+        minutes=SCHEDULED_MESSAGE_LATE_CUTOFF_MINUTES
+    )
+    if timezone_now() > latest_send_time:
+        raise JsonableError(_("Message could not be sent at the scheduled time."))
+
+    # Recheck that we have permission to send this message, in case
+    # permissions have changed since the message was scheduled.
+    if scheduled_message.stream is not None:
+        addressee = Addressee.for_stream(scheduled_message.stream, scheduled_message.topic_name())
+    else:
+        subscriber_ids = list(
+            Subscription.objects.filter(recipient=scheduled_message.recipient).values_list(
+                "user_profile_id", flat=True
+            )
+        )
+        addressee = Addressee.for_user_ids(subscriber_ids, scheduled_message.realm)
+
+    # Calling check_message again is important because permissions may
+    # have changed since the message was originally scheduled. This
+    # means that Markdown syntax referencing mutable organization data
+    # (for example, mentioning a user by name) will work (or not) as
+    # if the message was sent at the delivery time, not the sending
+    # time.
+    send_request = check_message(
+        scheduled_message.sender,
+        scheduled_message.sending_client,
+        addressee,
+        scheduled_message.content,
+        scheduled_message.realm,
+    )
+
+    sent_message_result = do_send_messages(
+        [send_request],
+        mark_as_read=[scheduled_message.sender_id] if scheduled_message.read_by_sender else [],
+    )[0]
+    scheduled_message.delivered_message_id = sent_message_result.message_id
+    scheduled_message.delivered = True
+    scheduled_message.save(update_fields=["delivered", "delivered_message_id"])
+    notify_remove_scheduled_message(scheduled_message.sender, scheduled_message.id)
+
+
+def send_failed_scheduled_message_notification(
+    user_profile: UserProfile, scheduled_message_id: int
+) -> None:
+    scheduled_message = access_scheduled_message(user_profile, scheduled_message_id)
+
+    with override_language(user_profile.default_language):
+        error_string = scheduled_message.failure_message
+        delivery_time_markdown = datetime_to_global_time(scheduled_message.scheduled_timestamp)
+
+        content = "".join(
+            [
+                _(
+                    "The message you scheduled for {delivery_datetime} was not sent because of the following error:"
+                ),
+                "\n\n",
+                "> {error_message}",
+                "\n\n",
+                _("[View scheduled messages](#scheduled)"),
+                "\n\n",
+            ]
+        )
+
+    content = content.format(
+        delivery_datetime=delivery_time_markdown,
+        error_message=error_string,
+    )
+
+    internal_send_private_message(
+        get_system_bot(settings.NOTIFICATION_BOT, user_profile.realm_id),
+        user_profile,
+        content,
+    )
+
+
+@transaction.atomic(durable=True)
+def try_deliver_one_scheduled_message() -> bool:
+    # Returns whether there was a scheduled message to attempt
+    # delivery on, regardless of whether delivery succeeded.
+    scheduled_message = (
+        ScheduledMessage.objects.filter(
+            scheduled_timestamp__lte=timezone_now(),
+            delivered=False,
+            failed=False,
+        )
+        .select_for_update(no_key=True)
+        .first()
+    )
+
+    if scheduled_message is None:
+        return False
+
+    logging.info(
+        "Sending scheduled message %s with date %s (sender: %s)",
+        scheduled_message.id,
+        scheduled_message.scheduled_timestamp,
+        scheduled_message.sender_id,
+    )
+
+    with override_language(scheduled_message.sender.default_language):
+        try:
+            send_scheduled_message(scheduled_message)
+        except Exception as e:
+            scheduled_message.refresh_from_db()
+            was_delivered = scheduled_message.delivered
+            scheduled_message.failed = True
+
+            if isinstance(e, JsonableError):
+                scheduled_message.failure_message = e.msg
+                logging.info("Failed with message: %s", e.msg)
+            else:
+                # An unexpected failure; store and send user a generic
+                # internal server error in notification message.
+                scheduled_message.failure_message = _("Internal server error")
+                logging.exception(
+                    "Unexpected error sending scheduled message %s (sent: %s)",
+                    scheduled_message.id,
+                    was_delivered,
+                    stack_info=True,
+                )
+
+            scheduled_message.save(update_fields=["failed", "failure_message"])
+
+            if (
+                not was_delivered
+                # Reminders have their own notification system.
+                and scheduled_message.delivery_type != ScheduledMessage.REMIND
+                # Do not send notification if either the realm or
+                # the sending user account has been deactivated.
+                and not isinstance(e, RealmDeactivatedError)
+                and not isinstance(e, UserDeactivatedError)
+                # Cross-realm system bots (welcome-bot, notification-bot,
+                # emailgateway) have no human behind them to read the
+                # notification, and notifying them leaves a stray
+                # notification-bot <-> system-bot DirectMessageGroup
+                # attached to no realm.
+                and not is_cross_realm_bot_email(scheduled_message.sender.delivery_email)
+            ):
+                notify_update_scheduled_message(scheduled_message.sender, scheduled_message)
+                send_failed_scheduled_message_notification(
+                    scheduled_message.sender, scheduled_message.id
+                )
+
+    return True

zerver/actions/submessage.py

@@ -0,0 +1,98 @@
+from django.utils.translation import gettext as _
+
+from zerver.actions.user_topics import do_set_user_topic_visibility_policy
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.message import (
+    event_recipient_ids_for_action_on_messages,
+    set_visibility_policy_possible,
+    should_change_visibility_policy,
+    visibility_policy_for_participation,
+)
+from zerver.lib.streams import access_stream_by_id
+from zerver.models import Realm, SubMessage, UserProfile
+from zerver.tornado.django_api import send_event_on_commit
+
+
+def verify_submessage_sender(
+    *,
+    message_id: int,
+    message_sender_id: int,
+    submessage_sender_id: int,
+) -> None:
+    """Even though our submessage architecture is geared toward
+    collaboration among all message readers, we still enforce
+    the first person to attach a submessage to the message
+    must be the original sender of the message.
+    """
+
+    if message_sender_id == submessage_sender_id:
+        return
+
+    if SubMessage.objects.filter(
+        message_id=message_id,
+        sender_id=message_sender_id,
+    ).exists():
+        return
+
+    raise JsonableError(_("You cannot attach a submessage to this message."))
+
+
+def do_add_submessage(
+    realm: Realm,
+    sender_id: int,
+    message_id: int,
+    msg_type: str,
+    content: str,
+) -> None:
+    """Should be called while holding a SELECT FOR UPDATE lock
+    (e.g. via access_message(..., lock_message=True)) on the
+    Message row, to prevent race conditions.
+    """
+    submessage = SubMessage(
+        sender_id=sender_id,
+        message_id=message_id,
+        msg_type=msg_type,
+        content=content,
+    )
+    submessage.save()
+
+    # Determine and set the visibility_policy depending on 'automatically_follow_topics_policy'
+    # and 'automatically_unmute_topics_policy'.
+    sender = submessage.sender
+    if set_visibility_policy_possible(
+        sender, submessage.message
+    ) and UserProfile.AUTOMATICALLY_CHANGE_VISIBILITY_POLICY_ON_PARTICIPATION in [
+        sender.automatically_follow_topics_policy,
+        sender.automatically_unmute_topics_in_muted_streams_policy,
+    ]:
+        stream_id = submessage.message.recipient.type_id
+        (stream, sub) = access_stream_by_id(sender, stream_id)
+        assert stream is not None
+        if sub:
+            new_visibility_policy = visibility_policy_for_participation(sender, sub.is_muted)
+            if new_visibility_policy and should_change_visibility_policy(
+                new_visibility_policy,
+                sender,
+                stream_id,
+                topic_name=submessage.message.topic_name(),
+            ):
+                do_set_user_topic_visibility_policy(
+                    user_profile=sender,
+                    stream=stream,
+                    topic_name=submessage.message.topic_name(),
+                    visibility_policy=new_visibility_policy,
+                )
+
+    event = dict(
+        type="submessage",
+        msg_type=msg_type,
+        message_id=message_id,
+        submessage_id=submessage.id,
+        sender_id=sender_id,
+        content=content,
+    )
+    target_user_ids = event_recipient_ids_for_action_on_messages(
+        [submessage.message.id], submessage.message.is_channel_message
+    )
+
+    send_event_on_commit(realm, event, target_user_ids)

zerver/actions/typing.py

@@ -0,0 +1,173 @@
+from typing import Literal
+
+from django.conf import settings
+from django.utils.translation import gettext as _
+
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.stream_subscription import get_active_subscriptions_for_stream_id
+from zerver.models import Realm, Stream, UserProfile
+from zerver.models.users import get_user_by_id_in_realm_including_cross_realm
+from zerver.tornado.django_api import send_event_rollback_unsafe
+
+
+def do_send_typing_notification(
+    realm: Realm, sender: UserProfile, recipient_user_profiles: list[UserProfile], operator: str
+) -> None:
+    sender_dict = {"user_id": sender.id, "email": sender.email}
+
+    # Include a list of recipients in the event body to help identify where the typing is happening
+    recipient_dicts = [
+        {"user_id": profile.id, "email": profile.email} for profile in recipient_user_profiles
+    ]
+    event = dict(
+        type="typing",
+        message_type="direct",
+        op=operator,
+        sender=sender_dict,
+        recipients=recipient_dicts,
+    )
+
+    # Only deliver the notification to active user recipients
+    user_ids_to_notify = [
+        user.id
+        for user in recipient_user_profiles
+        if user.is_active and user.receives_typing_notifications
+    ]
+
+    send_event_rollback_unsafe(realm, event, user_ids_to_notify)
+
+
+# check_send_typing_notification:
+# Checks the typing notification and sends it
+def check_send_typing_notification(sender: UserProfile, user_ids: list[int], operator: str) -> None:
+    realm = sender.realm
+
+    if sender.id not in user_ids:
+        user_ids.append(sender.id)
+
+    # If any of the user_ids being sent in are invalid, we will
+    # just reject the whole request, since a partial list of user_ids
+    # can create confusion related to direct message groups. Plus it's
+    # a good sign that a client is confused (or possibly even malicious)
+    # if we get bad user_ids.
+    user_profiles = []
+    for user_id in user_ids:
+        try:
+            # We include cross-bot realms as possible recipients,
+            # so that clients can know which direct message group
+            # conversation is relevant here.
+            user_profile = get_user_by_id_in_realm_including_cross_realm(user_id, sender.realm)
+        except UserProfile.DoesNotExist:
+            raise JsonableError(_("Invalid user ID {user_id}").format(user_id=user_id))
+        user_profiles.append(user_profile)
+
+    do_send_typing_notification(
+        realm=realm,
+        sender=sender,
+        recipient_user_profiles=user_profiles,
+        operator=operator,
+    )
+
+
+def do_send_stream_typing_notification(
+    sender: UserProfile, operator: str, stream: Stream, topic_name: str
+) -> None:
+    sender_dict = {"user_id": sender.id, "email": sender.email}
+
+    event = dict(
+        type="typing",
+        message_type="stream",
+        op=operator,
+        sender=sender_dict,
+        stream_id=stream.id,
+        topic=topic_name,
+    )
+
+    subscriptions_query = get_active_subscriptions_for_stream_id(
+        stream.id, include_deactivated_users=False
+    )
+
+    total_subscriptions = subscriptions_query.count()
+    if total_subscriptions > settings.MAX_STREAM_SIZE_FOR_TYPING_NOTIFICATIONS:
+        # TODO: Stream typing notifications are disabled in streams
+        # with too many subscribers for performance reasons.
+        return
+
+    user_ids_to_notify = set(
+        subscriptions_query.exclude(user_profile__long_term_idle=True)
+        .exclude(user_profile__receives_typing_notifications=False)
+        .values_list("user_profile_id", flat=True)
+    )
+
+    send_event_rollback_unsafe(sender.realm, event, user_ids_to_notify)
+
+
+def do_send_stream_message_edit_typing_notification(
+    sender: UserProfile,
+    channel_id: int,
+    message_id: int,
+    operator: Literal["start", "stop"],
+    topic_name: str,
+) -> None:
+    event = dict(
+        type="typing_edit_message",
+        op=operator,
+        sender_id=sender.id,
+        message_id=message_id,
+        recipient=dict(
+            type="channel",
+            channel_id=channel_id,
+            topic=topic_name,
+        ),
+    )
+
+    subscriptions_query = get_active_subscriptions_for_stream_id(
+        channel_id, include_deactivated_users=False
+    )
+
+    total_subscriptions = subscriptions_query.count()
+    if total_subscriptions > settings.MAX_STREAM_SIZE_FOR_TYPING_NOTIFICATIONS:
+        # TODO: Stream typing notifications are disabled in streams
+        # with too many subscribers for performance reasons.
+        return
+
+    # We don't notify long_term_idle subscribers.
+    user_ids_to_notify = set(
+        subscriptions_query.exclude(user_profile__long_term_idle=True)
+        .exclude(user_profile__receives_typing_notifications=False)
+        .values_list("user_profile_id", flat=True)
+    )
+
+    send_event_rollback_unsafe(sender.realm, event, user_ids_to_notify)
+
+
+def do_send_direct_message_edit_typing_notification(
+    sender: UserProfile,
+    user_ids: list[int],
+    message_id: int,
+    operator: Literal["start", "stop"],
+) -> None:
+    recipient_user_profiles = []
+    for user_id in user_ids:
+        user_profile = get_user_by_id_in_realm_including_cross_realm(user_id, sender.realm)
+        recipient_user_profiles.append(user_profile)
+
+    # Only deliver the notification to active user recipients
+    user_ids_to_notify = [
+        user.id
+        for user in recipient_user_profiles
+        if user.is_active and user.receives_typing_notifications
+    ]
+
+    event = dict(
+        type="typing_edit_message",
+        op=operator,
+        sender_id=sender.id,
+        message_id=message_id,
+        recipient=dict(
+            type="direct",
+            user_ids=user_ids_to_notify,
+        ),
+    )
+
+    send_event_rollback_unsafe(sender.realm, event, user_ids_to_notify)

zerver/actions/uploads.py

@@ -0,0 +1,143 @@
+import logging
+from dataclasses import dataclass
+from typing import Any
+
+from django.db import transaction
+
+from zerver.lib.attachments import get_old_unclaimed_attachments, validate_attachment_request
+from zerver.lib.markdown import MessageRenderingResult
+from zerver.lib.upload import claim_attachment, delete_message_attachments
+from zerver.models import (
+    ArchivedAttachment,
+    Attachment,
+    ImageAttachment,
+    Message,
+    ScheduledMessage,
+    Stream,
+    UserProfile,
+)
+from zerver.tornado.django_api import send_event_on_commit
+
+
+@dataclass
+class AttachmentChangeResult:
+    did_attachment_change: bool
+    detached_attachments: list[dict[str, Any]]
+
+
+def notify_attachment_update(
+    user_profile: UserProfile, op: str, attachment_dict: dict[str, Any]
+) -> None:
+    event = {
+        "type": "attachment",
+        "op": op,
+        "attachment": attachment_dict,
+        "upload_space_used": user_profile.realm.currently_used_upload_space_bytes(),
+    }
+    send_event_on_commit(user_profile.realm, event, [user_profile.id])
+
+
+def do_claim_attachments(
+    message: Message | ScheduledMessage, potential_path_ids: list[str]
+) -> bool:
+    claimed = False
+    for path_id in potential_path_ids:
+        user_profile = message.sender
+        is_message_realm_public = False
+        is_message_web_public = False
+        if isinstance(message, Message):
+            is_channel_message = message.is_channel_message
+        else:
+            assert isinstance(message, ScheduledMessage)
+            is_channel_message = message.is_channel_message()
+
+        if is_channel_message:
+            stream = Stream.objects.get(id=message.recipient.type_id)
+            is_message_realm_public = stream.is_public()
+            is_message_web_public = (
+                user_profile.realm.web_public_streams_enabled() and stream.is_web_public
+            )
+
+        if not validate_attachment_request(user_profile, path_id)[0]:
+            # Technically, there are 2 cases here:
+            # * The user put something in their message that has the form
+            # of an upload URL, but does not actually correspond to a previously
+            # uploaded file.  validate_attachment_request will return None.
+            # * The user is trying to send a link to a file they don't have permission to
+            # access themselves.  validate_attachment_request will return False.
+            #
+            # Either case is unusual and suggests a UI bug that got
+            # the user in this situation, so we log in these cases.
+            logging.warning(
+                "User %s tried to share upload %s in message %s, but lacks permission",
+                user_profile.id,
+                path_id,
+                message.id,
+            )
+            continue
+
+        claimed = True
+        attachment = claim_attachment(
+            path_id, message, is_message_realm_public, is_message_web_public
+        )
+        if not isinstance(message, ScheduledMessage):
+            # attachment update events don't say anything about scheduled messages,
+            # so sending an event is pointless.
+            notify_attachment_update(user_profile, "update", attachment.to_dict())
+    return claimed
+
+
+@transaction.atomic(durable=True)
+def do_delete_old_unclaimed_attachments(weeks_ago: int) -> None:
+    old_unclaimed_attachments, old_unclaimed_archived_attachments = get_old_unclaimed_attachments(
+        weeks_ago
+    )
+
+    with delete_message_attachments(delete_from=(ImageAttachment, Attachment)) as delete_one:
+        for path_id in (
+            old_unclaimed_attachments.values_list("path_id", flat=True)
+            .select_for_update(of=("self",), no_key=False)
+            .iterator()
+        ):
+            delete_one(path_id)
+    with delete_message_attachments(
+        delete_from=(ImageAttachment, ArchivedAttachment)
+    ) as delete_one:
+        for path_id in (
+            old_unclaimed_archived_attachments.values_list("path_id", flat=True)
+            .select_for_update(of=("self",), no_key=False)
+            .iterator()
+        ):
+            delete_one(path_id)
+
+
+def check_attachment_reference_change(
+    message: Message | ScheduledMessage, rendering_result: MessageRenderingResult
+) -> AttachmentChangeResult:
+    # For an unsaved message edit (message.* has been updated, but not
+    # saved to the database), adjusts Attachment data to correspond to
+    # the new content.
+    prev_attachments = {a.path_id for a in message.attachment_set.all()}
+    new_attachments = set(rendering_result.potential_attachment_path_ids)
+
+    if new_attachments == prev_attachments:
+        return AttachmentChangeResult(bool(prev_attachments), [])
+
+    to_remove = list(prev_attachments - new_attachments)
+    if len(to_remove) > 0:
+        attachments_to_update = Attachment.objects.filter(path_id__in=to_remove).select_for_update(
+            no_key=True
+        )
+        message.attachment_set.remove(*attachments_to_update)
+
+    sender = message.sender
+    detached_attachments_query = Attachment.objects.filter(
+        path_id__in=to_remove, messages__isnull=True, owner=sender
+    )
+    detached_attachments = [attachment.to_dict() for attachment in detached_attachments_query]
+
+    to_add = list(new_attachments - prev_attachments)
+    if len(to_add) > 0:
+        do_claim_attachments(message, to_add)
+
+    return AttachmentChangeResult(message.attachment_set.exists(), detached_attachments)

zerver/actions/user_activity.py

@@ -0,0 +1,31 @@
+from datetime import datetime
+
+from zerver.lib.queue import queue_json_publish_rollback_unsafe
+from zerver.lib.timestamp import datetime_to_timestamp
+from zerver.models import UserActivityInterval, UserProfile
+
+
+def do_update_user_activity_interval(user_profile: UserProfile, log_time: datetime) -> None:
+    effective_end = log_time + UserActivityInterval.MIN_INTERVAL_LENGTH
+    # This code isn't perfect, because with various races we might end
+    # up creating two overlapping intervals, but that shouldn't happen
+    # often, and can be corrected for in post-processing
+    last = UserActivityInterval.objects.filter(user_profile=user_profile).order_by("-end").first()
+    # Two intervals overlap iff each interval ends after the other
+    # begins.  In this case, we just extend the old interval to
+    # include the new interval.
+    if last is not None and log_time <= last.end and effective_end >= last.start:
+        last.end = max(last.end, effective_end)
+        last.start = min(last.start, log_time)
+        last.save(update_fields=["start", "end"])
+        return
+
+    # Otherwise, the intervals don't overlap, so we should make a new one
+    UserActivityInterval.objects.create(
+        user_profile=user_profile, start=log_time, end=effective_end
+    )
+
+
+def update_user_activity_interval(user_profile: UserProfile, log_time: datetime) -> None:
+    event = {"user_profile_id": user_profile.id, "time": datetime_to_timestamp(log_time)}
+    queue_json_publish_rollback_unsafe("user_activity_interval", event)

zerver/actions/user_groups.py

@@ -0,0 +1,777 @@
+from collections.abc import Mapping, Sequence
+from datetime import datetime
+from typing import TypedDict
+
+import django.db.utils
+from django.db import transaction
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.stream_subscription import get_user_ids_for_streams
+from zerver.lib.stream_traffic import get_streams_traffic
+from zerver.lib.streams import (
+    bulk_can_access_stream_metadata_user_ids,
+    get_anonymous_group_membership_dict_for_streams,
+    get_metadata_access_streams_via_group_ids,
+    send_stream_creation_event,
+    send_stream_deletion_event,
+)
+from zerver.lib.timestamp import datetime_to_timestamp
+from zerver.lib.types import UserGroupMembersData, UserGroupMembersDict
+from zerver.lib.user_counts import realm_user_count_by_role
+from zerver.lib.user_groups import (
+    convert_to_user_group_members_dict,
+    get_group_setting_value_for_api,
+    get_group_setting_value_for_audit_log_data,
+    get_recursive_supergroups_union_for_groups,
+    get_role_based_system_groups_dict,
+    set_defaults_for_group_settings,
+)
+from zerver.lib.workplace_users import check_any_group_used_for_workplace_users_group
+from zerver.models import (
+    GroupGroupMembership,
+    NamedUserGroup,
+    Realm,
+    RealmAuditLog,
+    UserGroup,
+    UserGroupMembership,
+    UserProfile,
+)
+from zerver.models.groups import SystemGroups
+from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.models.users import active_user_ids
+from zerver.tornado.django_api import send_event_on_commit
+
+
+class MemberGroupUserDict(TypedDict):
+    id: int
+    role: int
+    date_joined: datetime
+
+
+@transaction.atomic(savepoint=False)
+def create_user_group_in_database(
+    name: str,
+    members: list[UserProfile],
+    realm: Realm,
+    *,
+    acting_user: UserProfile | None,
+    description: str = "",
+    group_settings_map: Mapping[str, UserGroup] = {},
+    is_system_group: bool = False,
+) -> NamedUserGroup:
+    user_group = NamedUserGroup(
+        name=name,
+        realm=realm,
+        description=description,
+        is_system_group=is_system_group,
+        realm_for_sharding=realm,
+        creator=acting_user,
+    )
+
+    for setting_name, setting_value in group_settings_map.items():
+        setattr(user_group, setting_name, setting_value)
+
+    system_groups_name_dict = get_role_based_system_groups_dict(realm)
+    user_group = set_defaults_for_group_settings(
+        user_group, group_settings_map, system_groups_name_dict
+    )
+    user_group.save()
+
+    UserGroupMembership.objects.bulk_create(
+        UserGroupMembership(user_profile=member, user_group=user_group) for member in members
+    )
+
+    creation_time = timezone_now()
+    audit_log_entries = [
+        RealmAuditLog(
+            realm=realm,
+            acting_user=acting_user,
+            event_type=AuditLogEventType.USER_GROUP_CREATED,
+            event_time=creation_time,
+            modified_user_group=user_group,
+        ),
+    ] + [
+        RealmAuditLog(
+            realm=realm,
+            acting_user=acting_user,
+            event_type=AuditLogEventType.USER_GROUP_DIRECT_USER_MEMBERSHIP_ADDED,
+            event_time=creation_time,
+            modified_user=member,
+            modified_user_group=user_group,
+        )
+        for member in members
+    ]
+    RealmAuditLog.objects.bulk_create(audit_log_entries)
+    return user_group
+
+
+@transaction.atomic(savepoint=False)
+def update_users_in_full_members_system_group(
+    realm: Realm,
+    affected_user_ids: Sequence[int] = [],
+    *,
+    acting_user: UserProfile | None,
+    skip_workplace_users_count_audit_log: bool = False,
+) -> None:
+    full_members_system_group = NamedUserGroup.objects.get(
+        realm_for_sharding=realm, name=SystemGroups.FULL_MEMBERS, is_system_group=True
+    )
+    members_system_group = NamedUserGroup.objects.get(
+        realm_for_sharding=realm, name=SystemGroups.MEMBERS, is_system_group=True
+    )
+
+    full_member_group_users: list[MemberGroupUserDict] = list()
+    member_group_users: list[MemberGroupUserDict] = list()
+
+    if affected_user_ids:
+        full_member_group_users = list(
+            full_members_system_group.direct_members.filter(id__in=affected_user_ids).values(
+                "id", "role", "date_joined"
+            )
+        )
+        member_group_users = list(
+            members_system_group.direct_members.filter(id__in=affected_user_ids).values(
+                "id", "role", "date_joined"
+            )
+        )
+    else:
+        full_member_group_users = list(
+            full_members_system_group.direct_members.all().values("id", "role", "date_joined")
+        )
+        member_group_users = list(
+            members_system_group.direct_members.all().values("id", "role", "date_joined")
+        )
+
+    def is_provisional_member(user: MemberGroupUserDict) -> bool:
+        diff = (timezone_now() - user["date_joined"]).days
+        if diff < realm.waiting_period_threshold:
+            return True
+        return False
+
+    old_full_members = [
+        user
+        for user in full_member_group_users
+        if is_provisional_member(user) or user["role"] != UserProfile.ROLE_MEMBER
+    ]
+
+    full_member_group_user_ids = [user["id"] for user in full_member_group_users]
+    members_excluding_full_members = [
+        user for user in member_group_users if user["id"] not in full_member_group_user_ids
+    ]
+
+    new_full_members = [
+        user for user in members_excluding_full_members if not is_provisional_member(user)
+    ]
+
+    old_full_member_ids = [user["id"] for user in old_full_members]
+    new_full_member_ids = [user["id"] for user in new_full_members]
+
+    if len(old_full_members) > 0:
+        bulk_remove_members_from_user_groups(
+            [full_members_system_group],
+            old_full_member_ids,
+            acting_user=acting_user,
+            skip_workplace_users_count_audit_log=skip_workplace_users_count_audit_log,
+        )
+
+    if len(new_full_members) > 0:
+        bulk_add_members_to_user_groups(
+            [full_members_system_group],
+            new_full_member_ids,
+            acting_user=acting_user,
+            skip_workplace_users_count_audit_log=skip_workplace_users_count_audit_log,
+        )
+
+
+def promote_new_full_members() -> None:
+    for realm in Realm.objects.filter(deactivated=False).exclude(waiting_period_threshold=0):
+        update_users_in_full_members_system_group(realm, acting_user=None)
+
+
+def do_send_create_user_group_event(
+    user_group: NamedUserGroup,
+    member_ids: list[int],
+    direct_subgroup_ids: Sequence[int] = [],
+    *,
+    for_reactivation: bool = False,
+) -> None:
+    creator_id = user_group.creator_id
+    assert user_group.date_created is not None
+    date_created = datetime_to_timestamp(user_group.date_created)
+
+    setting_values = {}
+    for setting_name in NamedUserGroup.GROUP_PERMISSION_SETTINGS:
+        setting_values[setting_name] = convert_to_user_group_members_dict(
+            get_group_setting_value_for_api(getattr(user_group, setting_name))
+        )
+
+    event = dict(
+        type="user_group",
+        op="add",
+        group=dict(
+            name=user_group.name,
+            creator_id=creator_id,
+            date_created=date_created,
+            members=member_ids,
+            description=user_group.description,
+            id=user_group.id,
+            is_system_group=user_group.is_system_group,
+            direct_subgroup_ids=direct_subgroup_ids,
+            **setting_values,
+            deactivated=False,
+        ),
+        for_reactivation=for_reactivation,
+    )
+    send_event_on_commit(user_group.realm, event, active_user_ids(user_group.realm_id))
+
+
+def check_add_user_group_core(
+    realm: Realm,
+    name: str,
+    initial_members: list[UserProfile],
+    description: str = "",
+    group_settings_map: Mapping[str, UserGroup] = {},
+    *,
+    acting_user: UserProfile | None,
+) -> NamedUserGroup:
+    user_group = create_user_group_in_database(
+        name,
+        initial_members,
+        realm,
+        description=description,
+        group_settings_map=group_settings_map,
+        acting_user=acting_user,
+    )
+    do_send_create_user_group_event(user_group, [member.id for member in initial_members])
+    return user_group
+
+
+def check_add_user_group(
+    realm: Realm,
+    name: str,
+    initial_members: list[UserProfile],
+    description: str = "",
+    group_settings_map: Mapping[str, UserGroup] = {},
+    *,
+    acting_user: UserProfile | None,
+) -> NamedUserGroup:
+    try:
+        return check_add_user_group_core(
+            realm, name, initial_members, description, group_settings_map, acting_user=acting_user
+        )
+    except django.db.utils.IntegrityError:
+        raise JsonableError(_("User group '{group_name}' already exists.").format(group_name=name))
+
+
+def do_send_user_group_update_event(
+    user_group: NamedUserGroup, data: dict[str, str | int | UserGroupMembersDict]
+) -> None:
+    event = dict(type="user_group", op="update", group_id=user_group.id, data=data)
+    if "name" in data:
+        # This field will be popped eventually before sending the event
+        # to client, but is needed to make sure we do not send the
+        # name update event for deactivated groups to client with
+        # 'include_deactivated_groups' client capability set to false.
+        event["deactivated"] = user_group.deactivated
+    send_event_on_commit(user_group.realm, event, active_user_ids(user_group.realm_id))
+
+
+@transaction.atomic(savepoint=False)
+def do_update_user_group_name(
+    user_group: NamedUserGroup, name: str, *, acting_user: UserProfile | None
+) -> None:
+    try:
+        old_value = user_group.name
+        user_group.name = name
+        user_group.save(update_fields=["name"])
+        RealmAuditLog.objects.create(
+            realm=user_group.realm,
+            modified_user_group=user_group,
+            event_type=AuditLogEventType.USER_GROUP_NAME_CHANGED,
+            event_time=timezone_now(),
+            acting_user=acting_user,
+            extra_data={
+                RealmAuditLog.OLD_VALUE: old_value,
+                RealmAuditLog.NEW_VALUE: name,
+            },
+        )
+    except django.db.utils.IntegrityError:
+        raise JsonableError(_("User group '{group_name}' already exists.").format(group_name=name))
+    do_send_user_group_update_event(user_group, dict(name=name))
+
+
+@transaction.atomic(savepoint=False)
+def do_update_user_group_description(
+    user_group: NamedUserGroup, description: str, *, acting_user: UserProfile | None
+) -> None:
+    old_value = user_group.description
+    user_group.description = description
+    user_group.save(update_fields=["description"])
+    RealmAuditLog.objects.create(
+        realm=user_group.realm,
+        modified_user_group=user_group,
+        event_type=AuditLogEventType.USER_GROUP_DESCRIPTION_CHANGED,
+        event_time=timezone_now(),
+        acting_user=acting_user,
+        extra_data={
+            RealmAuditLog.OLD_VALUE: old_value,
+            RealmAuditLog.NEW_VALUE: description,
+        },
+    )
+    do_send_user_group_update_event(user_group, dict(description=description))
+
+
+def do_send_user_group_members_update_event(
+    event_name: str, user_group: NamedUserGroup, user_ids: list[int]
+) -> None:
+    event = dict(type="user_group", op=event_name, group_id=user_group.id, user_ids=user_ids)
+    send_event_on_commit(user_group.realm, event, active_user_ids(user_group.realm_id))
+
+
+@transaction.atomic(savepoint=False)
+def bulk_add_members_to_user_groups(
+    user_groups: list[NamedUserGroup],
+    user_profile_ids: list[int],
+    *,
+    acting_user: UserProfile | None,
+    skip_workplace_users_count_audit_log: bool = False,
+) -> None:
+    # All intended callers of this function involve a single user
+    # being added to one or more groups, or many users being added to
+    # a single group; but it's easy enough for the implementation to
+    # support both.
+
+    if len(user_groups) == 0 or len(user_profile_ids) == 0:
+        return
+
+    realm = user_groups[0].realm
+    supergroups = get_recursive_supergroups_union_for_groups(
+        [user_group.id for user_group in user_groups]
+    )
+    streams = list(
+        get_metadata_access_streams_via_group_ids([group.id for group in supergroups], realm)
+    )
+    old_stream_metadata_user_ids = bulk_can_access_stream_metadata_user_ids(streams)
+    memberships = [
+        UserGroupMembership(user_group_id=user_group.id, user_profile_id=user_id)
+        for user_id in user_profile_ids
+        for user_group in user_groups
+    ]
+    UserGroupMembership.objects.bulk_create(memberships)
+    now = timezone_now()
+    RealmAuditLog.objects.bulk_create(
+        RealmAuditLog(
+            realm=realm,
+            modified_user_id=user_id,
+            modified_user_group=user_group,
+            event_type=AuditLogEventType.USER_GROUP_DIRECT_USER_MEMBERSHIP_ADDED,
+            event_time=now,
+            acting_user=acting_user,
+        )
+        for user_id in user_profile_ids
+        for user_group in user_groups
+    )
+
+    if not skip_workplace_users_count_audit_log and check_any_group_used_for_workplace_users_group(
+        realm, user_groups
+    ):
+        RealmAuditLog.objects.create(
+            realm=realm,
+            acting_user=acting_user,
+            event_type=AuditLogEventType.WORKPLACE_USERS_COUNT_CHANGED,
+            event_time=now,
+            extra_data={
+                RealmAuditLog.ROLE_COUNT: realm_user_count_by_role(realm),
+                "trigger": "user_membership_changed",
+            },
+        )
+
+        from zerver.lib.remote_server import maybe_enqueue_audit_log_upload
+
+        maybe_enqueue_audit_log_upload(realm)
+
+    subscriber_ids_for_streams = get_user_ids_for_streams({stream.id for stream in streams})
+    new_stream_metadata_user_ids = bulk_can_access_stream_metadata_user_ids(streams)
+    recent_traffic = get_streams_traffic(realm, {stream.id for stream in streams})
+    anonymous_group_membership = get_anonymous_group_membership_dict_for_streams(streams)
+
+    for user_group in user_groups:
+        do_send_user_group_members_update_event("add_members", user_group, user_profile_ids)
+
+    for stream in streams:
+        user_ids_gaining_metadata_access = (
+            new_stream_metadata_user_ids[stream.id] - old_stream_metadata_user_ids[stream.id]
+        )
+        send_stream_creation_event(
+            realm,
+            stream,
+            list(user_ids_gaining_metadata_access),
+            recent_traffic,
+            anonymous_group_membership,
+        )
+        peer_add_event = dict(
+            type="subscription",
+            op="peer_add",
+            stream_ids=[stream.id],
+            user_ids=sorted(subscriber_ids_for_streams[stream.id]),
+        )
+        send_event_on_commit(
+            realm,
+            peer_add_event,
+            user_ids_gaining_metadata_access,
+        )
+
+
+@transaction.atomic(savepoint=False)
+def bulk_remove_members_from_user_groups(
+    user_groups: list[NamedUserGroup],
+    user_profile_ids: list[int],
+    *,
+    acting_user: UserProfile | None,
+    skip_workplace_users_count_audit_log: bool = False,
+) -> None:
+    # All intended callers of this function involve a single user
+    # being added to one or more groups, or many users being added to
+    # a single group; but it's easy enough for the implementation to
+    # support both.
+
+    if len(user_groups) == 0 or len(user_profile_ids) == 0:
+        return
+
+    realm = user_groups[0].realm
+    supergroups = get_recursive_supergroups_union_for_groups(
+        [user_group.id for user_group in user_groups]
+    )
+    streams = list(
+        get_metadata_access_streams_via_group_ids([group.id for group in supergroups], realm)
+    )
+    old_stream_metadata_user_ids = bulk_can_access_stream_metadata_user_ids(streams)
+
+    UserGroupMembership.objects.filter(
+        user_group__in=user_groups, user_profile_id__in=user_profile_ids
+    ).delete()
+    now = timezone_now()
+    RealmAuditLog.objects.bulk_create(
+        RealmAuditLog(
+            realm=user_group.realm,
+            modified_user_id=user_id,
+            modified_user_group=user_group,
+            event_type=AuditLogEventType.USER_GROUP_DIRECT_USER_MEMBERSHIP_REMOVED,
+            event_time=now,
+            acting_user=acting_user,
+        )
+        for user_id in user_profile_ids
+        for user_group in user_groups
+    )
+
+    if not skip_workplace_users_count_audit_log and check_any_group_used_for_workplace_users_group(
+        realm, user_groups
+    ):
+        RealmAuditLog.objects.create(
+            realm=realm,
+            acting_user=acting_user,
+            event_type=AuditLogEventType.WORKPLACE_USERS_COUNT_CHANGED,
+            event_time=now,
+            extra_data={
+                RealmAuditLog.ROLE_COUNT: realm_user_count_by_role(realm),
+                "trigger": "user_membership_changed",
+            },
+        )
+
+        from zerver.lib.remote_server import maybe_enqueue_audit_log_upload
+
+        maybe_enqueue_audit_log_upload(realm)
+
+    for user_group in user_groups:
+        do_send_user_group_members_update_event("remove_members", user_group, user_profile_ids)
+
+    new_stream_metadata_user_ids = bulk_can_access_stream_metadata_user_ids(streams)
+    for stream in streams:
+        user_ids_losing_metadata_access = (
+            old_stream_metadata_user_ids[stream.id] - new_stream_metadata_user_ids[stream.id]
+        )
+        send_stream_deletion_event(
+            realm,
+            list(user_ids_losing_metadata_access),
+            [stream],
+        )
+
+
+def do_send_subgroups_update_event(
+    event_name: str, user_group: NamedUserGroup, subgroup_ids: list[int]
+) -> None:
+    event = dict(
+        type="user_group", op=event_name, group_id=user_group.id, direct_subgroup_ids=subgroup_ids
+    )
+    send_event_on_commit(user_group.realm, event, active_user_ids(user_group.realm_id))
+
+
+@transaction.atomic(savepoint=False)
+def add_subgroups_to_user_group(
+    user_group: NamedUserGroup,
+    subgroups: list[NamedUserGroup],
+    *,
+    acting_user: UserProfile | None,
+) -> None:
+    if len(subgroups) == 0:
+        return
+
+    realm = user_group.realm
+    supergroups = get_recursive_supergroups_union_for_groups([user_group.id])
+    streams = list(
+        get_metadata_access_streams_via_group_ids([group.id for group in supergroups], realm)
+    )
+    old_stream_metadata_user_ids = bulk_can_access_stream_metadata_user_ids(streams)
+
+    group_memberships = [
+        GroupGroupMembership(supergroup=user_group, subgroup=subgroup) for subgroup in subgroups
+    ]
+    GroupGroupMembership.objects.bulk_create(group_memberships)
+
+    subgroup_ids = [subgroup.id for subgroup in subgroups]
+    now = timezone_now()
+    audit_log_entries = [
+        RealmAuditLog(
+            realm=user_group.realm,
+            modified_user_group=user_group,
+            event_type=AuditLogEventType.USER_GROUP_DIRECT_SUBGROUP_MEMBERSHIP_ADDED,
+            event_time=now,
+            acting_user=acting_user,
+            extra_data={"subgroup_ids": subgroup_ids},
+        ),
+        *(
+            RealmAuditLog(
+                realm=user_group.realm,
+                modified_user_group_id=subgroup_id,
+                event_type=AuditLogEventType.USER_GROUP_DIRECT_SUPERGROUP_MEMBERSHIP_ADDED,
+                event_time=now,
+                acting_user=acting_user,
+                extra_data={"supergroup_ids": [user_group.id]},
+            )
+            for subgroup_id in subgroup_ids
+        ),
+    ]
+    RealmAuditLog.objects.bulk_create(audit_log_entries)
+
+    if check_any_group_used_for_workplace_users_group(realm, [user_group]):
+        RealmAuditLog.objects.create(
+            realm=realm,
+            acting_user=acting_user,
+            event_type=AuditLogEventType.WORKPLACE_USERS_COUNT_CHANGED,
+            event_time=now,
+            extra_data={
+                RealmAuditLog.ROLE_COUNT: realm_user_count_by_role(realm),
+                "trigger": "subgroups_changed",
+            },
+        )
+
+        from zerver.lib.remote_server import maybe_enqueue_audit_log_upload
+
+        maybe_enqueue_audit_log_upload(realm)
+
+    subscriber_ids_for_streams = get_user_ids_for_streams({stream.id for stream in streams})
+    new_stream_metadata_user_ids = bulk_can_access_stream_metadata_user_ids(streams)
+    recent_traffic = get_streams_traffic(realm, {stream.id for stream in streams})
+    anonymous_group_membership = get_anonymous_group_membership_dict_for_streams(streams)
+
+    do_send_subgroups_update_event("add_subgroups", user_group, subgroup_ids)
+
+    for stream in streams:
+        user_ids_gaining_metadata_access = (
+            new_stream_metadata_user_ids[stream.id] - old_stream_metadata_user_ids[stream.id]
+        )
+        send_stream_creation_event(
+            realm,
+            stream,
+            list(user_ids_gaining_metadata_access),
+            recent_traffic,
+            anonymous_group_membership,
+        )
+        peer_add_event = dict(
+            type="subscription",
+            op="peer_add",
+            stream_ids=[stream.id],
+            user_ids=sorted(subscriber_ids_for_streams[stream.id]),
+        )
+        send_event_on_commit(
+            realm,
+            peer_add_event,
+            user_ids_gaining_metadata_access,
+        )
+
+
+@transaction.atomic(savepoint=False)
+def remove_subgroups_from_user_group(
+    user_group: NamedUserGroup,
+    subgroups: list[NamedUserGroup],
+    *,
+    acting_user: UserProfile | None,
+) -> None:
+    if len(subgroups) == 0:
+        return
+
+    realm = user_group.realm
+    supergroups = get_recursive_supergroups_union_for_groups([user_group.id])
+    streams = list(
+        get_metadata_access_streams_via_group_ids([group.id for group in supergroups], realm)
+    )
+    old_stream_metadata_user_ids = bulk_can_access_stream_metadata_user_ids(streams)
+
+    GroupGroupMembership.objects.filter(supergroup=user_group, subgroup__in=subgroups).delete()
+
+    subgroup_ids = [subgroup.id for subgroup in subgroups]
+    now = timezone_now()
+    audit_log_entries = [
+        RealmAuditLog(
+            realm=user_group.realm,
+            modified_user_group=user_group,
+            event_type=AuditLogEventType.USER_GROUP_DIRECT_SUBGROUP_MEMBERSHIP_REMOVED,
+            event_time=now,
+            acting_user=acting_user,
+            extra_data={"subgroup_ids": subgroup_ids},
+        ),
+        *(
+            RealmAuditLog(
+                realm=user_group.realm,
+                modified_user_group_id=subgroup_id,
+                event_type=AuditLogEventType.USER_GROUP_DIRECT_SUPERGROUP_MEMBERSHIP_REMOVED,
+                event_time=now,
+                acting_user=acting_user,
+                extra_data={"supergroup_ids": [user_group.id]},
+            )
+            for subgroup_id in subgroup_ids
+        ),
+    ]
+    RealmAuditLog.objects.bulk_create(audit_log_entries)
+
+    if check_any_group_used_for_workplace_users_group(realm, [user_group]):
+        RealmAuditLog.objects.create(
+            realm=realm,
+            acting_user=acting_user,
+            event_type=AuditLogEventType.WORKPLACE_USERS_COUNT_CHANGED,
+            event_time=now,
+            extra_data={
+                RealmAuditLog.ROLE_COUNT: realm_user_count_by_role(realm),
+                "trigger": "subgroups_changed",
+            },
+        )
+
+        from zerver.lib.remote_server import maybe_enqueue_audit_log_upload
+
+        maybe_enqueue_audit_log_upload(realm)
+
+    do_send_subgroups_update_event("remove_subgroups", user_group, subgroup_ids)
+
+    new_stream_metadata_user_ids = bulk_can_access_stream_metadata_user_ids(streams)
+    for stream in streams:
+        user_ids_losing_metadata_access = (
+            old_stream_metadata_user_ids[stream.id] - new_stream_metadata_user_ids[stream.id]
+        )
+        send_stream_deletion_event(
+            realm,
+            list(user_ids_losing_metadata_access),
+            [stream],
+        )
+
+
+@transaction.atomic(savepoint=False)
+def do_deactivate_user_group(
+    user_group: NamedUserGroup, *, acting_user: UserProfile | None
+) -> None:
+    user_group.deactivated = True
+    user_group.save(update_fields=["deactivated"])
+
+    now = timezone_now()
+    RealmAuditLog.objects.create(
+        realm=user_group.realm,
+        modified_user_group_id=user_group.id,
+        event_type=AuditLogEventType.USER_GROUP_DEACTIVATED,
+        event_time=now,
+        acting_user=acting_user,
+    )
+
+    do_send_user_group_update_event(user_group, dict(deactivated=True))
+
+    event = dict(type="user_group", op="remove", group_id=user_group.id)
+    send_event_on_commit(user_group.realm, event, active_user_ids(user_group.realm_id))
+
+
+@transaction.atomic(savepoint=False)
+def do_reactivate_user_group(
+    user_group: NamedUserGroup, *, acting_user: UserProfile | None
+) -> None:
+    user_group.deactivated = False
+    user_group.save(update_fields=["deactivated"])
+
+    now = timezone_now()
+    RealmAuditLog.objects.create(
+        realm=user_group.realm,
+        modified_user_group_id=user_group.id,
+        event_type=AuditLogEventType.USER_GROUP_REACTIVATED,
+        event_time=now,
+        acting_user=acting_user,
+    )
+
+    do_send_user_group_update_event(user_group, dict(deactivated=False))
+
+    direct_member_ids = list(user_group.direct_members.all().values_list("id", flat=True))
+    direct_subgroup_ids = list(user_group.direct_subgroups.all().values_list("id", flat=True))
+    do_send_create_user_group_event(
+        user_group, direct_member_ids, direct_subgroup_ids, for_reactivation=True
+    )
+
+
+@transaction.atomic(savepoint=False)
+def do_change_user_group_permission_setting(
+    user_group: NamedUserGroup,
+    setting_name: str,
+    setting_value_group: UserGroup,
+    *,
+    old_setting_api_value: int | UserGroupMembersData | None = None,
+    acting_user: UserProfile | None,
+) -> None:
+    old_value = getattr(user_group, setting_name)
+    setattr(user_group, setting_name, setting_value_group)
+    user_group.save()
+
+    if old_setting_api_value is None:
+        # Most production callers will have computed this as part of
+        # verifying whether there's an actual change to make, but it
+        # feels quite clumsy to have to pass it from unit tests, so we
+        # compute it here if not provided by the caller.
+        old_setting_api_value = get_group_setting_value_for_api(old_value)
+    new_setting_api_value = get_group_setting_value_for_api(setting_value_group)
+
+    if not hasattr(old_value, "named_user_group") and hasattr(
+        setting_value_group, "named_user_group"
+    ):
+        # We delete the UserGroup which the setting was set to
+        # previously if it does not have any linked NamedUserGroup
+        # object, as it is not used anywhere else. A new UserGroup
+        # object would be created if the setting is later set to
+        # a combination of users and groups.
+        old_value.delete()
+
+    RealmAuditLog.objects.create(
+        realm=user_group.realm,
+        acting_user=acting_user,
+        event_type=AuditLogEventType.USER_GROUP_GROUP_BASED_SETTING_CHANGED,
+        event_time=timezone_now(),
+        modified_user_group=user_group,
+        extra_data={
+            RealmAuditLog.OLD_VALUE: get_group_setting_value_for_audit_log_data(
+                old_setting_api_value
+            ),
+            RealmAuditLog.NEW_VALUE: get_group_setting_value_for_audit_log_data(
+                new_setting_api_value
+            ),
+            "property": setting_name,
+        },
+    )
+
+    event_data_dict: dict[str, str | int | UserGroupMembersDict] = {
+        setting_name: convert_to_user_group_members_dict(new_setting_api_value)
+    }
+    do_send_user_group_update_event(user_group, event_data_dict)

zerver/actions/user_settings.py

@@ -0,0 +1,723 @@
+from collections.abc import Iterable
+from datetime import datetime, timedelta
+from enum import Enum
+
+from django.conf import settings
+from django.db import transaction
+from django.db.models import F
+from django.utils.timezone import now as timezone_now
+
+from confirmation.models import Confirmation, create_confirmation_link
+from confirmation.settings import STATUS_REVOKED, STATUS_USED
+from zerver.actions.message_send import send_user_profile_update_notification
+from zerver.actions.presence import do_update_user_presence
+from zerver.lib.avatar import avatar_url, generate_and_upload_jdenticon_avatar
+from zerver.lib.cache import (
+    bulk_flush_users,
+    cache_delete,
+    delete_user_profile_caches,
+    user_profile_by_api_key_cache_key,
+)
+from zerver.lib.create_user import get_display_email_address
+from zerver.lib.i18n import get_language_name
+from zerver.lib.queue import queue_event_on_commit
+from zerver.lib.send_email import FromAddress, clear_scheduled_emails, send_email
+from zerver.lib.timezone import canonicalize_timezone
+from zerver.lib.types import UserProfileChangeDict
+from zerver.lib.upload import delete_avatar_image
+from zerver.lib.users import (
+    can_access_delivery_email,
+    check_bot_name_available,
+    check_full_name,
+    get_user_ids_who_can_access_user,
+    get_users_with_access_to_real_email,
+    user_access_restricted_in_realm,
+)
+from zerver.lib.utils import generate_api_key
+from zerver.models import (
+    Device,
+    Draft,
+    EmailChangeStatus,
+    Realm,
+    RealmAuditLog,
+    ScheduledEmail,
+    ScheduledMessageNotificationEmail,
+    UserPresence,
+    UserProfile,
+)
+from zerver.models.clients import get_client
+from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.models.users import get_user_profile_by_id
+from zerver.tornado.django_api import send_event_on_commit
+
+
+def send_user_email_update_event(user_profile: UserProfile) -> None:
+    payload = dict(user_id=user_profile.id, new_email=user_profile.email)
+    event = dict(type="realm_user", op="update", person=payload)
+    send_event_on_commit(
+        user_profile.realm,
+        event,
+        get_user_ids_who_can_access_user(user_profile),
+    )
+
+
+def send_delivery_email_update_events(
+    user_profile: UserProfile, old_visibility_setting: int, visibility_setting: int
+) -> None:
+    if not user_access_restricted_in_realm(user_profile):
+        active_users = user_profile.realm.get_active_users()
+    else:
+        # The get_user_ids_who_can_access_user returns user IDs and not
+        # user objects and we instead do one more query for UserProfile
+        # objects. We need complete UserProfile objects only for a couple
+        # of cases and it is not worth to query the whole UserProfile
+        # objects in all the cases and it is fine to do the extra query
+        # wherever needed.
+        user_ids_who_can_access_user = get_user_ids_who_can_access_user(user_profile)
+        active_users = UserProfile.objects.filter(
+            id__in=user_ids_who_can_access_user, is_active=True
+        )
+
+    delivery_email_now_visible_user_ids = []
+    delivery_email_now_invisible_user_ids = []
+
+    for active_user in active_users:
+        could_access_delivery_email_previously = can_access_delivery_email(
+            active_user, user_profile.id, old_visibility_setting
+        )
+        can_access_delivery_email_now = can_access_delivery_email(
+            active_user, user_profile.id, visibility_setting
+        )
+
+        if could_access_delivery_email_previously != can_access_delivery_email_now:
+            if can_access_delivery_email_now:
+                delivery_email_now_visible_user_ids.append(active_user.id)
+            else:
+                delivery_email_now_invisible_user_ids.append(active_user.id)
+
+    if delivery_email_now_visible_user_ids:
+        person = dict(user_id=user_profile.id, delivery_email=user_profile.delivery_email)
+        event = dict(type="realm_user", op="update", person=person)
+        send_event_on_commit(
+            user_profile.realm,
+            event,
+            delivery_email_now_visible_user_ids,
+        )
+    if delivery_email_now_invisible_user_ids:
+        person = dict(user_id=user_profile.id, delivery_email=None)
+        event = dict(type="realm_user", op="update", person=person)
+        send_event_on_commit(
+            user_profile.realm,
+            event,
+            delivery_email_now_invisible_user_ids,
+        )
+
+
+@transaction.atomic(savepoint=False)
+def do_change_user_delivery_email(
+    user_profile: UserProfile, new_email: str, *, acting_user: UserProfile | None
+) -> None:
+    delete_user_profile_caches([user_profile], user_profile.realm_id)
+
+    original_email = user_profile.delivery_email
+
+    user_profile.delivery_email = new_email
+    if user_profile.email_address_is_realm_public():
+        user_profile.email = new_email
+        user_profile.save(update_fields=["email", "delivery_email"])
+    else:
+        user_profile.save(update_fields=["delivery_email"])
+
+    # We notify all the users who have access to delivery email.
+    payload = dict(user_id=user_profile.id, delivery_email=new_email)
+    event = dict(type="realm_user", op="update", person=payload)
+    delivery_email_visible_user_ids = get_users_with_access_to_real_email(user_profile)
+
+    send_event_on_commit(user_profile.realm, event, delivery_email_visible_user_ids)
+
+    if user_profile.avatar_source == UserProfile.AVATAR_FROM_GRAVATAR:
+        # If the user is using Gravatar to manage their email address,
+        # their Gravatar just changed, and we need to notify other
+        # clients.
+        notify_avatar_url_change(user_profile)
+
+    if user_profile.email_address_is_realm_public():
+        # Additionally, if we're also changing the publicly visible
+        # email, we send a new_email event as well.
+        send_user_email_update_event(user_profile)
+
+    event_time = timezone_now()
+    RealmAuditLog.objects.create(
+        realm=user_profile.realm,
+        acting_user=acting_user,
+        modified_user=user_profile,
+        event_type=AuditLogEventType.USER_EMAIL_CHANGED,
+        event_time=event_time,
+        extra_data={RealmAuditLog.OLD_VALUE: original_email, RealmAuditLog.NEW_VALUE: new_email},
+    )
+
+
+def do_start_email_change_process(user_profile: UserProfile, new_email: str) -> None:
+    old_email = user_profile.delivery_email
+    obj = EmailChangeStatus.objects.create(
+        new_email=new_email,
+        old_email=old_email,
+        user_profile=user_profile,
+        realm=user_profile.realm,
+    )
+
+    # Deactivate existing email change requests
+    EmailChangeStatus.objects.filter(realm=user_profile.realm, user_profile=user_profile).exclude(
+        id=obj.id,
+    ).exclude(status=STATUS_USED).update(status=STATUS_REVOKED)
+
+    activation_url = create_confirmation_link(obj, Confirmation.EMAIL_CHANGE)
+    from zerver.context_processors import common_context
+
+    context = common_context(user_profile)
+    context.update(
+        old_email=old_email,
+        new_email=new_email,
+        activate_url=activation_url,
+        organization_host=user_profile.realm.host,
+    )
+    language = user_profile.default_language
+
+    email_template = "zerver/emails/confirm_new_email"
+
+    if old_email == "":
+        # The assertions here are to help document the only circumstance under which
+        # this condition should be possible.
+        assert (
+            user_profile.realm.demo_organization_scheduled_deletion_date is not None
+            and user_profile.is_realm_owner
+        )
+        email_template = "zerver/emails/confirm_demo_organization_email"
+
+    send_email(
+        template_prefix=email_template,
+        to_emails=[new_email],
+        from_name=FromAddress.security_email_from_name(language=language),
+        from_address=FromAddress.tokenized_no_reply_address(),
+        language=language,
+        context=context,
+        realm=user_profile.realm,
+    )
+
+
+def do_change_password(user_profile: UserProfile, password: str, commit: bool = True) -> None:
+    user_profile.set_password(password)
+    if commit:
+        user_profile.save(update_fields=["password"])
+
+    # Imported here to prevent import cycles
+    from zproject.backends import RateLimitedAuthenticationByUsername
+
+    RateLimitedAuthenticationByUsername(user_profile.delivery_email).clear_history()
+    event_time = timezone_now()
+    RealmAuditLog.objects.create(
+        realm=user_profile.realm,
+        acting_user=user_profile,
+        modified_user=user_profile,
+        event_type=AuditLogEventType.USER_PASSWORD_CHANGED,
+        event_time=event_time,
+    )
+
+
+@transaction.atomic(savepoint=False)
+def do_change_full_name(
+    user_profile: UserProfile, full_name: str, acting_user: UserProfile | None, notify: bool
+) -> None:
+    old_name = user_profile.full_name
+    if old_name == full_name:
+        return
+
+    user_profile.full_name = full_name
+    user_profile.save(update_fields=["full_name"])
+    event_time = timezone_now()
+    RealmAuditLog.objects.create(
+        realm=user_profile.realm,
+        acting_user=acting_user,
+        modified_user=user_profile,
+        event_type=AuditLogEventType.USER_FULL_NAME_CHANGED,
+        event_time=event_time,
+        extra_data={RealmAuditLog.OLD_VALUE: old_name, RealmAuditLog.NEW_VALUE: full_name},
+    )
+    payload = dict(user_id=user_profile.id, full_name=user_profile.full_name)
+    send_event_on_commit(
+        user_profile.realm,
+        dict(type="realm_user", op="update", person=payload),
+        get_user_ids_who_can_access_user(user_profile),
+    )
+
+    if notify:
+        changes: list[UserProfileChangeDict] = [
+            UserProfileChangeDict(
+                field_name="full name",
+                old_value=old_name,
+                new_value=full_name,
+            )
+        ]
+
+        send_user_profile_update_notification(user_profile, acting_user, changes)
+
+
+def check_change_full_name(
+    user_profile: UserProfile, full_name_raw: str, acting_user: UserProfile | None
+) -> str:
+    """Verifies that the user's proposed full name is valid.  The caller
+    is responsible for checking check permissions.  Returns the new
+    full name, which may differ from what was passed in (because this
+    function strips whitespace)."""
+    new_full_name = check_full_name(
+        full_name_raw=full_name_raw, user_profile=user_profile, realm=user_profile.realm
+    )
+    do_change_full_name(user_profile, new_full_name, acting_user, notify=True)
+    return new_full_name
+
+
+def check_change_bot_full_name(
+    user_profile: UserProfile, full_name_raw: str, acting_user: UserProfile
+) -> None:
+    new_full_name = check_full_name(
+        full_name_raw=full_name_raw, user_profile=user_profile, realm=user_profile.realm
+    )
+
+    if new_full_name == user_profile.full_name:
+        # Our web app will try to patch full_name even if the user didn't
+        # modify the name in the form.  We just silently ignore those
+        # situations.
+        return
+
+    check_bot_name_available(
+        realm_id=user_profile.realm_id,
+        full_name=new_full_name,
+        is_activation=False,
+    )
+    do_change_full_name(user_profile, new_full_name, acting_user, notify=False)
+
+
+@transaction.atomic(durable=True)
+def do_change_tos_version(user_profile: UserProfile, tos_version: str | None) -> None:
+    old_value = user_profile.tos_version
+    user_profile.tos_version = tos_version
+    user_profile.save(update_fields=["tos_version"])
+    event_time = timezone_now()
+    RealmAuditLog.objects.create(
+        realm=user_profile.realm,
+        acting_user=user_profile,
+        modified_user=user_profile,
+        event_type=AuditLogEventType.USER_TERMS_OF_SERVICE_VERSION_CHANGED,
+        event_time=event_time,
+        extra_data={
+            RealmAuditLog.OLD_VALUE: old_value,
+            RealmAuditLog.NEW_VALUE: tos_version,
+        },
+    )
+
+
+@transaction.atomic(durable=True)
+def do_regenerate_api_key(user_profile: UserProfile, acting_user: UserProfile) -> str:
+    old_api_key = user_profile.api_key
+    new_api_key = generate_api_key()
+    user_profile.api_key = new_api_key
+    user_profile.save(update_fields=["api_key"])
+
+    # We need to explicitly delete the old API key from our caches,
+    # because the on-save handler for flushing the UserProfile object
+    # in zerver/lib/cache.py only has access to the new API key.
+    cache_delete(user_profile_by_api_key_cache_key(old_api_key))
+
+    event_time = timezone_now()
+    RealmAuditLog.objects.create(
+        realm=user_profile.realm,
+        acting_user=acting_user,
+        modified_user=user_profile,
+        event_type=AuditLogEventType.USER_API_KEY_CHANGED,
+        event_time=event_time,
+    )
+
+    event = {"type": "clear_push_device_tokens", "user_profile_id": user_profile.id}
+    queue_event_on_commit("deferred_work", event)
+
+    # Delete all of the user's Device records to stop sending E2EE push notifications.
+    Device.objects.filter(user_id=user_profile.id).delete()
+
+    return new_api_key
+
+
+def bulk_regenerate_api_keys(user_profile_ids: Iterable[int]) -> None:
+    for user_profile_id in user_profile_ids:
+        user_profile = get_user_profile_by_id(user_profile_id)
+        do_regenerate_api_key(user_profile, user_profile)
+
+
+def notify_avatar_url_change(user_profile: UserProfile) -> None:
+    payload = dict(
+        avatar_source=user_profile.avatar_source,
+        avatar_url=avatar_url(user_profile),
+        avatar_url_medium=avatar_url(user_profile, medium=True),
+        avatar_version=user_profile.avatar_version,
+        # Even clients using client_gravatar don't need the email,
+        # since we're sending the URL anyway.
+        user_id=user_profile.id,
+    )
+
+    event = dict(type="realm_user", op="update", person=payload)
+    send_event_on_commit(
+        user_profile.realm,
+        event,
+        get_user_ids_who_can_access_user(user_profile),
+    )
+
+
+@transaction.atomic(savepoint=False)
+def do_change_avatar_fields(
+    user_profile: UserProfile,
+    avatar_source: str,
+    skip_notify: bool = False,
+    *,
+    acting_user: UserProfile | None,
+) -> None:
+    user_profile.avatar_source = avatar_source
+    user_profile.avatar_version += 1
+    user_profile.save(update_fields=["avatar_source", "avatar_version"])
+    event_time = timezone_now()
+    RealmAuditLog.objects.create(
+        realm=user_profile.realm,
+        modified_user=user_profile,
+        event_type=AuditLogEventType.USER_AVATAR_SOURCE_CHANGED,
+        extra_data={"avatar_source": avatar_source},
+        event_time=event_time,
+        acting_user=acting_user,
+    )
+
+    if not skip_notify:
+        notify_avatar_url_change(user_profile)
+
+
+def do_scrub_avatar_images(user: UserProfile, *, acting_user: UserProfile | None) -> None:
+    old_version = user.avatar_version
+    # Note: while scrubbing, no need to generate and upload
+    # avatar when default_avatar_source is jdenticon.
+    do_change_avatar_fields(
+        user, user.realm.default_avatar_source, skip_notify=True, acting_user=acting_user
+    )
+    for version in range(1, old_version + 1):
+        delete_avatar_image(user, version)
+
+
+def set_avatar_to_default(user_profile: UserProfile, *, acting_user: UserProfile | None) -> None:
+    default_avatar_source = user_profile.realm.default_avatar_source
+
+    if user_profile.avatar_source == default_avatar_source:  # nocoverage
+        return
+
+    if default_avatar_source == UserProfile.AVATAR_FROM_JDENTICON:
+        generate_and_upload_jdenticon_avatar(
+            user_profile, realm_uuid=str(user_profile.realm.uuid), future=True
+        )
+    do_change_avatar_fields(user_profile, default_avatar_source, acting_user=acting_user)
+
+
+def update_scheduled_email_notifications_time(
+    user_profile: UserProfile, old_batching_period: int, new_batching_period: int
+) -> None:
+    existing_scheduled_emails = ScheduledMessageNotificationEmail.objects.filter(
+        user_profile=user_profile
+    )
+
+    scheduled_timestamp_change = timedelta(seconds=new_batching_period) - timedelta(
+        seconds=old_batching_period
+    )
+
+    existing_scheduled_emails.update(
+        scheduled_timestamp=F("scheduled_timestamp") + scheduled_timestamp_change
+    )
+
+
+def do_change_user_setting(
+    user_profile: UserProfile,
+    setting_name: str,
+    setting_value: bool | str | int | Enum,
+    *,
+    acting_user: UserProfile | None,
+) -> None:
+    bulk_change_user_setting(
+        user_profile.realm, [user_profile], setting_name, setting_value, acting_user=acting_user
+    )
+
+
+@transaction.atomic(savepoint=False)
+def bulk_change_user_setting(
+    realm: Realm,
+    user_profiles: list[UserProfile],
+    setting_name: str,
+    setting_value: bool | str | int | Enum,
+    *,
+    acting_user: UserProfile | None,
+) -> None:
+    event_time = timezone_now()
+    audit_logs = []
+    old_values = {}
+
+    if isinstance(setting_value, Enum):
+        db_setting_value = setting_value.value
+        event_value: bool | str | int = setting_value.name
+    else:
+        db_setting_value = setting_value
+        event_value = db_setting_value
+
+    if setting_name == "timezone":
+        assert isinstance(db_setting_value, str)
+        assert isinstance(event_value, str)
+        db_setting_value = canonicalize_timezone(db_setting_value)
+    else:
+        property_type = UserProfile.property_types[setting_name]
+        if isinstance(setting_value, Enum):
+            assert isinstance(setting_value, property_type)
+            assert isinstance(event_value, str)
+        else:
+            assert isinstance(db_setting_value, property_type)
+            assert isinstance(event_value, property_type)
+
+    for user in user_profiles:
+        old_value = getattr(user, setting_name)
+        old_values[user.id] = old_value
+
+        setattr(user, setting_name, db_setting_value)
+
+        audit_logs.append(
+            RealmAuditLog(
+                realm=user.realm,
+                event_type=AuditLogEventType.USER_SETTING_CHANGED,
+                event_time=event_time,
+                acting_user=acting_user,
+                modified_user=user,
+                extra_data={
+                    RealmAuditLog.OLD_VALUE: old_value,
+                    RealmAuditLog.NEW_VALUE: db_setting_value,
+                    "property": setting_name,
+                },
+            )
+        )
+
+    UserProfile.objects.bulk_update(user_profiles, [setting_name])
+    RealmAuditLog.objects.bulk_create(audit_logs)
+
+    # Disabling digest emails should clear a user's email queue
+    if setting_name == "enable_digest_emails" and not db_setting_value:
+        assert len(user_profiles) == 1
+        user_profile = user_profiles[0]
+        clear_scheduled_emails([user_profile.id], ScheduledEmail.DIGEST)
+
+    if setting_name == "email_notifications_batching_period_seconds":
+        for user_profile in user_profiles:
+            old_value = old_values[user_profile.id]
+            assert isinstance(old_value, int)
+            assert isinstance(db_setting_value, int)
+            update_scheduled_email_notifications_time(user_profile, old_value, db_setting_value)
+
+    event = {
+        "type": "user_settings",
+        "op": "update",
+        "property": setting_name,
+        "value": event_value,
+    }
+
+    if setting_name == "default_language":
+        assert isinstance(db_setting_value, str)
+        event["language_name"] = get_language_name(db_setting_value)
+
+    transaction.on_commit(lambda: bulk_flush_users(user_profiles=user_profiles, realm=realm))
+
+    user_ids = [u.id for u in user_profiles]
+    send_event_on_commit(realm, event, user_ids)
+
+    if setting_name == "allow_private_data_export":
+        assert len(user_profiles) == 1
+        user_profile = user_profiles[0]
+        realm_export_event = {
+            "type": "realm_export_consent",
+            "user_id": user_profile.id,
+            "consented": event_value,
+        }
+        send_event_on_commit(
+            user_profile.realm,
+            realm_export_event,
+            list(realm.get_human_admin_users().values_list("id", flat=True)),
+        )
+
+    # Updates to the time zone display setting are sent to all users
+    if setting_name == "timezone":
+        assert len(user_profiles) == 1
+        user_profile = user_profiles[0]
+        payload = dict(
+            email=user_profile.email,
+            user_id=user_profile.id,
+            timezone=canonicalize_timezone(user_profile.timezone),
+        )
+        timezone_event = dict(type="realm_user", op="update", person=payload)
+        send_event_on_commit(
+            realm,
+            timezone_event,
+            get_user_ids_who_can_access_user(user_profile),
+        )
+
+    if setting_name == "email_address_visibility":
+        assert len(user_profiles) == 1
+        user_profile = user_profiles[0]
+        old_value = old_values[user_profile.id]
+        send_delivery_email_update_events(
+            user_profile, old_value, user_profile.email_address_visibility
+        )
+
+        if UserProfile.EMAIL_ADDRESS_VISIBILITY_EVERYONE not in [old_value, db_setting_value]:
+            # We use real email addresses on UserProfile.email only if
+            # EMAIL_ADDRESS_VISIBILITY_EVERYONE is configured, so
+            # changes between values that will not require changing
+            # that field, so we can save work and return here.
+            return
+
+        user_profile.email = get_display_email_address(user_profile)
+        user_profile.save(update_fields=["email"])
+
+        send_user_email_update_event(user_profile)
+        notify_avatar_url_change(user_profile)
+
+    if setting_name == "enable_drafts_synchronization" and db_setting_value is False:
+        # Delete all of the drafts from the backend but don't send delete events
+        # for them since all that's happened is that we stopped syncing changes,
+        # not deleted every previously synced draft - to do that use the DELETE
+        # endpoint.
+        Draft.objects.filter(user_profile__in=user_profiles).delete()
+
+    if setting_name == "presence_enabled":
+        # The presence_enabled setting's primary function is to stop
+        # doing presence updates for the user altogether.
+        #
+        # When a user toggles the presence_enabled setting, we
+        # immediately trigger a presence update, so all users see the
+        # user's current presence state as consistent with the new
+        # setting; not doing so can make it look like the settings
+        # change didn't have any effect.
+
+        assert len(user_profiles) == 1
+        user_profile = user_profiles[0]
+
+        if db_setting_value:
+            status = UserPresence.LEGACY_STATUS_ACTIVE_INT
+            presence_time = timezone_now()
+        else:
+            # We want to ensure the user's presence data is such that
+            # they will be treated as offline by correct clients,
+            # There are two cases:
+            #
+            # (1) If the user's presence was current, we backdate it
+            #     so that the user will, for as long as presence
+            #     remains disabled, appear to have been last online a
+            #     few minutes before they disabled presence.
+            #
+            # (2) If the user only has a presence older than what our
+            #     backdate would be, we keep the original value - it
+            #     already guarantees that the user will appear to be
+            #     offline.
+            backdated_presence_time = timezone_now() - timedelta(
+                # We add a small additional offset as a fudge factor in
+                # case of clock skew.
+                seconds=settings.OFFLINE_THRESHOLD_SECS
+                + settings.PRESENCE_UPDATE_MIN_FREQ_SECONDS
+                + 10
+            )
+            minimum_previous_presence_time = backdated_presence_time - timedelta(
+                seconds=settings.PRESENCE_UPDATE_MIN_FREQ_SECONDS + 10
+            )
+
+            try:
+                presence: UserPresence | None = UserPresence.objects.get(user_profile=user_profile)
+                assert presence is not None
+                assert presence.last_connected_time is not None
+
+                original_last_active_time = presence.last_active_time
+                if presence.last_connected_time <= backdated_presence_time:
+                    # This is case (2), so we don't need to do
+                    # anything. presence is already as intended.
+                    # last_active_time <= last_connected_time always holds, so
+                    # last_active_time <= backdated_presence_time is also ensured here.
+                    return
+
+                # do_update_user_presence will only send an event if
+                # the presence event it receives is sufficiently newer
+                # than whatever it had before, so we backdate the
+                # existing presence data if necessary to ensure that
+                # our new update event will be seen as new by clients.
+                presence_time = backdated_presence_time
+                presence.last_connected_time = min(
+                    minimum_previous_presence_time, presence.last_connected_time
+                )
+                update_fields = ["last_connected_time"]
+
+                if (
+                    original_last_active_time is None
+                    or original_last_active_time < backdated_presence_time
+                ):
+                    # If the user's last_active_time was old enough (or nonexistent), we don't want to perturb
+                    # it, as it's already in a correct state. Thus we only do_update_user_presence with an
+                    # IDLE status - which will only update the last_connected_time.
+                    status = UserPresence.LEGACY_STATUS_IDLE_INT
+                else:
+                    assert presence.last_active_time is not None
+                    presence.last_active_time = min(
+                        minimum_previous_presence_time, presence.last_active_time
+                    )
+                    update_fields.append("last_active_time")
+                    status = UserPresence.LEGACY_STATUS_ACTIVE_INT
+                presence.save(update_fields=update_fields)
+
+            except UserPresence.DoesNotExist:
+                # If the user has no presence data at all, that should
+                # logically get consistent treatment with having very
+                # old presence data (case (2) above). We want to leave
+                # their presence state intact - so just return without
+                # doing anything.
+                return
+
+        # do_update_user_presence doesn't allow being run inside another
+        # transaction.atomic block, so we need to use on_commit here to ensure
+        # it gets executed outside of our current transaction.
+        transaction.on_commit(
+            lambda: do_update_user_presence(
+                user_profile,
+                get_client("website"),
+                presence_time,
+                status,
+                force_send_update=True,
+            )
+        )
+
+
+@transaction.atomic(durable=True)
+def do_change_user_date_joined(user_profile: UserProfile, date_joined: datetime) -> None:
+    old_date_joined = user_profile.date_joined
+    user_profile.date_joined = date_joined
+    user_profile.save(update_fields=["date_joined"])
+
+    event_time = timezone_now()
+    RealmAuditLog.objects.create(
+        realm=user_profile.realm,
+        acting_user=user_profile,
+        modified_user=user_profile,
+        event_type=AuditLogEventType.USER_DATE_JOINED_CHANGED,
+        event_time=event_time,
+        extra_data={
+            RealmAuditLog.OLD_VALUE: old_date_joined.isoformat(),
+            RealmAuditLog.NEW_VALUE: date_joined.isoformat(),
+        },
+    )
+
+    payload = dict(user_id=user_profile.id, date_joined=date_joined.isoformat(timespec="minutes"))
+    send_event_on_commit(
+        user_profile.realm,
+        dict(type="realm_user", op="update", person=payload),
+        get_user_ids_who_can_access_user(user_profile),
+    )

zerver/actions/user_status.py

@@ -0,0 +1,55 @@
+from django.db import transaction
+
+from zerver.actions.user_settings import do_change_user_setting
+from zerver.lib.user_status import update_user_status
+from zerver.lib.users import get_user_ids_who_can_access_user
+from zerver.models import UserProfile
+from zerver.tornado.django_api import send_event_on_commit
+
+
+@transaction.atomic(durable=True)
+def do_update_user_status(
+    user_profile: UserProfile,
+    away: bool | None,
+    status_text: str | None,
+    client_id: int,
+    emoji_name: str | None,
+    emoji_code: str | None,
+    reaction_type: str | None,
+) -> None:
+    # Deprecated way for clients to access the user's `presence_enabled`
+    # setting, with away != presence_enabled. Can be removed when clients
+    # migrate "away" (also referred to as "unavailable") feature to directly
+    # use and update the user's presence_enabled setting.
+    if away is not None:
+        user_setting = "presence_enabled"
+        value = not away
+        do_change_user_setting(user_profile, user_setting, value, acting_user=user_profile)
+
+    realm = user_profile.realm
+
+    update_user_status(
+        user_profile_id=user_profile.id,
+        status_text=status_text,
+        client_id=client_id,
+        emoji_name=emoji_name,
+        emoji_code=emoji_code,
+        reaction_type=reaction_type,
+    )
+
+    event = dict(
+        type="user_status",
+        user_id=user_profile.id,
+    )
+
+    if away is not None:
+        event["away"] = away
+
+    if status_text is not None:
+        event["status_text"] = status_text
+
+    if emoji_name is not None:
+        event["emoji_name"] = emoji_name
+        event["emoji_code"] = emoji_code
+        event["reaction_type"] = reaction_type
+    send_event_on_commit(realm, event, get_user_ids_who_can_access_user(user_profile))

zerver/actions/user_topics.py

@@ -0,0 +1,87 @@
+from datetime import datetime
+from typing import Any
+
+from django.db import transaction
+from django.utils.timezone import now as timezone_now
+
+from zerver.lib.timestamp import datetime_to_timestamp
+from zerver.lib.topic import maybe_rename_general_chat_to_empty_topic
+from zerver.lib.user_topics import (
+    bulk_set_user_topic_visibility_policy_in_database,
+    get_topic_mutes,
+)
+from zerver.models import Stream, UserProfile
+from zerver.tornado.django_api import send_event_on_commit
+
+
+@transaction.atomic(savepoint=False)
+def bulk_do_set_user_topic_visibility_policy(
+    user_profiles: list[UserProfile],
+    stream: Stream,
+    topic_name: str,
+    *,
+    visibility_policy: int,
+    last_updated: datetime | None = None,
+    skip_muted_topics_event: bool = False,
+) -> None:
+    if last_updated is None:
+        last_updated = timezone_now()
+
+    topic_name = maybe_rename_general_chat_to_empty_topic(topic_name)
+
+    user_profiles_with_changed_user_topic_rows = bulk_set_user_topic_visibility_policy_in_database(
+        user_profiles,
+        stream.id,
+        topic_name,
+        visibility_policy=visibility_policy,
+        recipient_id=stream.recipient_id,
+        last_updated=last_updated,
+    )
+
+    # Users with requests to set the visibility_policy to its current value
+    # or to delete a UserTopic row that doesn't exist shouldn't
+    # send an unnecessary event.
+    if len(user_profiles_with_changed_user_topic_rows) == 0:
+        return
+
+    for user_profile in user_profiles_with_changed_user_topic_rows:
+        # This first muted_topics event is deprecated and will be removed
+        # once clients are migrated to handle the user_topic event type
+        # instead.
+        if not skip_muted_topics_event:
+            muted_topics_event = dict(
+                type="muted_topics", muted_topics=get_topic_mutes(user_profile)
+            )
+            send_event_on_commit(user_profile.realm, muted_topics_event, [user_profile.id])
+
+        user_topic_event: dict[str, Any] = {
+            "type": "user_topic",
+            "stream_id": stream.id,
+            "topic_name": topic_name,
+            "last_updated": datetime_to_timestamp(last_updated),
+            "visibility_policy": visibility_policy,
+        }
+
+        send_event_on_commit(user_profile.realm, user_topic_event, [user_profile.id])
+
+
+def do_set_user_topic_visibility_policy(
+    user_profile: UserProfile,
+    stream: Stream,
+    topic_name: str,
+    *,
+    visibility_policy: int,
+    last_updated: datetime | None = None,
+    skip_muted_topics_event: bool = False,
+) -> None:
+    # For conciseness, this function should be used when a single
+    # user_profile is involved. In the case of multiple user profiles,
+    # call 'bulk_do_set_user_topic_visibility_policy' directly.
+    bulk_do_set_user_topic_visibility_policy(
+        [user_profile],
+        stream,
+        topic_name,
+        visibility_policy=visibility_policy,
+        last_updated=last_updated,
+        skip_muted_topics_event=skip_muted_topics_event,
+    )

zerver/actions/video_calls.py

@@ -0,0 +1,17 @@
+from django.db import transaction
+
+from zerver.models import UserProfile
+from zerver.tornado.django_api import send_event_on_commit
+
+
+@transaction.atomic(durable=True)
+def do_set_video_call_provider_token(
+    user: UserProfile, token_key: str, /, token: dict[str, object] | None
+) -> None:
+    user.third_party_api_state[token_key] = token
+    user.save(update_fields=["third_party_api_state"])
+    send_event_on_commit(
+        user.realm,
+        dict(type=f"has_{token_key}_token", value=token is not None),
+        [user.id],
+    )