snapshot of https://github.com/zulip/zulip.git @ 48c69911c7dd4e03b20552eb3bd77ad8151c1357 (2026-09-15)

zerver/models/__init__.py

@@ -0,0 +1,76 @@
+from zerver.models import lookups as lookups
+from zerver.models.alert_words import AlertWord as AlertWord
+from zerver.models.bots import BotConfigData as BotConfigData
+from zerver.models.bots import BotStorageData as BotStorageData
+from zerver.models.bots import Service as Service
+from zerver.models.channel_folders import ChannelFolder as ChannelFolder
+from zerver.models.clients import Client as Client
+from zerver.models.custom_profile_fields import CustomProfileField as CustomProfileField
+from zerver.models.custom_profile_fields import CustomProfileFieldValue as CustomProfileFieldValue
+from zerver.models.devices import Device as Device
+from zerver.models.drafts import Draft as Draft
+from zerver.models.groups import GroupGroupMembership as GroupGroupMembership
+from zerver.models.groups import NamedUserGroup as NamedUserGroup
+from zerver.models.groups import UserGroup as UserGroup
+from zerver.models.groups import UserGroupMembership as UserGroupMembership
+from zerver.models.linkifiers import RealmFilter as RealmFilter
+from zerver.models.messages import AbstractAttachment as AbstractAttachment
+from zerver.models.messages import AbstractEmoji as AbstractEmoji
+from zerver.models.messages import AbstractMessage as AbstractMessage
+from zerver.models.messages import AbstractReaction as AbstractReaction
+from zerver.models.messages import AbstractSubMessage as AbstractSubMessage
+from zerver.models.messages import AbstractUserMessage as AbstractUserMessage
+from zerver.models.messages import ArchivedAttachment as ArchivedAttachment
+from zerver.models.messages import ArchivedMessage as ArchivedMessage
+from zerver.models.messages import ArchivedReaction as ArchivedReaction
+from zerver.models.messages import ArchivedSubMessage as ArchivedSubMessage
+from zerver.models.messages import ArchivedUserMessage as ArchivedUserMessage
+from zerver.models.messages import ArchiveTransaction as ArchiveTransaction
+from zerver.models.messages import Attachment as Attachment
+from zerver.models.messages import ImageAttachment as ImageAttachment
+from zerver.models.messages import Message as Message
+from zerver.models.messages import OnboardingUserMessage as OnboardingUserMessage
+from zerver.models.messages import Reaction as Reaction
+from zerver.models.messages import SubMessage as SubMessage
+from zerver.models.messages import UserMessage as UserMessage
+from zerver.models.muted_users import MutedUser as MutedUser
+from zerver.models.navigation_views import NavigationView as NavigationView
+from zerver.models.onboarding_steps import OnboardingStep as OnboardingStep
+from zerver.models.prereg_users import EmailChangeStatus as EmailChangeStatus
+from zerver.models.prereg_users import MultiuseInvite as MultiuseInvite
+from zerver.models.prereg_users import PreregistrationRealm as PreregistrationRealm
+from zerver.models.prereg_users import PreregistrationUser as PreregistrationUser
+from zerver.models.prereg_users import RealmReactivationStatus as RealmReactivationStatus
+from zerver.models.presence import UserPresence as UserPresence
+from zerver.models.presence import UserStatus as UserStatus
+from zerver.models.push_notifications import AbstractPushDeviceToken as AbstractPushDeviceToken
+from zerver.models.push_notifications import PushDeviceToken as PushDeviceToken
+from zerver.models.realm_audit_logs import AbstractRealmAuditLog as AbstractRealmAuditLog
+from zerver.models.realm_audit_logs import RealmAuditLog as RealmAuditLog
+from zerver.models.realm_emoji import RealmEmoji as RealmEmoji
+from zerver.models.realm_playgrounds import RealmPlayground as RealmPlayground
+from zerver.models.realms import Realm as Realm
+from zerver.models.realms import RealmAuthenticationMethod as RealmAuthenticationMethod
+from zerver.models.realms import RealmDomain as RealmDomain
+from zerver.models.realms import RealmExport as RealmExport
+from zerver.models.recipients import DirectMessageGroup as DirectMessageGroup
+from zerver.models.recipients import Recipient as Recipient
+from zerver.models.saved_snippets import SavedSnippet as SavedSnippet
+from zerver.models.scheduled_jobs import AbstractScheduledJob as AbstractScheduledJob
+from zerver.models.scheduled_jobs import MissedMessageEmailAddress as MissedMessageEmailAddress
+from zerver.models.scheduled_jobs import ScheduledEmail as ScheduledEmail
+from zerver.models.scheduled_jobs import ScheduledMessage as ScheduledMessage
+from zerver.models.scheduled_jobs import (
+    ScheduledMessageNotificationEmail as ScheduledMessageNotificationEmail,
+)
+from zerver.models.streams import ChannelEmailAddress as ChannelEmailAddress
+from zerver.models.streams import DefaultStream as DefaultStream
+from zerver.models.streams import DefaultStreamGroup as DefaultStreamGroup
+from zerver.models.streams import Stream as Stream
+from zerver.models.streams import Subscription as Subscription
+from zerver.models.user_activity import UserActivity as UserActivity
+from zerver.models.user_activity import UserActivityInterval as UserActivityInterval
+from zerver.models.user_topics import UserTopic as UserTopic
+from zerver.models.users import RealmUserDefault as RealmUserDefault
+from zerver.models.users import UserBaseSettings as UserBaseSettings
+from zerver.models.users import UserProfile as UserProfile

zerver/models/alert_words.py

@@ -0,0 +1,39 @@
+from django.db import models
+from django.db.models import CASCADE
+from django.db.models.signals import post_delete, post_save
+
+from zerver.lib.cache import (
+    cache_delete,
+    realm_alert_words_automaton_cache_key,
+    realm_alert_words_cache_key,
+)
+from zerver.models.realms import Realm
+from zerver.models.users import UserProfile
+
+
+class AlertWord(models.Model):
+    # Realm isn't necessary, but it's a nice denormalization.  Users
+    # never move to another realm, so it's static, and having Realm
+    # here optimizes the main query on this table, which is fetching
+    # all the alert words in a realm.
+    realm = models.ForeignKey(Realm, db_index=True, on_delete=CASCADE)
+    user_profile = models.ForeignKey(UserProfile, on_delete=CASCADE)
+    # Case-insensitive name for the alert word.
+    word = models.TextField()
+
+    class Meta:
+        unique_together = ("user_profile", "word")
+
+
+def flush_realm_alert_words(realm_id: int) -> None:
+    cache_delete(realm_alert_words_cache_key(realm_id))
+    cache_delete(realm_alert_words_automaton_cache_key(realm_id))
+
+
+def flush_alert_word(*, instance: AlertWord, **kwargs: object) -> None:
+    realm_id = instance.realm_id
+    flush_realm_alert_words(realm_id)
+
+
+post_save.connect(flush_alert_word, sender=AlertWord)
+post_delete.connect(flush_alert_word, sender=AlertWord)

zerver/models/bots.py

@@ -0,0 +1,80 @@
+from django.db import models
+from django.db.models import CASCADE
+
+from zerver.models.users import UserProfile
+
+# Interfaces for services
+# They provide additional functionality like parsing message to obtain query URL, data to be sent to URL,
+# and parsing the response.
+GENERIC_INTERFACE = "GenericService"
+SLACK_INTERFACE = "SlackOutgoingWebhookService"
+
+
+# A Service corresponds to either an outgoing webhook bot or an embedded bot.
+# The type of Service is determined by the bot_type field of the referenced
+# UserProfile.
+#
+# If the Service is an outgoing webhook bot:
+# - name is any human-readable identifier for the Service
+# - base_url is the address of the third-party site
+# - token is used for authentication with the third-party site
+#
+# If the Service is an embedded bot:
+# - name is the canonical name for the type of bot (e.g. 'xkcd' for an instance
+#   of the xkcd bot); multiple embedded bots can have the same name, but all
+#   embedded bots with the same name will run the same code
+# - base_url and token are currently unused
+class Service(models.Model):
+    name = models.CharField(max_length=UserProfile.MAX_NAME_LENGTH)
+    # Bot user corresponding to the Service.  The bot_type of this user
+    # determines the type of service.  If non-bot services are added later,
+    # user_profile can also represent the owner of the Service.
+    user_profile = models.ForeignKey(UserProfile, on_delete=CASCADE)
+    base_url = models.TextField()
+    token = models.TextField()
+    # Interface / API version of the service.
+    interface = models.PositiveSmallIntegerField(default=1)
+
+    # Valid interfaces are {generic, zulip_bot_service, slack}
+    GENERIC = 1
+    SLACK = 2
+
+    ALLOWED_INTERFACE_TYPES = [
+        GENERIC,
+        SLACK,
+    ]
+    # N.B. If we used Django's choice=... we would get this for free (kinda)
+    _interfaces: dict[int, str] = {
+        GENERIC: GENERIC_INTERFACE,
+        SLACK: SLACK_INTERFACE,
+    }
+
+    def interface_name(self) -> str:
+        # Raises KeyError if invalid
+        return self._interfaces[self.interface]
+
+
+def get_bot_services(user_profile_id: int) -> list[Service]:
+    return list(Service.objects.filter(user_profile_id=user_profile_id))
+
+
+def get_service_profile(user_profile_id: int, service_name: str) -> Service:
+    return Service.objects.get(user_profile_id=user_profile_id, name=service_name)
+
+
+class BotStorageData(models.Model):
+    bot_profile = models.ForeignKey(UserProfile, on_delete=CASCADE)
+    key = models.TextField(db_index=True)
+    value = models.TextField()
+
+    class Meta:
+        unique_together = ("bot_profile", "key")
+
+
+class BotConfigData(models.Model):
+    bot_profile = models.ForeignKey(UserProfile, on_delete=CASCADE)
+    key = models.TextField(db_index=True)
+    value = models.TextField()
+
+    class Meta:
+        unique_together = ("bot_profile", "key")

zerver/models/channel_folders.py

@@ -0,0 +1,32 @@
+from django.db import models
+from django.db.models import Q
+from django.db.models.functions import Lower
+from django.utils.timezone import now as timezone_now
+
+from zerver.models.realms import Realm
+from zerver.models.users import UserProfile
+
+
+class ChannelFolder(models.Model):
+    MAX_NAME_LENGTH = 60
+    MAX_DESCRIPTION_LENGTH = 1024
+
+    realm = models.ForeignKey(Realm, on_delete=models.CASCADE)
+    name = models.CharField(max_length=MAX_NAME_LENGTH)
+    description = models.CharField(max_length=MAX_DESCRIPTION_LENGTH, default="")
+    rendered_description = models.TextField(default="")
+    order = models.IntegerField(default=0)
+
+    date_created = models.DateTimeField(default=timezone_now)
+    creator = models.ForeignKey(UserProfile, null=True, on_delete=models.SET_NULL)
+    is_archived = models.BooleanField(default=False)
+
+    class Meta:
+        constraints = [
+            models.UniqueConstraint(
+                Lower("name"),
+                "realm",
+                condition=Q(is_archived=False),
+                name="unique_realm_folder_name_when_not_archived",
+            ),
+        ]

zerver/models/clients.py

@@ -0,0 +1,76 @@
+import hashlib
+
+from django.conf import settings
+from django.db import models
+from typing_extensions import override
+
+from zerver.lib import cache
+from zerver.lib.cache import cache_with_key
+
+
+class Client(models.Model):
+    MAX_NAME_LENGTH = 30
+    id = models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")
+    name = models.CharField(max_length=MAX_NAME_LENGTH, db_index=True, unique=True)
+
+    @override
+    def __str__(self) -> str:
+        return self.name
+
+    def default_read_by_sender(self) -> bool:
+        """Used to determine whether a message was sent by a full Zulip UI
+        style client (and thus whether the message should be treated
+        as sent by a human and automatically marked as read for the
+        sender).  The purpose of this distinction is to ensure that
+        message sent to the user by e.g. a Google Calendar integration
+        using the user's own API key don't get marked as read
+        automatically.
+        """
+        sending_client = self.name.lower()
+
+        return (
+            sending_client
+            in (
+                "zulipandroid",
+                "zulipios",
+                "zulipdesktop",
+                "zulipmobile",
+                "zulipelectron",
+                "zulipterminal",
+                "snipe",
+                "website",
+                "ios",
+                "android",
+            )
+            # Since the vast majority of messages are sent by humans
+            # in Zulip, treat test suite messages as such.
+            or (sending_client == "test suite" and settings.TEST_SUITE)
+        )
+
+
+get_client_cache: dict[str, Client] = {}
+
+
+def clear_client_cache() -> None:  # nocoverage
+    global get_client_cache
+    get_client_cache = {}
+
+
+def get_client(name: str) -> Client:
+    # Accessing KEY_PREFIX through the module is necessary
+    # because we need the updated value of the variable.
+    cache_name = cache.KEY_PREFIX + name[0 : Client.MAX_NAME_LENGTH]
+    if cache_name not in get_client_cache:
+        result = get_client_remote_cache(name)
+        get_client_cache[cache_name] = result
+    return get_client_cache[cache_name]
+
+
+def get_client_cache_key(name: str) -> str:
+    return f"get_client:{hashlib.sha1(name.encode()).hexdigest()}"
+
+
+@cache_with_key(get_client_cache_key, timeout=3600 * 24 * 7)
+def get_client_remote_cache(name: str) -> Client:
+    (client, _) = Client.objects.get_or_create(name=name[0 : Client.MAX_NAME_LENGTH])
+    return client

zerver/models/constants.py

@@ -0,0 +1,2 @@
+MAX_TOPIC_NAME_LENGTH = 60
+MAX_LANGUAGE_ID_LENGTH: int = 50

zerver/models/custom_profile_fields.py

@@ -0,0 +1,210 @@
+from collections.abc import Callable
+from typing import Any
+
+import orjson
+from django.core.exceptions import ValidationError
+from django.db import models
+from django.db.models import CASCADE, QuerySet
+from django.utils.translation import gettext as _
+from django.utils.translation import gettext_lazy
+from django_stubs_ext import StrPromise
+from typing_extensions import override
+
+from zerver.lib.types import (
+    ExtendedFieldElement,
+    ExtendedValidator,
+    FieldElement,
+    ProfileDataElementBase,
+    ProfileDataElementValue,
+    RealmUserValidator,
+    UserFieldElement,
+    Validator,
+)
+from zerver.lib.validator import (
+    check_date,
+    check_int,
+    check_list,
+    check_long_string,
+    check_short_string,
+    check_url,
+    validate_dropdown_field,
+)
+from zerver.models.realms import Realm
+from zerver.models.users import UserProfile
+
+
+def check_valid_user_ids(realm_id: int, val: object, allow_deactivated: bool = False) -> list[int]:
+    user_ids = check_list(check_int)("User IDs", val)
+    user_profiles = UserProfile.objects.filter(realm_id=realm_id, id__in=user_ids)
+
+    valid_users_ids = set(user_profiles.values_list("id", flat=True))
+    invalid_users_ids = [invalid_id for invalid_id in user_ids if invalid_id not in valid_users_ids]
+
+    if invalid_users_ids:
+        raise ValidationError(
+            _("Invalid user IDs: {invalid_ids}").format(
+                invalid_ids=", ".join(map(str, invalid_users_ids))
+            )
+        )
+
+    for user in user_profiles:
+        if not allow_deactivated and not user.is_active:
+            raise ValidationError(
+                _("User with ID {user_id} is deactivated").format(user_id=user.id)
+            )
+
+        if user.is_bot:
+            raise ValidationError(_("User with ID {user_id} is a bot").format(user_id=user.id))
+
+    return user_ids
+
+
+class CustomProfileField(models.Model):
+    """Defines a form field for the per-realm custom profile fields feature.
+
+    See CustomProfileFieldValue for an individual user's values for one of
+    these fields.
+    """
+
+    HINT_MAX_LENGTH = 80
+    NAME_MAX_LENGTH = 40
+    MAX_DISPLAY_IN_PROFILE_SUMMARY_FIELDS = 2
+
+    realm = models.ForeignKey(Realm, on_delete=CASCADE)
+    name = models.CharField(max_length=NAME_MAX_LENGTH)
+    hint = models.CharField(max_length=HINT_MAX_LENGTH, default="")
+
+    # Sort order for display of custom profile fields.
+    order = models.IntegerField(default=0)
+
+    # Whether the field should be displayed in smaller summary
+    # sections of a page displaying custom profile fields.
+    display_in_profile_summary = models.BooleanField(default=False)
+    required = models.BooleanField(default=False)
+
+    # Whether this field should be used in @-mention typeahead suggestions.
+    # Only applicable for EXTERNAL_ACCOUNT and SHORT_TEXT field types.
+    use_for_user_matching = models.BooleanField(default=False, db_default=False)
+
+    # Whether regular users can edit this field on their own account.
+    editable_by_user = models.BooleanField(default=True, db_default=True)
+
+    SHORT_TEXT = 1
+    PARAGRAPH = 2
+    DROPDOWN = 3
+    DATE = 4
+    URL = 5
+    USER = 6
+    EXTERNAL_ACCOUNT = 7
+    PRONOUNS = 8
+
+    # These are the fields whose validators require more than var_name
+    # and value argument. i.e. DROPDOWN require field_data, USER require
+    # realm as argument.
+    DROPDOWN_FIELD_TYPE_DATA: list[ExtendedFieldElement] = [
+        (DROPDOWN, gettext_lazy("Dropdown"), validate_dropdown_field, str, "DROPDOWN"),
+    ]
+    USER_FIELD_TYPE_DATA: list[UserFieldElement] = [
+        (USER, gettext_lazy("Users"), check_valid_user_ids, orjson.loads, "USER"),
+    ]
+
+    DROPDOWN_FIELD_VALIDATORS: dict[int, ExtendedValidator] = {
+        item[0]: item[2] for item in DROPDOWN_FIELD_TYPE_DATA
+    }
+    USER_FIELD_VALIDATORS: dict[int, RealmUserValidator] = {
+        item[0]: item[2] for item in USER_FIELD_TYPE_DATA
+    }
+
+    FIELD_TYPE_DATA: list[FieldElement] = [
+        # Type, display name, validator, converter, keyword
+        (SHORT_TEXT, gettext_lazy("Short text"), check_short_string, str, "SHORT_TEXT"),
+        (PARAGRAPH, gettext_lazy("Paragraph"), check_long_string, str, "PARAGRAPH"),
+        (DATE, gettext_lazy("Date"), check_date, str, "DATE"),
+        (URL, gettext_lazy("Link"), check_url, str, "URL"),
+        (
+            EXTERNAL_ACCOUNT,
+            gettext_lazy("External account"),
+            check_short_string,
+            str,
+            "EXTERNAL_ACCOUNT",
+        ),
+        (PRONOUNS, gettext_lazy("Pronouns"), check_short_string, str, "PRONOUNS"),
+    ]
+
+    ALL_FIELD_TYPES = sorted(
+        [*FIELD_TYPE_DATA, *DROPDOWN_FIELD_TYPE_DATA, *USER_FIELD_TYPE_DATA], key=lambda x: x[1]
+    )
+
+    FIELD_VALIDATORS: dict[int, Validator[ProfileDataElementValue]] = {
+        item[0]: item[2] for item in FIELD_TYPE_DATA
+    }
+    FIELD_CONVERTERS: dict[int, Callable[[Any], Any]] = {
+        item[0]: item[3] for item in ALL_FIELD_TYPES
+    }
+    FIELD_TYPE_CHOICES: list[tuple[int, StrPromise]] = [
+        (item[0], item[1]) for item in ALL_FIELD_TYPES
+    ]
+
+    field_type = models.PositiveSmallIntegerField(
+        choices=FIELD_TYPE_CHOICES,
+        default=SHORT_TEXT,
+    )
+
+    # A JSON blob of any additional data needed to define the field beyond
+    # type/name/hint.
+    #
+    # The format depends on the type.  Field types SHORT_TEXT, PARAGRAPH,
+    # DATE, URL, and USER leave this empty.  Fields of type DROPDOWN store the
+    # choices' descriptions.
+    #
+    # Note: There is no performance overhead of using TextField in PostgreSQL.
+    # See https://www.postgresql.org/docs/9.0/static/datatype-character.html
+    field_data = models.TextField(default="")
+
+    class Meta:
+        unique_together = ("realm", "name")
+
+    @override
+    def __str__(self) -> str:
+        return f"{self.realm!r} {self.name} {self.field_type} {self.order}"
+
+    def as_dict(self) -> ProfileDataElementBase:
+        data_as_dict: ProfileDataElementBase = {
+            "id": self.id,
+            "name": self.name,
+            "type": self.field_type,
+            "hint": self.hint,
+            "field_data": self.field_data,
+            "order": self.order,
+            "required": self.required,
+            "editable_by_user": self.editable_by_user,
+        }
+        if self.display_in_profile_summary:
+            data_as_dict["display_in_profile_summary"] = True
+        if self.use_for_user_matching:
+            data_as_dict["use_for_user_matching"] = True
+
+        return data_as_dict
+
+    def is_renderable(self) -> bool:
+        if self.field_type in [CustomProfileField.SHORT_TEXT, CustomProfileField.PARAGRAPH]:
+            return True
+        return False
+
+
+def custom_profile_fields_for_realm(realm_id: int) -> QuerySet[CustomProfileField]:
+    return CustomProfileField.objects.filter(realm=realm_id).order_by("order")
+
+
+class CustomProfileFieldValue(models.Model):
+    user_profile = models.ForeignKey(UserProfile, on_delete=CASCADE)
+    field = models.ForeignKey(CustomProfileField, on_delete=CASCADE)
+    value = models.TextField()
+    rendered_value = models.TextField(null=True, default=None)
+
+    class Meta:
+        unique_together = ("user_profile", "field")
+
+    @override
+    def __str__(self) -> str:
+        return f"{self.user_profile!r} {self.field!r} {self.value}"

zerver/models/devices.py

@@ -0,0 +1,70 @@
+from django.db import models
+from django.db.models import Q
+
+from zerver.lib.exceptions import (
+    InvalidBouncerPublicKeyError,
+    InvalidEncryptedPushRegistrationError,
+    RequestExpiredError,
+)
+from zerver.models.users import UserProfile
+
+
+class Device(models.Model):
+    """Core zulip server table storing logged-in devices.
+
+    Currently, only used by mobile apps for E2EE push notifications.
+    """
+
+    # The user on this server to whom this Device belongs.
+    user = models.ForeignKey(UserProfile, on_delete=models.CASCADE)
+
+    # Key to use to encrypt notifications for delivery to this device.
+    # Consists of a 1-byte prefix identifying the symmetric cryptosystem
+    # in use, followed by the secret key.
+    # Prefix                 Cryptosystem
+    #  0x31        libsodium's `crypto_secretbox_easy`
+    push_key = models.BinaryField(null=True)
+    # ID to reference the `push_key` - unsigned 32-bit integer.
+    push_key_id = models.PositiveBigIntegerField(null=True)
+
+    # ID to reference the token provided by FCM/APNs, registered to bouncer.
+    push_token_id = models.BigIntegerField(null=True)
+    # ID to reference the token provided by FCM/APNs, registration in progress to bouncer.
+    pending_push_token_id = models.BigIntegerField(null=True)
+
+    # The last time when `pending_push_token_id` was set to a new value.
+    push_token_last_updated_timestamp = models.DateTimeField(null=True)
+
+    class PushTokenKind(models.TextChoices):
+        APNS = "apns", "APNs"
+        FCM = "fcm", "FCM"
+
+    push_token_kind = models.CharField(max_length=4, choices=PushTokenKind.choices, null=True)
+
+    class PushRegistrationErrorCode(models.TextChoices):
+        INVALID_BOUNCER_PUBLIC_KEY = InvalidBouncerPublicKeyError.code.name
+        INVALID_ENCRYPTED_PUSH_REGISTRATION = InvalidEncryptedPushRegistrationError.code.name
+        REQUEST_EXPIRED = RequestExpiredError.code.name
+
+    # The error code returned when registration to bouncer fails.
+    push_registration_error_code = models.CharField(
+        max_length=100, choices=PushRegistrationErrorCode.choices, null=True
+    )
+
+    class Meta:
+        constraints = [
+            models.CheckConstraint(
+                condition=Q(push_key_id__lte=2**32 - 1),
+                name="push_key_id_lte_max_push_key_id",
+            )
+        ]
+        indexes = [
+            models.Index(
+                # Used in 'get_recipient_info', `do_clear_mobile_push_notifications_for_ids`,
+                # `prepare_payload_and_send_push_notifications`, `send_push_notifications`,
+                # and `send_e2ee_test_push_notification_api`.
+                fields=["user", "push_token_id"],
+                condition=Q(push_token_id__isnull=False),
+                name="zerver_device_user_push_token_id_idx",
+            ),
+        ]

zerver/models/drafts.py

@@ -0,0 +1,36 @@
+from typing import Any
+
+from django.db import models
+from typing_extensions import override
+
+from zerver.lib.display_recipient import get_recipient_ids
+from zerver.models.constants import MAX_TOPIC_NAME_LENGTH
+from zerver.models.recipients import Recipient
+from zerver.models.users import UserProfile
+
+
+class Draft(models.Model):
+    """Server-side storage model for storing drafts so that drafts can be synced across
+    multiple clients/devices.
+    """
+
+    user_profile = models.ForeignKey(UserProfile, on_delete=models.CASCADE)
+    recipient = models.ForeignKey(Recipient, null=True, on_delete=models.SET_NULL)
+    topic = models.CharField(max_length=MAX_TOPIC_NAME_LENGTH, db_index=True)
+    content = models.TextField()  # Length should not exceed MAX_MESSAGE_LENGTH
+    last_edit_time = models.DateTimeField(db_index=True)
+
+    @override
+    def __str__(self) -> str:
+        return f"{self.user_profile.email} / {self.id} / {self.last_edit_time}"
+
+    def to_dict(self) -> dict[str, Any]:
+        to, recipient_type_str = get_recipient_ids(self.recipient, self.user_profile_id)
+        return {
+            "id": self.id,
+            "type": recipient_type_str,
+            "to": to,
+            "topic": self.topic,
+            "content": self.content,
+            "timestamp": int(self.last_edit_time.timestamp()),
+        }

zerver/models/groups.py

@@ -0,0 +1,191 @@
+from django.db import models
+from django.db.models import CASCADE
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext_lazy
+
+from zerver.lib.cache import cache_with_key, get_realm_system_groups_cache_key
+from zerver.lib.types import GroupPermissionSetting
+from zerver.models.users import UserProfile
+
+
+class SystemGroups:
+    FULL_MEMBERS = "role:fullmembers"
+    EVERYONE_ON_INTERNET = "role:internet"
+    OWNERS = "role:owners"
+    ADMINISTRATORS = "role:administrators"
+    MODERATORS = "role:moderators"
+    MEMBERS = "role:members"
+    EVERYONE = "role:everyone"
+    NOBODY = "role:nobody"
+
+    GROUP_DISPLAY_NAME_MAP = {
+        NOBODY: gettext_lazy("Nobody"),
+        OWNERS: gettext_lazy("Owners"),
+        ADMINISTRATORS: gettext_lazy("Administrators"),
+        MODERATORS: gettext_lazy("Moderators"),
+        FULL_MEMBERS: gettext_lazy("Full members"),
+        MEMBERS: gettext_lazy("Members"),
+        EVERYONE: gettext_lazy("Everyone"),
+        EVERYONE_ON_INTERNET: gettext_lazy("Everyone on the internet"),
+    }
+
+
+class UserGroup(models.Model):
+    direct_members = models.ManyToManyField(
+        UserProfile, through="zerver.UserGroupMembership", related_name="direct_groups"
+    )
+    direct_subgroups = models.ManyToManyField(
+        "zerver.NamedUserGroup",
+        symmetrical=False,
+        through="zerver.GroupGroupMembership",
+        through_fields=("supergroup", "subgroup"),
+        related_name="direct_supergroups",
+    )
+    realm = models.ForeignKey("zerver.Realm", on_delete=CASCADE)
+
+
+class NamedUserGroup(UserGroup):
+    MAX_NAME_LENGTH = 100
+    INVALID_NAME_PREFIXES = ["@", "role:", "user:", "stream:", "channel:"]
+
+    # This field is automatically created by django, but we still need
+    # to add this here to keep mypy happy when accessing usergroup_ptr.
+    usergroup_ptr = models.OneToOneField(
+        auto_created=True,
+        on_delete=CASCADE,
+        parent_link=True,
+        primary_key=True,
+        serialize=False,
+        to=UserGroup,
+        # We are not using the auto-generated name here to avoid
+        # duplicate backward relation name because "can_mention_group"
+        # setting also points to a UserGroup object.
+        related_name="named_user_group",
+    )
+    name = models.CharField(max_length=MAX_NAME_LENGTH, db_column="name")
+    description = models.TextField(default="", db_column="description")
+    date_created = models.DateTimeField(default=timezone_now, null=True)
+    creator = models.ForeignKey(
+        UserProfile, null=True, on_delete=models.SET_NULL, related_name="+", db_column="creator_id"
+    )
+    is_system_group = models.BooleanField(default=False, db_column="is_system_group")
+
+    can_add_members_group = models.ForeignKey(
+        UserGroup, on_delete=models.RESTRICT, related_name="+"
+    )
+    can_join_group = models.ForeignKey(UserGroup, on_delete=models.RESTRICT, related_name="+")
+    can_leave_group = models.ForeignKey(UserGroup, on_delete=models.RESTRICT, related_name="+")
+    can_manage_group = models.ForeignKey(UserGroup, on_delete=models.RESTRICT, related_name="+")
+    can_mention_group = models.ForeignKey(
+        UserGroup, on_delete=models.RESTRICT, db_column="can_mention_group_id"
+    )
+    can_remove_members_group = models.ForeignKey(
+        UserGroup, on_delete=models.RESTRICT, related_name="+"
+    )
+
+    realm_for_sharding = models.ForeignKey("zerver.Realm", on_delete=CASCADE, db_column="realm_id")
+    deactivated = models.BooleanField(default=False, db_default=False)
+
+    # We do not have "Full members" and "Everyone on the internet"
+    # group here since there isn't a separate role value for full
+    # members and spectators.
+    SYSTEM_USER_GROUP_ROLE_MAP = {
+        UserProfile.ROLE_REALM_OWNER: {
+            "name": SystemGroups.OWNERS,
+            "description": "Owners of this organization",
+        },
+        UserProfile.ROLE_REALM_ADMINISTRATOR: {
+            "name": SystemGroups.ADMINISTRATORS,
+            "description": "Administrators of this organization, including owners",
+        },
+        UserProfile.ROLE_MODERATOR: {
+            "name": SystemGroups.MODERATORS,
+            "description": "Moderators of this organization, including administrators",
+        },
+        UserProfile.ROLE_MEMBER: {
+            "name": SystemGroups.MEMBERS,
+            "description": "Members of this organization, not including guests",
+        },
+        UserProfile.ROLE_GUEST: {
+            "name": SystemGroups.EVERYONE,
+            "description": "Everyone in this organization, including guests",
+        },
+    }
+
+    GROUP_PERMISSION_SETTINGS = {
+        "can_add_members_group": GroupPermissionSetting(
+            allow_nobody_group=True,
+            allow_everyone_group=False,
+            default_group_name="group_creator",
+            default_for_system_groups=SystemGroups.NOBODY,
+        ),
+        "can_join_group": GroupPermissionSetting(
+            allow_nobody_group=True,
+            allow_everyone_group=False,
+            default_group_name=SystemGroups.NOBODY,
+            default_for_system_groups=SystemGroups.NOBODY,
+        ),
+        "can_leave_group": GroupPermissionSetting(
+            allow_nobody_group=True,
+            allow_everyone_group=True,
+            default_group_name=SystemGroups.EVERYONE,
+            default_for_system_groups=SystemGroups.NOBODY,
+        ),
+        "can_manage_group": GroupPermissionSetting(
+            allow_nobody_group=True,
+            allow_everyone_group=False,
+            default_group_name="group_creator",
+            default_for_system_groups=SystemGroups.NOBODY,
+        ),
+        "can_mention_group": GroupPermissionSetting(
+            allow_nobody_group=True,
+            allow_everyone_group=True,
+            default_group_name=SystemGroups.EVERYONE,
+            default_for_system_groups=SystemGroups.NOBODY,
+        ),
+        "can_remove_members_group": GroupPermissionSetting(
+            allow_nobody_group=True,
+            allow_everyone_group=False,
+            default_group_name=SystemGroups.NOBODY,
+            default_for_system_groups=SystemGroups.NOBODY,
+        ),
+    }
+
+    MEMBERSHIP_MANAGEMENT_SETTINGS = [
+        "can_add_members_group",
+        "can_join_group",
+        "can_leave_group",
+        "can_manage_group",
+        "can_remove_members_group",
+    ]
+
+    class Meta:
+        unique_together = (("realm_for_sharding", "name"),)
+
+
+class UserGroupMembership(models.Model):
+    user_group = models.ForeignKey(UserGroup, on_delete=CASCADE, related_name="+")
+    user_profile = models.ForeignKey(UserProfile, on_delete=CASCADE, related_name="+")
+
+    class Meta:
+        unique_together = (("user_group", "user_profile"),)
+
+
+class GroupGroupMembership(models.Model):
+    supergroup = models.ForeignKey(UserGroup, on_delete=CASCADE, related_name="+")
+    subgroup = models.ForeignKey(NamedUserGroup, on_delete=CASCADE, related_name="+")
+
+    class Meta:
+        constraints = [
+            models.UniqueConstraint(
+                fields=["supergroup", "subgroup"], name="zerver_groupgroupmembership_uniq"
+            )
+        ]
+
+
+@cache_with_key(get_realm_system_groups_cache_key, timeout=3600 * 24 * 7)
+def get_realm_system_groups_name_dict(realm_id: int) -> dict[int, str]:
+    system_groups = NamedUserGroup.objects.filter(
+        realm_for_sharding_id=realm_id, is_system_group=True
+    ).values_list("id", "name")
+    return dict(system_groups)

zerver/models/linkifiers.py

@@ -0,0 +1,237 @@
+import re2
+import uri_template
+from django.core.exceptions import ValidationError
+from django.db import models
+from django.db.models import CASCADE
+from django.db.models.signals import post_delete, post_save
+from django.utils.translation import gettext as _
+from typing_extensions import override
+
+from zerver.lib import cache
+from zerver.lib.cache import cache_delete, cache_with_key
+from zerver.lib.exceptions import MissingDependentParameterError
+from zerver.lib.per_request_cache import cache_for_current_request, flush_per_request_cache
+from zerver.lib.types import LinkifierDict
+from zerver.models.realms import Realm
+
+
+def normalize_optional_text(value: str | None) -> str | None:
+    if value is None:
+        return None
+    value = value.strip()
+    return value or None
+
+
+# A function with the same name implements this on the JS side.
+# Please replicate any changes here to that function as well.
+def expand_reverse_template(
+    reverse_template: str, match: "re2._Match[str]", group_set: set[str]
+) -> str:
+    output: list[str] = []
+    index = 0
+    while index < len(reverse_template):
+        if reverse_template.startswith("{{", index):
+            output.append("{")
+            index += 2
+        elif reverse_template.startswith("}}", index):
+            output.append("}")
+            index += 2
+        elif reverse_template[index] == "{":
+            end_index = reverse_template.find("}", index + 1)
+            if end_index == -1:
+                raise ValidationError(_("Invalid auto-conversion field: missing '}' character."))
+            name = reverse_template[index + 1 : end_index]
+            if not name:
+                raise ValidationError(_("Invalid auto-conversion field: empty field name."))
+            if name not in group_set:
+                raise ValidationError(
+                    _(
+                        "Group %(name)r in auto-conversion field is not present in linkifier pattern."
+                    ),
+                    params={"name": name},
+                )
+            # For optional groups, match.group might return None, which we cannot
+            # append in the output to be .join(ed) later, so we add an empty string
+            # in that case.
+            output.append(match.group(name) or "")
+            index = end_index + 1
+        else:
+            output.append(reverse_template[index])
+            index += 1
+    return "".join(output)
+
+
+def filter_pattern_validator(value: str) -> "re2._Regexp[str]":
+    try:
+        # Do not write errors to stderr (this still raises exceptions)
+        options = re2.Options()
+        options.log_errors = False
+
+        regex = re2.compile(value, options=options)
+    except re2.error as e:
+        if len(e.args) >= 1:
+            if isinstance(e.args[0], str):  # nocoverage
+                raise ValidationError(_("Bad regular expression: {regex}").format(regex=e.args[0]))
+            if isinstance(e.args[0], bytes):
+                raise ValidationError(
+                    _("Bad regular expression: {regex}").format(regex=e.args[0].decode())
+                )
+        raise ValidationError(_("Unknown regular expression error"))  # nocoverage
+
+    return regex
+
+
+def url_template_validator(value: str) -> None:
+    """Validate as a URL template"""
+    if not uri_template.validate(value):
+        raise ValidationError(_("Invalid URL template."))
+
+
+class RealmFilter(models.Model):
+    """Realm-specific regular expressions to automatically linkify certain
+    strings inside the Markdown processor.  See "Custom filters" in the settings UI.
+    """
+
+    realm = models.ForeignKey(Realm, on_delete=CASCADE)
+    pattern = models.TextField()
+    url_template = models.TextField(validators=[url_template_validator])
+    example_input = models.TextField(blank=True, null=True)
+    reverse_template = models.TextField(blank=True, null=True)
+    alternative_url_templates = models.JSONField(default=list, blank=True)
+    # Linkifiers are applied in a message/topic in order; the processing order
+    # is important when there are overlapping patterns.
+    order = models.IntegerField(default=0)
+
+    class Meta:
+        unique_together = ("realm", "pattern")
+
+    @override
+    def __str__(self) -> str:
+        return f"{self.realm.string_id}: {self.pattern} {self.url_template}"
+
+    @override
+    def clean(self) -> None:
+        """Validate whether the set of parameters in the URL template
+        match the set of parameters in the regular expression.
+
+        Django's `full_clean` calls `clean_fields` followed by `clean` method
+        and stores all ValidationErrors from all stages to return as JSON.
+        """
+
+        # Extract variables present in the pattern
+        pattern = filter_pattern_validator(self.pattern)
+        group_set = set(pattern.groupindex.keys())
+
+        example_input = normalize_optional_text(self.example_input)
+        reverse_template = normalize_optional_text(self.reverse_template)
+
+        if reverse_template is not None and example_input is None:
+            raise MissingDependentParameterError(
+                required_parameter="Example text",
+                set_parameter="auto-conversion field",
+            )
+
+        if example_input is not None:
+            match = pattern.search(example_input)
+            if match is None:
+                raise ValidationError(_("Example input does not match the linkifier pattern."))
+            elif reverse_template is not None:
+                reversed_input = expand_reverse_template(reverse_template, match, group_set)
+                if reversed_input != example_input:
+                    raise ValidationError(_("Example text does not match auto-conversion field."))
+
+        # Do not continue the check if the url template is invalid to begin with.
+        # The ValidationError for invalid template will only be raised by the validator
+        # set on the url_template field instead of here to avoid duplicates.
+        if not uri_template.validate(self.url_template):
+            return
+
+        # Extract variables used in the URL template.
+        template_variables_set = set(uri_template.URITemplate(self.url_template).variable_names)
+
+        # Report patterns missing in linkifier pattern.
+        missing_in_pattern_set = template_variables_set - group_set
+        if len(missing_in_pattern_set) > 0:
+            name = min(missing_in_pattern_set)
+            raise ValidationError(
+                _("Group %(name)r in URL template is not present in linkifier pattern."),
+                params={"name": name},
+            )
+
+        missing_in_url_set = group_set - template_variables_set
+        # Report patterns missing in URL template.
+        if len(missing_in_url_set) > 0:
+            # We just report the first missing pattern here. Users can
+            # incrementally resolve errors if there are multiple
+            # missing patterns.
+            name = min(missing_in_url_set)
+            raise ValidationError(
+                _("Group %(name)r in linkifier pattern is not present in URL template."),
+                params={"name": name},
+            )
+
+        if self.url_template in self.alternative_url_templates:
+            raise ValidationError(
+                _("An alternative URL template cannot be the same as the URL template.")
+            )
+
+        if len(self.alternative_url_templates) != len(set(self.alternative_url_templates)):
+            raise ValidationError(_("Alternative URL templates must be unique."))
+
+        for alternative_template in self.alternative_url_templates:
+            url_template_validator(alternative_template)
+
+            alternative_template_variables = set(
+                uri_template.URITemplate(alternative_template).variable_names
+            )
+
+            missing_in_pattern = alternative_template_variables - group_set
+            if len(missing_in_pattern) > 0:
+                name = min(missing_in_pattern)
+                raise ValidationError(
+                    _(
+                        "Group %(name)r in alternative URL template is not present in linkifier pattern."
+                    ),
+                    params={"name": name},
+                )
+
+            missing_in_alternative = group_set - alternative_template_variables
+            if len(missing_in_alternative) > 0:
+                name = min(missing_in_alternative)
+                raise ValidationError(
+                    _(
+                        "Group %(name)r in linkifier pattern is not present in alternative URL template."
+                    ),
+                    params={"name": name},
+                )
+
+
+def get_linkifiers_cache_key(realm_id: int) -> str:
+    return f"{cache.KEY_PREFIX}:all_linkifiers_for_realm:{realm_id}"
+
+
+@cache_for_current_request
+@cache_with_key(get_linkifiers_cache_key, timeout=3600 * 24 * 7)
+def linkifiers_for_realm(realm_id: int) -> list[LinkifierDict]:
+    return [
+        LinkifierDict(
+            pattern=linkifier.pattern,
+            url_template=linkifier.url_template,
+            id=linkifier.id,
+            example_input=linkifier.example_input,
+            reverse_template=linkifier.reverse_template,
+            alternative_url_templates=linkifier.alternative_url_templates,
+        )
+        for linkifier in RealmFilter.objects.filter(realm_id=realm_id).order_by("order")
+    ]
+
+
+def flush_linkifiers(*, instance: RealmFilter, **kwargs: object) -> None:
+    realm_id = instance.realm_id
+    cache_delete(get_linkifiers_cache_key(realm_id))
+    flush_per_request_cache("linkifiers_for_realm")
+    flush_per_request_cache("topic_links")
+
+
+post_save.connect(flush_linkifiers, sender=RealmFilter)
+post_delete.connect(flush_linkifiers, sender=RealmFilter)

zerver/models/lookups.py

@@ -0,0 +1,30 @@
+from django.db import models
+from django.db.backends.base.base import BaseDatabaseWrapper
+from django.db.models.sql.compiler import SQLCompiler
+from typing_extensions import override
+
+
+@models.Field.register_lookup
+class AndZero(models.Lookup[int]):
+    lookup_name = "andz"
+
+    @override
+    def as_sql(
+        self, compiler: SQLCompiler, connection: BaseDatabaseWrapper
+    ) -> tuple[str, tuple[str | int, ...]]:  # nocoverage # currently only used in migrations
+        lhs, lhs_params = self.process_lhs(compiler, connection)
+        rhs, rhs_params = self.process_rhs(compiler, connection)
+        return f"{lhs} & {rhs} = 0", (*lhs_params, *rhs_params)
+
+
+@models.Field.register_lookup
+class AndNonZero(models.Lookup[int]):
+    lookup_name = "andnz"
+
+    @override
+    def as_sql(
+        self, compiler: SQLCompiler, connection: BaseDatabaseWrapper
+    ) -> tuple[str, tuple[str | int, ...]]:  # nocoverage # currently only used in migrations
+        lhs, lhs_params = self.process_lhs(compiler, connection)
+        rhs, rhs_params = self.process_rhs(compiler, connection)
+        return f"{lhs} & {rhs} != 0", (*lhs_params, *rhs_params)

zerver/models/messages.py

@@ -0,0 +1,851 @@
+import time
+from datetime import timedelta
+from typing import Any
+
+from bitfield import BitField
+from bitfield.types import Bit, BitHandler
+from django.contrib.postgres.indexes import GinIndex
+from django.contrib.postgres.search import SearchVectorField
+from django.db import models
+from django.db.models import CASCADE, F, Q, QuerySet
+from django.db.models.functions import Upper
+from django.db.models.signals import post_delete, post_save
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext_lazy
+from typing_extensions import override
+
+from zerver.lib.cache import flush_message, flush_submessage, flush_used_upload_space_cache
+from zerver.models.clients import Client
+from zerver.models.constants import MAX_TOPIC_NAME_LENGTH
+from zerver.models.realms import Realm
+from zerver.models.recipients import Recipient
+from zerver.models.users import UserProfile
+
+
+class AbstractMessage(models.Model):
+    id = models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")
+    sender = models.ForeignKey(UserProfile, on_delete=CASCADE)
+
+    # The target of the message is signified by the Recipient object.
+    # See the Recipient class for details.
+    recipient = models.ForeignKey(Recipient, on_delete=CASCADE)
+
+    # The realm containing the message. Usually this will be the same
+    # as the realm of the messages's sender; the exception to that is
+    # cross-realm bot users.
+    #
+    # Important for efficient indexes and sharding in multi-realm servers.
+    realm = models.ForeignKey(Realm, on_delete=CASCADE)
+
+    class MessageType(models.IntegerChoices):
+        NORMAL = 1
+        RESOLVE_TOPIC_NOTIFICATION = 2
+
+    # IMPORTANT: message.type is not to be confused with the
+    # "recipient type" ("channel" or "direct"), which is sometimes
+    # called message_type in the APIs, CountStats or some variable
+    # names. We intend to rename those to recipient_type.
+    #
+    # Type of the message, used to distinguish between "normal"
+    # messages and some special kind of messages, such as notification
+    # messages that may be sent by system bots.
+    type = models.PositiveSmallIntegerField(
+        choices=MessageType.choices,
+        default=MessageType.NORMAL,
+        # Note: db_default is a new feature in Django 5.0, so we don't use
+        # it across the codebase yet. It's useful here to simplify the
+        # associated database migration, so we're making use of it.
+        db_default=MessageType.NORMAL,
+    )
+
+    # Direct messages do not have topics in the API. Originally, all
+    # DMs had a topic of "" in the database. When we started using ""
+    # as the topic for "general chat", this caused problems with the
+    # PostgreSQL query planner. Because a large portion of all
+    # messages are DMs, PostgreSQL could do very inefficient table
+    # scans to fetch messages in general chat, ignoring the topic
+    # index, because it assumed most messages were in general chat.
+    #
+    # To avoid that query planner statistics problem, we use an
+    # unprintable character, which isn't permitted in actual topics,
+    # as the topic for DMs in the database. The "BEL" character is an
+    # arbitrary choice, but feels suitable given DMs trigger a
+    # notification sound.
+    DM_TOPIC = "\x07"
+
+    # The message's topic.
+    #
+    # Early versions of Zulip called this concept a "subject", as in an email
+    # "subject line", before changing to "topic" in 2013 (commit dac5a46fa).
+    # UI and user documentation now consistently say "topic".  New APIs and
+    # new code should generally also say "topic".
+    #
+    # See also the `topic_name` method on `Message`.
+    subject = models.CharField(max_length=MAX_TOPIC_NAME_LENGTH, db_index=True)
+
+    # The raw Markdown-format text (E.g., what the user typed into the compose box).
+    content = models.TextField()
+
+    # The HTML rendered content resulting from rendering the content
+    # with the Markdown processor.
+    rendered_content = models.TextField(null=True)
+    # A rarely-incremented version number, theoretically useful for
+    # tracking which messages have been already rerendered when making
+    # major changes to the markup rendering process.
+    rendered_content_version = models.IntegerField(null=True)
+
+    date_sent = models.DateTimeField("date sent", db_index=True)
+
+    # A Client object indicating what type of Zulip client sent this message.
+    sending_client = models.ForeignKey(Client, on_delete=CASCADE)
+
+    # The last time the message was modified by message editing or moving.
+    last_edit_time = models.DateTimeField(null=True)
+
+    # A JSON-encoded list of objects describing any past edits to this
+    # message, oldest first.
+    edit_history = models.TextField(null=True)
+
+    # Whether the message contains a (link to) an uploaded file.
+    has_attachment = models.BooleanField(default=False, db_index=True)
+    # Whether the message contains a visible image element.
+    has_image = models.BooleanField(default=False, db_index=True)
+    # Whether the message contains a link.
+    has_link = models.BooleanField(default=False, db_index=True)
+    # If the message is a channel message (as opposed to a DM or group-DM)
+    is_channel_message = models.BooleanField(default=True, db_index=True)
+
+    class Meta:
+        abstract = True
+
+    @override
+    def __str__(self) -> str:
+        if not self.is_channel_message:
+            return f"{self.recipient.label()} /  / {self.sender!r}"
+
+        return f"{self.recipient.label()} / {self.subject} / {self.sender!r}"
+
+
+class ArchiveTransaction(models.Model):
+    timestamp = models.DateTimeField(default=timezone_now, db_index=True)
+    # Marks if the data archived in this transaction has been restored:
+    restored = models.BooleanField(default=False, db_index=True)
+    restored_timestamp = models.DateTimeField(null=True, db_index=True)
+
+    # ArchiveTransaction objects are regularly deleted. This flag allows tagging
+    # an ArchiveTransaction as protected from such automated deletion.
+    protect_from_deletion = models.BooleanField(default=False, db_index=True)
+
+    type = models.PositiveSmallIntegerField(db_index=True)
+    # Valid types:
+    RETENTION_POLICY_BASED = 1  # Archiving was executed due to automated retention policies
+    MANUAL = 2  # Archiving was run manually, via move_messages_to_archive function
+
+    # ForeignKey to the realm with which objects archived in this transaction are associated.
+    # If type is set to MANUAL, this should be null.
+    realm = models.ForeignKey(Realm, null=True, on_delete=CASCADE)
+
+    @override
+    def __str__(self) -> str:
+        return "id: {id}, type: {type}, realm: {realm}, timestamp: {timestamp}".format(
+            id=self.id,
+            type="MANUAL" if self.type == self.MANUAL else "RETENTION_POLICY_BASED",
+            realm=self.realm.string_id if self.realm else None,
+            timestamp=self.timestamp,
+        )
+
+
+class ArchivedMessage(AbstractMessage):
+    """Used as a temporary holding place for deleted messages before they
+    are permanently deleted.  This is an important part of a robust
+    'message retention' feature.
+    """
+
+    archive_transaction = models.ForeignKey(ArchiveTransaction, on_delete=CASCADE)
+
+
+class Message(AbstractMessage):
+    # Recipient types used when a Message object is provided to
+    # Zulip clients via the API.
+    #
+    # A detail worth noting:
+    # * "direct" was introduced in 2023 with the goal of
+    #   deprecating the original "private" and becoming the
+    #   preferred way to indicate a personal or direct_message_group
+    #   Recipient type via the API.
+    API_RECIPIENT_TYPES = ["direct", "private", "stream", "channel"]
+
+    MAX_POSSIBLE_MESSAGE_ID = 2147483647
+
+    search_tsvector = SearchVectorField(null=True)
+
+    DEFAULT_SELECT_RELATED = ["sender", "realm", "recipient", "sending_client"]
+
+    # Name to be used for the empty topic with clients that have not
+    # yet migrated to have the `empty_topic_name` client capability.
+    EMPTY_TOPIC_FALLBACK_NAME = "general chat"
+
+    class Meta:
+        indexes = [
+            GinIndex("search_tsvector", fastupdate=False, name="zerver_message_search_tsvector"),
+            models.Index(
+                # For moving messages between streams or marking
+                # streams as read.  The "id" at the end makes it easy
+                # to scan the resulting messages in order, and perform
+                # batching.
+                "realm_id",
+                "recipient_id",
+                "id",
+                name="zerver_message_realm_recipient_id",
+            ),
+            models.Index(
+                # For generating digest emails and message archiving,
+                # which both group by stream.
+                "realm_id",
+                "recipient_id",
+                "date_sent",
+                name="zerver_message_realm_recipient_date_sent",
+            ),
+            models.Index(
+                # For exports, which want to limit both sender and
+                # receiver.  The prefix of this index (realm_id,
+                # sender_id) can be used for scrubbing users and/or
+                # deleting users' messages.
+                # Also used in send_welcome_bot_response
+                "realm_id",
+                "sender_id",
+                "recipient_id",
+                name="zerver_message_realm_sender_recipient",
+            ),
+            models.Index(
+                # For analytics and retention queries
+                "realm_id",
+                "date_sent",
+                name="zerver_message_realm_date_sent",
+            ),
+            models.Index(
+                # For users searching by topic (but not stream), which
+                # is done case-insensitively
+                "realm_id",
+                Upper("subject"),
+                F("id").desc(),
+                name="zerver_message_realm_upper_subject",
+                condition=Q(is_channel_message=True),
+            ),
+            models.Index(
+                # Most stream/topic searches are case-insensitive by
+                # topic name (e.g. messages_for_topic).  The "id" at
+                # the end makes it easy to scan the resulting messages
+                # in order, and perform batching.
+                "realm_id",
+                "recipient_id",
+                Upper("subject"),
+                F("id").desc(),
+                name="zerver_message_realm_recipient_upper_subject",
+                condition=Q(is_channel_message=True),
+            ),
+            models.Index(
+                # Used when determining recent topics (we post-process
+                # to merge and show the most recent case)
+                "realm_id",
+                "recipient_id",
+                "subject",
+                F("id").desc(),
+                name="zerver_message_realm_recipient_subject",
+                condition=Q(is_channel_message=True),
+            ),
+            models.Index(
+                # Only used by update_first_visible_message_id
+                "realm_id",
+                F("id").desc(),
+                name="zerver_message_realm_id",
+            ),
+            models.Index(
+                # Potentially useful for migrations that rewrite
+                # message edit history. Originally added for
+                # 0680_rename_general_chat_to_empty_string_topic,
+                # though that migration was adjusted in a way that no
+                # longer uses this.
+                fields=["id"],
+                condition=Q(edit_history__isnull=False),
+                name="zerver_message_edit_history_id",
+            ),
+        ]
+
+    def topic_name(self) -> str:
+        """
+        Please start using this helper to facilitate an
+        eventual switch over to a separate topic table.
+        """
+        return self.subject
+
+    def set_topic_name(self, topic_name: str) -> None:
+        self.subject = topic_name
+
+    def get_realm(self) -> Realm:
+        return self.realm
+
+    def save_rendered_content(self) -> None:
+        self.save(update_fields=["rendered_content", "rendered_content_version"])
+
+    @staticmethod
+    def need_to_render_content(
+        rendered_content: str | None,
+        rendered_content_version: int | None,
+        markdown_version: int,
+    ) -> bool:
+        return (
+            rendered_content is None
+            or rendered_content_version is None
+            or rendered_content_version < markdown_version
+        )
+
+    @staticmethod
+    def is_status_message(content: str, rendered_content: str) -> bool:
+        """
+        "status messages" start with /me and have special rendering:
+            /me loves chocolate -> Full Name loves chocolate
+        """
+        if content.startswith("/me "):
+            return True
+        return False
+
+
+def get_context_for_message(message: Message) -> QuerySet[Message]:
+    return Message.objects.filter(
+        # Uses index: zerver_message_realm_recipient_upper_subject
+        realm_id=message.realm_id,
+        recipient_id=message.recipient_id,
+        subject__iexact=message.subject,
+        is_channel_message=True,
+        id__lt=message.id,
+        date_sent__gt=message.date_sent - timedelta(minutes=15),
+    ).order_by("-id")[:10]
+
+
+post_save.connect(flush_message, sender=Message)
+
+
+class AbstractSubMessage(models.Model):
+    # We can send little text messages that are associated with a regular
+    # Zulip message.  These can be used for experimental widgets like embedded
+    # games, surveys, mini threads, etc.  These are designed to be pretty
+    # generic in purpose.
+
+    sender = models.ForeignKey(UserProfile, on_delete=CASCADE)
+    msg_type = models.TextField()
+    content = models.TextField()
+
+    class Meta:
+        abstract = True
+
+
+class SubMessage(AbstractSubMessage):
+    message = models.ForeignKey(Message, on_delete=CASCADE)
+
+    @staticmethod
+    def get_raw_db_rows(needed_ids: list[int]) -> list[dict[str, Any]]:
+        fields = ["id", "message_id", "sender_id", "msg_type", "content"]
+        query = SubMessage.objects.filter(message_id__in=needed_ids).values(*fields)
+        query = query.order_by("message_id", "id")
+        return list(query)
+
+
+class ArchivedSubMessage(AbstractSubMessage):
+    message = models.ForeignKey(ArchivedMessage, on_delete=CASCADE)
+
+
+post_save.connect(flush_submessage, sender=SubMessage)
+
+
+class AbstractEmoji(models.Model):
+    """For emoji reactions to messages (and potentially future reaction types).
+
+    Emoji are surprisingly complicated to implement correctly.  For details
+    on how this subsystem works, see:
+      https://zulip.readthedocs.io/en/latest/subsystems/emoji.html
+    """
+
+    user_profile = models.ForeignKey(UserProfile, on_delete=CASCADE)
+
+    # The user-facing name for an emoji reaction.  With emoji aliases,
+    # there may be multiple accepted names for a given emoji; this
+    # field encodes which one the user selected.
+    emoji_name = models.TextField()
+
+    UNICODE_EMOJI = "unicode_emoji"
+    REALM_EMOJI = "realm_emoji"
+    ZULIP_EXTRA_EMOJI = "zulip_extra_emoji"
+    REACTION_TYPES = (
+        (UNICODE_EMOJI, gettext_lazy("Unicode emoji")),
+        (REALM_EMOJI, gettext_lazy("Custom emoji")),
+        (ZULIP_EXTRA_EMOJI, gettext_lazy("Zulip extra emoji")),
+    )
+    reaction_type = models.CharField(default=UNICODE_EMOJI, choices=REACTION_TYPES, max_length=30)
+
+    # A string with the property that (realm, reaction_type,
+    # emoji_code) uniquely determines the emoji glyph.
+    #
+    # We cannot use `emoji_name` for this purpose, since the
+    # name-to-glyph mappings for unicode emoji change with time as we
+    # update our emoji database, and multiple custom emoji can have
+    # the same `emoji_name` in a realm (at most one can have
+    # `deactivated=False`). The format for `emoji_code` varies by
+    # `reaction_type`:
+    #
+    # * For Unicode emoji, a dash-separated hex encoding of the sequence of
+    #   Unicode codepoints that define this emoji in the Unicode
+    #   specification.  For examples, see "non_qualified" or "unified" in the
+    #   following data, with "non_qualified" taking precedence when both present:
+    #     https://raw.githubusercontent.com/iamcal/emoji-data/master/emoji_pretty.json
+    #
+    # * For user uploaded custom emoji (`reaction_type="realm_emoji"`), the stringified ID
+    #   of the RealmEmoji object, computed as `str(realm_emoji.id)`.
+    #
+    # * For "Zulip extra emoji" (like :zulip:), the name of the emoji (e.g. "zulip").
+    emoji_code = models.TextField()
+
+    class Meta:
+        abstract = True
+
+
+class AbstractReaction(AbstractEmoji):
+    class Meta:
+        abstract = True
+        unique_together = ("user_profile", "message", "reaction_type", "emoji_code")
+
+
+class Reaction(AbstractReaction):
+    message = models.ForeignKey(Message, on_delete=CASCADE)
+
+    @override
+    def __str__(self) -> str:
+        return f"{self.user_profile.email} / {self.message.id} / {self.emoji_name}"
+
+    @staticmethod
+    def get_raw_db_rows(needed_ids: list[int]) -> list[dict[str, Any]]:
+        fields = [
+            "message_id",
+            "emoji_name",
+            "emoji_code",
+            "reaction_type",
+            "user_profile_id",
+        ]
+        # The ordering is important here, as it makes it convenient
+        # for clients to display reactions in order without
+        # client-side sorting code.
+        return Reaction.objects.filter(message_id__in=needed_ids).values(*fields).order_by("id")
+
+
+class ArchivedReaction(AbstractReaction):
+    message = models.ForeignKey(ArchivedMessage, on_delete=CASCADE)
+
+
+# Whenever a message is sent, for each user subscribed to the
+# corresponding Recipient object (that is not long-term idle), we add
+# a row to the UserMessage table indicating that that user received
+# that message.  This table allows us to quickly query any user's last
+# 1000 messages to generate the home view and search exactly the
+# user's message history.
+#
+# The long-term idle optimization is extremely important for large,
+# open organizations, and is described in detail here:
+# https://zulip.readthedocs.io/en/latest/subsystems/sending-messages.html#soft-deactivation
+#
+# In particular, new messages to public streams will only generate
+# UserMessage rows for Members who are long_term_idle if they would
+# have nonzero flags for the message (E.g. a mention, alert word, or
+# mobile push notification).
+#
+# The flags field stores metadata like whether the user has read the
+# message, starred or collapsed the message, was mentioned in the
+# message, etc. We use of postgres partial indexes on flags to make
+# queries for "User X's messages with flag Y" extremely fast without
+# consuming much storage space.
+#
+# UserMessage is the largest table in many Zulip installations, even
+# though each row is only 4 integers.
+class AbstractUserMessage(models.Model):
+    id = models.BigAutoField(primary_key=True)
+
+    # We disable the index on this, because we provide a unique index
+    # on (user_profile_id, message_id) whose prefix can always be used
+    # instead of this index, and which is always going to produce
+    # sorted message-id rows.  Sometimes PostgreSQL would choose this
+    # non-sorted index and then have to perform an extra sort and
+    # limit after getting _all_ of the user's rows, which is quite
+    # wasteful.
+    user_profile = models.ForeignKey(UserProfile, on_delete=CASCADE, db_index=False)
+
+    # The order here is important!  It's the order of fields in the bitfield.
+    ALL_FLAGS = [
+        "read",
+        "starred",
+        "collapsed",
+        "mentioned",
+        "stream_wildcard_mentioned",
+        "topic_wildcard_mentioned",
+        "group_mentioned",
+        # These next 2 flags are from features that have since been removed.
+        # We've cleared these 2 flags in migration 0486.
+        "force_expand",
+        "force_collapse",
+        # Whether the message contains any of the user's alert words.
+        "has_alert_word",
+        # The historical flag is used to mark messages which the user
+        # did not receive when they were sent, but later added to
+        # their history via e.g. starring the message.  This is
+        # important accounting for the "Subscribed to stream" dividers.
+        "historical",
+        # Whether the message is a direct message; this flag is a
+        # denormalization of message.recipient.type to support an
+        # efficient index on UserMessage for a user's direct messages.
+        "is_private",
+        # Whether we've sent a push notification to the user's mobile
+        # devices for this message that has not been revoked.
+        "active_mobile_push_notification",
+        # Whether the user has hidden auto-generated link previews
+        # on this message.
+        "hide_link_previews",
+    ]
+    # Certain flags are used only for internal accounting within the
+    # Zulip backend, and don't make sense to expose to the API.
+    NON_API_FLAGS = {"is_private", "active_mobile_push_notification"}
+    # Certain additional flags are just set once when the UserMessage
+    # row is created.
+    NON_EDITABLE_FLAGS = {
+        # These flags are bookkeeping and don't make sense to edit.
+        "has_alert_word",
+        "mentioned",
+        "stream_wildcard_mentioned",
+        "topic_wildcard_mentioned",
+        "group_mentioned",
+        "historical",
+        # Unused flags can't be edited.
+        "force_expand",
+        "force_collapse",
+    }
+    flags: BitHandler = BitField(flags=ALL_FLAGS, default=0)
+
+    class Meta:
+        abstract = True
+        unique_together = ("user_profile", "message")
+
+    @staticmethod
+    def where_flag_is_present(flagattr: Bit) -> str:
+        # Use this for Django ORM queries to access starred messages.
+        # This custom SQL plays nice with our partial indexes.  Grep
+        # the code for example usage.
+        #
+        # The key detail is that e.g.
+        #   UserMessage.objects.filter(user_profile=user_profile, flags=UserMessage.flags.starred)
+        # will generate a query involving `flags & 2 = 2`, which doesn't match our index.
+        return f"flags & {1 << flagattr.number} <> 0"
+
+    @staticmethod
+    def where_flag_is_absent(flagattr: Bit) -> str:
+        return f"flags & {1 << flagattr.number} = 0"
+
+    @staticmethod
+    def where_unread() -> str:
+        return AbstractUserMessage.where_flag_is_absent(AbstractUserMessage.flags.read)
+
+    @staticmethod
+    def where_read() -> str:
+        return AbstractUserMessage.where_flag_is_present(AbstractUserMessage.flags.read)
+
+    @staticmethod
+    def where_starred() -> str:
+        return AbstractUserMessage.where_flag_is_present(AbstractUserMessage.flags.starred)
+
+    @staticmethod
+    def where_active_push_notification() -> str:
+        return AbstractUserMessage.where_flag_is_present(
+            AbstractUserMessage.flags.active_mobile_push_notification
+        )
+
+    def flags_list(self) -> list[str]:
+        flags = int(self.flags)
+        return self.flags_list_for_flags(flags)
+
+    @staticmethod
+    def flags_list_for_flags(val: int) -> list[str]:
+        """
+        This function is highly optimized, because it actually slows down
+        sending messages in a naive implementation.
+        """
+        flags = []
+        mask = 1
+        for flag in UserMessage.ALL_FLAGS:
+            if (val & mask) and flag not in AbstractUserMessage.NON_API_FLAGS:
+                flags.append(flag)
+            mask <<= 1
+        return flags
+
+
+class UserMessage(AbstractUserMessage):
+    message = models.ForeignKey(Message, on_delete=CASCADE)
+
+    class Meta(AbstractUserMessage.Meta):
+        indexes = [
+            models.Index(
+                "user_profile",
+                "message",
+                condition=Q(flags__andnz=AbstractUserMessage.flags.starred.mask),
+                name="zerver_usermessage_starred_message_id",
+            ),
+            models.Index(
+                "user_profile",
+                "message",
+                condition=Q(flags__andnz=AbstractUserMessage.flags.mentioned.mask),
+                name="zerver_usermessage_mentioned_message_id",
+            ),
+            models.Index(
+                "user_profile",
+                "message",
+                condition=Q(flags__andz=AbstractUserMessage.flags.read.mask),
+                name="zerver_usermessage_unread_message_id",
+            ),
+            models.Index(
+                "user_profile",
+                "message",
+                condition=Q(flags__andnz=AbstractUserMessage.flags.has_alert_word.mask),
+                name="zerver_usermessage_has_alert_word_message_id",
+            ),
+            models.Index(
+                "user_profile",
+                "message",
+                condition=Q(flags__andnz=AbstractUserMessage.flags.mentioned.mask)
+                | Q(flags__andnz=AbstractUserMessage.flags.stream_wildcard_mentioned.mask),
+                name="zerver_usermessage_wildcard_mentioned_message_id",
+            ),
+            models.Index(
+                "user_profile",
+                "message",
+                condition=Q(
+                    flags__andnz=AbstractUserMessage.flags.mentioned.mask
+                    | AbstractUserMessage.flags.stream_wildcard_mentioned.mask
+                    | AbstractUserMessage.flags.topic_wildcard_mentioned.mask
+                    | AbstractUserMessage.flags.group_mentioned.mask
+                ),
+                name="zerver_usermessage_any_mentioned_message_id",
+            ),
+            models.Index(
+                "user_profile",
+                "message",
+                condition=Q(flags__andnz=AbstractUserMessage.flags.is_private.mask),
+                name="zerver_usermessage_is_private_message_id",
+            ),
+            models.Index(
+                "user_profile",
+                "message",
+                condition=(
+                    Q(flags__andnz=AbstractUserMessage.flags.is_private.mask)
+                    & Q(flags__andz=AbstractUserMessage.flags.read.mask)
+                ),
+                name="zerver_usermessage_is_private_unread_message_id",
+            ),
+            models.Index(
+                "user_profile",
+                "message",
+                condition=Q(
+                    flags__andnz=AbstractUserMessage.flags.active_mobile_push_notification.mask
+                ),
+                name="zerver_usermessage_active_mobile_push_notification_id",
+            ),
+            models.Index(
+                "message",
+                condition=Q(
+                    flags__andnz=AbstractUserMessage.flags.active_mobile_push_notification.mask
+                ),
+                name="zerver_usermessage_message_active_mobile_push_notification_idx",
+            ),
+        ]
+
+    @override
+    def __str__(self) -> str:
+        recipient_string = self.message.recipient.label()
+        return f"{recipient_string} / {self.user_profile.email} ({self.flags_list()})"
+
+    @staticmethod
+    def select_for_update_query() -> QuerySet["UserMessage"]:
+        """This `SELECT FOR UPDATE OF zerver_usermessage` query ensures
+        consistent ordering on the row locks acquired by a bulk update
+        operation to modify message flags using bitand/bitor.
+
+        This consistent ordering is important to prevent deadlocks when
+        2 or more bulk updates to the same rows in the UserMessage table
+        race against each other (For example, if a client submits
+        simultaneous duplicate API requests to mark a certain set of
+        messages as read).
+
+        Note: Since we don't expect these UserMessage rows to be deleted by the
+        caller, a FOR NO KEY UPDATE lock might be sufficient here. However, we
+        don't expect the stronger FOR UPDATE lock to cause any issues,
+        so for now, we still pass no_key=False, acquiring the stronger lock.
+        """
+        return UserMessage.objects.select_for_update(of=("self",), no_key=False).order_by(
+            "message_id"
+        )
+
+    @staticmethod
+    def has_any_mentions(user_profile_id: int, message_id: int) -> bool:
+        # The query uses the 'zerver_usermessage_any_mentioned_message_id' index.
+        return UserMessage.objects.filter(
+            Q(
+                flags__andnz=UserMessage.flags.mentioned.mask
+                | UserMessage.flags.stream_wildcard_mentioned.mask
+                | UserMessage.flags.topic_wildcard_mentioned.mask
+                | UserMessage.flags.group_mentioned.mask
+            ),
+            user_profile_id=user_profile_id,
+            message_id=message_id,
+        ).exists()
+
+
+def get_usermessage_by_message_id(user_profile: UserProfile, message_id: int) -> UserMessage | None:
+    try:
+        return UserMessage.objects.get(user_profile=user_profile, message_id=message_id)
+    except UserMessage.DoesNotExist:
+        return None
+
+
+class ArchivedUserMessage(AbstractUserMessage):
+    """Used as a temporary holding place for deleted UserMessages objects
+    before they are permanently deleted.  This is an important part of
+    a robust 'message retention' feature.
+    """
+
+    message = models.ForeignKey(ArchivedMessage, on_delete=CASCADE)
+
+    @override
+    def __str__(self) -> str:
+        recipient_string = self.message.recipient.label()
+        return f"{recipient_string} / {self.user_profile.email} ({self.flags_list()})"
+
+
+class ImageAttachment(models.Model):
+    realm = models.ForeignKey(Realm, on_delete=CASCADE)
+    path_id = models.TextField(db_index=True, unique=True)
+    content_type = models.TextField(null=True)
+
+    original_width_px = models.IntegerField()
+    original_height_px = models.IntegerField()
+    frames = models.IntegerField()
+
+    # Contains a list of zerver.lib.thumbnail.StoredThumbnailFormat objects, serialized
+    thumbnail_metadata = models.JSONField(default=list, null=False)
+
+
+class AbstractAttachment(models.Model):
+    file_name = models.TextField(db_index=True)
+
+    # path_id is a storage location agnostic representation of the path of the file.
+    # If the path of a file is http://localhost:9991/user_uploads/a/b/abc/temp_file.py
+    # then its path_id will be a/b/abc/temp_file.py.
+    path_id = models.TextField(db_index=True, unique=True)
+    owner = models.ForeignKey(UserProfile, on_delete=CASCADE)
+    realm = models.ForeignKey(Realm, on_delete=CASCADE)
+
+    create_time = models.DateTimeField(
+        default=timezone_now,
+        db_index=True,
+    )
+    # Size of the uploaded file, in bytes
+    size = models.IntegerField()
+
+    content_type = models.TextField(null=True)
+
+    # The two fields below serve as caches to let us avoid looking up
+    # the corresponding messages/streams to check permissions before
+    # serving these files.
+    #
+    # For both fields, the `null` state is used when a change in
+    # message permissions mean that we need to determine their proper
+    # value.
+
+    # Whether this attachment has been posted to a public stream, and
+    # thus should be available to all non-guest users in the
+    # organization (even if they weren't a recipient of a message
+    # linking to it).
+    is_realm_public = models.BooleanField(default=False, null=True)
+    # Whether this attachment has been posted to a web-public stream,
+    # and thus should be available to everyone on the internet, even
+    # if the person isn't logged in.
+    is_web_public = models.BooleanField(default=False, null=True)
+
+    class Meta:
+        abstract = True
+
+    @override
+    def __str__(self) -> str:
+        return self.file_name
+
+
+class ArchivedAttachment(AbstractAttachment):
+    """Used as a temporary holding place for deleted Attachment objects
+    before they are permanently deleted.  This is an important part of
+    a robust 'message retention' feature.
+
+    Unlike the similar archive tables, ArchivedAttachment does not
+    have an ArchiveTransaction foreign key, and thus will not be
+    directly deleted by clean_archived_data. Instead, attachments that
+    were only referenced by now fully deleted messages will leave
+    ArchivedAttachment objects with empty `.messages`.
+
+    A second step, delete_old_unclaimed_attachments, will delete the
+    resulting orphaned ArchivedAttachment objects, along with removing
+    the associated uploaded files from storage.
+    """
+
+    messages = models.ManyToManyField(
+        ArchivedMessage, related_name="attachment_set", related_query_name="attachment"
+    )
+
+
+class Attachment(AbstractAttachment):
+    messages = models.ManyToManyField(Message)
+
+    # This is only present for Attachment and not ArchiveAttachment.
+    # because ScheduledMessage is not subject to archiving.
+    scheduled_messages = models.ManyToManyField("zerver.ScheduledMessage")
+
+    class Meta:
+        indexes = [
+            models.Index(
+                "realm",
+                "create_time",
+                name="zerver_attachment_realm_create_time",
+            ),
+        ]
+
+    def is_claimed(self) -> bool:
+        return self.messages.exists() or self.scheduled_messages.exists()
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "id": self.id,
+            "name": self.file_name,
+            "path_id": self.path_id,
+            "size": self.size,
+            "create_time": int(time.mktime(self.create_time.timetuple())),
+            "message_ids": [m.id for m in self.messages.all()],
+        }
+
+
+post_save.connect(flush_used_upload_space_cache, sender=Attachment)
+post_delete.connect(flush_used_upload_space_cache, sender=Attachment)
+
+
+class OnboardingUserMessage(models.Model):
+    """
+    Stores the message_id of new onboarding messages with the
+    flags data that should be copied while creating UserMessage
+    rows for a new user in 'add_new_user_history'.
+    """
+
+    realm = models.ForeignKey(Realm, on_delete=CASCADE)
+    message = models.ForeignKey(Message, on_delete=CASCADE)
+
+    ALL_FLAGS = ["read", "historical", "starred"]
+    flags: BitHandler = BitField(flags=ALL_FLAGS, default=0)

zerver/models/muted_users.py

@@ -0,0 +1,25 @@
+from django.db import models
+from django.db.models import CASCADE
+from django.db.models.signals import post_delete, post_save
+from django.utils.timezone import now as timezone_now
+from typing_extensions import override
+
+from zerver.lib.cache import flush_muting_users_cache
+from zerver.models.users import UserProfile
+
+
+class MutedUser(models.Model):
+    user_profile = models.ForeignKey(UserProfile, related_name="muter", on_delete=CASCADE)
+    muted_user = models.ForeignKey(UserProfile, related_name="muted", on_delete=CASCADE)
+    date_muted = models.DateTimeField(default=timezone_now)
+
+    class Meta:
+        unique_together = ("user_profile", "muted_user")
+
+    @override
+    def __str__(self) -> str:
+        return f"{self.user_profile.email} -> {self.muted_user.email}"
+
+
+post_save.connect(flush_muting_users_cache, sender=MutedUser)
+post_delete.connect(flush_muting_users_cache, sender=MutedUser)

zerver/models/navigation_views.py

@@ -0,0 +1,27 @@
+from django.db import models
+from django.db.models import CASCADE
+
+from zerver.models.users import UserProfile
+
+
+class NavigationView(models.Model):
+    """
+    Represents a user's configuration for pinned views in the left sidebar,
+    allowing views to be pinned or hidden.
+    """
+
+    # Reference to the user who owns this pinned view configuration.
+    user = models.ForeignKey(UserProfile, on_delete=CASCADE)
+
+    # A unique URL hash identifier for the view (e.g., 'inbox', 'narrow/has/reactions'),
+    # used for navigation and identification of the view.
+    fragment = models.TextField()
+
+    # Whether the view is pinned (True) or hidden in the menu (False).
+    is_pinned = models.BooleanField(default=False)
+
+    # Optional display name of the view, provided by the user for custom views.
+    name = models.TextField(blank=True, null=True)
+
+    class Meta:
+        unique_together = ("user", "fragment")

zerver/models/onboarding_steps.py

@@ -0,0 +1,14 @@
+from django.db import models
+from django.db.models import CASCADE
+from django.utils.timezone import now as timezone_now
+
+from zerver.models.users import UserProfile
+
+
+class OnboardingStep(models.Model):
+    user = models.ForeignKey(UserProfile, on_delete=CASCADE)
+    onboarding_step = models.CharField(max_length=40)
+    timestamp = models.DateTimeField(default=timezone_now)
+
+    class Meta:
+        unique_together = ("user", "onboarding_step")

zerver/models/prereg_users.py

@@ -0,0 +1,211 @@
+from django.contrib.contenttypes.fields import GenericRelation
+from django.core.serializers.json import DjangoJSONEncoder
+from django.db import models
+from django.db.models import CASCADE, Q, QuerySet
+from django.db.models.functions import Upper
+from django.utils.timezone import now as timezone_now
+
+from confirmation import settings as confirmation_settings
+from zerver.models.constants import MAX_LANGUAGE_ID_LENGTH
+from zerver.models.realms import Realm
+from zerver.models.users import UserProfile
+
+
+class PreregistrationRealm(models.Model):
+    """Data on a partially created realm entered by a user who has
+    completed the "new organization" form. Used to transfer the user's
+    selections from the pre-confirmation "new organization" form to
+    the post-confirmation user registration form.
+
+    Note that the values stored here may not match those of the
+    created realm (in the event the user creates a realm at all),
+    because we allow the user to edit these values in the registration
+    form (and in fact the user will be required to do so if the
+    `string_id` is claimed by another realm before registraiton is
+    completed).
+    """
+
+    name = models.CharField(max_length=Realm.MAX_REALM_NAME_LENGTH)
+    org_type = models.PositiveSmallIntegerField(
+        default=Realm.ORG_TYPES["unspecified"]["id"],
+        choices=[(t["id"], t["name"]) for t in Realm.ORG_TYPES.values()],
+    )
+    default_language = models.CharField(
+        default="en",
+        max_length=MAX_LANGUAGE_ID_LENGTH,
+    )
+    string_id = models.CharField(max_length=Realm.MAX_REALM_SUBDOMAIN_LENGTH)
+    email = models.EmailField()
+
+    confirmation = GenericRelation("confirmation.Confirmation", related_query_name="prereg_realm")
+    status = models.IntegerField(default=0)
+
+    # The Realm created upon completion of the registration
+    # for this PregistrationRealm
+    created_realm = models.ForeignKey(Realm, null=True, related_name="+", on_delete=models.SET_NULL)
+
+    # The UserProfile created upon completion of the registration
+    # for this PregistrationRealm
+    created_user = models.ForeignKey(
+        UserProfile, null=True, related_name="+", on_delete=models.SET_NULL
+    )
+
+    IMPORT_FROM_CHOICES = [
+        ("none", "Don't import"),
+        ("slack", "Import from Slack"),
+    ]
+    data_import_metadata = models.JSONField(default=dict, encoder=DjangoJSONEncoder)
+
+
+class PreregistrationUser(models.Model):
+    # Data on a partially created user, before the completion of
+    # registration.  This is used in at least three major code paths:
+    # * Realm creation, in which case realm is None.
+    #
+    # * Invitations, in which case referred_by will always be set.
+    #
+    # * Social authentication signup, where it's used to store data
+    #   from the authentication step and pass it to the registration
+    #   form.
+
+    email = models.EmailField()
+
+    confirmation = GenericRelation("confirmation.Confirmation", related_query_name="prereg_user")
+    # If the pre-registration process provides a suggested full name for this user,
+    # store it here to use it to prepopulate the full name field in the registration form:
+    full_name = models.CharField(max_length=UserProfile.MAX_NAME_LENGTH, null=True)
+    full_name_validated = models.BooleanField(default=False)
+    referred_by = models.ForeignKey(UserProfile, null=True, on_delete=CASCADE)
+    notify_referrer_on_join = models.BooleanField(default=True)
+    streams = models.ManyToManyField("zerver.Stream")
+    groups = models.ManyToManyField("zerver.NamedUserGroup")
+    invited_at = models.DateTimeField(auto_now=True)
+    realm_creation = models.BooleanField(default=False)
+
+    # Custom text to be sent to created users in a welcome bot message.
+    welcome_message_custom_text = models.TextField(null=True)
+
+    # Indicates whether the user needs a password.  Users who were
+    # created via SSO style auth (e.g. GitHub/Google) generally do not.
+    password_required = models.BooleanField(default=True)
+
+    # status: whether an object has been confirmed.
+    #   if confirmed, set to confirmation.settings.STATUS_USED
+    status = models.IntegerField(default=0)
+
+    # The realm should only ever be None for PreregistrationUser
+    # objects created as part of realm creation.
+    realm = models.ForeignKey(Realm, null=True, on_delete=CASCADE)
+
+    # These values should be consistent with the values
+    # in settings_config.user_role_values.
+    INVITE_AS = dict(
+        REALM_OWNER=100,
+        REALM_ADMIN=200,
+        MODERATOR=300,
+        MEMBER=400,
+        GUEST_USER=600,
+    )
+    invited_as = models.PositiveSmallIntegerField(default=INVITE_AS["MEMBER"])
+
+    multiuse_invite = models.ForeignKey("MultiuseInvite", null=True, on_delete=models.SET_NULL)
+
+    # The UserProfile created upon completion of the registration
+    # for this PregistrationUser
+    created_user = models.ForeignKey(
+        UserProfile, null=True, related_name="+", on_delete=models.SET_NULL
+    )
+
+    include_realm_default_subscriptions = models.BooleanField(default=True)
+
+    # Used in realm import flow to allow importer (the person
+    # whose email is set as PreregistrationRealm.email) to create
+    # a new user if an imported user with the matching
+    # email was not found.
+    is_realm_importer = models.BooleanField(default=False)
+
+    # Used in the SAML authentication flow to pass the external auth ID
+    # from the authentication step to user creation, so that an ExternalAuthID
+    # record can be created for the new user.
+    external_auth_method_name = models.CharField(max_length=100, null=True, default=None)
+    external_auth_id = models.CharField(max_length=255, null=True, default=None)
+
+    class Meta:
+        indexes = [
+            models.Index(Upper("email"), name="upper_preregistration_email_idx"),
+        ]
+
+
+def filter_to_valid_prereg_users(
+    query: QuerySet[PreregistrationUser],
+    invitations_only: bool = False,
+) -> QuerySet[PreregistrationUser]:
+    """
+    Filters out PreregistrationUser objects that have been used or
+    revoked, or whose confirmation link has expired.
+    """
+    used_value = confirmation_settings.STATUS_USED
+    revoked_value = confirmation_settings.STATUS_REVOKED
+
+    query = query.exclude(status__in=[used_value, revoked_value])
+
+    confirmation_q = Q(confirmation__expiry_date=None) | Q(
+        confirmation__expiry_date__gte=timezone_now()
+    )
+    if invitations_only:
+        # Imported here to avoid a circular import.
+        from confirmation.models import Confirmation
+
+        confirmation_q &= Q(confirmation__type=Confirmation.INVITATION)
+
+    return query.filter(confirmation_q)
+
+
+class MultiuseInvite(models.Model):
+    referred_by = models.ForeignKey(UserProfile, on_delete=CASCADE)
+    streams = models.ManyToManyField("zerver.Stream")
+    groups = models.ManyToManyField("zerver.NamedUserGroup")
+    realm = models.ForeignKey(Realm, on_delete=CASCADE)
+    invited_as = models.PositiveSmallIntegerField(default=PreregistrationUser.INVITE_AS["MEMBER"])
+    include_realm_default_subscriptions = models.BooleanField(default=True)
+
+    # Custom text to be sent to created users in a welcome bot message.
+    welcome_message_custom_text = models.TextField(null=True)
+
+    # status for tracking whether the invite has been revoked.
+    # If revoked, set to confirmation.settings.STATUS_REVOKED.
+    # STATUS_USED is not supported, because these objects are supposed
+    # to be usable multiple times.
+    status = models.IntegerField(default=0)
+
+
+class EmailChangeStatus(models.Model):
+    new_email = models.EmailField()
+    old_email = models.EmailField()
+    updated_at = models.DateTimeField(auto_now=True)
+    user_profile = models.ForeignKey(UserProfile, on_delete=CASCADE)
+
+    # status: whether an object has been confirmed.
+    #   if confirmed, set to confirmation.settings.STATUS_USED
+    status = models.IntegerField(default=0)
+
+    realm = models.ForeignKey(Realm, on_delete=CASCADE)
+
+
+class RealmReactivationStatus(models.Model):
+    # status: whether an object has been confirmed.
+    #   if confirmed, set to confirmation.settings.STATUS_USED
+    status = models.IntegerField(default=0)
+
+    realm = models.ForeignKey(Realm, on_delete=CASCADE)
+
+
+class RealmCreationStatus(models.Model):
+    # status: whether an object has been confirmed.
+    #   if confirmed, set to confirmation.settings.STATUS_USED
+    status = models.IntegerField(default=0)
+    date_created = models.DateTimeField(default=timezone_now)
+
+    # True just if we should presume the email address the user enters
+    # is theirs, and skip sending mail to it to confirm that.
+    presume_email_valid = models.BooleanField(default=False)

zerver/models/presence.py

@@ -0,0 +1,114 @@
+from django.db import models
+from django.db.models import CASCADE
+from django.utils.timezone import now as timezone_now
+
+from zerver.models.clients import Client
+from zerver.models.messages import AbstractEmoji
+from zerver.models.realms import Realm
+from zerver.models.users import UserProfile
+
+
+class UserPresence(models.Model):
+    """A record from the last time we heard from a given user on a given client.
+
+    This is the core table for Zulip's presence API, which consists of two components:
+    1) The update-presence endpoint, which is used for clients to update their presence
+       status and fetch latest presence data the whole realm:
+       https://zulip.com/api/update-presence
+    2) The events system is used to send limited immediate updates when
+       a user comes back online: https://zulip.com/api/get-events#presence
+
+    NOTE: Users can disable updates to this table (see UserProfile.presence_enabled),
+    so this cannot be used to determine if a user was recently active on Zulip.
+    The UserActivity table is recommended for that purpose.
+    """
+
+    user_profile = models.OneToOneField(UserProfile, on_delete=CASCADE, unique=True)
+
+    # Realm is just here as denormalization to optimize database
+    # queries to fetch all presence data for a given realm.
+    realm = models.ForeignKey(Realm, on_delete=CASCADE)
+
+    # The sequence ID within this realm for the last update to this user's presence;
+    # these IDs are generated by the PresenceSequence table and an important part
+    # of how we send incremental presence updates efficiently.
+    # To put it simply, every time we update a UserPresence row in a realm,
+    # the row gets last_update_id equal to 1 more than the previously updated
+    # row in that realm.
+    # This allows us to order UserPresence rows by when they were last updated.
+    # The API side of this functionality is documented at:
+    # https://zulip.com/api/update-presence#parameter-last_update_id
+    last_update_id = models.PositiveBigIntegerField(db_index=True, default=0)
+
+    # The last time the user had a client connected to Zulip,
+    # including idle clients where the user hasn't interacted with the
+    # system recently (and thus might be AFK).
+    last_connected_time = models.DateTimeField(default=timezone_now, db_index=True, null=True)
+    # The last time a client connected to Zulip reported that the user
+    # was actually present (E.g. via focusing a browser window or
+    # interacting with a computer running the desktop app)
+    last_active_time = models.DateTimeField(default=timezone_now, db_index=True, null=True)
+
+    # The following constants are used in the presence API for
+    # communicating whether a user is active (last_active_time recent)
+    # or idle (last_connected_time recent) or offline (neither
+    # recent).  They're no longer part of the data model.
+    LEGACY_STATUS_ACTIVE = "active"
+    LEGACY_STATUS_IDLE = "idle"
+    LEGACY_STATUS_ACTIVE_INT = 1
+    LEGACY_STATUS_IDLE_INT = 2
+
+    class Meta:
+        indexes = [
+            models.Index(
+                fields=["realm", "last_active_time"],
+                name="zerver_userpresence_realm_id_last_active_time_1c5aa9a2_idx",
+            ),
+            models.Index(
+                fields=["realm", "last_connected_time"],
+                name="zerver_userpresence_realm_id_last_connected_time_98d2fc9f_idx",
+            ),
+            models.Index(
+                fields=["realm", "last_update_id"],
+                name="zerver_userpresence_realm_last_update_id_idx",
+            ),
+        ]
+
+    @staticmethod
+    def status_from_string(status: str) -> int | None:
+        if status == "active":
+            return UserPresence.LEGACY_STATUS_ACTIVE_INT
+        elif status == "idle":
+            return UserPresence.LEGACY_STATUS_IDLE_INT
+
+        return None
+
+
+class PresenceSequence(models.Model):
+    """
+    This table is used to generate last_update_id values in the UserPresence table.
+
+    It serves as a per-realm sequence generator, while also facilitating
+    locking to avoid concurrency issues with setting last_update_id values.
+
+    Every realm has its unique row in this table, and when a UserPresence in the realm
+    is being updated, this row get locked against other UserPresence updates in the realm
+    to ensure sequential processing and set last_update_id values correctly.
+    """
+
+    realm = models.OneToOneField(Realm, on_delete=CASCADE)
+    last_update_id = models.PositiveBigIntegerField()
+
+
+class UserStatus(AbstractEmoji):
+    user_profile = models.OneToOneField(UserProfile, on_delete=CASCADE)
+
+    timestamp = models.DateTimeField()
+    client = models.ForeignKey(Client, on_delete=CASCADE)
+
+    # Override emoji_name and emoji_code field of (AbstractReaction model) to accept
+    # default value.
+    emoji_name = models.TextField(default="")
+    emoji_code = models.TextField(default="")
+
+    status_text = models.CharField(max_length=255, default="")

zerver/models/push_notifications.py

@@ -0,0 +1,58 @@
+from django.db import models
+from django.db.models import CASCADE, F, Q
+from django.db.models.functions import Lower
+
+from zerver.models.users import UserProfile
+
+
+class AbstractPushDeviceToken(models.Model):
+    APNS = 1
+    FCM = 2
+
+    KINDS = (
+        (APNS, "apns"),
+        # The string value in the database is "gcm" for legacy reasons.
+        # TODO: We should migrate it.
+        (FCM, "gcm"),
+    )
+
+    kind = models.PositiveSmallIntegerField(choices=KINDS)
+
+    # The token is a unique device-specific token that is
+    # sent to us from each device:
+    #   - APNS token if kind == APNS
+    #   - FCM registration id if kind == FCM
+    token = models.CharField(max_length=4096, db_index=True)
+
+    # TODO: last_updated should be renamed date_created, since it is
+    # no longer maintained as a last_updated value.
+    last_updated = models.DateTimeField(auto_now=True)
+
+    # [optional] Contains the app id of the device if it is an iOS device
+    ios_app_id = models.TextField(null=True)
+
+    class Meta:
+        abstract = True
+
+
+class PushDeviceToken(AbstractPushDeviceToken):
+    # The user whose device this is
+    user = models.ForeignKey(UserProfile, db_index=True, on_delete=CASCADE)
+
+    class Meta:
+        constraints = [
+            models.UniqueConstraint(
+                "user_id",
+                "kind",
+                Lower(F("token")),
+                name="zerver_pushdevicetoken_apns_user_kind_token",
+                condition=Q(kind=AbstractPushDeviceToken.APNS),
+            ),
+            models.UniqueConstraint(
+                "user_id",
+                "kind",
+                "token",
+                name="zerver_pushdevicetoken_fcm_user_kind_token",
+                condition=Q(kind=AbstractPushDeviceToken.FCM),
+            ),
+        ]

zerver/models/realm_audit_logs.py

@@ -0,0 +1,313 @@
+from enum import IntEnum, unique
+
+from django.core.serializers.json import DjangoJSONEncoder
+from django.db import models
+from django.db.models import CASCADE, Q
+from typing_extensions import override
+
+from zerver.models.channel_folders import ChannelFolder
+from zerver.models.groups import NamedUserGroup
+from zerver.models.realms import Realm
+from zerver.models.streams import Stream
+from zerver.models.users import UserProfile
+
+
+@unique
+class AuditLogEventType(IntEnum):
+    USER_CREATED = 101
+    USER_ACTIVATED = 102
+    USER_DEACTIVATED = 103
+    USER_REACTIVATED = 104
+    USER_ROLE_CHANGED = 105
+    USER_DELETED = 106
+    USER_DELETED_PRESERVING_MESSAGES = 107
+    USER_SPECIAL_PERMISSION_CHANGED = 108
+
+    USER_SOFT_ACTIVATED = 120
+    USER_SOFT_DEACTIVATED = 121
+    USER_PASSWORD_CHANGED = 122
+    USER_AVATAR_SOURCE_CHANGED = 123
+    USER_FULL_NAME_CHANGED = 124
+    USER_EMAIL_CHANGED = 125
+    USER_TERMS_OF_SERVICE_VERSION_CHANGED = 126
+    USER_API_KEY_CHANGED = 127
+    USER_BOT_OWNER_CHANGED = 128
+    USER_DEFAULT_SENDING_STREAM_CHANGED = 129
+    USER_DEFAULT_REGISTER_STREAM_CHANGED = 130
+    USER_DEFAULT_ALL_PUBLIC_STREAMS_CHANGED = 131
+    USER_SETTING_CHANGED = 132
+    USER_DIGEST_EMAIL_CREATED = 133
+    USER_IS_IMPORTED_STUB_CHANGED = 134
+    USER_DATE_JOINED_CHANGED = 135
+
+    REALM_DEACTIVATED = 201
+    REALM_REACTIVATED = 202
+    REALM_SCRUBBED = 203
+    REALM_PLAN_TYPE_CHANGED = 204
+    REALM_LOGO_CHANGED = 205
+    REALM_EXPORTED = 206
+    REALM_PROPERTY_CHANGED = 207
+    REALM_ICON_SOURCE_CHANGED = 208
+    REALM_DISCOUNT_CHANGED = 209
+    REALM_SPONSORSHIP_APPROVED = 210
+    REALM_BILLING_MODALITY_CHANGED = 211
+    REALM_REACTIVATION_EMAIL_SENT = 212
+    REALM_SPONSORSHIP_PENDING_STATUS_CHANGED = 213
+    REALM_SUBDOMAIN_CHANGED = 214
+    REALM_CREATED = 215
+    REALM_DEFAULT_USER_SETTINGS_CHANGED = 216
+    REALM_ORG_TYPE_CHANGED = 217
+    REALM_DOMAIN_ADDED = 218
+    REALM_DOMAIN_CHANGED = 219
+    REALM_DOMAIN_REMOVED = 220
+    REALM_PLAYGROUND_ADDED = 221
+    REALM_PLAYGROUND_REMOVED = 222
+    REALM_LINKIFIER_ADDED = 223
+    REALM_LINKIFIER_CHANGED = 224
+    REALM_LINKIFIER_REMOVED = 225
+    REALM_EMOJI_ADDED = 226
+    REALM_EMOJI_REMOVED = 227
+    REALM_LINKIFIERS_REORDERED = 228
+
+    # This event for a realm means that this server processed exported data
+    # (either from another Zulip server or a 3rd party app such as Slack),
+    # and imported the data as the given realm.
+    REALM_IMPORTED = 229
+    REALM_EXPORT_DELETED = 230
+
+    SUBSCRIPTION_CREATED = 301
+    SUBSCRIPTION_ACTIVATED = 302
+    SUBSCRIPTION_DEACTIVATED = 303
+    SUBSCRIPTION_PROPERTY_CHANGED = 304
+
+    USER_MUTED = 350
+    USER_UNMUTED = 351
+
+    STRIPE_CUSTOMER_CREATED = 401
+    STRIPE_CARD_CHANGED = 402
+    STRIPE_PLAN_CHANGED = 403
+    STRIPE_PLAN_QUANTITY_RESET = 404
+
+    CUSTOMER_CREATED = 501
+    CUSTOMER_PLAN_CREATED = 502
+    CUSTOMER_SWITCHED_FROM_MONTHLY_TO_ANNUAL_PLAN = 503
+    CUSTOMER_SWITCHED_FROM_ANNUAL_TO_MONTHLY_PLAN = 504
+    CUSTOMER_PROPERTY_CHANGED = 505
+    CUSTOMER_PLAN_PROPERTY_CHANGED = 506
+
+    CHANNEL_CREATED = 601
+    CHANNEL_DEACTIVATED = 602
+    CHANNEL_NAME_CHANGED = 603
+    CHANNEL_REACTIVATED = 604
+    CHANNEL_MESSAGE_RETENTION_DAYS_CHANGED = 605
+    CHANNEL_PROPERTY_CHANGED = 607
+    CHANNEL_GROUP_BASED_SETTING_CHANGED = 608
+    CHANNEL_FOLDER_CHANGED = 609
+
+    USER_GROUP_CREATED = 701
+    USER_GROUP_DELETED = 702
+    USER_GROUP_DIRECT_USER_MEMBERSHIP_ADDED = 703
+    USER_GROUP_DIRECT_USER_MEMBERSHIP_REMOVED = 704
+    USER_GROUP_DIRECT_SUBGROUP_MEMBERSHIP_ADDED = 705
+    USER_GROUP_DIRECT_SUBGROUP_MEMBERSHIP_REMOVED = 706
+    USER_GROUP_DIRECT_SUPERGROUP_MEMBERSHIP_ADDED = 707
+    USER_GROUP_DIRECT_SUPERGROUP_MEMBERSHIP_REMOVED = 708
+    # 709 to 719 reserved for membership changes
+    USER_GROUP_NAME_CHANGED = 720
+    USER_GROUP_DESCRIPTION_CHANGED = 721
+    USER_GROUP_GROUP_BASED_SETTING_CHANGED = 722
+    USER_GROUP_DEACTIVATED = 723
+    USER_GROUP_REACTIVATED = 724
+    WORKPLACE_USERS_COUNT_CHANGED = 725
+
+    SAVED_SNIPPET_CREATED = 800
+
+    CUSTOM_EMAIL_SENT = 810
+
+    NAVIGATION_VIEW_CREATED = 850
+    NAVIGATION_VIEW_UPDATED = 851
+    NAVIGATION_VIEW_DELETED = 852
+
+    CHANNEL_FOLDER_CREATED = 901
+    CHANNEL_FOLDER_NAME_CHANGED = 902
+    CHANNEL_FOLDER_DESCRIPTION_CHANGED = 903
+    CHANNEL_FOLDER_ARCHIVED = 904
+    CHANNEL_FOLDER_UNARCHIVED = 905
+
+    INVITATION_REVOKED = 1001
+
+    # The following values are only for remote server/realm logs.
+    # Values should be exactly 10000 greater than the corresponding
+    # value used for the same purpose in realm audit logs (e.g.,
+    # REALM_DEACTIVATED = 201, and REMOTE_SERVER_DEACTIVATED = 10201).
+    REMOTE_SERVER_DEACTIVATED = 10201
+    REMOTE_SERVER_REACTIVATED = 10202
+    REMOTE_SERVER_PLAN_TYPE_CHANGED = 10204
+    REMOTE_SERVER_DISCOUNT_CHANGED = 10209
+    REMOTE_SERVER_SPONSORSHIP_APPROVED = 10210
+    REMOTE_SERVER_BILLING_MODALITY_CHANGED = 10211
+    REMOTE_SERVER_SPONSORSHIP_PENDING_STATUS_CHANGED = 10213
+    REMOTE_SERVER_CREATED = 10215
+    REMOTE_SERVER_REGISTRATION_TRANSFERRED = 10216
+
+    # This value is for RemoteRealmAuditLog entries tracking changes to the
+    # RemoteRealm model resulting from modified realm information sent to us
+    # via send_server_data_to_push_bouncer.
+    REMOTE_REALM_VALUE_UPDATED = 20001
+    REMOTE_PLAN_TRANSFERRED_SERVER_TO_REALM = 20002
+    REMOTE_REALM_LOCALLY_DELETED = 20003
+    REMOTE_REALM_LOCALLY_DELETED_RESTORED = 20004
+
+
+class AbstractRealmAuditLog(models.Model):
+    """Defines fields common to RealmAuditLog and RemoteRealmAuditLog."""
+
+    event_time = models.DateTimeField(db_index=True)
+    # If True, event_time is an overestimate of the true time. Can be used
+    # by migrations when introducing a new event_type.
+    backfilled = models.BooleanField(default=False)
+
+    # Keys within extra_data, when extra_data is a json dict. Keys are strings because
+    # json keys must always be strings.
+    OLD_VALUE = "1"
+    NEW_VALUE = "2"
+    ROLE_COUNT = "10"
+    ROLE_COUNT_HUMANS = "11"
+    ROLE_COUNT_BOTS = "12"
+
+    extra_data = models.JSONField(default=dict, encoder=DjangoJSONEncoder)
+
+    # See AuditLogEventType class above.
+    event_type = models.PositiveSmallIntegerField()
+
+    # event_types synced from on-prem installations to Zulip Cloud when
+    # billing for mobile push notifications is enabled.  Every billing
+    # event_type should have ROLE_COUNT populated in extra_data.
+    SYNCED_BILLING_EVENTS = [
+        AuditLogEventType.USER_CREATED,
+        AuditLogEventType.USER_ACTIVATED,
+        AuditLogEventType.USER_DEACTIVATED,
+        AuditLogEventType.USER_REACTIVATED,
+        AuditLogEventType.USER_ROLE_CHANGED,
+        AuditLogEventType.REALM_DEACTIVATED,
+        AuditLogEventType.REALM_REACTIVATED,
+        AuditLogEventType.REALM_IMPORTED,
+        AuditLogEventType.WORKPLACE_USERS_COUNT_CHANGED,
+    ]
+
+    HOW_REALM_CREATOR_FOUND_ZULIP_OPTIONS = {
+        "existing_user": "At an organization that's using it",
+        "search_engine": "Search engine",
+        "ai_chatbot": "AI/LLM",
+        "review_site": "Review site",
+        "personal_recommendation": "Personal recommendation",
+        "hacker_news": "Hacker News",
+        "reddit": "Reddit",
+        "ad": "Advertisement",
+        "other": "Other",
+        "forgot": "Don't remember",
+        "refuse_to_answer": "Prefer not to say",
+    }
+
+    class Meta:
+        abstract = True
+
+
+class RealmAuditLog(AbstractRealmAuditLog):
+    """
+    RealmAuditLog tracks important changes to users, streams, and
+    realms in Zulip.  It is intended to support both
+    debugging/introspection (e.g. determining when a user's left a
+    given stream?) as well as help with some database migrations where
+    we might be able to do a better data backfill with it.  Here are a
+    few key details about how this works:
+
+    * acting_user is the user who initiated the state change
+    * modified_user (if present) is the user being modified
+    * modified_stream (if present) is the stream being modified
+    * modified_user_group (if present) is the user group being modified
+
+    For example:
+    * When a user subscribes another user to a stream, modified_user,
+      acting_user, and modified_stream will all be present and different.
+    * When an administrator changes an organization's realm icon,
+      acting_user is that administrator and modified_user,
+      modified_stream and modified_user_group will be None.
+    """
+
+    realm = models.ForeignKey(Realm, on_delete=CASCADE)
+    acting_user = models.ForeignKey(
+        UserProfile,
+        null=True,
+        related_name="+",
+        on_delete=CASCADE,
+    )
+    modified_user = models.ForeignKey(
+        UserProfile,
+        null=True,
+        related_name="+",
+        on_delete=CASCADE,
+    )
+    modified_stream = models.ForeignKey(
+        Stream,
+        null=True,
+        on_delete=CASCADE,
+    )
+    modified_user_group = models.ForeignKey(
+        NamedUserGroup,
+        null=True,
+        on_delete=CASCADE,
+    )
+    modified_channel_folder = models.ForeignKey(
+        ChannelFolder,
+        null=True,
+        on_delete=CASCADE,
+    )
+    event_last_message_id = models.IntegerField(null=True)
+    scrubbed = models.BooleanField(db_default=False, default=False)
+
+    class Meta:
+        ordering = ["id"]
+        indexes = [
+            models.Index(
+                name="zerver_realmauditlog_realm__event_type__event_time",
+                fields=["realm", "event_type", "event_time"],
+            ),
+            models.Index(
+                name="zerver_realmauditlog_user_subscriptions_idx",
+                fields=["modified_user", "modified_stream"],
+                condition=Q(
+                    event_type__in=[
+                        AuditLogEventType.SUBSCRIPTION_CREATED,
+                        AuditLogEventType.SUBSCRIPTION_ACTIVATED,
+                        AuditLogEventType.SUBSCRIPTION_DEACTIVATED,
+                    ]
+                ),
+            ),
+            models.Index(
+                # Used in analytics/lib/counts.py for computing active users for realm_active_humans
+                name="zerver_realmauditlog_user_activations_idx",
+                fields=["modified_user", "event_time"],
+                condition=Q(
+                    event_type__in=[
+                        AuditLogEventType.USER_CREATED,
+                        AuditLogEventType.USER_ACTIVATED,
+                        AuditLogEventType.USER_DEACTIVATED,
+                        AuditLogEventType.USER_REACTIVATED,
+                    ]
+                ),
+            ),
+        ]
+
+    @override
+    def __str__(self) -> str:
+        event_type_name = AuditLogEventType(self.event_type).name
+        if self.modified_user is not None:
+            return f"{event_type_name} {self.event_time} (id={self.id}): {self.modified_user!r}"
+        if self.modified_stream is not None:
+            return f"{event_type_name} {self.event_time} (id={self.id}): {self.modified_stream!r}"
+        if self.modified_user_group is not None:
+            return (
+                f"{event_type_name} {self.event_time} (id={self.id}): {self.modified_user_group!r}"
+            )
+        return f"{event_type_name} {self.event_time} (id={self.id}): {self.realm!r}"

zerver/models/realm_emoji.py

@@ -0,0 +1,138 @@
+from typing import TypedDict
+
+from django.core.validators import MinLengthValidator, RegexValidator
+from django.db import models
+from django.db.models import CASCADE, Q
+from django.db.models.signals import post_delete, post_save
+from django.utils.translation import gettext_lazy
+from typing_extensions import override
+
+from zerver.lib.cache import cache_delete, cache_with_key
+from zerver.models.realms import Realm
+
+
+class EmojiInfo(TypedDict):
+    id: str
+    name: str
+    source_url: str
+    deactivated: bool
+    author_id: int | None
+    still_url: str | None
+
+
+def get_all_custom_emoji_for_realm_cache_key(realm_id: int) -> str:
+    return f"realm_emoji:{realm_id}"
+
+
+class RealmEmoji(models.Model):
+    author = models.ForeignKey(
+        "UserProfile",
+        on_delete=CASCADE,
+    )
+    realm = models.ForeignKey(Realm, on_delete=CASCADE)
+    name = models.TextField(
+        validators=[
+            MinLengthValidator(1),
+            # The second part of the regex (negative lookbehind) disallows names
+            # ending with one of the punctuation characters.
+            RegexValidator(
+                regex=r"^[0-9a-z.\-_]+(?<![.\-_])$",
+                message=gettext_lazy("Invalid characters in emoji name"),
+            ),
+        ]
+    )
+
+    # The basename of the custom emoji's filename; see PATH_ID_TEMPLATE for the full path.
+    file_name = models.TextField(db_index=True, null=True, blank=True)
+
+    # Whether this custom emoji is an animated image.
+    is_animated = models.BooleanField(default=False)
+
+    deactivated = models.BooleanField(default=False)
+
+    PATH_ID_TEMPLATE = "{realm_id}/emoji/images/{emoji_file_name}"
+    STILL_PATH_ID_TEMPLATE = "{realm_id}/emoji/images/still/{emoji_filename_without_extension}.png"
+
+    class Meta:
+        constraints = [
+            models.UniqueConstraint(
+                fields=["realm", "name"],
+                condition=Q(deactivated=False),
+                name="unique_realm_emoji_when_false_deactivated",
+            ),
+        ]
+
+    @override
+    def __str__(self) -> str:
+        return f"{self.realm.string_id}: {self.id} {self.name} {self.deactivated} {self.file_name}"
+
+
+def emoji_as_emojiinfo(realm_emoji: RealmEmoji) -> EmojiInfo:
+    from zerver.lib.upload import get_emoji_url
+
+    author_id = realm_emoji.author_id
+    assert realm_emoji.file_name is not None
+    emoji_url = get_emoji_url(realm_emoji.file_name, realm_emoji.realm_id)
+
+    emoji_dict: EmojiInfo = dict(
+        id=str(realm_emoji.id),
+        name=realm_emoji.name,
+        source_url=emoji_url,
+        deactivated=realm_emoji.deactivated,
+        author_id=author_id,
+        still_url=None,
+    )
+
+    if realm_emoji.is_animated:
+        # For animated emoji, we include still_url with a static
+        # version of the image, so that clients can display the
+        # emoji in a less distracting (not animated) fashion when
+        # desired.
+        emoji_dict["still_url"] = get_emoji_url(
+            realm_emoji.file_name, realm_emoji.realm_id, still=True
+        )
+
+    return emoji_dict
+
+
+def get_all_custom_emoji_for_realm_uncached(realm_id: int) -> dict[str, EmojiInfo]:
+    # RealmEmoji objects with file_name=None are still in the process
+    # of being uploaded, and we expect to be cleaned up by a
+    # try/finally block if the upload fails, so it's correct to
+    # exclude them.
+    query = RealmEmoji.objects.filter(realm_id=realm_id).exclude(file_name=None)
+
+    d = {}
+    for realm_emoji in query.iterator():
+        d[str(realm_emoji.id)] = emoji_as_emojiinfo(realm_emoji)
+
+    return d
+
+
+@cache_with_key(get_all_custom_emoji_for_realm_cache_key, timeout=3600 * 24 * 7)
+def get_all_custom_emoji_for_realm(realm_id: int) -> dict[str, EmojiInfo]:
+    return get_all_custom_emoji_for_realm_uncached(realm_id)
+
+
+def get_name_keyed_dict_for_active_realm_emoji(realm_id: int) -> dict[str, EmojiInfo]:
+    # It's important to use the cached version here.
+    realm_emojis = get_all_custom_emoji_for_realm(realm_id)
+    return {row["name"]: row for row in realm_emojis.values() if not row["deactivated"]}
+
+
+def flush_realm_emoji(*, instance: RealmEmoji, **kwargs: object) -> None:
+    if instance.file_name is None:
+        # Because we construct RealmEmoji.file_name using the ID for
+        # the RealmEmoji object, it will always have file_name=None,
+        # and then it'll be updated with the actual filename as soon
+        # as the upload completes successfully.
+        #
+        # Doing nothing when file_name=None is the best option, since
+        # such an object shouldn't have been cached yet, and this
+        # function will be called again when file_name is set.
+        return
+    cache_delete(get_all_custom_emoji_for_realm_cache_key(instance.realm_id))
+
+
+post_save.connect(flush_realm_emoji, sender=RealmEmoji)
+post_delete.connect(flush_realm_emoji, sender=RealmEmoji)

zerver/models/realm_playgrounds.py

@@ -0,0 +1,91 @@
+import uri_template
+from django.core.exceptions import ValidationError
+from django.core.validators import RegexValidator
+from django.db import models
+from django.db.models import CASCADE
+from django.utils.translation import gettext as _
+from typing_extensions import override
+
+from zerver.lib.types import RealmPlaygroundDict
+from zerver.models.linkifiers import url_template_validator
+from zerver.models.realms import Realm
+
+PLAYGROUND_LANGUAGE_REGEX = r"[a-zA-Z0-9_+-./#]+"
+
+
+class RealmPlayground(models.Model):
+    """Server side storage model to store playground information needed by our
+    'view code in playground' feature in code blocks.
+    """
+
+    MAX_PYGMENTS_LANGUAGE_LENGTH = 40
+
+    RESTRICTED_KEYWORDS = {"latex", "math", "quote", "spoiler"}
+
+    realm = models.ForeignKey(Realm, on_delete=CASCADE)
+    url_template = models.TextField(validators=[url_template_validator])
+
+    # User-visible display name used when configuring playgrounds in the settings page and
+    # when displaying them in the playground links popover.
+    name = models.TextField(db_index=True)
+
+    # This stores the pygments lexer subclass names and not the aliases themselves.
+    pygments_language = models.CharField(
+        db_index=True,
+        max_length=MAX_PYGMENTS_LANGUAGE_LENGTH,
+        # We validate to see if this conforms to the character set allowed for a
+        # language in the code block.
+        validators=[
+            RegexValidator(
+                regex=rf"^{PLAYGROUND_LANGUAGE_REGEX}$",
+                message=_("Invalid characters in pygments language"),
+            )
+        ],
+    )
+
+    class Meta:
+        unique_together = (("realm", "pygments_language", "name"),)
+
+    @override
+    def __str__(self) -> str:
+        return f"{self.realm.string_id}: {self.pygments_language} {self.name}"
+
+    @override
+    def clean(self) -> None:
+        """Validate whether the URL template is valid for the playground,
+        ensuring that "code" is the sole variable present in it.
+
+        Django's `full_clean` calls `clean_fields` followed by `clean` method
+        and stores all ValidationErrors from all stages to return as JSON.
+        """
+
+        # Do not continue the check if the url template is invalid to begin
+        # with. The ValidationError for invalid template will only be raised by
+        # the validator set on the url_template field instead of here to avoid
+        # duplicates.
+        if not uri_template.validate(self.url_template):
+            return
+
+        # Extract variables used in the URL template.
+        template_variables = set(uri_template.URITemplate(self.url_template).variable_names)
+
+        if "code" not in template_variables:
+            raise ValidationError(_('Missing the required variable "code" in the URL template'))
+
+        # The URL template should only contain a single variable, which is "code".
+        if len(template_variables) != 1:
+            raise ValidationError(
+                _('"code" should be the only variable present in the URL template'),
+            )
+
+
+def get_realm_playgrounds(realm: Realm) -> list[RealmPlaygroundDict]:
+    return [
+        RealmPlaygroundDict(
+            id=playground.id,
+            name=playground.name,
+            pygments_language=playground.pygments_language,
+            url_template=playground.url_template,
+        )
+        for playground in RealmPlayground.objects.filter(realm=realm).all()
+    ]

zerver/models/recipients.py

@@ -0,0 +1,241 @@
+import hashlib
+from collections import defaultdict
+from typing import TYPE_CHECKING
+
+from django.db import connection, models, transaction
+from django.db.models import QuerySet
+from psycopg2.sql import SQL, Identifier
+from typing_extensions import override
+
+from zerver.lib.display_recipient import get_display_recipient
+
+if TYPE_CHECKING:
+    from zerver.models import Subscription
+
+
+class Recipient(models.Model):
+    """Represents an audience that can potentially receive messages in Zulip.
+
+    This table essentially functions as a generic foreign key that
+    allows Message.recipient_id to be a simple ForeignKey representing
+    the audience for a message, while supporting the different types
+    of audiences Zulip supports for a message.
+
+    Recipient has just two attributes: The enum type, and a type_id,
+    which is the ID of the Stream/DirectMessageGroup object containing
+    all the metadata for the audience. There are 2 recipient types:
+
+    1. Stream message: The type_id is the ID of the associated Stream.
+    2. Direct message: The type_id is the ID of the associated
+       DirectMessageGroup object, which encodes the set of users in
+       the conversation. The set of users is usually retrieved via the
+       Subscription table. See the DirectMessageGroup model for
+       details.
+
+    See also the Subscription model, which stores which UserProfile
+    objects are subscribed to which Recipient objects.
+    """
+
+    id = models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")
+    type_id = models.IntegerField(db_index=True)
+    type = models.PositiveSmallIntegerField(db_index=True)
+    # Valid types are {stream, direct_message_group}
+
+    # The type for stream messages.
+    STREAM = 2
+    # The type for direct messages (both 1:1 and group).
+    DIRECT_MESSAGE_GROUP = 3
+
+    class Meta:
+        unique_together = ("type", "type_id")
+
+    @override
+    def __str__(self) -> str:
+        return f"{self.label()} ({self.type_id}, {self.type})"
+
+    def label(self) -> str:
+        from zerver.models import Stream
+
+        if self.type == Recipient.STREAM:
+            return Stream.objects.get(id=self.type_id).name
+        else:
+            return str(get_display_recipient(self))
+
+
+def get_direct_message_group_user_ids(recipient: Recipient) -> QuerySet["Subscription", int]:
+    from zerver.models import Subscription
+
+    assert recipient.type == Recipient.DIRECT_MESSAGE_GROUP
+
+    return (
+        Subscription.objects.filter(
+            recipient=recipient,
+        )
+        .order_by("user_profile_id")
+        .values_list("user_profile_id", flat=True)
+    )
+
+
+def bulk_get_direct_message_group_user_ids(recipient_ids: list[int]) -> dict[int, set[int]]:
+    """
+    Takes a list of direct_message_group type recipient_ids, returns
+    a dictmapping recipient id to list of user ids in the direct
+    message group.
+
+    We rely on our caller to pass us recipient_ids that correspond
+    to direct_message_group, but technically this function is valid
+    for any typeof subscription.
+    """
+    from zerver.models import Subscription
+
+    if not recipient_ids:
+        return {}
+
+    subscriptions = Subscription.objects.filter(
+        recipient_id__in=recipient_ids,
+    ).only("user_profile_id", "recipient_id")
+
+    result_dict: dict[int, set[int]] = defaultdict(set)
+    for subscription in subscriptions:
+        result_dict[subscription.recipient_id].add(subscription.user_profile_id)
+
+    return result_dict
+
+
+class DirectMessageGroup(models.Model):
+    """
+    Represents a group of individuals who may have a
+    group direct message conversation together.
+
+    The membership of the DirectMessageGroup is stored in the Subscription
+    table just like with Streams - for each user in the DirectMessageGroup,
+    there is a Subscription object tied to the UserProfile and the
+    DirectMessageGroup's recipient object.
+
+    A hash of the list of user IDs is stored in the huddle_hash field
+    below, to support efficiently mapping from a set of users to the
+    corresponding DirectMessageGroup object.
+    """
+
+    # We omit db_index and unique, as that also results in a useless
+    # varchar_pattern_ops index; see Meta, below.
+    huddle_hash = models.CharField(max_length=40)
+    # Foreign key to the Recipient object for this DirectMessageGroup.
+    recipient = models.OneToOneField(Recipient, null=True, on_delete=models.SET_NULL)
+
+    group_size = models.IntegerField()
+
+    # TODO: The model still uses the old "zerver_huddle" database table.
+    # As a part of the migration of "Huddle" to "DirectMessageGroup"
+    # it needs to be renamed to "zerver_directmessagegroup".
+    class Meta:
+        db_table = "zerver_huddle"
+        constraints = [
+            models.UniqueConstraint(fields=["huddle_hash"], name="zerver_huddle_huddle_hash_key")
+        ]
+
+
+def get_direct_message_group_hash(id_list: list[int]) -> str:
+    id_list = sorted(set(id_list))
+    hash_key = ",".join(str(x) for x in id_list)
+    return hashlib.sha1(hash_key.encode()).hexdigest()
+
+
+def get_or_create_direct_message_group(id_list: list[int]) -> DirectMessageGroup:
+    """
+    Takes a list of user IDs and returns the DirectMessageGroup
+    object for the group consisting of these users. If the
+    DirectMessageGroup object does not yet exist, it will be
+    transparently created.
+    """
+    from zerver.models import Subscription, UserProfile
+
+    assert len(id_list) == len(set(id_list))
+    direct_message_group_hash = get_direct_message_group_hash(id_list)
+    with transaction.atomic(savepoint=False):
+        (direct_message_group, created) = DirectMessageGroup.objects.get_or_create(
+            huddle_hash=direct_message_group_hash,
+            group_size=len(id_list),
+        )
+
+        if not created:
+            return direct_message_group
+
+        recipient = Recipient.objects.create(
+            type_id=direct_message_group.id, type=Recipient.DIRECT_MESSAGE_GROUP
+        )
+
+        # The equivalent effect of doing
+        # direct_message_group.save(update_fields=["recipient"]) is handled
+        # in the following raw query via UPDATE to save us a query.
+        direct_message_group.recipient = recipient
+
+        # In a single raw query, we do 2 required things after creating
+        # a new DirectMessageGroup object:
+        # 1. Update the recipient field to reflect the above assignment in db.
+        # 2. Bulk create subscriptions for the participants.
+        query = SQL(
+            """
+            UPDATE {dmg_table}
+            SET recipient_id = %(recipient_id)s
+            WHERE {dmg_table}.id = %(dmg_id)s;
+
+            INSERT INTO {subscription_table}
+                (
+                    user_profile_id,
+                    is_user_active,
+                    recipient_id,
+                    active,
+                    is_muted,
+                    color,
+                    pin_to_top
+                )
+            SELECT
+                {user_profile_table}.id,
+                {user_profile_table}.is_active,
+                %(recipient_id)s,
+                %(active)s,
+                %(is_muted)s,
+                %(color)s,
+                %(pin_to_top)s
+            FROM {user_profile_table}
+            WHERE {user_profile_table}.id = ANY(%(participant_ids)s);
+        """
+        ).format(
+            dmg_table=Identifier(DirectMessageGroup._meta.db_table),
+            subscription_table=Identifier(Subscription._meta.db_table),
+            user_profile_table=Identifier(UserProfile._meta.db_table),
+        )
+        with connection.cursor() as cursor:
+            cursor.execute(
+                query,
+                {
+                    "recipient_id": recipient.id,
+                    "dmg_id": direct_message_group.id,
+                    "active": Subscription._meta.get_field("active").get_default(),
+                    "is_muted": Subscription._meta.get_field("is_muted").get_default(),
+                    "color": Subscription._meta.get_field("color").get_default(),
+                    "pin_to_top": Subscription._meta.get_field("pin_to_top").get_default(),
+                    "participant_ids": id_list,
+                },
+            )
+
+        return direct_message_group
+
+
+def get_direct_message_group(id_list: list[int]) -> DirectMessageGroup | None:
+    """
+    Takes a list of user IDs and returns the DirectMessageGroup
+    object for the group consisting of these users if exists. If
+    the DirectMessageGroup object does not yet exist, it will
+    return None.
+    """
+    assert len(id_list) == len(set(id_list))
+    try:
+        direct_message_group_hash = get_direct_message_group_hash(id_list)
+        return DirectMessageGroup.objects.get(
+            huddle_hash=direct_message_group_hash,
+            group_size=len(id_list),
+        )
+    except DirectMessageGroup.DoesNotExist:
+        return None

zerver/models/saved_snippets.py

@@ -0,0 +1,26 @@
+from typing import Any
+
+from django.conf import settings
+from django.db import models
+from django.utils.timezone import now as timezone_now
+
+from zerver.models.realms import Realm
+from zerver.models.users import UserProfile
+
+
+class SavedSnippet(models.Model):
+    MAX_TITLE_LENGTH = 60
+
+    realm = models.ForeignKey(Realm, on_delete=models.CASCADE)
+    user_profile = models.ForeignKey(UserProfile, on_delete=models.CASCADE)
+    title = models.TextField(max_length=MAX_TITLE_LENGTH)
+    content = models.TextField(max_length=settings.MAX_MESSAGE_LENGTH)
+    date_created = models.DateTimeField(default=timezone_now)
+
+    def to_api_dict(self) -> dict[str, Any]:
+        return {
+            "id": self.id,
+            "title": self.title,
+            "content": self.content,
+            "date_created": int(self.date_created.timestamp()),
+        }

zerver/models/scheduled_jobs.py

@@ -0,0 +1,285 @@
+from typing import TypedDict
+
+from django.conf import settings
+from django.db import models
+from django.db.models import CASCADE, Q
+from django.utils.timezone import now as timezone_now
+from typing_extensions import override
+
+from zerver.lib.display_recipient import get_recipient_ids
+from zerver.lib.timestamp import datetime_to_timestamp
+from zerver.models.clients import Client
+from zerver.models.constants import MAX_TOPIC_NAME_LENGTH
+from zerver.models.groups import NamedUserGroup
+from zerver.models.messages import Message
+from zerver.models.realms import Realm
+from zerver.models.recipients import Recipient
+from zerver.models.streams import Stream
+from zerver.models.users import UserProfile
+
+
+class AbstractScheduledJob(models.Model):
+    scheduled_timestamp = models.DateTimeField(db_index=True)
+    # JSON representation of arguments to consumer
+    data = models.TextField()
+    realm = models.ForeignKey(Realm, on_delete=CASCADE)
+
+    class Meta:
+        abstract = True
+
+
+class ScheduledEmail(AbstractScheduledJob):
+    # Exactly one of users or address should be set. These are
+    # duplicate values, used to efficiently filter the set of
+    # ScheduledEmails for use in clear_scheduled_emails; the
+    # recipients used for actually sending messages are stored in the
+    # data field of AbstractScheduledJob.
+    users = models.ManyToManyField(UserProfile)
+    # Just the address part of a full "name <address>" email address
+    address = models.EmailField(null=True, db_index=True)
+
+    # Valid types are below
+    WELCOME = 1
+    DIGEST = 2
+    INVITATION_REMINDER = 3
+    type = models.PositiveSmallIntegerField()
+
+    @override
+    def __str__(self) -> str:
+        return f"{self.type} {self.address or list(self.users.all())} {self.scheduled_timestamp}"
+
+
+class MissedMessageEmailAddress(models.Model):
+    message = models.ForeignKey(Message, on_delete=CASCADE)
+    user_profile = models.ForeignKey(UserProfile, on_delete=CASCADE)
+    email_token = models.CharField(max_length=34, unique=True, db_index=True)
+
+    # Timestamp of when the missed message address generated.
+    timestamp = models.DateTimeField(db_index=True, default=timezone_now)
+    # Number of times the missed message address has been used.
+    times_used = models.PositiveIntegerField(default=0, db_index=True)
+
+    @override
+    def __str__(self) -> str:
+        return settings.EMAIL_GATEWAY_PATTERN % (self.email_token,)
+
+    def increment_times_used(self) -> None:
+        self.times_used += 1
+        self.save(update_fields=["times_used"])
+
+
+class NotificationTriggers:
+    # "direct_message" is for 1:1 and group direct messages
+    DIRECT_MESSAGE = "direct_message"
+    MENTION = "mentioned"
+    TOPIC_WILDCARD_MENTION = "topic_wildcard_mentioned"
+    STREAM_WILDCARD_MENTION = "stream_wildcard_mentioned"
+    STREAM_PUSH = "stream_push_notify"
+    STREAM_EMAIL = "stream_email_notify"
+    FOLLOWED_TOPIC_PUSH = "followed_topic_push_notify"
+    FOLLOWED_TOPIC_EMAIL = "followed_topic_email_notify"
+    TOPIC_WILDCARD_MENTION_IN_FOLLOWED_TOPIC = "topic_wildcard_mentioned_in_followed_topic"
+    STREAM_WILDCARD_MENTION_IN_FOLLOWED_TOPIC = "stream_wildcard_mentioned_in_followed_topic"
+
+
+class ScheduledMessageNotificationEmail(models.Model):
+    """Stores planned outgoing message notification emails. They may be
+    processed earlier should Zulip choose to batch multiple messages
+    in a single email, but typically will be processed just after
+    scheduled_timestamp.
+    """
+
+    user_profile = models.ForeignKey(UserProfile, on_delete=CASCADE)
+    message = models.ForeignKey(Message, on_delete=CASCADE)
+
+    EMAIL_NOTIFICATION_TRIGGER_CHOICES = [
+        (NotificationTriggers.DIRECT_MESSAGE, "Direct message"),
+        (NotificationTriggers.MENTION, "Mention"),
+        (NotificationTriggers.TOPIC_WILDCARD_MENTION, "Topic wildcard mention"),
+        (NotificationTriggers.STREAM_WILDCARD_MENTION, "Stream wildcard mention"),
+        (NotificationTriggers.STREAM_EMAIL, "Stream notifications enabled"),
+        (NotificationTriggers.FOLLOWED_TOPIC_EMAIL, "Followed topic notifications enabled"),
+        (
+            NotificationTriggers.TOPIC_WILDCARD_MENTION_IN_FOLLOWED_TOPIC,
+            "Topic wildcard mention in followed topic",
+        ),
+        (
+            NotificationTriggers.STREAM_WILDCARD_MENTION_IN_FOLLOWED_TOPIC,
+            "Stream wildcard mention in followed topic",
+        ),
+    ]
+
+    trigger = models.TextField(choices=EMAIL_NOTIFICATION_TRIGGER_CHOICES)
+    mentioned_user_group = models.ForeignKey(NamedUserGroup, null=True, on_delete=CASCADE)
+
+    # Timestamp for when the notification should be processed and sent.
+    # Calculated from the time the event was received and the batching period.
+    scheduled_timestamp = models.DateTimeField(db_index=True)
+
+
+class APIScheduledStreamMessageDict(TypedDict):
+    scheduled_message_id: int
+    to: int
+    type: str
+    content: str
+    rendered_content: str
+    topic: str
+    scheduled_delivery_timestamp: int
+    failed: bool
+
+
+class APIScheduledDirectMessageDict(TypedDict):
+    scheduled_message_id: int
+    to: list[int]
+    type: str
+    content: str
+    rendered_content: str
+    scheduled_delivery_timestamp: int
+    failed: bool
+
+
+class APIReminderDirectMessageDict(TypedDict):
+    reminder_id: int
+    to: list[int]
+    type: str
+    content: str
+    rendered_content: str
+    scheduled_delivery_timestamp: int
+    failed: bool
+    reminder_target_message_id: int
+
+
+class ScheduledMessage(models.Model):
+    sender = models.ForeignKey(UserProfile, on_delete=CASCADE)
+    recipient = models.ForeignKey(Recipient, on_delete=CASCADE)
+    subject = models.CharField(max_length=MAX_TOPIC_NAME_LENGTH)
+    content = models.TextField()
+    rendered_content = models.TextField()
+    sending_client = models.ForeignKey(Client, on_delete=CASCADE)
+    stream = models.ForeignKey(Stream, null=True, on_delete=CASCADE)
+    realm = models.ForeignKey(Realm, on_delete=CASCADE)
+    scheduled_timestamp = models.DateTimeField(db_index=True)
+    read_by_sender = models.BooleanField()
+    delivered = models.BooleanField(default=False)
+    delivered_message = models.ForeignKey(Message, null=True, on_delete=CASCADE)
+    has_attachment = models.BooleanField(default=False, db_index=True)
+    request_timestamp = models.DateTimeField(default=timezone_now)
+    # Only used for REMIND delivery_type messages.
+    reminder_target_message_id = models.IntegerField(null=True)
+    reminder_note = models.TextField(null=True)
+
+    # Metadata for messages that failed to send when their scheduled
+    # moment arrived.
+    failed = models.BooleanField(default=False)
+    failure_message = models.TextField(null=True)
+
+    SEND_LATER = 1
+    REMIND = 2
+
+    DELIVERY_TYPES = (
+        (SEND_LATER, "send_later"),
+        (REMIND, "remind"),
+    )
+
+    delivery_type = models.PositiveSmallIntegerField(
+        choices=DELIVERY_TYPES,
+        default=SEND_LATER,
+    )
+
+    class Meta:
+        indexes = [
+            # We expect a large number of delivered scheduled messages
+            # to accumulate over time. This first index is for the
+            # deliver_scheduled_messages worker.
+            models.Index(
+                name="zerver_unsent_scheduled_messages_by_time",
+                fields=["scheduled_timestamp"],
+                condition=Q(
+                    delivered=False,
+                    failed=False,
+                ),
+            ),
+            # This index is for displaying scheduled messages to the
+            # user themself via the API; we don't filter failed
+            # messages since we will want to display those so that
+            # failures don't just disappear into a black hole.
+            models.Index(
+                name="zerver_realm_unsent_scheduled_messages_by_user",
+                fields=["realm_id", "sender", "delivery_type", "scheduled_timestamp"],
+                condition=Q(
+                    delivered=False,
+                ),
+            ),
+        ]
+
+    @override
+    def __str__(self) -> str:
+        if self.recipient.type != Recipient.STREAM:
+            return f"{self.recipient.label()} {self.sender!r} {self.scheduled_timestamp}"
+
+        return f"{self.recipient.label()} {self.subject} {self.sender!r} {self.scheduled_timestamp}"
+
+    def topic_name(self) -> str:
+        return self.subject
+
+    def set_topic_name(self, topic_name: str) -> None:
+        self.subject = topic_name
+
+    def is_channel_message(self) -> bool:
+        return self.recipient.type == Recipient.STREAM
+
+    def to_dict(self) -> APIScheduledStreamMessageDict | APIScheduledDirectMessageDict:
+        recipient, recipient_type_str = get_recipient_ids(self.recipient, self.sender.id)
+
+        if recipient_type_str == "private":
+            # The topic for direct messages should always be "\x07".
+            assert self.topic_name() == Message.DM_TOPIC
+
+            return APIScheduledDirectMessageDict(
+                scheduled_message_id=self.id,
+                to=recipient,
+                type=recipient_type_str,
+                content=self.content,
+                rendered_content=self.rendered_content,
+                scheduled_delivery_timestamp=datetime_to_timestamp(self.scheduled_timestamp),
+                failed=self.failed,
+            )
+
+        # The recipient for stream messages should always just be the unique stream ID.
+        assert len(recipient) == 1
+
+        return APIScheduledStreamMessageDict(
+            scheduled_message_id=self.id,
+            to=recipient[0],
+            type=recipient_type_str,
+            content=self.content,
+            rendered_content=self.rendered_content,
+            topic=self.topic_name(),
+            scheduled_delivery_timestamp=datetime_to_timestamp(self.scheduled_timestamp),
+            failed=self.failed,
+        )
+
+    def to_reminder_dict(self) -> APIReminderDirectMessageDict:
+        assert self.reminder_target_message_id is not None
+        recipient, recipient_type_str = get_recipient_ids(self.recipient, self.sender.id)
+        assert recipient_type_str == "private"
+        return APIReminderDirectMessageDict(
+            reminder_id=self.id,
+            to=recipient,
+            type=recipient_type_str,
+            content=self.content,
+            rendered_content=self.rendered_content,
+            scheduled_delivery_timestamp=datetime_to_timestamp(self.scheduled_timestamp),
+            failed=self.failed,
+            reminder_target_message_id=self.reminder_target_message_id,
+        )
+
+
+EMAIL_TYPES = {
+    "account_registered": ScheduledEmail.WELCOME,
+    "onboarding_zulip_topics": ScheduledEmail.WELCOME,
+    "onboarding_zulip_guide": ScheduledEmail.WELCOME,
+    "onboarding_team_to_zulip": ScheduledEmail.WELCOME,
+    "digest": ScheduledEmail.DIGEST,
+    "invitation_reminder": ScheduledEmail.INVITATION_REMINDER,
+}

zerver/models/streams.py

@@ -0,0 +1,508 @@
+import secrets
+from enum import Enum
+from typing import Any
+
+from django.db import models
+from django.db.models import CASCADE, Q, QuerySet
+from django.db.models.functions import Upper
+from django.db.models.signals import post_delete, post_save
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext_lazy
+from typing_extensions import override
+
+from zerver.lib.cache import flush_stream
+from zerver.lib.types import GroupPermissionSetting
+from zerver.models.channel_folders import ChannelFolder
+from zerver.models.groups import SystemGroups, UserGroup
+from zerver.models.realms import Realm
+from zerver.models.recipients import Recipient
+from zerver.models.users import UserProfile
+
+
+def generate_email_token_for_stream() -> str:
+    return secrets.token_hex(16)
+
+
+class StreamTopicsPolicyEnum(Enum):
+    inherit = 1
+    allow_empty_topic = 2
+    disable_empty_topic = 3
+    empty_topic_only = 4
+
+
+class Stream(models.Model):
+    MAX_NAME_LENGTH = 60
+    MAX_DESCRIPTION_LENGTH = 1024
+
+    name = models.CharField(max_length=MAX_NAME_LENGTH, db_index=True)
+    realm = models.ForeignKey(Realm, db_index=True, on_delete=CASCADE)
+    date_created = models.DateTimeField(default=timezone_now)
+    creator = models.ForeignKey(UserProfile, null=True, on_delete=models.SET_NULL)
+    deactivated = models.BooleanField(default=False)
+    description = models.CharField(max_length=MAX_DESCRIPTION_LENGTH, default="")
+    rendered_description = models.TextField(default="")
+
+    # Total number of non-deactivated users who are subscribed to the channel.
+    # It's obvious to be a positive field but also in case it becomes negative
+    # we know immediately that something is wrong as it raises IntegrityError.
+    subscriber_count = models.PositiveIntegerField(default=0, db_default=0)
+
+    # Foreign key to the Recipient object for STREAM type messages to this stream.
+    recipient = models.OneToOneField(Recipient, null=True, on_delete=models.SET_NULL)
+
+    folder = models.ForeignKey(ChannelFolder, null=True, on_delete=models.SET_NULL)
+
+    # Various permission policy configurations
+    PERMISSION_POLICIES: dict[str, dict[str, Any]] = {
+        "web_public": {
+            "invite_only": False,
+            "history_public_to_subscribers": True,
+            "is_web_public": True,
+            "policy_name": gettext_lazy("Web-public"),
+        },
+        "public": {
+            "invite_only": False,
+            "history_public_to_subscribers": True,
+            "is_web_public": False,
+            "policy_name": gettext_lazy("Public"),
+        },
+        "private_shared_history": {
+            "invite_only": True,
+            "history_public_to_subscribers": True,
+            "is_web_public": False,
+            "policy_name": gettext_lazy("Private, shared history"),
+        },
+        "private_protected_history": {
+            "invite_only": True,
+            "history_public_to_subscribers": False,
+            "is_web_public": False,
+            "policy_name": gettext_lazy("Private, protected history"),
+        },
+    }
+    invite_only = models.BooleanField(default=False)
+    history_public_to_subscribers = models.BooleanField(default=True)
+
+    # Whether this stream's content should be published by the web-public archive features
+    is_web_public = models.BooleanField(default=False)
+
+    # These values are used to map the can_send_message_group setting value
+    # to the corresponding stream_post_policy value for legacy API clients
+    # in get_stream_post_policy_value_based_on_group_setting defined in
+    # zerver/lib/streams.py.
+    STREAM_POST_POLICY_EVERYONE = 1
+    STREAM_POST_POLICY_ADMINS = 2
+    STREAM_POST_POLICY_RESTRICT_NEW_MEMBERS = 3
+    STREAM_POST_POLICY_MODERATORS = 4
+
+    STREAM_POST_POLICY_TYPES = [
+        STREAM_POST_POLICY_EVERYONE,
+        STREAM_POST_POLICY_ADMINS,
+        STREAM_POST_POLICY_RESTRICT_NEW_MEMBERS,
+        STREAM_POST_POLICY_MODERATORS,
+    ]
+
+    SYSTEM_GROUPS_ENUM_MAP = {
+        SystemGroups.EVERYONE: STREAM_POST_POLICY_EVERYONE,
+        SystemGroups.ADMINISTRATORS: STREAM_POST_POLICY_ADMINS,
+        SystemGroups.FULL_MEMBERS: STREAM_POST_POLICY_RESTRICT_NEW_MEMBERS,
+        SystemGroups.MODERATORS: STREAM_POST_POLICY_MODERATORS,
+    }
+
+    # For old messages being automatically deleted.
+    # Value NULL means "use retention policy of the realm".
+    # Value -1 means "disable retention policy for this stream unconditionally".
+    # Non-negative values have the natural meaning of "archive messages older than <value> days".
+    MESSAGE_RETENTION_SPECIAL_VALUES_MAP = {
+        "unlimited": -1,
+        "realm_default": None,
+    }
+    message_retention_days = models.IntegerField(null=True, default=None)
+
+    # When True, new subscriptions default to push_notifications=True instead of
+    # inheriting from the user's account-level default.
+    default_push_notifications = models.BooleanField(default=False, db_default=False)
+
+    # on_delete field for group value settings is set to RESTRICT
+    # because we don't want to allow deleting a user group in case it
+    # is referenced by the respective setting. We are not using PROTECT
+    # since we want to allow deletion of user groups when the realm
+    # itself is deleted.
+    can_add_subscribers_group = models.ForeignKey(
+        UserGroup, on_delete=models.RESTRICT, related_name="+"
+    )
+    can_administer_channel_group = models.ForeignKey(
+        UserGroup, on_delete=models.RESTRICT, related_name="+"
+    )
+    can_create_topic_group = models.ForeignKey(
+        UserGroup, on_delete=models.RESTRICT, related_name="+"
+    )
+    can_delete_any_message_group = models.ForeignKey(
+        UserGroup, on_delete=models.RESTRICT, related_name="+"
+    )
+    can_delete_own_message_group = models.ForeignKey(
+        UserGroup, on_delete=models.RESTRICT, related_name="+"
+    )
+    can_move_messages_out_of_channel_group = models.ForeignKey(
+        UserGroup, on_delete=models.RESTRICT, related_name="+"
+    )
+    can_move_messages_within_channel_group = models.ForeignKey(
+        UserGroup, on_delete=models.RESTRICT, related_name="+"
+    )
+    can_remove_subscribers_group = models.ForeignKey(UserGroup, on_delete=models.RESTRICT)
+    can_send_message_group = models.ForeignKey(
+        UserGroup, on_delete=models.RESTRICT, related_name="+"
+    )
+    can_subscribe_group = models.ForeignKey(UserGroup, on_delete=models.RESTRICT, related_name="+")
+    can_resolve_topics_group = models.ForeignKey(
+        UserGroup, on_delete=models.RESTRICT, related_name="+"
+    )
+
+    # The very first message ID in the stream.  Used to help clients
+    # determine whether they might need to display "show all topics" for a
+    # stream based on what messages they have cached.
+    first_message_id = models.IntegerField(null=True, db_index=True)
+
+    LAST_ACTIVITY_DAYS_BEFORE_FOR_ACTIVE = 180
+
+    # Whether a message has been sent to this stream in the last X days.
+    is_recently_active = models.BooleanField(default=True, db_default=True)
+
+    topics_policy = models.PositiveSmallIntegerField(default=StreamTopicsPolicyEnum.inherit.value)
+
+    stream_permission_group_settings = {
+        "can_add_subscribers_group": GroupPermissionSetting(
+            allow_nobody_group=True,
+            allow_everyone_group=False,
+            default_group_name=SystemGroups.NOBODY,
+        ),
+        "can_administer_channel_group": GroupPermissionSetting(
+            allow_nobody_group=True,
+            allow_everyone_group=False,
+            default_group_name="channel_creator",
+        ),
+        "can_create_topic_group": GroupPermissionSetting(
+            allow_nobody_group=True,
+            allow_everyone_group=True,
+            default_group_name=SystemGroups.EVERYONE,
+        ),
+        "can_delete_any_message_group": GroupPermissionSetting(
+            allow_nobody_group=True,
+            allow_everyone_group=True,
+            default_group_name=SystemGroups.NOBODY,
+        ),
+        "can_delete_own_message_group": GroupPermissionSetting(
+            allow_nobody_group=True,
+            allow_everyone_group=True,
+            default_group_name=SystemGroups.NOBODY,
+        ),
+        "can_move_messages_out_of_channel_group": GroupPermissionSetting(
+            allow_nobody_group=True,
+            allow_everyone_group=True,
+            default_group_name=SystemGroups.NOBODY,
+        ),
+        "can_move_messages_within_channel_group": GroupPermissionSetting(
+            allow_nobody_group=True,
+            allow_everyone_group=True,
+            default_group_name=SystemGroups.NOBODY,
+        ),
+        "can_remove_subscribers_group": GroupPermissionSetting(
+            allow_nobody_group=True,
+            allow_everyone_group=True,
+            default_group_name=SystemGroups.ADMINISTRATORS,
+        ),
+        "can_send_message_group": GroupPermissionSetting(
+            allow_nobody_group=True,
+            allow_everyone_group=True,
+            default_group_name=SystemGroups.EVERYONE,
+        ),
+        "can_subscribe_group": GroupPermissionSetting(
+            allow_nobody_group=True,
+            allow_everyone_group=False,
+            default_group_name=SystemGroups.NOBODY,
+        ),
+        "can_resolve_topics_group": GroupPermissionSetting(
+            allow_nobody_group=True,
+            allow_everyone_group=True,
+            default_group_name=SystemGroups.NOBODY,
+        ),
+    }
+
+    stream_permission_group_settings_requiring_content_access = [
+        "can_add_subscribers_group",
+        "can_subscribe_group",
+    ]
+    assert set(stream_permission_group_settings_requiring_content_access).issubset(
+        stream_permission_group_settings.keys()
+    )
+
+    stream_permission_group_settings_granting_metadata_access = [
+        "can_add_subscribers_group",
+        "can_administer_channel_group",
+        "can_subscribe_group",
+    ]
+    assert set(stream_permission_group_settings_granting_metadata_access).issubset(
+        stream_permission_group_settings.keys()
+    )
+
+    class Meta:
+        indexes = [
+            models.Index(Upper("name"), name="upper_stream_name_idx"),
+        ]
+
+    @override
+    def __str__(self) -> str:
+        return self.name
+
+    def is_public(self) -> bool:
+        return not self.invite_only
+
+    def is_history_realm_public(self) -> bool:
+        return self.is_public()
+
+    def is_history_public_to_subscribers(self) -> bool:
+        return self.history_public_to_subscribers
+
+    # Stream fields included whenever a Stream object is provided to
+    # Zulip clients via the API.  A few details worth noting:
+    # * "id" is represented as "stream_id" in most API interfaces.
+    # * "deactivated" streams are filtered from the API entirely.
+    # * "realm" and "recipient" are not exposed to clients via the API.
+    API_FIELDS = [
+        "creator_id",
+        "date_created",
+        "deactivated",
+        "default_push_notifications",
+        "description",
+        "first_message_id",
+        "folder_id",
+        "history_public_to_subscribers",
+        "id",
+        "invite_only",
+        "is_web_public",
+        "message_retention_days",
+        "name",
+        "rendered_description",
+        "subscriber_count",
+        "can_add_subscribers_group_id",
+        "can_administer_channel_group_id",
+        "can_create_topic_group_id",
+        "can_delete_any_message_group_id",
+        "can_delete_own_message_group_id",
+        "can_move_messages_out_of_channel_group_id",
+        "can_move_messages_within_channel_group_id",
+        "can_send_message_group_id",
+        "can_remove_subscribers_group_id",
+        "can_subscribe_group_id",
+        "can_resolve_topics_group_id",
+        "is_recently_active",
+        "topics_policy",
+    ]
+
+
+post_save.connect(flush_stream, sender=Stream)
+post_delete.connect(flush_stream, sender=Stream)
+
+
+def get_realm_stream(stream_name: str, realm_id: int) -> Stream:
+    return Stream.objects.get(name__iexact=stream_name.strip(), realm_id=realm_id)
+
+
+def get_active_streams(realm: Realm) -> QuerySet[Stream]:
+    """
+    Return all streams (including invite-only streams) that have not been deactivated.
+    """
+    return Stream.objects.filter(realm=realm, deactivated=False)
+
+
+def get_all_streams(realm: Realm, include_archived_channels: bool = True) -> QuerySet[Stream]:
+    """
+    Return all streams for `include_archived_channels`= true (including invite-only and deactivated streams).
+    """
+    if not include_archived_channels:
+        return get_active_streams(realm)
+
+    return Stream.objects.filter(realm=realm)
+
+
+def get_stream(stream_name: str, realm: Realm) -> Stream:
+    """
+    Callers that don't have a Realm object already available should use
+    get_realm_stream directly, to avoid unnecessarily fetching the
+    Realm object.
+    """
+    return get_realm_stream(stream_name, realm.id)
+
+
+def get_stream_by_id_in_realm(stream_id: int, realm: Realm) -> Stream:
+    return Stream.objects.select_related("realm", "recipient").get(id=stream_id, realm=realm)
+
+
+def bulk_get_streams(realm: Realm, stream_names: set[str]) -> dict[str, Any]:
+    def fetch_streams_by_name(stream_names: set[str]) -> QuerySet[Stream]:
+        #
+        # This should be just
+        #
+        # Stream.objects.select_related().filter(name__iexact__in=stream_names,
+        #                                        realm_id=realm_id)
+        #
+        # But chaining __in and __iexact doesn't work with Django's
+        # ORM, so we have the following hack to construct the relevant where clause
+        where_clause = (
+            "upper(zerver_stream.name::text) IN (SELECT upper(name) FROM unnest(%s) AS name)"
+        )
+        return (
+            get_all_streams(realm, include_archived_channels=True)
+            .select_related("can_send_message_group", "can_send_message_group__named_user_group")
+            .extra(where=[where_clause], params=(list(stream_names),))  # noqa: S610
+        )
+
+    if not stream_names:
+        return {}
+    streams = list(fetch_streams_by_name(stream_names))
+    return {stream.name.lower(): stream for stream in streams}
+
+
+class Subscription(models.Model):
+    """Keeps track of which users are part of the
+    audience for a given Recipient object.
+
+    For 1:1 and group direct message Recipient objects, only the
+    user_profile and recipient fields have any meaning, defining the
+    immutable set of users who are in the audience for that Recipient.
+
+    For Recipient objects associated with a Stream, the remaining
+    fields in this model describe the user's subscription to that stream.
+    """
+
+    user_profile = models.ForeignKey(UserProfile, on_delete=CASCADE)
+    recipient = models.ForeignKey(Recipient, on_delete=CASCADE)
+
+    # Whether the user has since unsubscribed.  We mark Subscription
+    # objects as inactive, rather than deleting them, when a user
+    # unsubscribes, so we can preserve user customizations like
+    # notification settings, stream color, etc., if the user later
+    # resubscribes.
+    active = models.BooleanField(default=True)
+    # This is a denormalization designed to improve the performance of
+    # bulk queries of Subscription objects, Whether the subscribed user
+    # is active tends to be a key condition in those queries.
+    # We intentionally don't specify a default value to promote thinking
+    # about this explicitly, as in some special cases, such as data import,
+    # we may be creating Subscription objects for a user that's deactivated.
+    is_user_active = models.BooleanField()
+
+    # Whether this user had muted this stream.
+    is_muted = models.BooleanField(default=False)
+
+    DEFAULT_STREAM_COLOR = "#c2c2c2"
+    color = models.CharField(max_length=10, default=DEFAULT_STREAM_COLOR)
+    pin_to_top = models.BooleanField(default=False)
+
+    # These fields are stream-level overrides for the user's default
+    # configuration for notification, configured in UserProfile.  The
+    # default, None, means we just inherit the user-level default.
+    desktop_notifications = models.BooleanField(null=True, default=None)
+    audible_notifications = models.BooleanField(null=True, default=None)
+    push_notifications = models.BooleanField(null=True, default=None)
+    email_notifications = models.BooleanField(null=True, default=None)
+    wildcard_mentions_notify = models.BooleanField(null=True, default=None)
+
+    class Meta:
+        unique_together = ("user_profile", "recipient")
+        indexes = [
+            models.Index(
+                fields=("recipient", "user_profile"),
+                name="zerver_subscription_recipient_id_user_profile_id_idx",
+                condition=Q(active=True, is_user_active=True),
+            ),
+        ]
+
+    @override
+    def __str__(self) -> str:
+        return f"{self.user_profile!r} -> {self.recipient!r}"
+
+    # Subscription fields included whenever a Subscription object is provided to
+    # Zulip clients via the API.  A few details worth noting:
+    # * These fields will generally be merged with Stream.API_FIELDS
+    #   data about the stream.
+    # * "user_profile" is usually implied as full API access to Subscription
+    #   is primarily done for the current user; API access to other users'
+    #   subscriptions is generally limited to boolean yes/no.
+    # * "id" and "recipient_id" are not included as they are not used
+    #   in the Zulip API; it's an internal implementation detail.
+    #   Subscription objects are always looked up in the API via
+    #   (user_profile, stream) pairs.
+    # * "active" is often excluded in API use cases where it is implied.
+    # * "is_muted" often needs to be copied to not "in_home_view" for
+    #   backwards-compatibility.
+    API_FIELDS = [
+        "audible_notifications",
+        "color",
+        "desktop_notifications",
+        "email_notifications",
+        "is_muted",
+        "pin_to_top",
+        "push_notifications",
+        "wildcard_mentions_notify",
+    ]
+
+
+class DefaultStream(models.Model):
+    realm = models.ForeignKey(Realm, on_delete=CASCADE)
+    stream = models.ForeignKey(Stream, on_delete=CASCADE)
+
+    class Meta:
+        unique_together = ("realm", "stream")
+
+
+class DefaultStreamGroup(models.Model):
+    MAX_NAME_LENGTH = 60
+
+    name = models.CharField(max_length=MAX_NAME_LENGTH, db_index=True)
+    realm = models.ForeignKey(Realm, on_delete=CASCADE)
+    streams = models.ManyToManyField("zerver.Stream")
+    description = models.CharField(max_length=1024, default="")
+
+    class Meta:
+        unique_together = ("realm", "name")
+
+    def to_dict(self) -> dict[str, Any]:
+        return dict(
+            name=self.name,
+            id=self.id,
+            description=self.description,
+            streams=[stream.id for stream in self.streams.all().order_by("name")],
+        )
+
+
+def get_default_stream_groups(realm: Realm) -> QuerySet[DefaultStreamGroup]:
+    return DefaultStreamGroup.objects.filter(realm=realm)
+
+
+class ChannelEmailAddress(models.Model):
+    realm = models.ForeignKey(Realm, on_delete=CASCADE)
+    channel = models.ForeignKey(Stream, on_delete=CASCADE)
+    creator = models.ForeignKey(UserProfile, null=True, on_delete=CASCADE, related_name="+")
+    sender = models.ForeignKey(UserProfile, on_delete=CASCADE, related_name="+")
+
+    # Used by the e-mail forwarder. The e-mail RFC specifies a maximum
+    # e-mail length of 254, and our max stream length is 30, so we
+    # have plenty of room for the token.
+    email_token = models.CharField(
+        max_length=32,
+        default=generate_email_token_for_stream,
+        unique=True,
+        db_index=True,
+    )
+
+    date_created = models.DateTimeField(default=timezone_now)
+    deactivated = models.BooleanField(default=False)
+
+    class Meta:
+        unique_together = ("channel", "creator", "sender")
+        indexes = [
+            models.Index(
+                fields=("realm", "channel"),
+                name="zerver_channelemailaddress_realm_id_channel_id_idx",
+            ),
+        ]

zerver/models/user_activity.py

@@ -0,0 +1,46 @@
+from datetime import timedelta
+
+from django.db import models
+from django.db.models import CASCADE
+
+from zerver.models.clients import Client
+from zerver.models.users import UserProfile
+
+
+class UserActivity(models.Model):
+    """Data table recording the last time each user hit Zulip endpoints
+    via which Clients; unlike UserPresence, these data are not exposed
+    to users via the Zulip API.
+
+    Useful for debugging as well as to answer analytics questions like
+    "How many users have accessed the Zulip mobile app in the last
+    month?" or "Which users/organizations have recently used API
+    endpoint X that is about to be desupported" for communications
+    and database migration purposes.
+    """
+
+    user_profile = models.ForeignKey(UserProfile, on_delete=CASCADE)
+    client = models.ForeignKey(Client, on_delete=CASCADE)
+    query = models.CharField(max_length=50, db_index=True)
+
+    count = models.IntegerField()
+    last_visit = models.DateTimeField("last visit")
+
+    class Meta:
+        unique_together = ("user_profile", "client", "query")
+
+
+class UserActivityInterval(models.Model):
+    MIN_INTERVAL_LENGTH = timedelta(minutes=15)
+
+    user_profile = models.ForeignKey(UserProfile, on_delete=CASCADE)
+    start = models.DateTimeField("start time", db_index=True)
+    end = models.DateTimeField("end time", db_index=True)
+
+    class Meta:
+        indexes = [
+            models.Index(
+                fields=["user_profile", "end"],
+                name="zerver_useractivityinterval_user_profile_id_end_bb3bfc37_idx",
+            ),
+        ]

zerver/models/user_topics.py

@@ -0,0 +1,87 @@
+from datetime import datetime, timezone
+
+from django.db import models
+from django.db.models import CASCADE
+from django.db.models.functions import Lower, Upper
+from typing_extensions import override
+
+from zerver.models.constants import MAX_TOPIC_NAME_LENGTH
+from zerver.models.recipients import Recipient
+from zerver.models.streams import Stream
+from zerver.models.users import UserProfile
+
+
+class UserTopic(models.Model):
+    user_profile = models.ForeignKey(UserProfile, on_delete=CASCADE)
+    stream = models.ForeignKey(Stream, on_delete=CASCADE)
+    recipient = models.ForeignKey(Recipient, on_delete=CASCADE)
+    topic_name = models.CharField(max_length=MAX_TOPIC_NAME_LENGTH)
+    # The default value for last_updated is a few weeks before tracking
+    # of when topics were muted was first introduced.  It's designed
+    # to be obviously incorrect so that one can tell it's backfilled data.
+    last_updated = models.DateTimeField(default=datetime(2020, 1, 1, 0, 0, tzinfo=timezone.utc))
+
+    class VisibilityPolicy(models.IntegerChoices):
+        # A normal muted topic. No notifications and unreads hidden.
+        MUTED = 1, "Muted topic"
+
+        # This topic will behave like an unmuted topic in an unmuted stream even if it
+        # belongs to a muted stream.
+        UNMUTED = 2, "Unmuted topic in muted stream"
+
+        # This topic will behave like `UNMUTED`, plus some additional
+        # display and/or notifications priority that is TBD and likely to
+        # be configurable; see #6027. Not yet implemented.
+        FOLLOWED = 3, "Followed topic"
+
+        # Implicitly, if a UserTopic does not exist, the (user, topic)
+        # pair should have normal behavior for that (user, stream) pair.
+
+        # We use this in our code to represent the condition in the comment above.
+        INHERIT = 0, "User's default policy for the stream."
+
+    visibility_policy = models.SmallIntegerField(
+        choices=VisibilityPolicy.choices, default=VisibilityPolicy.MUTED
+    )
+
+    class Meta:
+        constraints = [
+            models.UniqueConstraint(
+                "user_profile",
+                "stream",
+                Lower("topic_name"),
+                name="usertopic_case_insensitive_topic_uniq",
+            ),
+        ]
+
+        indexes = [
+            models.Index("stream", Upper("topic_name"), name="zerver_mutedtopic_stream_topic"),
+            # This index is designed to optimize queries fetching the
+            # set of users who have special policy for a stream,
+            # e.g. for the send-message code paths.
+            models.Index(
+                fields=("stream", "topic_name", "visibility_policy", "user_profile"),
+                name="zerver_usertopic_stream_topic_user_visibility_idx",
+            ),
+            # This index is useful for handling API requests fetching the
+            # muted topics for a given user or user/stream pair.
+            models.Index(
+                fields=("user_profile", "visibility_policy", "stream", "topic_name"),
+                name="zerver_usertopic_user_visibility_idx",
+            ),
+            # Supports the per-message visibility policy lookup in
+            # matching_user_topic_exists_q, which has the recipient
+            # rather than the channel and matches topic names
+            # case-insensitively, so the index above cannot serve it.
+            models.Index(
+                "user_profile",
+                "visibility_policy",
+                "recipient",
+                Upper("topic_name"),
+                name="zerver_usertopic_user_visibility_recipient_idx",
+            ),
+        ]
+
+    @override
+    def __str__(self) -> str:
+        return f"({self.user_profile.email}, {self.stream.name}, {self.topic_name}, {self.last_updated})"