snapshot of https://github.com/zulip/zulip.git @ 48c69911c7dd4e03b20552eb3bd77ad8151c1357 (2026-09-15)

zerver/lib/digest.py

@@ -0,0 +1,560 @@
+import functools
+import heapq
+import logging
+from collections import defaultdict
+from collections.abc import Collection, Iterator
+from dataclasses import dataclass
+from datetime import datetime, timedelta, timezone
+from typing import Any, TypeAlias
+
+from django.conf import settings
+from django.db import transaction
+from django.db.models import Exists, OuterRef, QuerySet
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+from markupsafe import Markup
+
+from confirmation.models import one_click_unsubscribe_link
+from zerver.context_processors import common_context
+from zerver.lib.email_notifications import (
+    MessageListPayload,
+    build_message_list,
+    get_channel_privacy_icon,
+    message_content_allowed_in_missedmessage_emails,
+)
+from zerver.lib.logging_util import log_to_file
+from zerver.lib.message import get_last_message_id
+from zerver.lib.queue import queue_json_publish_rollback_unsafe
+from zerver.lib.send_email import FromAddress, send_future_email
+from zerver.lib.url_encoding import stream_narrow_url
+from zerver.models import (
+    Message,
+    Realm,
+    RealmAuditLog,
+    Recipient,
+    Stream,
+    Subscription,
+    UserActivityInterval,
+    UserMessage,
+    UserProfile,
+    UserTopic,
+)
+from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.models.streams import get_active_streams
+
+logger = logging.getLogger(__name__)
+log_to_file(logger, settings.DIGEST_LOG_PATH)
+
+DIGEST_CUTOFF = 5
+MAX_HOT_TOPICS_TO_BE_INCLUDED_IN_DIGEST = 4
+
+TopicKey: TypeAlias = tuple[int, str]
+
+
+@dataclass
+class DigestTeaserData:
+    participants: list[str]
+    count: int
+    first_few_messages: MessageListPayload
+
+
+class DigestTopic:
+    def __init__(self, topic_key: TopicKey) -> None:
+        self.topic_key = topic_key
+        self.human_senders: set[str] = set()
+        self.sample_messages: list[Message] = []
+        self.num_human_messages = 0
+
+    def stream_id(self) -> int:
+        # topic_key is (stream_id, topic_name)
+        return self.topic_key[0]
+
+    def topic_name(self) -> str:
+        return self.topic_key[1]
+
+    def add_message(self, message: Message) -> None:
+        if len(self.sample_messages) < 2:
+            self.sample_messages.append(message)
+
+        if not message.sender.is_bot:
+            self.human_senders.add(message.sender.full_name)
+            self.num_human_messages += 1
+
+    def length(self) -> int:
+        return self.num_human_messages
+
+    def diversity(self) -> int:
+        return len(self.human_senders)
+
+    def teaser_data(self, user: UserProfile, stream_id_map: dict[int, Stream]) -> DigestTeaserData:
+        teaser_count = self.num_human_messages - len(self.sample_messages)
+        first_few_messages = build_message_list(
+            user=user,
+            messages=self.sample_messages,
+            stream_id_map=stream_id_map,
+        )
+        return DigestTeaserData(
+            participants=sorted(self.human_senders),
+            count=teaser_count,
+            first_few_messages=first_few_messages,
+        )
+
+
+# Digests accumulate 2 types of interesting traffic for a user:
+# 1. New streams
+# 2. Interesting stream traffic, as determined by the longest and most
+#    diversely comment upon topics.
+
+
+# Changes to this should also be reflected in
+# zerver/worker/digest_emails.py:DigestWorker.consume()
+def queue_digest_user_ids(user_ids: list[int], cutoff: datetime) -> None:
+    # Convert cutoff to epoch seconds for transit.
+    event = {"user_ids": user_ids, "cutoff": cutoff.strftime("%s")}
+    queue_json_publish_rollback_unsafe("digest_emails", event)
+
+
+def enqueue_emails(cutoff: datetime) -> None:
+    if not settings.SEND_DIGEST_EMAILS:
+        return
+
+    weekday = timezone_now().weekday()
+    for realm in Realm.objects.filter(
+        deactivated=False, digest_emails_enabled=True, digest_weekday=weekday
+    ).exclude(string_id__in=settings.SYSTEM_ONLY_REALMS):
+        _enqueue_emails_for_realm(realm, cutoff)
+
+
+def _enqueue_emails_for_realm(realm: Realm, cutoff: datetime) -> None:
+    # This should only be called directly by tests.  Use enqueue_emails
+    # to process all realms that are set up for processing on any given day.
+    twelve_hours_ago = timezone_now() - timedelta(hours=12)
+
+    target_users = (
+        UserProfile.objects.filter(
+            realm=realm,
+            is_active=True,
+            is_bot=False,
+            enable_digest_emails=True,
+        )
+        .alias(
+            recent_activity=Exists(
+                UserActivityInterval.objects.filter(user_profile_id=OuterRef("id"), end__gt=cutoff)
+            )
+        )
+        .filter(recent_activity=False)
+        .alias(
+            sent_recent_digest=Exists(
+                RealmAuditLog.objects.filter(
+                    realm_id=realm.id,
+                    event_type=AuditLogEventType.USER_DIGEST_EMAIL_CREATED,
+                    event_time__gt=twelve_hours_ago,
+                    modified_user_id=OuterRef("id"),
+                )
+            )
+        )
+        .filter(sent_recent_digest=False)
+    )
+
+    user_ids = list(target_users.order_by("id").values_list("id", flat=True))
+
+    # We process batches of 30.  We want a big enough batch
+    # to amortize work, but not so big that a single item
+    # from the queue takes too long to process.
+    chunk_size = 30
+    for i in range(0, len(user_ids), chunk_size):
+        chunk_user_ids = user_ids[i : i + chunk_size]
+        queue_digest_user_ids(chunk_user_ids, cutoff)
+        logger.info(
+            "Queuing user_ids for potential digest: %s",
+            chunk_user_ids,
+        )
+
+
+last_realm_id: int | None = None
+last_cutoff: float | None = None
+
+
+def maybe_clear_recent_topics_cache(realm_id: int, cutoff: float) -> None:
+    # As an optimization, we clear the digest caches when we switch to
+    # a new realm or cutoff value.  Since these values are part of the
+    # cache key, this is not necessary for correctness -- it merely
+    # helps reduce the memory footprint of the cache.
+    global last_realm_id, last_cutoff
+    if last_realm_id != realm_id or last_cutoff != cutoff:
+        logger.info("Flushing stream cache: %s", get_recent_topics.cache_info())
+        get_recent_topics.cache_clear()
+    last_realm_id = realm_id
+    last_cutoff = cutoff
+
+
+# We cache both by stream-id and cutoff, which ensures the per-stream
+# cache also does not contain data from old digests
+@functools.lru_cache(maxsize=5000)
+def get_recent_topics(
+    realm_id: int,
+    stream_id: int,
+    cutoff_date: datetime,
+) -> list[DigestTopic]:
+    # Gather information about topic conversations, then
+    # classify by:
+    #   * topic length
+    #   * number of senders
+
+    messages = (
+        # Uses index: zerver_message_realm_recipient_date_sent
+        Message.objects.filter(
+            realm_id=realm_id,
+            recipient__type=Recipient.STREAM,
+            recipient__type_id=stream_id,
+            date_sent__gt=cutoff_date,
+        )
+        .order_by(
+            "id",  # we will sample the first few messages
+        )
+        .select_related(
+            "recipient",  # build_message_list looks up recipient.type
+            "sender",  # we need the sender's full name
+        )
+        .defer(
+            # This construction, to only fetch the sender's full_name and is_bot,
+            # is because `.only()` doesn't work with select_related tables.
+            *{
+                f"sender__{f.name}"
+                for f in UserProfile._meta.fields
+                if f.name not in {"full_name", "is_bot"}
+            }
+        )
+    )
+
+    digest_topic_map: dict[TopicKey, DigestTopic] = {}
+    for message in messages:
+        topic_key = (stream_id, message.topic_name())
+
+        if topic_key not in digest_topic_map:
+            digest_topic_map[topic_key] = DigestTopic(topic_key)
+
+        digest_topic_map[topic_key].add_message(message)
+
+    topics = list(digest_topic_map.values())
+
+    return topics
+
+
+def get_hot_topics(
+    all_topics: list[DigestTopic],
+    stream_ids: set[int],
+) -> list[DigestTopic]:
+    topics = [topic for topic in all_topics if topic.stream_id() in stream_ids]
+
+    hot_topics = heapq.nlargest(2, topics, key=DigestTopic.diversity)
+
+    for topic in heapq.nlargest(
+        MAX_HOT_TOPICS_TO_BE_INCLUDED_IN_DIGEST, topics, key=DigestTopic.length
+    ):
+        if topic not in hot_topics:
+            hot_topics.append(topic)
+        if len(hot_topics) == MAX_HOT_TOPICS_TO_BE_INCLUDED_IN_DIGEST:
+            break
+
+    return hot_topics
+
+
+def get_recently_created_streams(realm: Realm, threshold: datetime) -> list[Stream]:
+    fields = ["id", "name", "is_web_public", "invite_only"]
+    return list(get_active_streams(realm).filter(date_created__gt=threshold).only(*fields))
+
+
+def gather_new_streams(
+    realm: Realm,
+    recently_created_streams: list[Stream],  # streams only need id and name
+    can_access_public: bool,
+) -> tuple[int, dict[str, list[Markup] | list[str]]]:
+    if can_access_public:
+        new_streams = [stream for stream in recently_created_streams if not stream.invite_only]
+    else:
+        new_streams = [stream for stream in recently_created_streams if stream.is_web_public]
+
+    channels_html = []
+    channels_plain = []
+
+    for stream in new_streams:
+        narrow_url = stream_narrow_url(realm, stream)
+        channel_link = Markup(
+            "<a href='{narrow_url}'>{channel_privacy_icon}{stream_name}</a>"
+        ).format(
+            narrow_url=narrow_url,
+            channel_privacy_icon=get_channel_privacy_icon(stream),
+            stream_name=stream.name,
+        )
+        channels_html.append(channel_link)
+        channels_plain.append(stream.name)
+
+    return len(new_streams), {"html": channels_html, "plain": channels_plain}
+
+
+def get_min_recent_message_id(realm_id: int, threshold: datetime) -> int | None:
+    # ID of the realm's earliest message at or after the threshold,
+    # or None if there are none.
+    return (
+        # Uses index: zerver_message_realm_date_sent
+        Message.objects.filter(realm_id=realm_id, date_sent__gte=threshold)
+        .order_by("date_sent")
+        .values_list("id", flat=True)
+        .first()
+    )
+
+
+def get_new_messages_count(user: UserProfile, min_recent_message_id: int | None) -> int:
+    if min_recent_message_id is None:
+        # No message was sent since the threshold.
+        return 0
+
+    # We filter on `message_id` for performance, so the planner drives
+    # from the unique (user_profile_id, message_id) index on `UserMessage`.
+    #
+    # Since we want "messages the user received since threshold",
+    # the intuition is to filter on `user_profile` and `message__date_sent`.
+    # When filtering without `message_id`, the planner drives from Message's
+    # `date_sent` index, which is not scoped to a user: it reads every message
+    # sent since threshold and joins against all UserMessage rows for the user
+    # to determine whether they received each message - which is too slow.
+    count = UserMessage.objects.filter(
+        user_profile=user,
+        message_id__gte=min_recent_message_id,
+        message__sender__is_bot=False,
+    ).count()
+    return count
+
+
+def enough_traffic(
+    hot_conversations: int, new_streams: int, new_messages: int, show_message_content: bool
+) -> bool:
+    if new_streams > 0:
+        return True
+    if not show_message_content:
+        return new_messages > 0
+    return hot_conversations > 0
+
+
+def get_user_stream_map(user_ids: list[int], cutoff_date: datetime) -> dict[int, set[int]]:
+    """Skipping streams where the user's subscription status has changed
+    when constructing digests is critical to ensure correctness for
+    streams without shared history, guest users, and long-term idle
+    users, because it means that every user has the same view of the
+    history of a given stream whose message history is being included
+    (and thus we can share a lot of work).
+
+    The downside is that newly created streams are never included in
+    the first digest email after their creation.  Should we wish to
+    change that, we will need to be very careful to avoid creating
+    bugs for any of those classes of users.
+    """
+    events = [
+        AuditLogEventType.SUBSCRIPTION_CREATED,
+        AuditLogEventType.SUBSCRIPTION_ACTIVATED,
+        AuditLogEventType.SUBSCRIPTION_DEACTIVATED,
+    ]
+    # This uses the zerver_realmauditlog_user_subscriptions_idx
+    # partial index on RealmAuditLog which is specifically for those
+    # three event types.
+    rows = (
+        Subscription.objects.filter(
+            user_profile_id__in=user_ids,
+            recipient__type=Recipient.STREAM,
+            active=True,
+            is_muted=False,
+        )
+        .alias(
+            was_modified=Exists(
+                RealmAuditLog.objects.filter(
+                    modified_stream_id=OuterRef("recipient__type_id"),
+                    modified_user_id=OuterRef("user_profile_id"),
+                    event_time__gt=cutoff_date,
+                    event_type__in=events,
+                )
+            )
+        )
+        .filter(was_modified=False)
+        .values("user_profile_id", "recipient__type_id")
+    )
+
+    # maps user_id -> {stream_id, stream_id, ...}
+    dct: dict[int, set[int]] = defaultdict(set)
+    for row in rows:
+        dct[row["user_profile_id"]].add(row["recipient__type_id"])
+
+    return dct
+
+
+def get_user_muted_topics_map(user_ids: list[int]) -> dict[int, set[tuple[int, str]]]:
+    # maps user_id -> {(stream_id, lowercased topic_name), ...};
+    # topic names are matched case-insensitively.
+    rows = UserTopic.objects.filter(
+        user_profile_id__in=user_ids,
+        visibility_policy=UserTopic.VisibilityPolicy.MUTED,
+    ).values("user_profile_id", "stream_id", "topic_name")
+
+    dct: dict[int, set[tuple[int, str]]] = defaultdict(set)
+    for row in rows:
+        dct[row["user_profile_id"]].add((row["stream_id"], row["topic_name"].lower()))
+
+    return dct
+
+
+def get_slim_stream_id_map(stream_ids: set[int]) -> dict[int, Stream]:
+    # "slim" because it only fetches the names of the stream objects,
+    # suitable for passing into build_message_list.
+    streams = Stream.objects.filter(id__in=stream_ids).only("id", "name")
+    return {stream.id: stream for stream in streams}
+
+
+def bulk_get_digest_context(
+    users: Collection[UserProfile] | QuerySet[UserProfile], cutoff: float
+) -> Iterator[tuple[UserProfile, dict[str, Any]]]:
+    # We expect a non-empty list of users all from the same realm.
+    assert users
+    realm = next(iter(users)).realm
+    for user in users:
+        assert user.realm_id == realm.id
+
+    # Convert from epoch seconds to a datetime object.
+    cutoff_date = datetime.fromtimestamp(int(cutoff), tz=timezone.utc)
+
+    maybe_clear_recent_topics_cache(realm.id, cutoff)
+
+    recently_created_streams = get_recently_created_streams(realm, cutoff_date)
+
+    user_ids = [user.id for user in users]
+    user_stream_map = get_user_stream_map(user_ids, cutoff_date)
+    stream_ids = set().union(*user_stream_map.values())
+    stream_id_map = get_slim_stream_id_map(stream_ids)
+    user_muted_topics_map = get_user_muted_topics_map(user_ids)
+
+    # ID of the realm's earliest message at or after the `cutoff`.
+    # Shared across all users, and computed lazily since it is only
+    # needed for users who have message content hidden in emails.
+    min_recent_message_id: int | None = None
+    have_min_recent_message_id = False
+
+    for user in users:
+        context = common_context(user)
+
+        # Start building email template data.
+        unsubscribe_link = one_click_unsubscribe_link(user, "digest")
+        context.update(unsubscribe_link=unsubscribe_link)
+
+        # Gather new streams.
+        new_streams_count, new_streams = gather_new_streams(
+            realm=realm,
+            recently_created_streams=recently_created_streams,
+            can_access_public=user.can_access_public_streams(),
+        )
+
+        context["new_streams_count"] = new_streams_count
+        context[
+            "message_content_disabled_by_realm"
+        ] = not realm.message_content_allowed_in_email_notifications
+        context[
+            "message_content_disabled_by_user"
+        ] = not user.message_content_in_email_notifications
+
+        if not message_content_allowed_in_missedmessage_emails(user):
+            # Count new messages when message content is hidden in email notifications.
+            if not have_min_recent_message_id:
+                min_recent_message_id = get_min_recent_message_id(realm.id, cutoff_date)
+                have_min_recent_message_id = True
+            context["new_messages_count"] = get_new_messages_count(user, min_recent_message_id)
+            context["hot_conversations"] = []
+            context["show_message_content"] = False
+        else:
+            # Otherwise, get context data for hot conversations.
+            stream_ids = user_stream_map[user.id]
+            muted_topics = user_muted_topics_map[user.id]
+            recent_topics = []
+            for stream_id in stream_ids:
+                recent_topics += [
+                    digest_topic
+                    for digest_topic in get_recent_topics(realm.id, stream_id, cutoff_date)
+                    # Topics the user has muted should not be featured
+                    # in their digest.
+                    if (digest_topic.stream_id(), digest_topic.topic_name().lower())
+                    not in muted_topics
+                ]
+            hot_topics = get_hot_topics(recent_topics, stream_ids)
+
+            context["hot_conversations"] = [
+                hot_topic.teaser_data(user, stream_id_map) for hot_topic in hot_topics
+            ]
+            context["new_channels"] = new_streams
+            context["new_messages_count"] = 0
+            context["show_message_content"] = True
+
+        yield user, context
+
+
+def get_digest_context(user: UserProfile, cutoff: float) -> dict[str, Any]:
+    for ignored, context in bulk_get_digest_context([user], cutoff):
+        return context
+    raise AssertionError("Unreachable")
+
+
+@transaction.atomic(durable=True)
+def bulk_handle_digest_email(user_ids: list[int], cutoff: float) -> None:
+    # We go directly to the database to get user objects,
+    # since inactive users are likely to not be in the cache.
+    users = (
+        UserProfile.objects.filter(id__in=user_ids, is_active=True, realm__deactivated=False)
+        .order_by("id")
+        .select_related("realm")
+    )
+    digest_users = []
+
+    for user, context in bulk_get_digest_context(users, cutoff):
+        # We don't want to send emails containing almost no information.
+        if not enough_traffic(
+            len(context["hot_conversations"]),
+            context["new_streams_count"],
+            context["new_messages_count"],
+            context["show_message_content"],
+        ):
+            continue
+
+        digest_users.append(user)
+        logger.info("Enqueuing digest email for user %s", user.id)
+
+        # Send now, as a ScheduledEmail
+        send_future_email(
+            "zerver/emails/digest",
+            user.realm,
+            to_user_ids=[user.id],
+            from_name=_("{service_name} digest").format(service_name=settings.INSTALLATION_NAME),
+            from_address=FromAddress.no_reply_placeholder,
+            context=context,
+        )
+
+    bulk_write_realm_audit_logs(digest_users)
+
+
+def bulk_write_realm_audit_logs(users: list[UserProfile]) -> None:
+    if not users:
+        return
+
+    # We write RealmAuditLog rows for auditing, and we will also
+    # use these rows during the next run to possibly exclude the
+    # users (if insufficient time has passed).
+    last_message_id = get_last_message_id()
+    now = timezone_now()
+
+    log_rows = [
+        RealmAuditLog(
+            realm_id=user.realm_id,
+            modified_user_id=user.id,
+            event_last_message_id=last_message_id,
+            event_time=now,
+            event_type=AuditLogEventType.USER_DIGEST_EMAIL_CREATED,
+        )
+        for user in users
+    ]
+
+    RealmAuditLog.objects.bulk_create(log_rows)