snapshot of https://github.com/zulip/zulip.git @ 48c69911c7dd4e03b20552eb3bd77ad8151c1357 (2026-09-15)

zerver/worker/base.py

@@ -0,0 +1,332 @@
+# Documented in https://zulip.readthedocs.io/en/latest/subsystems/queuing.html
+import logging
+import os
+import signal
+import time
+from abc import ABC, abstractmethod
+from collections import deque
+from collections.abc import Callable, MutableSequence
+from types import FrameType
+from typing import Any, TypeVar
+
+import orjson
+import sentry_sdk
+from django.conf import settings
+from django.db import connection
+from typing_extensions import override
+
+from zerver.lib.context_managers import lockfile
+from zerver.lib.db_connections import reset_queries
+from zerver.lib.partial import partial
+from zerver.lib.per_request_cache import flush_per_request_caches
+from zerver.lib.pysa import mark_sanitized
+from zerver.lib.queue import SimpleQueueClient
+
+logger = logging.getLogger(__name__)
+
+
+class WorkerTimeoutError(Exception):
+    def __init__(self, queue_name: str, limit: int, event_count: int) -> None:
+        self.queue_name = queue_name
+        self.limit = limit
+        self.event_count = event_count
+
+    @override
+    def __str__(self) -> str:
+        return f"Timed out in {self.queue_name} after {self.limit * self.event_count} seconds processing {self.event_count} events"
+
+
+class InterruptConsumeError(Exception):
+    """
+    This exception is to be thrown inside event consume function
+    if the intention is to simply interrupt the processing
+    of the current event and normally continue the work of the queue.
+    """
+
+
+class WorkerDeclarationError(Exception):
+    pass
+
+
+ConcreteQueueWorker = TypeVar("ConcreteQueueWorker", bound="QueueProcessingWorker")
+
+
+def assign_queue(
+    queue_name: str,
+    enabled: bool = True,
+    is_test_queue: bool = False,
+) -> Callable[[type[ConcreteQueueWorker]], type[ConcreteQueueWorker]]:
+    def decorate(clazz: type[ConcreteQueueWorker]) -> type[ConcreteQueueWorker]:
+        clazz.queue_name = queue_name
+        if enabled:
+            register_worker(queue_name, clazz, is_test_queue)
+        return clazz
+
+    return decorate
+
+
+worker_classes: dict[str, type["QueueProcessingWorker"]] = {}
+test_queues: set[str] = set()
+
+
+def register_worker(
+    queue_name: str, clazz: type["QueueProcessingWorker"], is_test_queue: bool = False
+) -> None:
+    worker_classes[queue_name] = clazz
+    if is_test_queue:
+        test_queues.add(queue_name)
+
+
+def check_and_send_restart_signal() -> None:
+    try:
+        if not connection.is_usable():
+            logging.warning("*** Sending self SIGUSR1 to trigger a restart.")
+            os.kill(os.getpid(), signal.SIGUSR1)
+    except Exception:
+        pass
+
+
+class QueueProcessingWorker(ABC):
+    queue_name: str
+    MAX_CONSUME_SECONDS: int | None = 30
+    CONSUME_ITERATIONS_BEFORE_UPDATE_STATS_NUM = 50
+    MAX_SECONDS_BEFORE_UPDATE_STATS = 30
+
+    # How many un-acknowledged events the worker should have on hand,
+    # fetched from the rabbitmq server.  Larger values may be more
+    # performant, but if queues are large, cause more network IO at
+    # startup and steady-state memory.
+    PREFETCH = 100
+
+    def __init__(
+        self,
+        threaded: bool = False,
+        disable_timeout: bool = False,
+        worker_num: int | None = None,
+    ) -> None:
+        self.q: SimpleQueueClient | None = None
+        self.threaded = threaded
+        self.disable_timeout = disable_timeout
+        self.worker_num = worker_num
+        if not hasattr(self, "queue_name"):
+            raise WorkerDeclarationError("Queue worker declared without queue_name")
+
+        self.initialize_statistics()
+
+    def initialize_statistics(self) -> None:
+        self.queue_last_emptied_timestamp = time.time()
+        self.consumed_since_last_emptied = 0
+        self.recent_consume_times: MutableSequence[tuple[int, float]] = deque(maxlen=50)
+        self.consume_iteration_counter = 0
+        self.idle = True
+        self.last_statistics_update_time = 0.0
+
+        self.update_statistics()
+
+    @sentry_sdk.trace
+    def update_statistics(self) -> None:
+        total_seconds = sum(seconds for _, seconds in self.recent_consume_times)
+        total_events = sum(events_number for events_number, _ in self.recent_consume_times)
+        if total_events == 0:
+            recent_average_consume_time = None
+        else:
+            recent_average_consume_time = total_seconds / total_events
+        stats_dict = dict(
+            update_time=time.time(),
+            recent_average_consume_time=recent_average_consume_time,
+            queue_last_emptied_timestamp=self.queue_last_emptied_timestamp,
+            consumed_since_last_emptied=self.consumed_since_last_emptied,
+        )
+
+        os.makedirs(settings.QUEUE_STATS_DIR, exist_ok=True)
+
+        fname = f"{self.queue_name}.stats"
+        fn = os.path.join(settings.QUEUE_STATS_DIR, fname)
+        with lockfile(fn + ".lock"):
+            tmp_fn = fn + ".tmp"
+            with open(tmp_fn, "wb") as f:
+                f.write(
+                    orjson.dumps(stats_dict, option=orjson.OPT_APPEND_NEWLINE | orjson.OPT_INDENT_2)
+                )
+            os.rename(tmp_fn, fn)
+        self.last_statistics_update_time = time.time()
+
+    def get_remaining_local_queue_size(self) -> int:
+        if self.q is not None:
+            return self.q.local_queue_size()
+        else:
+            # This is a special case that will happen if we're operating without
+            # using RabbitMQ (e.g. in tests). In that case there's no queuing to speak of
+            # and the only reasonable size to return is 0.
+            return 0
+
+    @abstractmethod
+    def consume(self, data: dict[str, Any]) -> None:
+        pass
+
+    def do_consume(
+        self, consume_func: Callable[[list[dict[str, Any]]], None], events: list[dict[str, Any]]
+    ) -> None:
+        consume_time_seconds: float | None = None
+        with sentry_sdk.start_transaction(
+            op="task",
+            name=f"consume {self.queue_name}",
+            custom_sampling_context={"queue": self.queue_name},
+        ):
+            if sentry_sdk.is_initialized():
+                sentry_sdk.add_breadcrumb(
+                    type="debug",
+                    category="queue_processor",
+                    message=f"Consuming {self.queue_name}",
+                    data={
+                        "events": events,
+                        "local_queue_size": self.get_remaining_local_queue_size(),
+                    },
+                )
+            try:
+                if self.idle:
+                    # We're reactivating after having gone idle due to emptying the queue.
+                    # We should update the stats file to keep it fresh and to make it clear
+                    # that the queue started processing, in case the event we're about to process
+                    # makes us freeze.
+                    self.idle = False
+                    self.update_statistics()
+
+                time_start = time.time()
+                if self.MAX_CONSUME_SECONDS and not self.threaded and not self.disable_timeout:
+                    try:
+                        signal.signal(
+                            signal.SIGALRM,
+                            partial(self.timer_expired, self.MAX_CONSUME_SECONDS, events),
+                        )
+                        try:
+                            signal.alarm(self.MAX_CONSUME_SECONDS * len(events))
+                            consume_func(events)
+                        finally:
+                            signal.alarm(0)
+                    finally:
+                        signal.signal(signal.SIGALRM, signal.SIG_DFL)
+                else:
+                    consume_func(events)
+                consume_time_seconds = time.time() - time_start
+                self.consumed_since_last_emptied += len(events)
+            except Exception as e:
+                self._handle_consume_exception(events, e)
+            finally:
+                flush_per_request_caches()
+                reset_queries()
+
+                with sentry_sdk.start_span(name="statistics"):
+                    if consume_time_seconds is not None:
+                        self.recent_consume_times.append((len(events), consume_time_seconds))
+
+                    remaining_local_queue_size = self.get_remaining_local_queue_size()
+                    if remaining_local_queue_size == 0:
+                        self.queue_last_emptied_timestamp = time.time()
+                        self.consumed_since_last_emptied = 0
+                        # We've cleared all the events from the queue, so we don't
+                        # need to worry about the small overhead of doing a disk write.
+                        # We take advantage of this to update the stats file to keep it fresh,
+                        # especially since the queue might go idle until new events come in.
+                        self.update_statistics()
+                        self.idle = True
+                    else:
+                        self.consume_iteration_counter += 1
+                        if (
+                            self.consume_iteration_counter
+                            >= self.CONSUME_ITERATIONS_BEFORE_UPDATE_STATS_NUM
+                            or time.time() - self.last_statistics_update_time
+                            >= self.MAX_SECONDS_BEFORE_UPDATE_STATS
+                        ):
+                            self.consume_iteration_counter = 0
+                            self.update_statistics()
+
+    def consume_single_event(self, event: dict[str, Any]) -> None:
+        consume_func = lambda events: self.consume(events[0])
+        self.do_consume(consume_func, [event])
+
+    def timer_expired(
+        self, limit: int, events: list[dict[str, Any]], signal: int, frame: FrameType | None
+    ) -> None:
+        raise WorkerTimeoutError(self.queue_name, limit, len(events))
+
+    def _handle_consume_exception(self, events: list[dict[str, Any]], exception: Exception) -> None:
+        if isinstance(exception, InterruptConsumeError):
+            # The exception signals that no further error handling
+            # is needed and the worker can proceed.
+            return
+
+        with sentry_sdk.new_scope() as scope:
+            scope.set_context(
+                "events",
+                {
+                    "data": events,
+                    "queue_name": self.queue_name,
+                },
+            )
+            if isinstance(exception, WorkerTimeoutError):
+                scope.fingerprint = ["worker-timeout", self.queue_name]
+                logging.error(exception, exc_info=exception, stack_info=True)
+            else:
+                logging.error(
+                    "Problem handling data on queue %s",
+                    self.queue_name,
+                    exc_info=exception,
+                    stack_info=True,
+                )
+
+        if not os.path.exists(settings.QUEUE_ERROR_DIR):
+            os.mkdir(settings.QUEUE_ERROR_DIR)  # nocoverage
+        # Use 'mark_sanitized' to prevent Pysa from detecting this false positive
+        # flow. 'queue_name' is always a constant string.
+        fname = mark_sanitized(f"{self.queue_name}.errors")
+        fn = os.path.join(settings.QUEUE_ERROR_DIR, fname)
+        line = f"{time.asctime()}\t{orjson.dumps(events).decode()}\n"
+        lock_fn = fn + ".lock"
+        with lockfile(lock_fn), open(fn, "a") as f:
+            f.write(line)
+        check_and_send_restart_signal()
+
+    def setup(self) -> None:
+        self.q = SimpleQueueClient(prefetch=self.PREFETCH)
+
+    def start(self) -> None:
+        assert self.q is not None
+        self.initialize_statistics()
+        self.q.start_json_consumer(
+            self.queue_name,
+            lambda events: self.consume_single_event(events[0]),
+        )
+
+    def stop(self) -> None:  # nocoverage
+        assert self.q is not None
+        self.q.stop_consuming()
+
+
+class LoopQueueProcessingWorker(QueueProcessingWorker):
+    sleep_delay = 1
+    batch_size = 100
+
+    @override
+    def setup(self) -> None:
+        self.q = SimpleQueueClient(prefetch=max(self.PREFETCH, self.batch_size))
+
+    @override
+    def start(self) -> None:  # nocoverage
+        assert self.q is not None
+        self.initialize_statistics()
+        self.q.start_json_consumer(
+            self.queue_name,
+            lambda events: self.do_consume(self.consume_batch, events),
+            batch_size=self.batch_size,
+            timeout=self.sleep_delay,
+        )
+
+    @abstractmethod
+    def consume_batch(self, events: list[dict[str, Any]]) -> None:
+        pass
+
+    @override
+    def consume(self, event: dict[str, Any]) -> None:
+        """In LoopQueueProcessingWorker, consume is used just for automated tests"""
+        self.consume_batch([event])

zerver/worker/deferred_email_senders.py

@@ -0,0 +1,8 @@
+# Documented in https://zulip.readthedocs.io/en/latest/subsystems/queuing.html
+from zerver.worker.base import assign_queue
+from zerver.worker.email_senders_base import EmailSendingWorker
+
+
+@assign_queue("deferred_email_senders")
+class DeferredEmailSenderWorker(EmailSendingWorker):
+    pass

zerver/worker/deferred_work.py

@@ -0,0 +1,209 @@
+# Documented in https://zulip.readthedocs.io/en/latest/subsystems/queuing.html
+import logging
+import tempfile
+import time
+from typing import Any
+
+from django.conf import settings
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+from django.utils.translation import override as override_language
+from typing_extensions import override
+
+from zerver.actions.data_import import import_slack_data
+from zerver.actions.message_flags import do_mark_stream_messages_as_read
+from zerver.actions.message_send import internal_send_private_message
+from zerver.actions.realm_export import notify_realm_export
+from zerver.actions.realm_settings import scrub_deactivated_realm
+from zerver.lib.export import export_realm_wrapper, export_tarball_prefix
+from zerver.lib.push_notifications import clear_push_device_tokens
+from zerver.lib.queue import retry_event
+from zerver.lib.remote_server import (
+    PushNotificationBouncerRetryLaterError,
+    send_server_data_to_push_bouncer,
+)
+from zerver.lib.soft_deactivation import reactivate_user_if_soft_deactivated
+from zerver.lib.upload import handle_reupload_emojis_event
+from zerver.models import Realm, RealmAuditLog, RealmExport
+from zerver.models.users import get_system_bot, get_user_profile_by_id
+from zerver.worker.base import QueueProcessingWorker, assign_queue
+
+logger = logging.getLogger(__name__)
+
+
+@assign_queue("deferred_work")
+class DeferredWorker(QueueProcessingWorker):
+    """This queue processor is intended for cases where we want to trigger a
+    potentially expensive, not urgent, job to be run on a separate
+    thread from the Django worker that initiated it (E.g. so we that
+    can provide a low-latency HTTP response or avoid risk of request
+    timeouts for an operation that could in rare cases take minutes).
+    """
+
+    # Because these operations have no SLO, and can take minutes,
+    # remove any processing timeouts
+    MAX_CONSUME_SECONDS = None
+
+    @override
+    def consume(self, event: dict[str, Any]) -> None:
+        start = time.time()
+        if event["type"] == "mark_stream_messages_as_read":
+            user_profile = get_user_profile_by_id(event["user_profile_id"])
+            logger.info(
+                "Marking messages as read for user %s, stream_recipient_ids %s",
+                user_profile.id,
+                event["stream_recipient_ids"],
+            )
+
+            for recipient_id in event["stream_recipient_ids"]:
+                count = do_mark_stream_messages_as_read(user_profile, recipient_id)
+                logger.info(
+                    "Marked %s messages as read for user %s, stream_recipient_id %s",
+                    count,
+                    user_profile.id,
+                    recipient_id,
+                )
+        elif event["type"] == "clear_push_device_tokens":
+            logger.info(
+                "Clearing push device tokens for user_profile_id %s",
+                event["user_profile_id"],
+            )
+            try:
+                clear_push_device_tokens(event["user_profile_id"])
+            except PushNotificationBouncerRetryLaterError:
+
+                def failure_processor(event: dict[str, Any]) -> None:
+                    logger.warning(
+                        "Maximum retries exceeded for trigger:%s event:clear_push_device_tokens",
+                        event["user_profile_id"],
+                    )
+
+                retry_event(self.queue_name, event, failure_processor)
+        elif event["type"] == "realm_export":
+            user_profile = get_user_profile_by_id(event["user_profile_id"])
+            realm = user_profile.realm
+            output_dir = tempfile.mkdtemp(prefix=export_tarball_prefix(realm))
+            export_event = None
+
+            if "realm_export_id" in event:
+                export_row = RealmExport.objects.get(id=event["realm_export_id"])
+            else:
+                # Handle existing events in the queue before we switched to RealmExport model.
+                export_event = RealmAuditLog.objects.get(id=event["id"])
+                extra_data = export_event.extra_data
+
+                if extra_data.get("export_row_id") is not None:
+                    export_row = RealmExport.objects.get(id=extra_data["export_row_id"])
+                else:
+                    export_row = RealmExport.objects.create(
+                        realm=realm,
+                        type=RealmExport.EXPORT_PUBLIC,
+                        acting_user=user_profile,
+                        status=RealmExport.REQUESTED,
+                        date_requested=event["time"],
+                    )
+                    export_event.extra_data = {"export_row_id": export_row.id}
+                    export_event.save(update_fields=["extra_data"])
+
+            if export_row.status != RealmExport.REQUESTED:
+                logger.error(
+                    "Marking export for realm %s as failed due to retry -- possible OOM during export?",
+                    realm.string_id,
+                )
+                export_row.status = RealmExport.FAILED
+                export_row.date_failed = timezone_now()
+                export_row.save(update_fields=["status", "date_failed"])
+                notify_realm_export(realm)
+                return
+
+            logger.info(
+                "Starting realm export for realm %s into %s, initiated by user_profile_id %s",
+                realm.string_id,
+                output_dir,
+                user_profile.id,
+            )
+
+            try:
+                export_realm_wrapper(
+                    export_row=export_row,
+                    output_dir=output_dir,
+                    processes=1 if self.threaded else 6,
+                    upload=True,
+                )
+            except Exception:
+                logging.exception(
+                    "Data export for %s failed after %s",
+                    realm.string_id,
+                    time.time() - start,
+                    stack_info=True,
+                )
+                notify_realm_export(realm)
+                return
+
+            # We create RealmAuditLog entry in 'export_realm_wrapper'.
+            # Delete the old entry created before we switched to RealmExport model.
+            if export_event:
+                export_event.delete()
+
+            # Send a direct message notification letting the user who
+            # triggered the export know the export finished.
+            with override_language(user_profile.default_language):
+                content = _(
+                    "Your data export is complete. [View and download exports]({export_settings_link})."
+                ).format(export_settings_link="/#organization/data-exports-admin")
+            internal_send_private_message(
+                sender=get_system_bot(settings.NOTIFICATION_BOT, realm.id),
+                recipient_user=user_profile,
+                content=content,
+            )
+
+            # For future frontend use, also notify administrator
+            # clients that the export happened.
+            notify_realm_export(realm)
+            logging.info(
+                "Completed data export for %s in %s",
+                realm.string_id,
+                time.time() - start,
+            )
+        elif event["type"] == "reupload_realm_emoji":
+            # This is a special event queued by the migration for reuploading emojis.
+            # We don't want to run the necessary code in the actual migration, so it simply
+            # queues the necessary event, and the actual work is done here in the queue worker.
+            realm = Realm.objects.get(id=event["realm_id"])
+            logger.info("Processing reupload_realm_emoji event for realm %s", realm.id)
+            handle_reupload_emojis_event(realm, logger)
+        elif event["type"] == "soft_reactivate":
+            # Soft reactivations are normally enqueued here. A server can
+            # opt into a dedicated soft_reactivation queue via the
+            # dedicated_soft_reactivation_queue setting; even then, do not
+            # remove this handler. It remains the default path, and it
+            # drains any soft_reactivate events already in this queue when
+            # that option is enabled -- stale deferred_work events can
+            # linger across upgrades for a long time.
+            logger.info(
+                "Starting soft reactivation for user_profile_id %s",
+                event["user_profile_id"],
+            )
+            user_profile = get_user_profile_by_id(event["user_profile_id"])
+            reactivate_user_if_soft_deactivated(user_profile)
+        elif event["type"] == "push_bouncer_update_for_realm":
+            # In the future we may use the realm_id to send only that single realm's info.
+            realm_id = event["realm_id"]
+            logger.info("Updating push bouncer with metadata on behalf of realm %s", realm_id)
+            send_server_data_to_push_bouncer(consider_usage_statistics=False)
+        elif event["type"] == "scrub_deactivated_realm":
+            realms_to_scrub = Realm.objects.filter(
+                deactivated=True,
+                scheduled_deletion_date__lte=timezone_now(),
+            )
+            for realm in realms_to_scrub:
+                scrub_deactivated_realm(realm)
+        elif event["type"] == "import_slack_data":
+            import_slack_data(event)
+
+        end = time.time()
+        logger.info(
+            "deferred_work processed %s event (%dms)",
+            event["type"],
+            (end - start) * 1000,
+        )

zerver/worker/digest_emails.py

@@ -0,0 +1,25 @@
+# Documented in https://zulip.readthedocs.io/en/latest/subsystems/queuing.html
+import logging
+from collections.abc import Mapping
+from typing import Any
+
+from typing_extensions import override
+
+from zerver.lib.digest import bulk_handle_digest_email
+from zerver.worker.base import QueueProcessingWorker, assign_queue
+
+logger = logging.getLogger(__name__)
+
+
+@assign_queue("digest_emails")
+class DigestWorker(QueueProcessingWorker):  # nocoverage
+    # Who gets a digest is entirely determined by the enqueue_digest_emails
+    # management command, not here.
+    @override
+    def consume(self, event: Mapping[str, Any]) -> None:
+        if "user_ids" in event:
+            user_ids = event["user_ids"]
+        else:
+            # legacy code may have enqueued a single id
+            user_ids = [event["user_profile_id"]]
+        bulk_handle_digest_email(user_ids, event["cutoff"])

zerver/worker/email_mirror.py

@@ -0,0 +1,38 @@
+# Documented in https://zulip.readthedocs.io/en/latest/subsystems/queuing.html
+import base64
+import email.parser
+import email.policy
+import logging
+from collections.abc import Mapping
+from email.message import EmailMessage
+from typing import Any
+
+from typing_extensions import override
+
+from zerver.lib.email_mirror import process_message as mirror_email
+from zerver.worker.base import QueueProcessingWorker, WorkerTimeoutError, assign_queue
+
+logger = logging.getLogger(__name__)
+
+
+@assign_queue("email_mirror")
+class MirrorWorker(QueueProcessingWorker):
+    MAX_CONSUME_SECONDS = 5
+
+    @override
+    def consume(self, event: Mapping[str, Any]) -> None:
+        rcpt_to = event["rcpt_to"]
+        content = base64.b64decode(event["msg_base64"])
+        msg = email.parser.BytesParser(_class=EmailMessage, policy=email.policy.default).parsebytes(
+            content
+        )
+        try:
+            mirror_email(msg, rcpt_to=rcpt_to)
+        except WorkerTimeoutError:  # nocoverage
+            logging.error(
+                "Timed out ingesting message-id %s to %s (%d bytes) -- dropping!",
+                msg["Message-ID"] or "<?>",
+                rcpt_to,
+                len(content),
+            )
+            return

zerver/worker/email_senders.py

@@ -0,0 +1,8 @@
+# Documented in https://zulip.readthedocs.io/en/latest/subsystems/queuing.html
+from zerver.worker.base import assign_queue
+from zerver.worker.email_senders_base import EmailSendingWorker
+
+
+@assign_queue("email_senders")
+class ImmediateEmailSenderWorker(EmailSendingWorker):
+    pass

zerver/worker/email_senders_base.py

@@ -0,0 +1,92 @@
+# Documented in https://zulip.readthedocs.io/en/latest/subsystems/queuing.html
+import copy
+import logging
+from collections.abc import Callable
+from functools import wraps
+from typing import Any
+
+from django.core.mail import get_connection
+from typing_extensions import override
+
+from zerver.lib.queue import retry_event
+from zerver.lib.send_email import (
+    EmailNotDeliveredError,
+    handle_send_email_format_changes,
+    send_immediate_email,
+)
+from zerver.models import Realm
+from zerver.worker.base import ConcreteQueueWorker, LoopQueueProcessingWorker
+from zproject.email_backends import PersistentSMTPEmailBackend
+
+logger = logging.getLogger(__name__)
+
+
+# If you change the function on which this decorator is used be careful that the new
+# function doesn't delete the "failed_tries" attribute of "data" which is needed for
+# "retry_event" to work correctly; see EmailSendingWorker for an example with deepcopy.
+def retry_send_email_failures(
+    func: Callable[[ConcreteQueueWorker, dict[str, Any]], None],
+) -> Callable[[ConcreteQueueWorker, dict[str, Any]], None]:
+    @wraps(func)
+    def wrapper(worker: ConcreteQueueWorker, data: dict[str, Any]) -> None:
+        try:
+            func(worker, data)
+        except EmailNotDeliveredError as e:
+            error_class_name = type(e).__name__
+
+            def on_failure(event: dict[str, Any]) -> None:
+                logging.exception(
+                    "Event %r failed due to exception %s", event, error_class_name, stack_info=True
+                )
+
+            retry_event(worker.queue_name, data, on_failure)
+
+    return wrapper
+
+
+class EmailSendingWorker(LoopQueueProcessingWorker):
+    def __init__(
+        self,
+        threaded: bool = False,
+        disable_timeout: bool = False,
+        worker_num: int | None = None,
+    ) -> None:
+        super().__init__(threaded, disable_timeout, worker_num)
+        connection = get_connection()
+        if isinstance(connection, PersistentSMTPEmailBackend):
+            self.connection: PersistentSMTPEmailBackend | None = connection
+        else:
+            self.connection = None
+
+    @retry_send_email_failures
+    def send_email(self, event: dict[str, Any]) -> None:
+        # Copy the event, so that we don't pass the `failed_tries'
+        # data to send_email (which neither takes that
+        # argument nor needs that data).
+        copied_event = copy.deepcopy(event)
+        if "failed_tries" in copied_event:
+            del copied_event["failed_tries"]
+        handle_send_email_format_changes(copied_event)
+        if "realm_id" in copied_event:
+            # "realm" does not serialize over the queue, so we send the realm_id
+            if copied_event.get("realm_id") is not None:
+                copied_event["realm"] = Realm.objects.get(id=copied_event["realm_id"])
+            del copied_event["realm_id"]
+        if self.connection is not None:
+            self.connection.ensure_connected()
+            send_immediate_email(**copied_event, connection=self.connection)
+        else:
+            send_immediate_email(**copied_event)
+
+    @override
+    def consume_batch(self, events: list[dict[str, Any]]) -> None:
+        for event in events:
+            self.send_email(event)
+
+    @override
+    def stop(self) -> None:  # nocoverage
+        try:
+            if self.connection is not None:
+                self.connection.close()
+        finally:
+            super().stop()

zerver/worker/embed_links.py

@@ -0,0 +1,89 @@
+# Documented in https://zulip.readthedocs.io/en/latest/subsystems/queuing.html
+import logging
+import time
+from collections.abc import Mapping
+from types import FrameType
+from typing import Any
+
+from django.db import transaction
+from typing_extensions import override
+
+from zerver.actions.message_edit import do_update_embedded_data
+from zerver.actions.message_send import render_incoming_message
+from zerver.lib.mention import MentionBackend, MentionData
+from zerver.lib.url_preview import preview as url_preview
+from zerver.lib.url_preview.types import UrlEmbedData
+from zerver.models import Message, Realm
+from zerver.worker.base import InterruptConsumeError, QueueProcessingWorker, assign_queue
+
+logger = logging.getLogger(__name__)
+
+
+@assign_queue("embed_links")
+class FetchLinksEmbedData(QueueProcessingWorker):
+    # This is a slow queue with network requests, so a disk write is negligible.
+    # Update stats file after every consume call.
+    CONSUME_ITERATIONS_BEFORE_UPDATE_STATS_NUM = 1
+
+    @override
+    def consume(self, event: Mapping[str, Any]) -> None:
+        url_embed_data: dict[str, UrlEmbedData | None] = {}
+        for url in event["urls"]:
+            start_time = time.time()
+            url_embed_data[url] = url_preview.get_link_embed_data(url)
+            logging.info(
+                "Time spent on get_link_embed_data for %s: %s", url, time.time() - start_time
+            )
+
+        # Ideally, we should use `durable=True` here. However, in the
+        # `test_message_update_race_condition` test, this function is not called
+        # as the outermost transaction. As a result, it's acceptable to make an
+        # exception and not use `durable=True` in this case.
+        #
+        # For more details on why the `consume` method is called directly in tests, see:
+        # https://zulip.readthedocs.io/en/latest/subsystems/queuing.html#publishing-events-into-a-queue
+        with transaction.atomic(savepoint=False):
+            try:
+                message = Message.objects.select_for_update(no_key=True).get(id=event["message_id"])
+            except Message.DoesNotExist:
+                # Message may have been deleted
+                return
+
+            # If the message changed, we will run this task after updating the message
+            # in zerver.actions.message_edit.check_update_message
+            if message.content != event["message_content"]:
+                return
+
+            # Fetch the realm whose settings we're using for rendering
+            realm = Realm.objects.get(id=event["message_realm_id"])
+
+            # If rendering fails, the called code will raise a JsonableError.
+            mention_data = MentionData(
+                mention_backend=MentionBackend(message.realm_id),
+                content=message.content,
+                message_sender=message.sender,
+            )
+            rendering_result = render_incoming_message(
+                message,
+                message.content,
+                realm,
+                url_embed_data=url_embed_data,
+                mention_data=mention_data,
+            )
+            do_update_embedded_data(message.sender, message, rendering_result, mention_data)
+
+    @override
+    def timer_expired(
+        self, limit: int, events: list[dict[str, Any]], signal: int, frame: FrameType | None
+    ) -> None:
+        assert len(events) == 1
+        event = events[0]
+
+        logging.warning(
+            "Timed out in %s after %s seconds while fetching URLs for message %s: %s",
+            self.queue_name,
+            limit,
+            event["message_id"],
+            event["urls"],
+        )
+        raise InterruptConsumeError

zerver/worker/embedded_bots.py

@@ -0,0 +1,61 @@
+# Documented in https://zulip.readthedocs.io/en/latest/subsystems/queuing.html
+import logging
+from collections.abc import Mapping
+from typing import Any
+
+from typing_extensions import override
+from zulip_bots.lib import extract_query_without_mention
+
+from zerver.lib.bot_lib import (
+    EmbeddedBotHandler,
+    EmbeddedBotQuitError,
+    do_flag_message_triggered_bots_messages_as_processed,
+    get_bot_handler,
+)
+from zerver.models import UserProfile
+from zerver.models.bots import get_bot_services
+from zerver.models.users import get_user_profile_by_id
+from zerver.worker.base import QueueProcessingWorker, assign_queue
+
+logger = logging.getLogger(__name__)
+
+
+@assign_queue("embedded_bots")
+class EmbeddedBotWorker(QueueProcessingWorker):
+    def get_bot_api_client(self, user_profile: UserProfile) -> EmbeddedBotHandler:
+        return EmbeddedBotHandler(user_profile)
+
+    @override
+    def consume(self, event: Mapping[str, Any]) -> None:
+        user_profile_id = event["user_profile_id"]
+        user_profile = get_user_profile_by_id(user_profile_id)
+
+        message: dict[str, Any] = event["message"]
+
+        # TODO: Do we actually want to allow multiple Services per bot user?
+        services = get_bot_services(user_profile_id)
+        for service in services:
+            bot_handler = get_bot_handler(str(service.name))
+            if bot_handler is None:
+                logging.error(
+                    "Error: User %s has bot with invalid embedded bot service %s",
+                    user_profile_id,
+                    service.name,
+                )
+                continue
+            try:
+                if hasattr(bot_handler, "initialize"):
+                    bot_handler.initialize(self.get_bot_api_client(user_profile))
+                if event["trigger"] == "mention":
+                    message["content"] = extract_query_without_mention(
+                        message=message,
+                        client=self.get_bot_api_client(user_profile),
+                    )
+                    assert message["content"] is not None
+                bot_handler.handle_message(
+                    message=message,
+                    bot_handler=self.get_bot_api_client(user_profile),
+                )
+            except EmbeddedBotQuitError as e:
+                logging.warning("%s", e)
+        do_flag_message_triggered_bots_messages_as_processed(user_profile, [message["id"]])

zerver/worker/missedmessage_emails.py

@@ -0,0 +1,288 @@
+# Documented in https://zulip.readthedocs.io/en/latest/subsystems/queuing.html
+import logging
+import threading
+from collections import defaultdict
+from datetime import timedelta
+from typing import Any
+
+import sentry_sdk
+from django.conf import settings
+from django.db import transaction
+from django.db.utils import IntegrityError
+from django.utils.timezone import now as timezone_now
+from typing_extensions import override
+
+from zerver.lib.db_connections import reset_queries
+from zerver.lib.email_notifications import MissedMessageData, handle_missedmessage_emails
+from zerver.lib.per_request_cache import flush_per_request_caches
+from zerver.models import ScheduledMessageNotificationEmail
+from zerver.models.users import get_user_profile_by_id
+from zerver.worker.base import QueueProcessingWorker, assign_queue
+
+logger = logging.getLogger(__name__)
+
+
+@assign_queue("missedmessage_emails")
+class MissedMessageWorker(QueueProcessingWorker):
+    # Aggregate all messages received over the last several seconds
+    # (configurable by each recipient) to let someone finish sending a
+    # batch of messages and/or editing them before they are sent out
+    # as emails to recipients.
+    #
+    # The batch interval is best-effort -- we poll at most every
+    # CHECK_FREQUENCY_SECONDS, to avoid excessive activity.
+    CHECK_FREQUENCY_SECONDS = 5
+
+    worker_thread: threading.Thread | None = None
+
+    # This condition variable mediates the stopping and has_timeout
+    # pieces of state, below it.
+    cv = threading.Condition()
+    stopping = False
+    has_timeout = False
+
+    # The main thread, which handles the RabbitMQ connection and creates
+    # database rows from them.
+    @override
+    @sentry_sdk.trace
+    def consume(self, event: dict[str, Any]) -> None:
+        logging.debug("Processing missedmessage_emails event: %s", event)
+        # When we consume an event, check if there are existing pending emails
+        # for that user, and if so use the same scheduled timestamp.
+
+        user_profile_id: int = event["user_profile_id"]
+        user_profile = get_user_profile_by_id(user_profile_id)
+        batch_duration_seconds = user_profile.email_notifications_batching_period_seconds
+        batch_duration = timedelta(seconds=batch_duration_seconds)
+
+        try:
+            pending_email = ScheduledMessageNotificationEmail.objects.filter(
+                user_profile_id=user_profile_id
+            )[0]
+            scheduled_timestamp = pending_email.scheduled_timestamp
+        except IndexError:
+            scheduled_timestamp = timezone_now() + batch_duration
+
+        with self.cv:
+            # We now hold the lock, so there are three places the
+            # worker thread can be:
+            #
+            #  1. In maybe_send_batched_emails, and will have to take
+            #     the lock (and thus block insertions of new rows
+            #     here) to decide if there are any rows and if it thus
+            #     needs a timeout.
+            #
+            #  2. In the cv.wait_for with a timeout because there were
+            #     rows already.  There's nothing for us to do, since
+            #     the newly-inserted row will get checked upon that
+            #     timeout.
+            #
+            #  3. In the cv.wait_for without a timeout, because there
+            #     weren't any rows (which we're about to change).
+            #
+            # Notifying in (1) is irrelevant, since the thread is not
+            # waiting.  If we over-notify by doing so for both (2) and
+            # (3), the behaviour is correct but slightly inefficient,
+            # as the thread will be needlessly awoken and will just
+            # re-wait.  However, if we fail to awake case (3), the
+            # worker thread will never wake up, and the
+            # ScheduledMessageNotificationEmail internal queue will
+            # back up.
+            #
+            # Use the self.has_timeout property (which is protected by
+            # the lock) to determine which of cases (2) or (3) we are
+            # in, and as such if we need to notify after making the
+            # row.
+            try:
+                ScheduledMessageNotificationEmail.objects.create(
+                    user_profile_id=user_profile_id,
+                    message_id=event["message_id"],
+                    trigger=event["trigger"],
+                    scheduled_timestamp=scheduled_timestamp,
+                    mentioned_user_group_id=event.get("mentioned_user_group_id"),
+                )
+                if not self.has_timeout:
+                    self.cv.notify()
+            except IntegrityError:
+                logging.debug(
+                    "ScheduledMessageNotificationEmail row could not be created. The message may have been deleted. Skipping event."
+                )
+
+    @override
+    def start(self) -> None:
+        with self.cv:
+            self.stopping = False
+
+        # Do nothing to process events on staging servers, since we do
+        # not support running multiple copies of this worker.
+        if settings.STAGING:
+            with self.cv:
+                self.cv.wait()
+            return
+
+        self.worker_thread = threading.Thread(target=self.work)
+        self.worker_thread.start()
+        super().start()
+
+    def work(self) -> None:
+        backoff = 1
+        while True:
+            with sentry_sdk.start_transaction(
+                op="task",
+                name=f"{self.queue_name} worker thread",
+                custom_sampling_context={"queue": self.queue_name},
+            ):
+                flush_per_request_caches()
+                reset_queries()
+                try:
+                    finished = self.background_loop()
+                    if finished:
+                        break
+                    # Success running the background loop; reset our backoff
+                    backoff = 1
+                except Exception:
+                    logging.exception(
+                        "Exception in MissedMessage background worker; restarting the loop",
+                        stack_info=True,
+                    )
+
+                    # We want to sleep, with backoff, before retrying
+                    # the background loop; there may be
+                    # non-recoverable errors which cause immediate
+                    # exceptions, and we should avoid fast
+                    # crash-looping.  Instead of using time.sleep,
+                    # which would block this thread and delay attempts
+                    # to exit, we wait on the condition variable.
+                    # With has_timeout set, this will only be notified
+                    # by .stop(), below.
+                    #
+                    # Generally, delays in this background process are
+                    # acceptable, so long as they at least
+                    # occasionally retry.
+                    with self.cv:
+                        self.has_timeout = True
+                        self.cv.wait(timeout=backoff)
+                    backoff = min(30, backoff * 2)
+
+    def background_loop(self) -> bool:
+        with self.cv:
+            if self.stopping:
+                return True
+            # There are three conditions which we wait for:
+            #
+            #  1. We are being explicitly asked to stop; see the
+            #     notify() call in stop()
+            #
+            #  2. We have no ScheduledMessageNotificationEmail
+            #     objects currently (has_timeout = False) and the
+            #     first one was just enqueued; see the notify()
+            #     call in consume().  We break out so that we can
+            #     come back around the loop and re-wait with a
+            #     timeout (see next condition).
+            #
+            #  3. One or more ScheduledMessageNotificationEmail
+            #     exist in the database, so we need to re-check
+            #     them regularly; this happens by hitting the
+            #     timeout and calling maybe_send_batched_emails().
+            #     There is no explicit notify() for this.
+            timeout: int | None = None
+            if ScheduledMessageNotificationEmail.objects.exists():
+                timeout = self.CHECK_FREQUENCY_SECONDS
+            self.has_timeout = timeout is not None
+
+            def wait_condition() -> bool:
+                if self.stopping:
+                    # Condition (1)
+                    return True
+                if timeout is None:
+                    # Condition (2).  We went to sleep with no
+                    # ScheduledMessageNotificationEmail existing,
+                    # and one has just been made.  We re-check
+                    # that is still true now that we have the
+                    # lock, and if we see it, we stop waiting.
+                    return ScheduledMessageNotificationEmail.objects.exists()
+                # This should only happen at the start or end of
+                # the wait, when we haven't been notified, but are
+                # re-checking the condition.
+                return False
+
+            with sentry_sdk.start_span(name="condvar wait") as span:
+                span.set_data("timeout", timeout)
+                was_notified = self.cv.wait_for(wait_condition, timeout=timeout)
+                span.set_data("was_notified", was_notified)
+
+        # Being notified means that we are in conditions (1) or
+        # (2), above.  In neither case do we need to look at if
+        # there are batches to send -- (2) means that the
+        # ScheduledMessageNotificationEmail was _just_ created, so
+        # there is no need to check it now.
+        if not was_notified:
+            self.maybe_send_batched_emails()
+
+        return False
+
+    @sentry_sdk.trace
+    def maybe_send_batched_emails(self) -> None:
+        current_time = timezone_now()
+
+        with transaction.atomic(durable=True):
+            events_to_process = ScheduledMessageNotificationEmail.objects.filter(
+                scheduled_timestamp__lte=current_time
+            ).select_for_update(
+                # We intend to delete these rows later, so we need a
+                # FOR UPDATE lock.
+                no_key=False,
+                # If another worker (or message deletion) has locked
+                # one of the rows, we can look at it later (if it
+                # still exists).  This avoids deadlocks.
+                skip_locked=True,
+            )
+
+            # Batch the entries by user
+            events_by_recipient: dict[int, dict[int, MissedMessageData]] = defaultdict(dict)
+            seen_events = []
+            for event in events_to_process:
+                seen_events.append(event.id)
+                events_by_recipient[event.user_profile_id][event.message_id] = MissedMessageData(
+                    trigger=event.trigger, mentioned_user_group_id=event.mentioned_user_group_id
+                )
+
+            for user_profile_id, events in events_by_recipient.items():
+                logging.info(
+                    "Batch-processing %s missedmessage_emails events for user %s",
+                    len(events),
+                    user_profile_id,
+                )
+                with sentry_sdk.start_span(name="sending missedmessage_emails to user") as span:
+                    span.set_data("user_profile_id", user_profile_id)
+                    span.set_data("event_count", len(events))
+                    try:
+                        # Because we process events in batches, an
+                        # escaped exception here would lead to
+                        # duplicate messages being sent for other
+                        # users in the same events_to_process batch,
+                        # and no guarantee of forward progress.
+                        handle_missedmessage_emails(user_profile_id, events)
+                    except Exception:
+                        logging.exception(
+                            "Failed to process %d missedmessage_emails for user %s",
+                            len(events),
+                            user_profile_id,
+                            stack_info=True,
+                        )
+
+            # These were already locked, above.
+            ScheduledMessageNotificationEmail.objects.filter(id__in=seen_events).delete()
+
+    @override
+    def stop(self) -> None:
+        with self.cv:
+            self.stopping = True
+            self.cv.notify()
+        if self.worker_thread is not None:
+            self.worker_thread.join()
+
+        if settings.STAGING:
+            return
+
+        super().stop()

zerver/worker/missedmessage_mobile_notifications.py

@@ -0,0 +1,72 @@
+# Documented in https://zulip.readthedocs.io/en/latest/subsystems/queuing.html
+import logging
+from typing import Any
+
+from django.conf import settings
+from typing_extensions import override
+
+from zerver.lib.push_notifications import (
+    handle_push_notification,
+    handle_remove_push_notification,
+    initialize_push_notifications,
+)
+from zerver.lib.push_registration import handle_register_push_device_to_bouncer
+from zerver.lib.queue import retry_event
+from zerver.lib.remote_server import PushNotificationBouncerRetryLaterError
+from zerver.worker.base import QueueProcessingWorker, assign_queue
+
+logger = logging.getLogger(__name__)
+
+
+@assign_queue("missedmessage_mobile_notifications")
+class PushNotificationsWorker(QueueProcessingWorker):
+    # The use of aioapns in the backend means that we cannot use
+    # SIGALRM to limit how long a consume takes, as SIGALRM does not
+    # play well with asyncio.
+    MAX_CONSUME_SECONDS = None
+
+    @override
+    def __init__(
+        self,
+        threaded: bool = False,
+        disable_timeout: bool = False,
+        worker_num: int | None = None,
+    ) -> None:
+        if settings.MOBILE_NOTIFICATIONS_SHARDS > 1 and worker_num is not None:  # nocoverage
+            self.queue_name += f"_shard{worker_num}"
+        super().__init__(threaded, disable_timeout, worker_num)
+
+    @override
+    def start(self) -> None:
+        # initialize_push_notifications doesn't strictly do anything
+        # beyond printing some logging warnings if push notifications
+        # are not available in the current configuration.
+        initialize_push_notifications()
+        super().start()
+
+    @override
+    def consume(self, event: dict[str, Any]) -> None:
+        try:
+            event_type = event.get("type")
+            if event_type == "register_push_device_to_bouncer":
+                handle_register_push_device_to_bouncer(event["payload"])
+            elif event_type == "remove":
+                message_ids = event["message_ids"]
+                handle_remove_push_notification(event["user_profile_id"], message_ids)
+            else:
+                handle_push_notification(event["user_profile_id"], event)
+        except PushNotificationBouncerRetryLaterError:
+
+            def failure_processor(event: dict[str, Any]) -> None:
+                if event_type == "register_push_device_to_bouncer":
+                    logger.warning(
+                        "Maximum retries exceeded for device_id:%s event:register_push_device_to_bouncer",
+                        event["payload"]["device_id"],
+                    )
+                else:
+                    logger.warning(
+                        "Maximum retries exceeded for trigger:%s event:push_notification",
+                        event["user_profile_id"],
+                    )
+
+            retry_event(self.queue_name, event, failure_processor)

zerver/worker/outgoing_webhooks.py

@@ -0,0 +1,30 @@
+# Documented in https://zulip.readthedocs.io/en/latest/subsystems/queuing.html
+import logging
+from typing import Any
+
+from typing_extensions import override
+
+from zerver.lib.bot_lib import do_flag_message_triggered_bots_messages_as_processed
+from zerver.lib.outgoing_webhook import do_rest_call, get_outgoing_webhook_service_handler
+from zerver.models.bots import get_bot_services
+from zerver.models.users import get_user_profile_by_id
+from zerver.worker.base import QueueProcessingWorker, assign_queue
+
+logger = logging.getLogger(__name__)
+
+
+@assign_queue("outgoing_webhooks")
+class OutgoingWebhookWorker(QueueProcessingWorker):
+    @override
+    def consume(self, event: dict[str, Any]) -> None:
+        message = event["message"]
+        event["command"] = message["content"]
+        bot_profile = get_user_profile_by_id(event["user_profile_id"])
+
+        services = get_bot_services(event["user_profile_id"])
+        for service in services:
+            event["service_name"] = str(service.name)
+            service_handler = get_outgoing_webhook_service_handler(service)
+            do_rest_call(service.base_url, event, service_handler)
+
+        do_flag_message_triggered_bots_messages_as_processed(bot_profile, [message["id"]])

zerver/worker/queue_processors.py

@@ -0,0 +1,44 @@
+# Documented in https://zulip.readthedocs.io/en/latest/subsystems/queuing.html
+import importlib
+import pkgutil
+
+from django.conf import settings
+
+import zerver.worker
+from zerver.worker.base import QueueProcessingWorker, test_queues, worker_classes
+
+
+def get_worker(
+    queue_name: str,
+    *,
+    threaded: bool = False,
+    disable_timeout: bool = False,
+    worker_num: int | None = None,
+) -> QueueProcessingWorker:
+    if queue_name in {"test", "noop", "noop_batch"}:
+        import_module = "zerver.worker.test"
+    else:
+        import_module = f"zerver.worker.{queue_name}"
+
+    importlib.import_module(import_module)
+    return worker_classes[queue_name](
+        threaded=threaded, disable_timeout=disable_timeout, worker_num=worker_num
+    )
+
+
+def get_active_worker_queues(only_test_queues: bool = False) -> list[str]:
+    """Returns the worker queues that should run on this server, honoring
+    configuration that gates a queue on a setting."""
+    for module_info in pkgutil.iter_modules(zerver.worker.__path__, "zerver.worker."):
+        importlib.import_module(module_info.name)
+
+    queues = [
+        queue_name
+        for queue_name in worker_classes
+        if bool(queue_name in test_queues) == only_test_queues
+    ]
+    if not settings.DEDICATED_SOFT_REACTIVATION_QUEUE:
+        # Soft reactivations share the deferred_work queue unless a server
+        # opts into a dedicated queue, so its worker is not run otherwise.
+        queues = [queue_name for queue_name in queues if queue_name != "soft_reactivation"]
+    return queues

zerver/worker/soft_reactivation.py

@@ -0,0 +1,41 @@
+# Documented in https://zulip.readthedocs.io/en/latest/subsystems/queuing.html
+import logging
+from typing import Any
+
+from typing_extensions import override
+
+from zerver.lib.soft_deactivation import reactivate_user_if_soft_deactivated
+from zerver.models.users import get_user_profile_by_id
+from zerver.worker.base import QueueProcessingWorker, assign_queue
+
+logger = logging.getLogger(__name__)
+
+
+@assign_queue("soft_reactivation")
+class SoftReactivationWorker(QueueProcessingWorker):
+    """Backfills the UserMessage rows skipped while a long-term-idle user was
+    soft-deactivated, in response to a signal that the user is likely about to
+    return (a notification or password-reset email).
+
+    It's undesirable for these jobs to share the deferred_work queue with jobs
+    such as realm exports, which can take many minutes: a prolonged delay in
+    processing a soft reactivation gives the returning user a bad experience,
+    and can race with the synchronous reactivation that runs when the user
+    loads the app. A server can opt into this dedicated queue via the
+    DEDICATED_SOFT_REACTIVATION_QUEUE setting.
+    """
+
+    # Soft reactivation has always run without a per-event timeout, as part of
+    # the deferred_work queue, and the backfill for a very idle user can
+    # legitimately take a while; preserve that rather than risk killing a job
+    # partway through.
+    MAX_CONSUME_SECONDS = None
+
+    @override
+    def consume(self, event: dict[str, Any]) -> None:
+        logger.info(
+            "Starting soft reactivation for user_profile_id %s",
+            event["user_profile_id"],
+        )
+        user_profile = get_user_profile_by_id(event["user_profile_id"])
+        reactivate_user_if_soft_deactivated(user_profile)

zerver/worker/test.py

@@ -0,0 +1,87 @@
+# Documented in https://zulip.readthedocs.io/en/latest/subsystems/queuing.html
+import logging
+import time
+from collections.abc import Mapping, Sequence
+from typing import Any
+
+import orjson
+from django.conf import settings
+from typing_extensions import override
+
+from zerver.worker.base import LoopQueueProcessingWorker, QueueProcessingWorker, assign_queue
+
+logger = logging.getLogger(__name__)
+
+
+@assign_queue("test", is_test_queue=True)
+class TestWorker(QueueProcessingWorker):
+    # This worker allows you to test the queue worker infrastructure without
+    # creating significant side effects.  It can be useful in development or
+    # for troubleshooting prod/staging.  It pulls a message off the test queue
+    # and appends it to a file in /var/log/zulip.
+    @override
+    def consume(self, event: Mapping[str, Any]) -> None:  # nocoverage
+        fn = settings.ZULIP_WORKER_TEST_FILE
+        message = orjson.dumps(event)
+        logging.info("TestWorker should append this message to %s: %s", fn, message.decode())
+        with open(fn, "ab") as f:
+            f.write(message + b"\n")
+
+
+@assign_queue("noop", is_test_queue=True)
+class NoopWorker(QueueProcessingWorker):
+    """Used to profile the queue processing framework, in zilencer's queue_rate."""
+
+    def __init__(
+        self,
+        threaded: bool = False,
+        disable_timeout: bool = False,
+        worker_num: int | None = None,
+        max_consume: int = 1000,
+        slow_queries: Sequence[int] = [],
+    ) -> None:
+        super().__init__(threaded, disable_timeout, worker_num)
+        self.consumed = 0
+        self.max_consume = max_consume
+        self.slow_queries: set[int] = set(slow_queries)
+
+    @override
+    def consume(self, event: Mapping[str, Any]) -> None:
+        self.consumed += 1
+        if self.consumed in self.slow_queries:
+            logging.info("Slow request...")
+            time.sleep(60)
+            logging.info("Done!")
+        if self.consumed >= self.max_consume:
+            self.stop()
+
+
+@assign_queue("noop_batch", is_test_queue=True)
+class BatchNoopWorker(LoopQueueProcessingWorker):
+    """Used to profile the queue processing framework, in zilencer's queue_rate."""
+
+    batch_size = 100
+
+    def __init__(
+        self,
+        threaded: bool = False,
+        disable_timeout: bool = False,
+        max_consume: int = 1000,
+        slow_queries: Sequence[int] = [],
+    ) -> None:
+        super().__init__(threaded, disable_timeout)
+        self.consumed = 0
+        self.max_consume = max_consume
+        self.slow_queries: set[int] = set(slow_queries)
+
+    @override
+    def consume_batch(self, events: list[dict[str, Any]]) -> None:
+        event_numbers = set(range(self.consumed + 1, self.consumed + 1 + len(events)))
+        found_slow = self.slow_queries & event_numbers
+        if found_slow:
+            logging.info("%d slow requests...", len(found_slow))
+            time.sleep(60 * len(found_slow))
+            logging.info("Done!")
+        self.consumed += len(events)
+        if self.consumed >= self.max_consume:
+            self.stop()

zerver/worker/thumbnail.py

@@ -0,0 +1,255 @@
+import logging
+import time
+from dataclasses import asdict, dataclass
+from io import BytesIO
+from typing import Any
+
+import pyvips
+from django.db import transaction
+from typing_extensions import override
+
+from zerver.actions.message_edit import do_update_embedded_data
+from zerver.lib.mime_types import guess_type
+from zerver.lib.thumbnail import (
+    IMAGE_MAX_ANIMATED_PIXELS,
+    MarkdownImageMetadata,
+    StoredThumbnailFormat,
+    get_default_thumbnail_url,
+    get_image_thumbnail_path,
+    get_transcoded_format,
+    missing_thumbnails,
+    rewrite_thumbnailed_images,
+)
+from zerver.lib.upload import save_attachment_contents, store_message_attachment
+from zerver.models import ArchivedMessage, ImageAttachment, Message
+from zerver.worker.base import QueueProcessingWorker, assign_queue
+
+logger = logging.getLogger(__name__)
+
+
+@assign_queue("thumbnail")
+class ThumbnailWorker(QueueProcessingWorker):
+    @override
+    def consume(self, event: dict[str, Any]) -> None:
+        start_time = time.perf_counter()
+        with transaction.atomic(savepoint=False):
+            logger.info("Starting thumbnailing for %s", event.get("path_id", f"id: {event['id']}"))
+            try:
+                # This lock prevents us from racing with the on-demand
+                # rendering that can be triggered if a request is made
+                # directly to a thumbnail URL we have not made yet.
+                # This may mean that we may generate 0 thumbnail
+                # images once we get the lock.
+                # If the image data is bad and thumbnailing fails, the
+                # ImageAttachment row might be deleted; so we need a FOR UPDATE
+                # lock.
+                row = ImageAttachment.objects.select_for_update(of=("self",), no_key=False).get(
+                    id=event["id"]
+                )
+            except ImageAttachment.DoesNotExist:  # nocoverage
+                logger.info(
+                    "ImageAttachment row %d missing [%dms]",
+                    event["id"],
+                    (time.perf_counter() - start_time) * 1000,
+                )
+                return
+            lock_time = time.perf_counter() - start_time
+            result = ensure_thumbnails(row)
+            commit_time = time.perf_counter()
+        end_time = time.perf_counter()
+        logger.info(
+            "Processed %d thumbnails [%dms; lock=%dms, fetch=%dms, thumb=%dms, upload=%dms, rewrite=%dms, commit=%dms]",
+            result.generated_thumbnail_count,
+            (end_time - start_time) * 1000,
+            lock_time * 1000,
+            result.fetching_seconds * 1000,
+            result.processing_seconds * 1000,
+            result.uploading_seconds * 1000,
+            result.message_update_seconds * 1000,
+            (end_time - commit_time) * 1000,
+        )
+
+
+@dataclass
+class ThumbnailingResult:
+    generated_thumbnail_count: int = 0
+    fetching_seconds: float = 0
+    processing_seconds: float = 0
+    uploading_seconds: float = 0
+    message_update_seconds: float = 0
+
+
+def ensure_thumbnails(image_attachment: ImageAttachment) -> ThumbnailingResult:
+    needed_thumbnails = missing_thumbnails(image_attachment)
+
+    if not needed_thumbnails:
+        return ThumbnailingResult()
+
+    written_images = 0
+    with BytesIO() as f:
+        start_time = time.perf_counter()
+        save_attachment_contents(image_attachment.path_id, f)
+        image_bytes = f.getvalue()
+        fetching_seconds = time.perf_counter() - start_time
+    try:
+        # TODO: We could save some computational time by using the same
+        # bytes if multiple resolutions are larger than the source
+        # image.  That is, if the input is 10x10, a 100x100.jpg is
+        # going to be the same as a 200x200.jpg, since those set the
+        # max dimensions, and we do not scale up.
+        processing_seconds = 0.0
+        uploading_seconds = 0.0
+        for thumbnail_format in needed_thumbnails:
+            # This will scale to fit within the given dimensions; it
+            # may be smaller in one or more of them.
+            logger.debug(
+                "Resizing to %d x %d, from %d x %d",
+                thumbnail_format.max_width,
+                thumbnail_format.max_height,
+                image_attachment.original_width_px,
+                image_attachment.original_height_px,
+            )
+            load_opts = ""
+            if image_attachment.frames > 1:
+                # If the original has multiple frames, we want to load
+                # one of them if we're outputting to a static format,
+                # otherwise we load them all.
+                if thumbnail_format.animated:
+                    # We compute how many frames to thumbnail based on
+                    # how many frames it will take us to get to
+                    # IMAGE_MAX_ANIMATED_PIXELS
+                    pixels_per_frame = (
+                        image_attachment.original_width_px * image_attachment.original_height_px
+                    )
+                    if pixels_per_frame * image_attachment.frames < IMAGE_MAX_ANIMATED_PIXELS:
+                        load_opts = "n=-1"
+                    else:
+                        load_opts = f"n={IMAGE_MAX_ANIMATED_PIXELS // pixels_per_frame}"
+                else:
+                    load_opts = "n=1"
+            start_time = time.perf_counter()
+            resized = pyvips.Image.thumbnail_buffer(
+                image_bytes,
+                thumbnail_format.max_width,
+                height=thumbnail_format.max_height,
+                option_string=load_opts,
+                size=pyvips.Size.DOWN,
+            )
+            thumbnailed_bytes = resized.write_to_buffer(
+                f".{thumbnail_format.extension}[{thumbnail_format.opts}]"
+            )
+            processing_seconds += time.perf_counter() - start_time
+            content_type = guess_type(f"image.{thumbnail_format.extension}")[0]
+            assert content_type is not None
+            thumbnail_path = get_image_thumbnail_path(image_attachment, thumbnail_format)
+            logger.debug("Uploading %d bytes to %s", len(thumbnailed_bytes), thumbnail_path)
+            start_time = time.perf_counter()
+            store_message_attachment(
+                thumbnail_path,
+                str(thumbnail_format),
+                content_type,
+                thumbnailed_bytes,
+                None,
+                None,
+            )
+            uploading_seconds += time.perf_counter() - start_time
+            height = resized.get("page-height") if thumbnail_format.animated else resized.height
+            image_attachment.thumbnail_metadata.append(
+                asdict(
+                    StoredThumbnailFormat(
+                        extension=thumbnail_format.extension,
+                        content_type=content_type,
+                        max_width=thumbnail_format.max_width,
+                        max_height=thumbnail_format.max_height,
+                        animated=thumbnail_format.animated,
+                        width=resized.width,
+                        height=height,
+                        byte_size=len(thumbnailed_bytes),
+                    )
+                )
+            )
+            written_images += 1
+
+    except pyvips.Error as e:
+        logger.exception(
+            "Failed to thumbnail %s, %d byte %s @ %dx%d: %s",
+            image_attachment.path_id,
+            len(image_bytes),
+            image_attachment.content_type,
+            image_attachment.original_width_px,
+            image_attachment.original_height_px,
+            e.message,
+        )
+
+        if written_images == 0 and len(image_attachment.thumbnail_metadata) == 0:
+            # We have never thumbnailed this -- it most likely had
+            # bad data.  Remove the ImageAttachment row, since it is
+            # not valid for thumbnailing.
+            start_time = time.perf_counter()
+            update_message_rendered_content(
+                image_attachment.realm_id, image_attachment.path_id, None
+            )
+            message_update_seconds = time.perf_counter() - start_time
+            image_attachment.delete()
+            return ThumbnailingResult(
+                0, fetching_seconds, processing_seconds, uploading_seconds, message_update_seconds
+            )
+        else:  # nocoverage
+            # TODO: Clean up any dangling thumbnails we may have
+            # produced?  Seems unlikely that we'd fail on one size,
+            # but not another, but anything's possible.
+            pass
+
+    image_attachment.save(update_fields=["thumbnail_metadata"])
+    start_time = time.perf_counter()
+    url, is_animated = get_default_thumbnail_url(image_attachment)
+    update_message_rendered_content(
+        image_attachment.realm_id,
+        image_attachment.path_id,
+        MarkdownImageMetadata(
+            url=url,
+            is_animated=is_animated,
+            original_width_px=image_attachment.original_width_px,
+            original_height_px=image_attachment.original_height_px,
+            original_content_type=image_attachment.content_type,
+            transcoded_image=get_transcoded_format(image_attachment),
+        ),
+    )
+    message_update_seconds = time.perf_counter() - start_time
+    return ThumbnailingResult(
+        written_images,
+        fetching_seconds,
+        processing_seconds,
+        uploading_seconds,
+        message_update_seconds,
+    )
+
+
+def update_message_rendered_content(
+    realm_id: int, path_id: str, image_data: MarkdownImageMetadata | None
+) -> None:
+    for message_class in (Message, ArchivedMessage):
+        messages_with_image = (
+            message_class.objects.filter(realm_id=realm_id, attachment__path_id=path_id)
+            .select_for_update(of=("self",), no_key=True)
+            .order_by("id")
+        )
+        for message in messages_with_image:
+            assert message.rendered_content is not None
+            rendered_content = rewrite_thumbnailed_images(
+                message.rendered_content,
+                {} if image_data is None else {path_id: image_data},
+                {path_id} if image_data is None else set(),
+            )[0]
+            if rendered_content is None:
+                # There were no updates -- for instance, if we re-run
+                # ensure_thumbnails on an ImageAttachment we already
+                # ran it on once.  Do not bother to no-op update
+                # clients.
+                continue
+            if isinstance(message, Message):
+                # Perform a silent update push to the clients
+                do_update_embedded_data(message.sender, message, rendered_content)
+            else:
+                message.rendered_content = rendered_content
+                message.save(update_fields=["rendered_content"])

zerver/worker/user_activity.py

@@ -0,0 +1,96 @@
+# Documented in https://zulip.readthedocs.io/en/latest/subsystems/queuing.html
+import logging
+from typing import Any
+
+from django.conf import settings
+from django.db import connection
+from psycopg2.sql import SQL, Literal
+from typing_extensions import override
+
+from zerver.worker.base import LoopQueueProcessingWorker, assign_queue
+
+logger = logging.getLogger(__name__)
+
+
+@assign_queue("user_activity")
+class UserActivityWorker(LoopQueueProcessingWorker):
+    """The UserActivity queue is perhaps our highest-traffic queue, and
+    requires some care to ensure it performs adequately.
+
+    We use a LoopQueueProcessingWorker as a performance optimization
+    for managing the queue.  The structure of UserActivity records is
+    such that they are easily deduplicated before being sent to the
+    database; we take advantage of that to make this queue highly
+    effective at dealing with a backlog containing many similar
+    events.  Such a backlog happen in a few ways:
+
+    * In abuse/DoS situations, if a client is sending huge numbers of
+      similar requests to the server.
+    * If the queue ends up with several minutes of backlog e.g. due to
+      downtime of the queue processor, many clients will have several
+      common events from doing an action multiple times.
+
+    """
+
+    client_id_map: dict[str, int] = {}
+
+    @override
+    def __init__(
+        self,
+        threaded: bool = False,
+        disable_timeout: bool = False,
+        worker_num: int | None = None,
+    ) -> None:
+        if settings.USER_ACTIVITY_SHARDS > 1 and worker_num is not None:  # nocoverage
+            self.queue_name += f"_shard{worker_num}"
+        super().__init__(threaded, disable_timeout, worker_num)
+
+    @override
+    def start(self) -> None:
+        # For our unit tests to make sense, we need to clear this on startup.
+        self.client_id_map = {}
+        super().start()
+
+    @override
+    def consume_batch(self, user_activity_events: list[dict[str, Any]]) -> None:
+        uncommitted_events: dict[tuple[int, int, str], tuple[int, float]] = {}
+
+        # First, we drain the queue of all user_activity events and
+        # deduplicate them for insertion into the database.
+        for event in user_activity_events:
+            user_profile_id = event["user_profile_id"]
+            client_id = event["client_id"]
+
+            key_tuple = (user_profile_id, client_id, event["query"])
+            if key_tuple not in uncommitted_events:
+                uncommitted_events[key_tuple] = (1, event["time"])
+            else:
+                count, event_time = uncommitted_events[key_tuple]
+                uncommitted_events[key_tuple] = (count + 1, max(event_time, event["time"]))
+
+        rows = []
+        for key_tuple, value_tuple in uncommitted_events.items():
+            user_profile_id, client_id, query = key_tuple
+            count, event_time = value_tuple
+            rows.append(
+                SQL("({},{},{},{},to_timestamp({}))").format(
+                    Literal(user_profile_id),
+                    Literal(client_id),
+                    Literal(query),
+                    Literal(count),
+                    Literal(event_time),
+                )
+            )
+
+        # Perform a single bulk UPSERT for all of the rows
+        sql_query = SQL(
+            """
+            INSERT INTO zerver_useractivity(user_profile_id, client_id, query, count, last_visit)
+            VALUES {rows}
+            ON CONFLICT (user_profile_id, client_id, query) DO UPDATE SET
+                count = zerver_useractivity.count + excluded.count,
+                last_visit = greatest(zerver_useractivity.last_visit, excluded.last_visit)
+            """
+        ).format(rows=SQL(", ").join(rows))
+        with connection.cursor() as cursor:
+            cursor.execute(sql_query)

zerver/worker/user_activity_interval.py

@@ -0,0 +1,22 @@
+# Documented in https://zulip.readthedocs.io/en/latest/subsystems/queuing.html
+import logging
+from collections.abc import Mapping
+from typing import Any
+
+from typing_extensions import override
+
+from zerver.actions.user_activity import do_update_user_activity_interval
+from zerver.lib.timestamp import timestamp_to_datetime
+from zerver.models.users import get_user_profile_by_id
+from zerver.worker.base import QueueProcessingWorker, assign_queue
+
+logger = logging.getLogger(__name__)
+
+
+@assign_queue("user_activity_interval")
+class UserActivityIntervalWorker(QueueProcessingWorker):
+    @override
+    def consume(self, event: Mapping[str, Any]) -> None:
+        user_profile = get_user_profile_by_id(event["user_profile_id"])
+        log_time = timestamp_to_datetime(event["time"])
+        do_update_user_activity_interval(user_profile, log_time)