snapshot of https://github.com/zulip/zulip.git @ 48c69911c7dd4e03b20552eb3bd77ad8151c1357 (2026-09-15)

zerver/tornado/application.py

@@ -0,0 +1,25 @@
+import tornado.web
+from django.conf import settings
+from django.core.handlers.base import BaseHandler
+from tornado import autoreload
+
+from zerver.lib.queue import TornadoQueueClient
+from zerver.tornado.handlers import AsyncDjangoHandler
+
+
+def setup_tornado_rabbitmq(queue_client: TornadoQueueClient) -> None:  # nocoverage
+    # When tornado is shut down, disconnect cleanly from RabbitMQ
+    autoreload.add_reload_hook(queue_client.close)
+
+
+def create_tornado_application(*, autoreload: bool = False) -> tornado.web.Application:
+    django_handler = BaseHandler()
+    django_handler.load_middleware()
+
+    return tornado.web.Application(
+        [(tornado.routing.AnyMatches(), AsyncDjangoHandler, dict(django_handler=django_handler))],
+        debug=settings.DEBUG,
+        autoreload=autoreload,
+        # Disable Tornado's own request logging, since we have our own
+        log_function=lambda x: None,
+    )

zerver/tornado/descriptors.py

@@ -0,0 +1,32 @@
+from typing import TYPE_CHECKING, Optional
+
+from django.conf import settings
+
+if TYPE_CHECKING:
+    from zerver.tornado.event_queue import ClientDescriptor
+
+descriptors_by_handler_id: dict[int, "ClientDescriptor"] = {}
+
+
+def get_descriptor_by_handler_id(handler_id: int) -> Optional["ClientDescriptor"]:
+    return descriptors_by_handler_id.get(handler_id)
+
+
+def set_descriptor_by_handler_id(handler_id: int, client_descriptor: "ClientDescriptor") -> None:
+    descriptors_by_handler_id[handler_id] = client_descriptor
+
+
+def clear_descriptor_by_handler_id(handler_id: int) -> None:
+    del descriptors_by_handler_id[handler_id]
+
+
+current_port: int | None = None
+
+
+def is_current_port(port: int) -> int | None:
+    return settings.TEST_SUITE or current_port == port
+
+
+def set_current_port(port: int) -> None:
+    global current_port
+    current_port = port

zerver/tornado/django_api.py

@@ -0,0 +1,254 @@
+from collections import defaultdict
+from collections.abc import Iterable, Mapping, Sequence
+from dataclasses import dataclass
+from functools import lru_cache
+from typing import Any, Literal
+from urllib.parse import urlsplit
+
+import orjson
+import requests
+from django.conf import settings
+from django.db import transaction
+from requests.adapters import ConnectionError, HTTPAdapter
+from requests.models import PreparedRequest, Response
+from typing_extensions import override
+from urllib3.util import Retry
+
+from zerver.lib.partial import partial
+from zerver.lib.queue import queue_json_publish_rollback_unsafe
+from zerver.models import Client, Realm, UserProfile
+from zerver.models.users import get_user_profile_narrow_by_id
+from zerver.tornado.sharding import (
+    get_realm_tornado_ports,
+    get_tornado_url,
+    get_user_id_tornado_port,
+    get_user_tornado_port,
+    notify_tornado_queue_name,
+)
+
+
+class TornadoAdapter(HTTPAdapter):
+    def __init__(self) -> None:
+        # All of the POST requests we make to Tornado are safe to
+        # retry; allow retries of them, which is not the default.
+        retry_methods = Retry.DEFAULT_ALLOWED_METHODS | {"POST"}
+        retry = Retry(total=3, backoff_factor=1, allowed_methods=retry_methods)
+        super().__init__(max_retries=retry)
+
+    @override
+    def send(
+        self,
+        request: PreparedRequest,
+        stream: bool = False,
+        timeout: float | tuple[float, float] | tuple[float, None] | None = 0.5,
+        verify: bool | str = True,
+        cert: bytes | str | tuple[bytes | str, bytes | str] | None = None,
+        proxies: Mapping[str, str] | None = None,
+    ) -> Response:
+        # Don't talk to Tornado through proxies, which only allow
+        # requests to external hosts.
+        proxies = {}
+        try:
+            resp = super().send(
+                request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies
+            )
+        except ConnectionError:
+            parsed_url = urlsplit(request.url)
+            logfile = (
+                f"tornado-{parsed_url.port}.log"
+                if settings.TORNADO_PROCESSES > 1
+                else "tornado.log"
+            )
+            raise ConnectionError(
+                f"Django cannot connect to Tornado server ({request.url}); "
+                f"check {settings.ERROR_FILE_LOG_PATH} and {logfile}"
+            )
+        resp.raise_for_status()
+        return resp
+
+
+@lru_cache(None)
+def requests_client() -> requests.Session:
+    c = requests.Session()
+    adapter = TornadoAdapter()
+    for scheme in ("https://", "http://"):
+        c.mount(scheme, adapter)
+    return c
+
+
+@dataclass
+class EventQueueData:
+    queue_id: str
+    idle_queue_timeout_secs: int
+
+
+def request_event_queue(
+    user_profile: UserProfile,
+    user_client: Client,
+    apply_markdown: bool,
+    client_gravatar: bool,
+    slim_presence: bool,
+    idle_queue_timeout: int | Literal["mobile"] | None,
+    event_types: Sequence[str] | None = None,
+    all_public_streams: bool = False,
+    narrow: Iterable[Sequence[str]] = [],
+    bulk_message_deletion: bool = False,
+    stream_typing_notifications: bool = False,
+    pronouns_field_type_supported: bool = True,
+    linkifier_url_template: bool = False,
+    user_list_incomplete: bool = False,
+    include_deactivated_groups: bool = False,
+    archived_channels: bool = False,
+    empty_topic_name: bool = False,
+    simplified_presence_events: bool = False,
+    individual_emoji_changes: bool = False,
+) -> EventQueueData | None:
+    if not settings.USING_TORNADO:
+        return None
+
+    # We make sure to pre-fill the narrow user cache, to save
+    # session-based Tornado (/json/events) from having to go to the
+    # database.
+    get_user_profile_narrow_by_id(user_profile.id)
+
+    tornado_url = get_tornado_url(get_user_tornado_port(user_profile))
+    req = {
+        "dont_block": "true",
+        "apply_markdown": orjson.dumps(apply_markdown),
+        "client_gravatar": orjson.dumps(client_gravatar),
+        "slim_presence": orjson.dumps(slim_presence),
+        "all_public_streams": orjson.dumps(all_public_streams),
+        "client": "internal",
+        "user_profile_id": user_profile.id,
+        "user_client": user_client.name,
+        "narrow": orjson.dumps(narrow),
+        "secret": settings.SHARED_SECRET,
+        "bulk_message_deletion": orjson.dumps(bulk_message_deletion),
+        "stream_typing_notifications": orjson.dumps(stream_typing_notifications),
+        "pronouns_field_type_supported": orjson.dumps(pronouns_field_type_supported),
+        "linkifier_url_template": orjson.dumps(linkifier_url_template),
+        "user_list_incomplete": orjson.dumps(user_list_incomplete),
+        "include_deactivated_groups": orjson.dumps(include_deactivated_groups),
+        "archived_channels": orjson.dumps(archived_channels),
+        "empty_topic_name": orjson.dumps(empty_topic_name),
+        "simplified_presence_events": orjson.dumps(simplified_presence_events),
+        "individual_emoji_changes": orjson.dumps(individual_emoji_changes),
+    }
+
+    if idle_queue_timeout is not None:
+        req["idle_queue_timeout"] = orjson.dumps(idle_queue_timeout)
+
+    if event_types is not None:
+        req["event_types"] = orjson.dumps(event_types)
+
+    resp = requests_client().post(tornado_url + "/api/v1/events/internal", data=req)
+    result = resp.json()
+    return EventQueueData(
+        queue_id=result["queue_id"],
+        idle_queue_timeout_secs=result["idle_queue_timeout_secs"],
+    )
+
+
+def get_user_events(
+    user_profile: UserProfile, queue_id: str, last_event_id: int
+) -> list[dict[str, Any]]:
+    if not settings.USING_TORNADO:
+        return []
+
+    # Pre-fill the narrow user cache, to save session-based Tornado
+    # (/json/events) from having to go to the database.  This is
+    # almost certainly filled already, from above, but there is little
+    # harm in forcing it.
+    get_user_profile_narrow_by_id(user_profile.id)
+
+    tornado_url = get_tornado_url(get_user_tornado_port(user_profile))
+    post_data: dict[str, Any] = {
+        "queue_id": queue_id,
+        "last_event_id": last_event_id,
+        "dont_block": "true",
+        "user_profile_id": user_profile.id,
+        "secret": settings.SHARED_SECRET,
+        "client": "internal",
+    }
+    resp = requests_client().post(tornado_url + "/api/v1/events/internal", data=post_data)
+    return resp.json()["events"]
+
+
+def send_notification_http(port: int, data: Mapping[str, Any]) -> None:
+    if not settings.USING_TORNADO or settings.RUNNING_INSIDE_TORNADO:
+        # To allow the backend test suite to not require a separate
+        # Tornado process, we simply call the process_notification
+        # handler directly rather than making the notify_tornado HTTP
+        # request.  It would perhaps be better to instead implement
+        # this via some sort of `responses` module configuration, but
+        # perhaps it's more readable to have the logic live here.
+        #
+        # We use an import local to this function to prevent this hack
+        # from creating import cycles.
+        from zerver.tornado.event_queue import process_notification
+
+        process_notification(data)
+    else:
+        # This codepath is only used when running full-stack puppeteer
+        # tests, which don't have RabbitMQ but do have a separate
+        # Tornado process.
+        tornado_url = get_tornado_url(port)
+        requests_client().post(
+            tornado_url + "/api/internal/notify_tornado",
+            data=dict(data=orjson.dumps(data), secret=settings.SHARED_SECRET),
+        )
+
+
+# The core function for sending an event from Django to Tornado (which
+# will then push it to web and mobile clients for the target users).
+#
+# One should generally use `send_event_on_commit` unless there's a strong
+# reason to use `send_event_rollback_unsafe` directly, as it doesn't wait
+# for the db transaction (within which it gets called, if any) to commit
+# and sends event irrespective of commit or rollback.
+#
+# By convention, `send_event_rollback_unsafe` / `send_event_on_commit`
+# should only be called from zerver/actions/*.py, which helps make it
+# easy to find event generation code.
+#
+# Every call point should be covered by a test in `test_events.py`,
+# with the schema verified in `zerver/lib/event_schema.py`.
+#
+# See https://zulip.readthedocs.io/en/latest/subsystems/events-system.html
+def send_event_rollback_unsafe(
+    realm: Realm, event: Mapping[str, Any], users: Iterable[int] | Iterable[Mapping[str, Any]]
+) -> None:
+    """`users` is a list of user IDs, or in some special cases like message
+    send/update or embeds, dictionaries containing extra data."""
+    realm_ports = get_realm_tornado_ports(realm)
+    if len(realm_ports) == 1:
+        port_user_map = {realm_ports[0]: list(users)}
+    else:
+        port_user_map = defaultdict(list)
+        for user in users:
+            user_id = user if isinstance(user, int) else user["id"]
+            port_user_map[get_user_id_tornado_port(realm_ports, user_id)].append(user)
+
+    for port, port_users in port_user_map.items():
+        queue_json_publish_rollback_unsafe(
+            notify_tornado_queue_name(port),
+            dict(event=event, users=port_users),
+            partial(send_notification_http, port),
+        )
+
+
+def send_event_on_commit(
+    realm: Realm, event: Mapping[str, Any], users: Iterable[int] | Iterable[Mapping[str, Any]]
+) -> None:
+    if not settings.USING_RABBITMQ:
+        # In tests, round-trip the event through JSON, as happens with
+        # RabbitMQ.  zerver.lib.queue also enforces this, but the
+        # on-commit nature of the event sending makes it difficult to
+        # trace which event was at fault -- so we also check it
+        # immediately, here.
+        try:
+            event = orjson.loads(orjson.dumps(event))
+        except TypeError:
+            print(event)
+            raise
+    transaction.on_commit(lambda: send_event_rollback_unsafe(realm, event, users))

zerver/tornado/exceptions.py

@@ -0,0 +1,17 @@
+from django.utils.translation import gettext as _
+from typing_extensions import override
+
+from zerver.lib.exceptions import ErrorCode, JsonableError
+
+
+class BadEventQueueIdError(JsonableError):
+    code = ErrorCode.BAD_EVENT_QUEUE_ID
+    data_fields = ["queue_id"]
+
+    def __init__(self, queue_id: str) -> None:
+        self.queue_id: str = queue_id
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return _("Bad event queue ID: {queue_id}")

zerver/tornado/handlers.py

@@ -0,0 +1,286 @@
+import logging
+from collections.abc import Collection
+from contextlib import suppress
+from typing import Any, Optional
+from urllib.parse import unquote
+
+import tornado.web
+from asgiref.sync import sync_to_async
+from django import http
+from django.core import signals
+from django.core.handlers import base
+from django.core.handlers.wsgi import WSGIRequest, get_script_name
+from django.http import HttpRequest, HttpResponse
+from django.urls import set_script_prefix
+from django.utils.cache import patch_vary_headers
+from tornado.iostream import StreamClosedError
+from tornado.wsgi import WSGIContainer
+from typing_extensions import override
+
+from zerver.lib.response import AsynchronousResponse, json_response
+from zerver.tornado.descriptors import get_descriptor_by_handler_id
+
+current_handler_id = 0
+handlers: dict[int, "AsyncDjangoHandler"] = {}
+fake_wsgi_container = WSGIContainer(lambda environ, start_response: [])
+
+
+def get_handler_by_id(handler_id: int) -> Optional["AsyncDjangoHandler"]:
+    return handlers.get(handler_id)
+
+
+def allocate_handler_id(handler: "AsyncDjangoHandler") -> int:
+    global current_handler_id
+    handlers[current_handler_id] = handler
+    handler_id = current_handler_id
+    current_handler_id += 1
+    return handler_id
+
+
+def clear_handler_by_id(handler_id: int) -> None:
+    handlers.pop(handler_id, None)
+
+
+def handler_stats_string() -> str:
+    return f"{len(handlers)} handlers, latest ID {current_handler_id}"
+
+
+def finish_handler(handler_id: int, event_queue_id: str, contents: list[dict[str, Any]]) -> None:
+    try:
+        # We do the import during runtime to avoid cyclic dependency
+        # with zerver.lib.request
+        from zerver.lib.request import RequestNotes
+        from zerver.middleware import async_request_timer_restart
+
+        # The request handler may have been GC'd by a
+        # on_connection_close elsewhere already, so we have to check
+        # it is still around.
+        handler = get_handler_by_id(handler_id)
+        if handler is None:
+            return
+        request = handler._request
+        assert request is not None
+
+        # We call async_request_timer_restart here in case we are
+        # being finished without any events (because another
+        # get_events request has supplanted this request)
+        async_request_timer_restart(request)
+        log_data = RequestNotes.get_notes(request).log_data
+        assert log_data is not None
+        if len(contents) != 1:
+            log_data["extra"] = f"[{event_queue_id}/1]"
+        else:
+            log_data["extra"] = "[{}/1/{}]".format(event_queue_id, contents[0]["type"])
+
+        tornado.ioloop.IOLoop.current().add_callback(
+            handler.zulip_finish,
+            dict(result="success", msg="", events=contents, queue_id=event_queue_id),
+            request,
+        )
+    except Exception as e:
+        if not (
+            (isinstance(e, OSError) and str(e) == "Stream is closed")
+            or (isinstance(e, AssertionError) and str(e) == "Request closed")
+        ):
+            logging.exception(
+                "Got error finishing handler for queue %s", event_queue_id, stack_info=True
+            )
+
+
+class AsyncDjangoHandler(tornado.web.RequestHandler):
+    handler_id: int
+
+    SUPPORTED_METHODS: Collection[str] = {"GET", "POST", "DELETE"}  # type: ignore[assignment]  # https://github.com/tornadoweb/tornado/pull/3354
+
+    @override
+    def initialize(self, django_handler: base.BaseHandler) -> None:
+        self.django_handler = django_handler
+
+        # Prevent Tornado from automatically finishing the request
+        self._auto_finish = False
+
+        # Handler IDs are allocated here, and the handler ID map must
+        # be cleared when the handler finishes its response.  See
+        # on_finish and on_connection_close.
+        self.handler_id = allocate_handler_id(self)
+
+        self._request: HttpRequest | None = None
+
+    @override
+    def on_finish(self) -> None:
+        # Note that this only runs on _successful_ requests.  If the
+        # client closes the connection, see on_connection_close,
+        # below.
+        clear_handler_by_id(self.handler_id)
+
+    @override
+    def __repr__(self) -> str:
+        descriptor = get_descriptor_by_handler_id(self.handler_id)
+        return f"AsyncDjangoHandler<{self.handler_id}, {descriptor}>"
+
+    async def convert_tornado_request_to_django_request(self) -> HttpRequest:
+        # This takes the WSGI environment that Tornado received (which
+        # fully describes the HTTP request that was sent to Tornado)
+        # and pass it to Django's WSGIRequest to generate a Django
+        # HttpRequest object with the original Tornado request's HTTP
+        # headers, parameters, etc.
+        environ = fake_wsgi_container.environ(self.request)
+        environ["PATH_INFO"] = unquote(environ["PATH_INFO"])
+
+        # Django WSGIRequest setup code that should match logic from
+        # Django's WSGIHandler.__call__ before the call to
+        # `get_response()`.
+        set_script_prefix(get_script_name(environ))
+        await signals.request_started.asend(sender=type(self.django_handler))
+        self._request = WSGIRequest(environ)
+
+        # We do the import during runtime to avoid cyclic dependency
+        from zerver.lib.request import RequestNotes
+
+        # Provide a way for application code to access this handler
+        # given the HttpRequest object.
+        RequestNotes.get_notes(self._request).tornado_handler_id = self.handler_id
+
+        return self._request
+
+    async def write_django_response_as_tornado_response(self, response: HttpResponse) -> None:
+        # This takes a Django HttpResponse and copies its HTTP status
+        # code, headers, cookies, and content onto this
+        # tornado.web.RequestHandler (which is how Tornado prepares a
+        # response to write).
+
+        # Copy the HTTP status code.
+        self.set_status(response.status_code)
+
+        # Copy the HTTP headers (iterating through a Django
+        # HttpResponse is the way to access its headers as key/value pairs)
+        for h in response.items():
+            self.set_header(h[0], h[1])
+
+        # Copy any cookies
+        if not hasattr(self, "_new_cookies"):
+            self._new_cookies: list[http.cookie.SimpleCookie] = []
+        self._new_cookies.append(response.cookies)
+
+        # Copy the response content
+        self.write(response.content)
+
+        # Close the connection.
+        # While writing the response, we might realize that the
+        # user already closed the connection; that is fine.
+        with suppress(StreamClosedError):
+            await self.finish()
+
+    @override
+    async def get(self, *args: Any, **kwargs: Any) -> None:
+        request = await self.convert_tornado_request_to_django_request()
+        response = await sync_to_async(
+            lambda: self.django_handler.get_response(request), thread_sensitive=True
+        )()
+
+        try:
+            if isinstance(response, AsynchronousResponse):
+                # We import async_request_timer_restart during runtime
+                # to avoid cyclic dependency with zerver.lib.request
+                from zerver.middleware import async_request_timer_stop
+
+                # For asynchronous requests, this is where we exit
+                # without returning the HttpResponse that Django
+                # generated back to the user in order to long-poll the
+                # connection.  We save some timers here in order to
+                # support accurate accounting of the total resources
+                # consumed by the request when it eventually returns a
+                # response and is logged.
+                async_request_timer_stop(request)
+            else:
+                # For normal/synchronous requests that don't end up
+                # long-polling, we just need to write the HTTP
+                # response that Django prepared for us via Tornado.
+                assert isinstance(response, HttpResponse)
+                await self.write_django_response_as_tornado_response(response)
+        finally:
+            # Tell Django that we're done processing this request on
+            # the Django side; this triggers cleanup work like
+            # resetting the urlconf and any cache/database
+            # connections.
+            await sync_to_async(response.close, thread_sensitive=True)()
+
+    @override
+    async def post(self, *args: Any, **kwargs: Any) -> None:
+        await self.get(*args, **kwargs)
+
+    @override
+    async def delete(self, *args: Any, **kwargs: Any) -> None:
+        await self.get(*args, **kwargs)
+
+    @override
+    def on_connection_close(self) -> None:
+        # Register a Tornado handler that runs when client-side
+        # connections are closed to notify the events system.
+
+        # If the client goes away, garbage collect the handler (with
+        # attached request information).
+        clear_handler_by_id(self.handler_id)
+        client_descriptor = get_descriptor_by_handler_id(self.handler_id)
+        if client_descriptor is not None:
+            client_descriptor.disconnect_handler(client_closed=True)
+
+    async def zulip_finish(self, result_dict: dict[str, Any], old_request: HttpRequest) -> None:
+        # Function called when we want to break a long-polled
+        # get_events request and return a response to the client.
+
+        # Marshall the response data from result_dict.
+        if result_dict["result"] == "error":
+            self.set_status(400)
+
+        # The `result` dictionary contains the data we want to return
+        # to the client.  We want to do so in a proper Tornado HTTP
+        # response after running the Django response middleware (which
+        # does things like log the request, add rate-limit headers,
+        # etc.).  The Django middleware API expects to receive a fresh
+        # HttpRequest object, and so to minimize hacks, our strategy
+        # is to create a duplicate Django HttpRequest object, tagged
+        # to automatically return our data in its response, and call
+        # Django's main self.get_response() handler to generate an
+        # HttpResponse with all Django middleware run.
+        request = await self.convert_tornado_request_to_django_request()
+
+        # We import RequestNotes during runtime to avoid
+        # cyclic import
+        from zerver.lib.request import RequestNotes
+
+        request_notes = RequestNotes.get_notes(request)
+        old_request_notes = RequestNotes.get_notes(old_request)
+
+        # Add to this new HttpRequest logging data from the processing of
+        # the original request; we will need these for logging.
+        request_notes.log_data = old_request_notes.log_data
+        request_notes.ratelimits_applied += old_request_notes.ratelimits_applied
+        request_notes.requester_for_logs = old_request_notes.requester_for_logs
+        request.user = old_request.user
+        request_notes.client = old_request_notes.client
+        request_notes.client_name = old_request_notes.client_name
+        request_notes.client_version = old_request_notes.client_version
+
+        # The saved_response attribute, if present, causes
+        # rest_dispatch to return the response immediately before
+        # doing any work.  This arrangement allows Django's full
+        # request/middleware system to run unmodified while avoiding
+        # running expensive things like Zulip's authentication code a
+        # second time.
+        request_notes.saved_response = json_response(
+            res_type=result_dict["result"], data=result_dict, status=self.get_status()
+        )
+
+        response = await sync_to_async(
+            lambda: self.django_handler.get_response(request), thread_sensitive=True
+        )()
+        try:
+            # Explicitly mark requests as varying by cookie, since the
+            # middleware will not have seen a session access
+            patch_vary_headers(response, ("Cookie",))
+            assert isinstance(response, HttpResponse)
+            await self.write_django_response_as_tornado_response(response)
+        finally:
+            # Tell Django we're done processing this request
+            await sync_to_async(response.close, thread_sensitive=True)()

zerver/tornado/ioloop_logging.py

@@ -0,0 +1,3 @@
+# This is used for a somewhat hacky way of passing the port number
+# into this early-initialized module.
+logging_data: dict[str, str] = {}

zerver/tornado/sharding.py

@@ -0,0 +1,52 @@
+import json
+import os
+import re
+from re import Pattern
+
+from django.conf import settings
+
+from zerver.models import Realm, UserProfile
+
+shard_map: dict[str, int | list[int]] = {}
+shard_regexes: list[tuple[Pattern[str], int | list[int]]] = []
+if os.path.exists("/etc/zulip/sharding.json"):
+    with open("/etc/zulip/sharding.json") as f:
+        data = json.loads(f.read())
+        shard_map = data.get(
+            "shard_map",
+            data,  # backwards compatibility
+        )
+        shard_regexes = [
+            (re.compile(regex, re.IGNORECASE), port)
+            for regex, port in data.get("shard_regexes", [])
+        ]
+
+
+def get_realm_tornado_ports(realm: Realm) -> list[int]:
+    if realm.host in shard_map:
+        ports = shard_map[realm.host]
+        return [ports] if isinstance(ports, int) else ports
+
+    for regex, ports in shard_regexes:
+        if regex.match(realm.host):
+            return [ports] if isinstance(ports, int) else ports
+
+    return [settings.TORNADO_PORTS[0]]
+
+
+def get_user_id_tornado_port(realm_ports: list[int], user_id: int) -> int:
+    return realm_ports[user_id % len(realm_ports)]
+
+
+def get_user_tornado_port(user: UserProfile) -> int:
+    return get_user_id_tornado_port(get_realm_tornado_ports(user.realm), user.id)
+
+
+def get_tornado_url(port: int) -> str:
+    return f"http://127.0.0.1:{port}"
+
+
+def notify_tornado_queue_name(port: int) -> str:
+    if settings.TORNADO_PROCESSES == 1:
+        return "notify_tornado"
+    return f"notify_tornado_port_{port}"

zerver/tornado/views.py

@@ -0,0 +1,298 @@
+import time
+from collections.abc import Callable
+from typing import Annotated, Any, Literal, TypeVar
+
+from asgiref.sync import async_to_sync
+from django.conf import settings
+from django.http import HttpRequest, HttpResponse
+from django.utils.translation import gettext as _
+from pydantic import BaseModel, Json, PositiveInt, StringConstraints, model_validator
+from typing_extensions import ParamSpec
+
+from zerver.decorator import internal_api_view, process_client
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.queue import get_queue_client
+from zerver.lib.request import RequestNotes
+from zerver.lib.response import AsynchronousResponse, json_response, json_success
+from zerver.lib.sessions import narrow_request_user
+from zerver.lib.typed_endpoint import ApiParamConfig, DocumentationStatus, typed_endpoint
+from zerver.models import UserProfile
+from zerver.models.clients import get_client
+from zerver.tornado.descriptors import is_current_port
+from zerver.tornado.event_queue import (
+    access_client_descriptor,
+    fetch_events,
+    process_notification,
+    send_web_reload_client_events,
+)
+from zerver.tornado.sharding import get_user_tornado_port, notify_tornado_queue_name
+
+P = ParamSpec("P")
+T = TypeVar("T")
+
+
+def in_tornado_thread(f: Callable[P, T]) -> Callable[P, T]:
+    async def wrapped(*args: P.args, **kwargs: P.kwargs) -> T:
+        return f(*args, **kwargs)
+
+    return async_to_sync(wrapped)
+
+
+# Registered as handler404/handler500 by zproject/tornado_urls.py.
+def json_not_found(request: HttpRequest, exception: Exception) -> HttpResponse:
+    return json_response(res_type="error", msg=_("Not found"), status=404)
+
+
+def json_internal_server_error(request: HttpRequest) -> HttpResponse:
+    return json_response(res_type="error", msg=_("Internal server error"), status=500)
+
+
+@internal_api_view(True)
+@typed_endpoint
+def notify(request: HttpRequest, *, data: Json[dict[str, Any]]) -> HttpResponse:
+    # Only the puppeteer full-stack tests use this endpoint; it
+    # injects an event, as if read from RabbitMQ.
+    in_tornado_thread(process_notification)(data)
+    return json_success(request)
+
+
+@internal_api_view(True)
+@typed_endpoint
+def web_reload_clients(
+    request: HttpRequest,
+    *,
+    client_count: Json[int] | None = None,
+    immediate: Json[bool] = False,
+) -> HttpResponse:
+    sent_events = in_tornado_thread(send_web_reload_client_events)(
+        immediate=immediate, count=client_count
+    )
+    return json_success(
+        request,
+        {
+            "sent_events": sent_events,
+            "complete": client_count is None or client_count != sent_events,
+        },
+    )
+
+
+@typed_endpoint
+def cleanup_event_queue(
+    request: HttpRequest, user_profile: UserProfile, *, queue_id: str
+) -> HttpResponse:
+    log_data = RequestNotes.get_notes(request).log_data
+    assert log_data is not None
+    log_data["extra"] = f"[{queue_id}]"
+
+    user_port = get_user_tornado_port(user_profile)
+    if not is_current_port(user_port):
+        # X-Accel-Redirect is not supported for HTTP DELETE requests,
+        # so we notify the shard hosting the acting user's queues via
+        # enqueuing a special event.
+        #
+        # TODO: Because we return a 200 before confirming that the
+        # event queue had been actually deleted by the process hosting
+        # the queue, there's a race where a `GET /events` request can
+        # succeed after getting a 200 from this endpoint.
+        assert settings.USING_RABBITMQ
+        get_queue_client().json_publish(
+            notify_tornado_queue_name(user_port),
+            {"users": [user_profile.id], "event": {"type": "cleanup_queue", "queue_id": queue_id}},
+        )
+        return json_success(request)
+
+    client = access_client_descriptor(user_profile.id, queue_id)
+    in_tornado_thread(client.cleanup)()
+    return json_success(request)
+
+
+@internal_api_view(True)
+@typed_endpoint
+def get_events_internal(request: HttpRequest, *, user_profile_id: Json[int]) -> HttpResponse:
+    user_profile = narrow_request_user(request, user_id=user_profile_id)
+    assert isinstance(user_profile, UserProfile)
+    assert is_current_port(get_user_tornado_port(user_profile))
+
+    process_client(request, user_profile, client_name="internal")
+    return get_events_backend(request, user_profile)
+
+
+def get_events(request: HttpRequest, user_profile: UserProfile) -> HttpResponse:
+    user_port = get_user_tornado_port(user_profile)
+    if not is_current_port(user_port):
+        # When a single realm is split across multiple Tornado shards,
+        # any `GET /events` requests that are routed to the wrong
+        # shard are redirected to the shard hosting the relevant
+        # user's queues. We use X-Accel-Redirect for this purpose,
+        # which is efficient and keeps this redirect invisible to
+        # clients.
+        return HttpResponse(
+            "",
+            headers={"X-Accel-Redirect": f"/internal/tornado/{user_port}{request.get_full_path()}"},
+        )
+
+    return get_events_backend(request, user_profile)
+
+
+class UserClient(BaseModel):
+    id: int
+    name: Annotated[str, StringConstraints(max_length=30)]
+
+    @model_validator(mode="before")
+    @classmethod
+    def convert_term(cls, elem: str) -> dict[str, Any]:
+        client = get_client(elem)
+        return {"id": client.id, "name": client.name}
+
+
+@typed_endpoint
+def get_events_backend(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    # user_client is intended only for internal Django=>Tornado requests
+    # and thus shouldn't be documented for external use.
+    user_client: Annotated[
+        UserClient | None,
+        ApiParamConfig(documentation_status=DocumentationStatus.INTENTIONALLY_UNDOCUMENTED),
+    ] = None,
+    last_event_id: Json[int] | None = None,
+    queue_id: str | None = None,
+    # apply_markdown, client_gravatar, all_public_streams, and various
+    # other parameters are only used when registering a new queue via this
+    # endpoint.  This is a feature used primarily by get_events_internal
+    # and not expected to be used by third-party clients.
+    apply_markdown: Annotated[
+        Json[bool],
+        ApiParamConfig(documentation_status=DocumentationStatus.INTENTIONALLY_UNDOCUMENTED),
+    ] = False,
+    client_gravatar: Annotated[
+        Json[bool],
+        ApiParamConfig(documentation_status=DocumentationStatus.INTENTIONALLY_UNDOCUMENTED),
+    ] = False,
+    slim_presence: Annotated[
+        Json[bool],
+        ApiParamConfig(documentation_status=DocumentationStatus.INTENTIONALLY_UNDOCUMENTED),
+    ] = False,
+    all_public_streams: Annotated[
+        Json[bool],
+        ApiParamConfig(documentation_status=DocumentationStatus.INTENTIONALLY_UNDOCUMENTED),
+    ] = False,
+    event_types: Annotated[
+        Json[list[str]] | None,
+        ApiParamConfig(documentation_status=DocumentationStatus.INTENTIONALLY_UNDOCUMENTED),
+    ] = None,
+    dont_block: Json[bool] = False,
+    narrow: Annotated[
+        Json[list[list[str]]] | None,
+        ApiParamConfig(documentation_status=DocumentationStatus.INTENTIONALLY_UNDOCUMENTED),
+    ] = None,
+    idle_queue_timeout: Annotated[
+        Json[PositiveInt | Literal["mobile"]] | None,
+        ApiParamConfig(documentation_status=DocumentationStatus.INTENTIONALLY_UNDOCUMENTED),
+    ] = None,
+    bulk_message_deletion: Annotated[
+        Json[bool],
+        ApiParamConfig(documentation_status=DocumentationStatus.INTENTIONALLY_UNDOCUMENTED),
+    ] = False,
+    stream_typing_notifications: Annotated[
+        Json[bool],
+        ApiParamConfig(documentation_status=DocumentationStatus.INTENTIONALLY_UNDOCUMENTED),
+    ] = False,
+    pronouns_field_type_supported: Annotated[
+        Json[bool],
+        ApiParamConfig(documentation_status=DocumentationStatus.INTENTIONALLY_UNDOCUMENTED),
+    ] = True,
+    linkifier_url_template: Annotated[
+        Json[bool],
+        ApiParamConfig(documentation_status=DocumentationStatus.INTENTIONALLY_UNDOCUMENTED),
+    ] = False,
+    user_list_incomplete: Annotated[
+        Json[bool],
+        ApiParamConfig(documentation_status=DocumentationStatus.INTENTIONALLY_UNDOCUMENTED),
+    ] = False,
+    include_deactivated_groups: Annotated[
+        Json[bool],
+        ApiParamConfig(documentation_status=DocumentationStatus.INTENTIONALLY_UNDOCUMENTED),
+    ] = False,
+    archived_channels: Annotated[
+        Json[bool],
+        ApiParamConfig(documentation_status=DocumentationStatus.INTENTIONALLY_UNDOCUMENTED),
+    ] = False,
+    empty_topic_name: Annotated[
+        Json[bool],
+        ApiParamConfig(documentation_status=DocumentationStatus.INTENTIONALLY_UNDOCUMENTED),
+    ] = False,
+    simplified_presence_events: Annotated[
+        Json[bool],
+        ApiParamConfig(documentation_status=DocumentationStatus.INTENTIONALLY_UNDOCUMENTED),
+    ] = False,
+    individual_emoji_changes: Annotated[
+        Json[bool],
+        ApiParamConfig(documentation_status=DocumentationStatus.INTENTIONALLY_UNDOCUMENTED),
+    ] = False,
+) -> HttpResponse:
+    if narrow is None:
+        narrow = []
+    if all_public_streams and not user_profile.can_access_public_streams():
+        raise JsonableError(_("User not authorized for this query"))
+
+    # Extract the Tornado handler from the request
+    handler_id = RequestNotes.get_notes(request).tornado_handler_id
+    assert handler_id is not None
+
+    if user_client is None:
+        valid_user_client = RequestNotes.get_notes(request).client
+        assert valid_user_client is not None
+        valid_user_client_name = valid_user_client.name
+    else:
+        valid_user_client_name = user_client.name
+
+    new_queue_data = None
+    if queue_id is None:
+        new_queue_data = dict(
+            user_profile_id=user_profile.id,
+            realm_id=user_profile.realm_id,
+            event_types=event_types,
+            client_type_name=valid_user_client_name,
+            apply_markdown=apply_markdown,
+            client_gravatar=client_gravatar,
+            slim_presence=slim_presence,
+            all_public_streams=all_public_streams,
+            queue_timeout=idle_queue_timeout,
+            last_connection_time=time.time(),
+            narrow=narrow,
+            bulk_message_deletion=bulk_message_deletion,
+            stream_typing_notifications=stream_typing_notifications,
+            pronouns_field_type_supported=pronouns_field_type_supported,
+            linkifier_url_template=linkifier_url_template,
+            user_list_incomplete=user_list_incomplete,
+            include_deactivated_groups=include_deactivated_groups,
+            archived_channels=archived_channels,
+            empty_topic_name=empty_topic_name,
+            simplified_presence_events=simplified_presence_events,
+            individual_emoji_changes=individual_emoji_changes,
+        )
+
+    result = in_tornado_thread(fetch_events)(
+        user_profile_id=user_profile.id,
+        queue_id=queue_id,
+        last_event_id=last_event_id,
+        client_type_name=valid_user_client_name,
+        dont_block=dont_block,
+        handler_id=handler_id,
+        new_queue_data=new_queue_data,
+    )
+    if "extra_log_data" in result:
+        log_data = RequestNotes.get_notes(request).log_data
+        assert log_data is not None
+        log_data["extra"] = result["extra_log_data"]
+
+    if result["type"] == "async":
+        # Return an AsynchronousResponse; this will result in
+        # Tornado discarding the response and instead long-polling the
+        # request.  See zulip_finish for more design details.
+        return AsynchronousResponse()
+    if result["type"] == "error":
+        raise result["exception"]
+    return json_success(request, data=result["response"])