snapshot of https://github.com/zulip/zulip.git @ 48c69911c7dd4e03b20552eb3bd77ad8151c1357 (2026-09-15)

zerver/views/alert_words.py

@@ -0,0 +1,41 @@
+from typing import Annotated
+
+from django.http import HttpRequest, HttpResponse
+from pydantic import Json, StringConstraints
+
+from zerver.actions.alert_words import do_add_alert_words, do_remove_alert_words
+from zerver.lib.alert_words import user_alert_words
+from zerver.lib.response import json_success
+from zerver.lib.typed_endpoint import typed_endpoint
+from zerver.models import UserProfile
+
+
+def list_alert_words(request: HttpRequest, user_profile: UserProfile) -> HttpResponse:
+    return json_success(request, data={"alert_words": user_alert_words(user_profile)})
+
+
+def clean_alert_words(alert_words: list[str]) -> list[str]:
+    alert_words = [w.strip() for w in alert_words]
+    return [w for w in alert_words if w != ""]
+
+
+@typed_endpoint
+def add_alert_words(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    alert_words: Json[list[Annotated[str, StringConstraints(max_length=100)]]],
+) -> HttpResponse:
+    do_add_alert_words(user_profile, clean_alert_words(alert_words))
+    return json_success(request, data={"alert_words": user_alert_words(user_profile)})
+
+
+@typed_endpoint
+def remove_alert_words(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    alert_words: Json[list[str]],
+) -> HttpResponse:
+    do_remove_alert_words(user_profile, alert_words)
+    return json_success(request, data={"alert_words": user_alert_words(user_profile)})

zerver/views/antispam.py

@@ -0,0 +1,45 @@
+import logging
+from datetime import timedelta
+
+from altcha.v1 import ChallengeOptions, create_challenge
+from django.conf import settings
+from django.http import HttpRequest, HttpResponseBase
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+
+from zerver.lib.captcha import captcha_enabled
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.response import json_success
+from zerver.lib.typed_endpoint import typed_endpoint_without_parameters
+
+
+@typed_endpoint_without_parameters
+def get_challenge(
+    request: HttpRequest,
+) -> HttpResponseBase:
+    if not captcha_enabled():  # nocoverage
+        raise JsonableError(_("Challenges are not enabled."))
+
+    now = timezone_now()
+    expires = now + timedelta(minutes=1)
+    try:
+        challenge = create_challenge(
+            ChallengeOptions(
+                hmac_key=settings.ALTCHA_HMAC_KEY,
+                max_number=500000,
+                expires=expires,
+            )
+        )
+        session_challenges = request.session.get("altcha_challenges", [])
+        # We prune out expired challenges not for correctness (the
+        # expiration is validated separately) but to prevent this from
+        # growing without bound
+        session_challenges = [(c, e) for (c, e) in session_challenges if e > now.timestamp()]
+        request.session["altcha_challenges"] = [
+            *session_challenges,
+            (challenge.challenge, expires.timestamp()),
+        ]
+        return json_success(request, data=challenge.to_dict())
+    except Exception:  # nocoverage
+        logging.exception("Error while generating challenge")
+        raise JsonableError(_("Failed to generate challenge"))

zerver/views/attachments.py

@@ -0,0 +1,25 @@
+from django.db import transaction
+from django.http import HttpRequest, HttpResponse
+
+from zerver.actions.uploads import notify_attachment_update
+from zerver.lib.attachments import access_attachment_by_id, remove_attachment, user_attachments
+from zerver.lib.response import json_success
+from zerver.models import UserProfile
+
+
+def list_by_user(request: HttpRequest, user_profile: UserProfile) -> HttpResponse:
+    return json_success(
+        request,
+        data={
+            "attachments": user_attachments(user_profile),
+            "upload_space_used": user_profile.realm.currently_used_upload_space_bytes(),
+        },
+    )
+
+
+@transaction.atomic(durable=True)
+def remove(request: HttpRequest, user_profile: UserProfile, attachment_id: int) -> HttpResponse:
+    attachment = access_attachment_by_id(user_profile, attachment_id, needs_owner=True)
+    remove_attachment(user_profile, attachment)
+    notify_attachment_update(user_profile, "remove", {"id": attachment_id})
+    return json_success(request)

zerver/views/channel_folders.py

@@ -0,0 +1,105 @@
+from dataclasses import asdict
+from typing import Annotated
+
+from django.http import HttpRequest, HttpResponse
+from django.utils.translation import gettext as _
+from pydantic import Json, StringConstraints
+
+from zerver.actions.channel_folders import (
+    check_add_channel_folder,
+    do_archive_channel_folder,
+    do_change_channel_folder_description,
+    do_change_channel_folder_name,
+    do_unarchive_channel_folder,
+    try_reorder_realm_channel_folders,
+)
+from zerver.decorator import require_realm_admin
+from zerver.lib.channel_folders import (
+    check_channel_folder_in_use,
+    check_channel_folder_name,
+    get_channel_folder_by_id,
+    get_channel_folders_in_realm,
+)
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.response import json_success
+from zerver.lib.typed_endpoint import PathOnly, typed_endpoint
+from zerver.models.channel_folders import ChannelFolder
+from zerver.models.users import UserProfile
+
+
+@require_realm_admin
+@typed_endpoint
+def create_channel_folder(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    description: Annotated[str, StringConstraints(max_length=ChannelFolder.MAX_DESCRIPTION_LENGTH)],
+    name: Annotated[str, StringConstraints(max_length=ChannelFolder.MAX_NAME_LENGTH)],
+) -> HttpResponse:
+    realm = user_profile.realm
+    check_channel_folder_name(name, realm)
+    channel_folder = check_add_channel_folder(realm, name, description, acting_user=user_profile)
+
+    return json_success(request, data={"channel_folder_id": channel_folder.id})
+
+
+@typed_endpoint
+def get_channel_folders(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    include_archived: Json[bool] = False,
+) -> HttpResponse:
+    channel_folders = get_channel_folders_in_realm(user_profile.realm, include_archived)
+    return json_success(
+        request,
+        data={"channel_folders": [asdict(folder) for folder in channel_folders]},
+    )
+
+
+@require_realm_admin
+@typed_endpoint
+def reorder_realm_channel_folders(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    order: Json[list[int]],
+) -> HttpResponse:
+    try_reorder_realm_channel_folders(user_profile.realm, order)
+    return json_success(request)
+
+
+@require_realm_admin
+@typed_endpoint
+def update_channel_folder(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    channel_folder_id: PathOnly[int],
+    description: Annotated[
+        str | None, StringConstraints(max_length=ChannelFolder.MAX_DESCRIPTION_LENGTH)
+    ] = None,
+    is_archived: Json[bool] | None = None,
+    name: Annotated[str | None, StringConstraints(max_length=ChannelFolder.MAX_NAME_LENGTH)] = None,
+) -> HttpResponse:
+    channel_folder = get_channel_folder_by_id(channel_folder_id, user_profile.realm)
+
+    if name is not None and channel_folder.name != name:
+        check_channel_folder_name(name, user_profile.realm)
+        do_change_channel_folder_name(channel_folder, name, acting_user=user_profile)
+
+    if description is not None and channel_folder.description != description:
+        do_change_channel_folder_description(channel_folder, description, acting_user=user_profile)
+
+    if is_archived is not None and channel_folder.is_archived != is_archived:
+        if is_archived:
+            if check_channel_folder_in_use(channel_folder):
+                raise JsonableError(
+                    _("You need to remove all the channels from this folder to archive it.")
+                )
+
+            do_archive_channel_folder(channel_folder, acting_user=user_profile)
+        else:
+            do_unarchive_channel_folder(channel_folder, acting_user=user_profile)
+
+    return json_success(request)

zerver/views/compatibility.py

@@ -0,0 +1,30 @@
+from django.http import HttpRequest, HttpResponse
+from django.utils.translation import gettext as _
+
+from zerver.lib.compatibility import find_mobile_os, version_lt
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.response import json_success
+from zerver.lib.user_agent import parse_user_agent
+
+# Zulip Mobile release 16.2.96 was made 2018-08-22.  It fixed a
+# bug in our Android code that causes spammy, obviously-broken
+# notifications once the "remove_push_notification" feature is
+# enabled on the user's Zulip server.
+android_min_app_version = "16.2.96"
+
+
+def check_global_compatibility(request: HttpRequest) -> HttpResponse:
+    if "User-Agent" not in request.headers:
+        raise JsonableError(_("User-Agent header missing from request"))
+
+    # This string should not be tagged for translation, since old
+    # clients are checking for an extra string.
+    legacy_compatibility_error_message = "Client is too old"
+    user_agent = parse_user_agent(request.headers["User-Agent"])
+    if user_agent["name"] == "ZulipInvalid":
+        raise JsonableError(legacy_compatibility_error_message)
+    if user_agent["name"] == "ZulipMobile":
+        user_os = find_mobile_os(request.headers["User-Agent"])
+        if user_os == "android" and version_lt(user_agent["version"], android_min_app_version):
+            raise JsonableError(legacy_compatibility_error_message)
+    return json_success(request)

zerver/views/custom_profile_fields.py

@@ -0,0 +1,348 @@
+from typing import Annotated
+
+import orjson
+from django.core.exceptions import ValidationError
+from django.db import IntegrityError, transaction
+from django.http import HttpRequest, HttpResponse
+from django.utils.translation import gettext as _
+from pydantic import Json, StringConstraints
+
+from zerver.actions.custom_profile_fields import (
+    check_remove_custom_profile_field_value,
+    do_remove_realm_custom_profile_field,
+    do_update_user_custom_profile_data_if_changed,
+    try_add_realm_custom_profile_field,
+    try_add_realm_default_custom_profile_field,
+    try_reorder_realm_custom_profile_fields,
+    try_update_realm_custom_profile_field,
+)
+from zerver.decorator import human_users_only, require_realm_admin
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.external_accounts import validate_external_account_field_data
+from zerver.lib.response import json_success
+from zerver.lib.typed_endpoint import PathOnly, typed_endpoint
+from zerver.lib.types import ProfileDataElementUpdateDict, ProfileFieldData
+from zerver.lib.users import validate_user_custom_profile_data
+from zerver.lib.validator import check_capped_string, validate_custom_profile_field_choices
+from zerver.models import CustomProfileField, Realm, UserProfile
+from zerver.models.custom_profile_fields import custom_profile_fields_for_realm
+
+
+def list_realm_custom_profile_fields(
+    request: HttpRequest, user_profile: UserProfile
+) -> HttpResponse:
+    fields = custom_profile_fields_for_realm(user_profile.realm_id)
+    return json_success(request, data={"custom_fields": [f.as_dict() for f in fields]})
+
+
+hint_validator = check_capped_string(CustomProfileField.HINT_MAX_LENGTH)
+name_validator = check_capped_string(CustomProfileField.NAME_MAX_LENGTH)
+
+
+def validate_field_name_and_hint(name: str, hint: str) -> None:
+    if not name.strip():
+        raise JsonableError(_("Label cannot be blank."))
+
+    try:
+        hint_validator("hint", hint)
+        name_validator("name", name)
+    except ValidationError as error:
+        raise JsonableError(error.message)
+
+
+def validate_custom_field_data(field_type: int, field_data: ProfileFieldData) -> None:
+    try:
+        if field_type == CustomProfileField.DROPDOWN:
+            # Choice type field must have at least have one choice
+            if len(field_data) < 1:
+                raise JsonableError(_("Field must have at least one choice."))
+            validate_custom_profile_field_choices(field_data)
+        elif field_type == CustomProfileField.EXTERNAL_ACCOUNT:
+            validate_external_account_field_data(field_data)
+    except ValidationError as error:
+        raise JsonableError(error.message)
+
+
+def validate_display_in_profile_summary_field(
+    field_type: int, display_in_profile_summary: bool
+) -> None:
+    if not display_in_profile_summary:
+        return
+
+    # The USER field type will require some further client support.
+    if field_type == CustomProfileField.USER:
+        raise JsonableError(_("Field type not supported for display in profile summary."))
+
+
+def validate_use_for_user_matching_field(field_type: int, use_for_user_matching: bool) -> None:
+    if not use_for_user_matching:
+        return
+
+    # Only SHORT_TEXT and EXTERNAL_ACCOUNT field types are supported for user matching.
+    if field_type not in (CustomProfileField.SHORT_TEXT, CustomProfileField.EXTERNAL_ACCOUNT):
+        raise JsonableError(_("Field type not supported for use for user matching."))
+
+
+def is_default_external_field(field_type: int, field_data: ProfileFieldData) -> bool:
+    if field_type != CustomProfileField.EXTERNAL_ACCOUNT:
+        return False
+    if field_data["subtype"] == "custom":
+        return False
+    return True
+
+
+def validate_custom_profile_field(
+    name: str,
+    hint: str,
+    field_type: int,
+    field_data: ProfileFieldData,
+    display_in_profile_summary: bool,
+    use_for_user_matching: bool,
+) -> None:
+    # Validate field data
+    validate_custom_field_data(field_type, field_data)
+
+    if not is_default_external_field(field_type, field_data):
+        # If field is default external field then we will fetch all data
+        # from our default field dictionary, so no need to validate name or hint
+        # Validate field name, hint if not default external account field
+        validate_field_name_and_hint(name, hint)
+
+    field_types = [i[0] for i in CustomProfileField.FIELD_TYPE_CHOICES]
+    if field_type not in field_types:
+        raise JsonableError(_("Invalid field type."))
+
+    validate_display_in_profile_summary_field(field_type, display_in_profile_summary)
+
+    validate_use_for_user_matching_field(field_type, use_for_user_matching)
+
+
+def validate_custom_profile_field_update(
+    field: CustomProfileField,
+    display_in_profile_summary: bool | None = None,
+    field_data: ProfileFieldData | None = None,
+    name: str | None = None,
+    hint: str | None = None,
+    use_for_user_matching: bool | None = None,
+) -> None:
+    if name is None:
+        name = field.name
+    if hint is None:
+        hint = field.hint
+    if field_data is None:
+        if field.field_data == "":
+            # We're passing this just for validation, sinec the function won't
+            # accept a string. This won't change the actual value.
+            field_data = {}
+        else:
+            field_data = orjson.loads(field.field_data)
+    if display_in_profile_summary is None:
+        display_in_profile_summary = field.display_in_profile_summary
+    if use_for_user_matching is None:
+        use_for_user_matching = field.use_for_user_matching
+
+    assert field_data is not None
+    validate_custom_profile_field(
+        name, hint, field.field_type, field_data, display_in_profile_summary, use_for_user_matching
+    )
+
+
+def update_only_display_in_profile_summary(
+    existing_field: CustomProfileField,
+    requested_field_data: ProfileFieldData | None = None,
+    requested_name: str | None = None,
+    requested_hint: str | None = None,
+) -> bool:
+    if (
+        (requested_name is not None and requested_name != existing_field.name)
+        or (requested_hint is not None and requested_hint != existing_field.hint)
+        or (
+            requested_field_data is not None
+            and requested_field_data != orjson.loads(existing_field.field_data)
+        )
+    ):
+        return False
+    return True
+
+
+def display_in_profile_summary_limit_reached(
+    realm: Realm, profile_field_id: int | None = None
+) -> bool:
+    query = CustomProfileField.objects.filter(realm=realm, display_in_profile_summary=True)
+    if profile_field_id is not None:
+        query = query.exclude(id=profile_field_id)
+    return query.count() >= CustomProfileField.MAX_DISPLAY_IN_PROFILE_SUMMARY_FIELDS
+
+
+@require_realm_admin
+@typed_endpoint
+def create_realm_custom_profile_field(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    display_in_profile_summary: Json[bool] = False,
+    editable_by_user: Json[bool] = True,
+    field_data: Json[ProfileFieldData] | None = None,
+    field_type: Json[int],
+    hint: str = "",
+    name: Annotated[str, StringConstraints(strip_whitespace=True)] = "",
+    required: Json[bool] = False,
+    use_for_user_matching: Json[bool] = False,
+) -> HttpResponse:
+    if field_data is None:
+        field_data = {}
+    if display_in_profile_summary and display_in_profile_summary_limit_reached(user_profile.realm):
+        raise JsonableError(
+            _("Only 2 custom profile fields can be displayed in the profile summary.")
+        )
+
+    validate_custom_profile_field(
+        name, hint, field_type, field_data, display_in_profile_summary, use_for_user_matching
+    )
+    try:
+        if is_default_external_field(field_type, field_data):
+            field_subtype = field_data["subtype"]
+            assert isinstance(field_subtype, str)
+            field = try_add_realm_default_custom_profile_field(
+                realm=user_profile.realm,
+                field_subtype=field_subtype,
+                display_in_profile_summary=display_in_profile_summary,
+                required=required,
+                editable_by_user=editable_by_user,
+                use_for_user_matching=use_for_user_matching,
+            )
+            return json_success(request, data={"id": field.id})
+        else:
+            field = try_add_realm_custom_profile_field(
+                realm=user_profile.realm,
+                name=name,
+                field_data=field_data,
+                field_type=field_type,
+                hint=hint,
+                display_in_profile_summary=display_in_profile_summary,
+                required=required,
+                editable_by_user=editable_by_user,
+                use_for_user_matching=use_for_user_matching,
+            )
+            return json_success(request, data={"id": field.id})
+    except IntegrityError:
+        raise JsonableError(_("A field with that label already exists."))
+
+
+@require_realm_admin
+def delete_realm_custom_profile_field(
+    request: HttpRequest, user_profile: UserProfile, field_id: int
+) -> HttpResponse:
+    try:
+        field = CustomProfileField.objects.get(realm_id=user_profile.realm_id, id=field_id)
+    except CustomProfileField.DoesNotExist:
+        raise JsonableError(_("Field id {id} not found.").format(id=field_id))
+
+    do_remove_realm_custom_profile_field(realm=user_profile.realm, field=field)
+    return json_success(request)
+
+
+@require_realm_admin
+@typed_endpoint
+def update_realm_custom_profile_field(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    display_in_profile_summary: Json[bool] | None = None,
+    editable_by_user: Json[bool] | None = None,
+    field_data: Json[ProfileFieldData] | None = None,
+    field_id: PathOnly[int],
+    hint: str | None = None,
+    name: Annotated[str, StringConstraints(strip_whitespace=True)] | None = None,
+    required: Json[bool] | None = None,
+    use_for_user_matching: Json[bool] | None = None,
+) -> HttpResponse:
+    realm = user_profile.realm
+    try:
+        field = CustomProfileField.objects.get(realm=realm, id=field_id)
+    except CustomProfileField.DoesNotExist:
+        raise JsonableError(_("Field id {id} not found.").format(id=field_id))
+
+    if display_in_profile_summary and display_in_profile_summary_limit_reached(
+        user_profile.realm, field.id
+    ):
+        raise JsonableError(
+            _("Only 2 custom profile fields can be displayed in the profile summary.")
+        )
+
+    if (
+        field.field_type == CustomProfileField.EXTERNAL_ACCOUNT
+        # HACK: Allow changing the display_in_profile_summary property
+        # of default external account types, but not any others.
+        #
+        # TODO: Make the name/hint/field_data parameters optional, and
+        # explicitly require that the client passes None for all of them for this case.
+        # Right now, for name/hint/field_data we allow the client to send the existing
+        # values for the respective fields. After this TODO is done, we will only allow
+        # the client to pass None values if the field is unchanged.
+        and is_default_external_field(field.field_type, orjson.loads(field.field_data))
+        and not update_only_display_in_profile_summary(field, field_data, name, hint)
+    ):
+        raise JsonableError(_("Default custom field cannot be updated."))
+
+    validate_custom_profile_field_update(
+        field, display_in_profile_summary, field_data, name, hint, use_for_user_matching
+    )
+    try:
+        try_update_realm_custom_profile_field(
+            realm=realm,
+            field=field,
+            name=name,
+            hint=hint,
+            field_data=field_data,
+            display_in_profile_summary=display_in_profile_summary,
+            required=required,
+            editable_by_user=editable_by_user,
+            use_for_user_matching=use_for_user_matching,
+        )
+    except IntegrityError:
+        raise JsonableError(_("A field with that label already exists."))
+    return json_success(request)
+
+
+@require_realm_admin
+@typed_endpoint
+def reorder_realm_custom_profile_fields(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    order: Json[list[int]],
+) -> HttpResponse:
+    try_reorder_realm_custom_profile_fields(user_profile.realm, order)
+    return json_success(request)
+
+
+@human_users_only
+@typed_endpoint
+def remove_user_custom_profile_data(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    data: Json[list[int]],
+) -> HttpResponse:
+    with transaction.atomic(durable=True):
+        for field_id in data:
+            check_remove_custom_profile_field_value(
+                user_profile, field_id, acting_user=user_profile, notify=False
+            )
+    return json_success(request)
+
+
+@human_users_only
+@typed_endpoint
+def update_user_custom_profile_data(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    data: Json[list[ProfileDataElementUpdateDict]],
+) -> HttpResponse:
+    validate_user_custom_profile_data(user_profile.realm.id, data, acting_user=user_profile)
+    with transaction.atomic(durable=True):
+        do_update_user_custom_profile_data_if_changed(user_profile, data, user_profile, notify=True)
+    # We need to call this explicitly otherwise constraints are not check
+    return json_success(request)

zerver/views/development/cache.py

@@ -0,0 +1,23 @@
+import os
+
+from django.http import HttpRequest, HttpResponse
+from django.views.decorators.csrf import csrf_exempt
+
+from zerver.decorator import require_post
+from zerver.lib.cache import get_cache_backend
+from zerver.lib.per_request_cache import flush_per_request_caches
+from zerver.lib.response import json_success
+from zerver.models.clients import clear_client_cache
+
+ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../../")
+
+
+# This is used only by the Puppeteer tests to clear all the cache after each run.
+@csrf_exempt
+@require_post
+def remove_caches(request: HttpRequest) -> HttpResponse:  # nocoverage
+    cache = get_cache_backend(None)
+    cache.clear()
+    clear_client_cache()
+    flush_per_request_caches()
+    return json_success(request)

zerver/views/development/camo.py

@@ -0,0 +1,20 @@
+from urllib.parse import urljoin
+
+from django.http import HttpRequest, HttpResponse, HttpResponseForbidden
+from django.shortcuts import redirect
+from django.utils.http import url_has_allowed_host_and_scheme
+
+from zerver.lib.camo import is_camo_url_valid
+
+
+def handle_camo_url(
+    request: HttpRequest, digest: str, received_url: str
+) -> HttpResponse:  # nocoverage
+    original_url = bytes.fromhex(received_url).decode()
+    if is_camo_url_valid(digest, original_url):
+        original_url = urljoin("/", original_url)
+        if url_has_allowed_host_and_scheme(original_url, allowed_hosts=None):
+            return redirect(original_url)
+        return HttpResponseForbidden("<p>Not a valid URL.</p>")
+    else:
+        return HttpResponseForbidden("<p>Not a valid URL.</p>")

zerver/views/development/dev_login.py

@@ -0,0 +1,165 @@
+from typing import Any
+
+from django.conf import settings
+from django.contrib.auth import authenticate
+from django.http import HttpRequest, HttpResponse, HttpResponseRedirect
+from django.utils.translation import gettext as _
+from django.views.decorators.csrf import csrf_exempt
+
+from zerver.context_processors import get_realm_from_request
+from zerver.decorator import do_login, require_post
+from zerver.lib.exceptions import (
+    AuthenticationFailedError,
+    InvalidSubdomainError,
+    JsonableError,
+    RealmDeactivatedError,
+    UserDeactivatedError,
+)
+from zerver.lib.response import json_success
+from zerver.lib.subdomains import get_subdomain
+from zerver.lib.typed_endpoint import typed_endpoint
+from zerver.lib.validator import validate_login_email
+from zerver.models import Realm, UserProfile
+from zerver.models.realms import get_realm
+from zerver.views.auth import get_safe_redirect_to
+from zerver.views.errors import config_error
+from zproject.backends import dev_auth_enabled
+
+
+def get_dev_users(realm: Realm | None = None, extra_users_count: int = 10) -> list[UserProfile]:
+    # Development environments usually have only a few users, but
+    # it still makes sense to limit how many extra users we render to
+    # support performance testing with DevAuthBackend.
+    if realm is not None:
+        users_query = UserProfile.objects.select_related("realm").filter(
+            is_bot=False, is_active=True, realm=realm
+        )
+    else:
+        users_query = UserProfile.objects.select_related("realm").filter(
+            is_bot=False, is_active=True
+        )
+
+    shakespearian_users = users_query.exclude(email__startswith="extrauser").order_by("email")
+    extra_users = users_query.filter(email__startswith="extrauser").order_by("email")
+    # Limit the number of extra users we offer by default
+    extra_users = extra_users[0:extra_users_count]
+    users = [*shakespearian_users, *extra_users]
+    return users
+
+
+def add_dev_login_context(realm: Realm | None, context: dict[str, Any]) -> None:
+    users = get_dev_users(realm)
+    context["current_realm"] = realm
+    context["all_realms"] = Realm.objects.filter(deactivated_redirect__isnull=True)
+
+    def sort(lst: list[UserProfile]) -> list[UserProfile]:
+        return sorted(lst, key=lambda u: u.delivery_email)
+
+    context["direct_owners"] = sort([u for u in users if u.is_realm_owner])
+    context["direct_admins"] = sort([u for u in users if u.is_realm_admin and not u.is_realm_owner])
+    context["guest_users"] = sort([u for u in users if u.is_guest])
+    context["direct_moderators"] = sort([u for u in users if u.role == UserProfile.ROLE_MODERATOR])
+    context["direct_users"] = sort([u for u in users if not (u.is_guest or u.is_moderator)])
+
+
+@csrf_exempt
+@typed_endpoint
+def dev_direct_login(
+    request: HttpRequest,
+    *,
+    next: str = "/",
+) -> HttpResponse:
+    # This function allows logging in without a password and should only be called
+    # in development environments.  It may be called if the DevAuthBackend is included
+    # in settings.AUTHENTICATION_BACKENDS
+    if (not dev_auth_enabled()) or settings.PRODUCTION:
+        # This check is probably not required, since authenticate would fail without
+        # an enabled DevAuthBackend.
+        return config_error(request, "dev_not_supported")
+
+    subdomain = get_subdomain(request)
+    realm = get_realm(subdomain)
+
+    if request.POST.get("prefers_web_public_view") == "Anonymous login":
+        redirect_to = get_safe_redirect_to(next, realm.url)
+        return HttpResponseRedirect(redirect_to)
+
+    email = request.POST["direct_email"]
+    user_profile = authenticate(dev_auth_username=email, realm=realm)
+    if user_profile is None:
+        return config_error(request, "dev_not_supported")
+    assert isinstance(user_profile, UserProfile)
+    do_login(request, user_profile)
+
+    redirect_to = get_safe_redirect_to(next, user_profile.realm.url)
+    return HttpResponseRedirect(redirect_to)
+
+
+def check_dev_auth_backend() -> None:
+    if settings.PRODUCTION:
+        raise JsonableError(_("Endpoint not available in production."))
+    if not dev_auth_enabled():
+        raise JsonableError(_("DevAuthBackend not enabled."))
+
+
+@csrf_exempt
+@require_post
+@typed_endpoint
+def api_dev_fetch_api_key(request: HttpRequest, *, username: str) -> HttpResponse:
+    """This function allows logging in without a password on the Zulip
+    mobile apps when connecting to a Zulip development environment.  It
+    requires DevAuthBackend to be included in settings.AUTHENTICATION_BACKENDS.
+    """
+    check_dev_auth_backend()
+
+    # Django invokes authenticate methods by matching arguments, and this
+    # authentication flow will not invoke LDAP authentication because of
+    # this condition of Django so no need to check if LDAP backend is
+    # enabled.
+    validate_login_email(username)
+    realm = get_realm_from_request(request)
+    if realm is None:
+        raise InvalidSubdomainError
+    return_data: dict[str, bool] = {}
+    user_profile = authenticate(dev_auth_username=username, realm=realm, return_data=return_data)
+    if return_data.get("inactive_realm"):
+        raise RealmDeactivatedError
+    if return_data.get("inactive_user"):
+        raise UserDeactivatedError
+    if return_data.get("invalid_subdomain"):  # nocoverage
+        raise InvalidSubdomainError
+    if user_profile is None:
+        # Since we're not actually checking passwords, this condition
+        # is when one's attempting to send an email address that
+        # doesn't have an account, i.e. it's definitely invalid username.
+        raise AuthenticationFailedError
+    assert isinstance(user_profile, UserProfile)
+
+    do_login(request, user_profile)
+    api_key = user_profile.api_key
+    return json_success(
+        request,
+        data={"api_key": api_key, "email": user_profile.delivery_email, "user_id": user_profile.id},
+    )
+
+
+@csrf_exempt
+def api_dev_list_users(request: HttpRequest) -> HttpResponse:
+    check_dev_auth_backend()
+
+    users = get_dev_users()
+    return json_success(
+        request,
+        data=dict(
+            direct_admins=[
+                dict(email=u.delivery_email, realm_url=u.realm.url)
+                for u in users
+                if u.is_realm_admin
+            ],
+            direct_users=[
+                dict(email=u.delivery_email, realm_url=u.realm.url)
+                for u in users
+                if not u.is_realm_admin
+            ],
+        ),
+    )

zerver/views/development/email_log.py

@@ -0,0 +1,167 @@
+import os
+from contextlib import suppress
+from urllib.parse import urlencode
+
+import orjson
+from django.conf import settings
+from django.http import HttpRequest, HttpResponse
+from django.shortcuts import redirect, render
+from django.views.decorators.http import require_safe
+
+from confirmation.models import Confirmation
+from zerver.actions.realm_settings import do_send_realm_reactivation_email
+from zerver.actions.user_settings import do_change_user_delivery_email
+from zerver.actions.users import change_user_is_active
+from zerver.lib.email_notifications import enqueue_welcome_emails, send_account_registered_email
+from zerver.lib.response import json_success
+from zerver.lib.typed_endpoint import typed_endpoint
+from zerver.models import Realm
+from zerver.models.realms import get_realm
+from zerver.models.streams import get_realm_stream
+from zerver.models.users import get_user_by_delivery_email
+from zerver.views.invite import INVITATION_LINK_VALIDITY_MINUTES
+from zproject.email_backends import get_forward_address, set_forward_address
+
+ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../")
+
+
+@typed_endpoint
+def email_page(request: HttpRequest, *, forward_address: str | None = None) -> HttpResponse:
+    if request.method == "POST":
+        assert forward_address is not None
+        set_forward_address(forward_address)
+        return json_success(request)
+    try:
+        with open(settings.EMAIL_CONTENT_LOG_PATH, "r+") as f:
+            content = f.read()
+    except FileNotFoundError:
+        content = ""
+    return render(
+        request,
+        "zerver/development/email_log.html",
+        {"log": content, "forward_address": get_forward_address()},
+    )
+
+
+def clear_emails(request: HttpRequest) -> HttpResponse:
+    with suppress(FileNotFoundError):
+        os.remove(settings.EMAIL_CONTENT_LOG_PATH)
+    return redirect(email_page)
+
+
+@require_safe
+def generate_all_emails(request: HttpRequest) -> HttpResponse:
+    # We import Django test client and override_settings inside
+    # the view function, because it isn't needed in production
+    # elsewhere, and not importing it saves ~50ms of unnecessary
+    # manage.py startup time.
+
+    from django.test import Client, override_settings
+
+    # override_settings is used as a hacky workaround to disable
+    # rate limiting; this is fine because this view only runs in
+    # the development environment.
+    with override_settings(RATE_LIMITING=False):
+        client = Client()
+
+        # write fake data for all variables
+        registered_email = "hamlet@zulip.com"
+        unregistered_email_1 = "new-person@zulip.com"
+        unregistered_email_2 = "new-person-2@zulip.com"
+        invite_expires_in_minutes = INVITATION_LINK_VALIDITY_MINUTES
+        realm = get_realm("zulip")
+        other_realm = Realm.objects.exclude(string_id="zulip").first()
+        user = get_user_by_delivery_email(registered_email, realm)
+
+        # Password reset emails
+        # active account in realm
+        result = client.post(
+            "/accounts/password/reset/", {"email": registered_email}, HTTP_HOST=realm.host
+        )
+        assert result.status_code == 302
+        # deactivated user
+        change_user_is_active(user, False)
+        result = client.post(
+            "/accounts/password/reset/", {"email": registered_email}, HTTP_HOST=realm.host
+        )
+        assert result.status_code == 302
+        change_user_is_active(user, True)
+        # account on different realm
+        assert other_realm is not None
+        result = client.post(
+            "/accounts/password/reset/", {"email": registered_email}, HTTP_HOST=other_realm.host
+        )
+        assert result.status_code == 302
+        # no account anywhere
+        result = client.post(
+            "/accounts/password/reset/", {"email": unregistered_email_1}, HTTP_HOST=realm.host
+        )
+        assert result.status_code == 302
+
+        # Confirm account email
+        result = client.post(
+            "/accounts/home/", {"email": unregistered_email_1}, HTTP_HOST=realm.host
+        )
+        assert result.status_code == 302
+
+        # Find account email
+        result = client.post("/accounts/find/", {"emails": registered_email}, HTTP_HOST=realm.host)
+        assert result.status_code == 200
+
+        # New login email
+        logged_in = client.login(dev_auth_username=registered_email, realm=realm)
+        assert logged_in
+
+        # New user invite and reminder emails
+        stream = get_realm_stream("Denmark", user.realm.id)
+        result = client.post(
+            "/json/invites",
+            {
+                "invitee_emails": unregistered_email_2,
+                "invite_expires_in_minutes": invite_expires_in_minutes,
+                "stream_ids": orjson.dumps([stream.id]).decode(),
+            },
+            HTTP_HOST=realm.host,
+        )
+        assert result.status_code == 200
+
+        # Verification for new email
+        result = client.patch(
+            "/json/settings",
+            urlencode({"email": "hamlets-new@zulip.com"}),
+            content_type="application/x-www-form-urlencoded",
+            HTTP_HOST=realm.host,
+        )
+        assert result.status_code == 200
+
+        # Email change successful
+        key = (
+            Confirmation.objects.filter(type=Confirmation.EMAIL_CHANGE)
+            .latest("id")
+            .confirmation_key
+        )
+        user_profile = get_user_by_delivery_email(registered_email, realm)
+        result = client.post("/accounts/confirm_new_email/", {"key": key})
+        assert result.status_code == 200
+
+        # Reset the email value so we can run this again
+        do_change_user_delivery_email(user_profile, registered_email, acting_user=None)
+
+        # Initial email with new account information for normal user
+        send_account_registered_email(user_profile)
+
+        # Onboarding emails for normal user
+        enqueue_welcome_emails(user_profile)
+
+        # Initial email with new account information for admin user
+        send_account_registered_email(get_user_by_delivery_email("iago@zulip.com", realm))
+
+        # Onboarding emails for admin user
+        enqueue_welcome_emails(
+            get_user_by_delivery_email("iago@zulip.com", realm), realm_creation=True
+        )
+
+        # Realm reactivation email
+        do_send_realm_reactivation_email(realm, acting_user=None)
+
+        return redirect(email_page)

zerver/views/development/help.py

@@ -0,0 +1,57 @@
+import os
+
+import werkzeug
+from django.conf import settings
+from django.http import HttpRequest, HttpResponse
+from django.shortcuts import render
+
+
+def help_dev_mode_view(request: HttpRequest, subpath: str = "") -> HttpResponse:
+    """
+    Dev only view that displays help information for setting up the
+    help center dev server in the default `run-dev` mode where the
+    help center server is not running. Also serves raw MDX content when
+    `raw` query param is passed is passed.
+    """
+
+    def read_mdx_file(filename: str) -> HttpResponse:
+        file_path = os.path.join(
+            settings.DEPLOY_ROOT, "starlight_help", "src", "content", "docs", f"{filename}.mdx"
+        )
+        try:
+            with open(file_path, encoding="utf-8") as f:
+                content = f.read()
+            return HttpResponse(content, content_type="text/plain")
+        except OSError:
+            return HttpResponse("Error reading MDX file", status=500)
+
+    mdx_file_exists = False
+    is_requesting_raw_file = request.GET.get("raw") == ""
+
+    if subpath:
+        subpath = werkzeug.utils.secure_filename(subpath)
+        raw_url = f"/help/{subpath}?raw"
+        mdx_path = os.path.join(
+            settings.DEPLOY_ROOT, "starlight_help", "src", "content", "docs", f"{subpath}.mdx"
+        )
+        mdx_file_exists = os.path.exists(mdx_path) and "/include/" not in mdx_path
+        if mdx_file_exists and is_requesting_raw_file:
+            return read_mdx_file(subpath)
+    else:
+        if request.path.endswith("/"):
+            raw_url = "/help/?raw"
+        else:
+            raw_url = "/help?raw"
+        mdx_file_exists = True
+        if is_requesting_raw_file:
+            return read_mdx_file("index")
+
+    return render(
+        request,
+        "zerver/development/dev_help.html",
+        {
+            "subpath": subpath,
+            "mdx_file_exists": mdx_file_exists,
+            "raw_url": raw_url,
+        },
+    )

zerver/views/development/integrations.py

@@ -0,0 +1,158 @@
+import os
+from contextlib import suppress
+from typing import TYPE_CHECKING, Any
+
+import orjson
+from django.http import HttpRequest, HttpResponse
+from django.http.response import HttpResponseBase
+from django.shortcuts import render
+from django.test import Client
+from pydantic import Json
+
+from zerver.lib.exceptions import JsonableError, ResourceNotFoundError
+from zerver.lib.integrations import INCOMING_WEBHOOK_INTEGRATIONS
+from zerver.lib.response import json_success
+from zerver.lib.typed_endpoint import PathOnly, typed_endpoint
+from zerver.lib.webhooks.common import call_fixture_to_headers, standardize_headers
+from zerver.models import UserProfile
+from zerver.models.realms import get_realm
+
+if TYPE_CHECKING:
+    from django.test.client import _MonkeyPatchedWSGIResponse as TestHttpResponse
+
+ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../../")
+
+
+def get_webhook_integrations() -> list[str]:
+    return [integration.name for integration in INCOMING_WEBHOOK_INTEGRATIONS]
+
+
+def get_valid_integration_name(name: str) -> str | None:
+    for integration_name in get_webhook_integrations():
+        if name == integration_name:
+            return integration_name
+    return None
+
+
+def dev_panel(request: HttpRequest) -> HttpResponse:
+    integrations = get_webhook_integrations()
+    bots = UserProfile.objects.filter(is_bot=True, bot_type=UserProfile.INCOMING_WEBHOOK_BOT)
+    context = {
+        "integrations": integrations,
+        "bots": bots,
+        # We set isolated_page to avoid clutter from footer/header.
+        "isolated_page": True,
+    }
+    return render(request, "zerver/development/integrations_dev_panel.html", context)
+
+
+def send_webhook_fixture_message(
+    url: str, body: str, is_json: bool, custom_headers: dict[str, Any]
+) -> "TestHttpResponse":
+    client = Client()
+    realm = get_realm("zulip")
+    standardized_headers = standardize_headers(custom_headers)
+    http_host = standardized_headers.pop("HTTP_HOST", realm.host)
+    if is_json:
+        content_type = standardized_headers.pop("HTTP_CONTENT_TYPE", "application/json")
+    else:
+        content_type = standardized_headers.pop("HTTP_CONTENT_TYPE", "text/plain")
+    return client.post(
+        url,
+        body,
+        content_type=content_type,
+        follow=False,
+        secure=False,
+        headers=None,
+        query_params=None,
+        HTTP_HOST=http_host,
+        **standardized_headers,
+    )
+
+
+@typed_endpoint
+def get_fixtures(request: HttpRequest, *, integration_name: PathOnly[str]) -> HttpResponse:
+    valid_integration_name = get_valid_integration_name(integration_name)
+    if not valid_integration_name:
+        raise ResourceNotFoundError(f'"{integration_name}" is not a valid webhook integration.')
+
+    fixtures = {}
+    fixtures_dir = os.path.join(ZULIP_PATH, f"zerver/webhooks/{valid_integration_name}/fixtures")
+    if not os.path.exists(fixtures_dir):
+        msg = f'The integration "{valid_integration_name}" does not have fixtures.'
+        raise ResourceNotFoundError(msg)
+
+    for fixture in os.listdir(fixtures_dir):
+        fixture_path = os.path.join(fixtures_dir, fixture)
+        with open(fixture_path) as f:
+            body = f.read()
+        # The file extension will be used to determine the type.
+        with suppress(orjson.JSONDecodeError):
+            body = orjson.loads(body)
+
+        headers_raw = call_fixture_to_headers(
+            valid_integration_name, "".join(fixture.split(".")[:-1])
+        )
+
+        def fix_name(header: str) -> str:  # nocoverage
+            return header.removeprefix("HTTP_")  # HTTP_ is a prefix intended for Django.
+
+        headers = {fix_name(k): v for k, v in headers_raw.items()}
+        fixtures[fixture] = {"body": body, "headers": headers}
+
+    return json_success(request, data={"fixtures": fixtures})
+
+
+@typed_endpoint
+def check_send_webhook_fixture_message(
+    request: HttpRequest,
+    *,
+    url: str,
+    body: str,
+    is_json: Json[bool],
+    custom_headers: str,
+) -> HttpResponseBase:
+    try:
+        custom_headers_dict = orjson.loads(custom_headers)
+    except orjson.JSONDecodeError as ve:  # nocoverage
+        raise JsonableError(f"Custom HTTP headers are not in a valid JSON format. {ve}")  # nolint
+
+    response = send_webhook_fixture_message(url, body, is_json, custom_headers_dict)
+    if response.status_code == 200:
+        responses = [{"status_code": response.status_code, "message": response.content.decode()}]
+        return json_success(request, data={"responses": responses})
+    else:  # nocoverage
+        return response
+
+
+@typed_endpoint
+def send_all_webhook_fixture_messages(
+    request: HttpRequest, *, url: str, integration_name: str
+) -> HttpResponse:
+    valid_integration_name = get_valid_integration_name(integration_name)
+    if not valid_integration_name:  # nocoverage
+        raise ResourceNotFoundError(f'"{integration_name}" is not a valid webhook integration.')
+
+    fixtures_dir = os.path.join(ZULIP_PATH, f"zerver/webhooks/{valid_integration_name}/fixtures")
+    if not os.path.exists(fixtures_dir):
+        msg = f'The integration "{valid_integration_name}" does not have fixtures.'
+        raise ResourceNotFoundError(msg)
+
+    responses = []
+    for fixture in os.listdir(fixtures_dir):
+        fixture_path = os.path.join(fixtures_dir, fixture)
+        with open(fixture_path) as f:
+            content = f.read()
+        x = fixture.split(".")
+        fixture_name, fixture_format = "".join(_ for _ in x[:-1]), x[-1]
+        headers = call_fixture_to_headers(valid_integration_name, fixture_name)
+        is_json = fixture_format == "json"
+        response = send_webhook_fixture_message(url, content, is_json, headers)
+        responses.append(
+            {
+                "status_code": response.status_code,
+                "fixture_name": fixture,
+                "message": response.content.decode(),
+            }
+        )
+    return json_success(request, data={"responses": responses})

zerver/views/development/registration.py

@@ -0,0 +1,104 @@
+from typing import TYPE_CHECKING, Any, cast
+
+from django.conf import settings
+from django.http import HttpRequest, HttpResponse, HttpResponseRedirect
+from django.views.decorators.csrf import csrf_exempt
+
+from confirmation.models import Confirmation, create_confirmation_link
+from zerver.context_processors import get_realm_from_request
+from zerver.lib.response import json_success
+from zerver.models import Realm, UserProfile
+from zerver.views.auth import (
+    create_preregistration_realm,
+    create_preregistration_user,
+    redirect_and_log_into_subdomain,
+)
+from zerver.views.registration import accounts_register, create_demo_helper
+from zproject.backends import ExternalAuthResult
+
+if TYPE_CHECKING:
+    from django.http.request import _ImmutableQueryDict
+
+
+# This is used only by the Puppeteer test in realm-creation.ts.
+def confirmation_key(request: HttpRequest) -> HttpResponse:
+    return json_success(request, data=request.session["confirmation_key"])
+
+
+def modify_postdata(request: HttpRequest, **kwargs: Any) -> None:
+    new_post = request.POST.copy()
+    for key, value in kwargs.items():
+        new_post[key] = value
+    new_post._mutable = False
+    request.POST = cast("_ImmutableQueryDict", new_post)
+
+
+@csrf_exempt
+def register_development_user(request: HttpRequest) -> HttpResponse:
+    realm = get_realm_from_request(request)
+    if realm is None:  # nocoverage
+        return HttpResponseRedirect(
+            f"{settings.EXTERNAL_URI_SCHEME}{settings.REALM_HOSTS['zulip']}/devtools/register_user/",
+            status=307,
+        )
+
+    count = UserProfile.objects.count()
+    name = f"user-{count}"
+    email = f"{name}@zulip.com"
+    prereg = create_preregistration_user(email, realm, password_required=False)
+    activation_url = create_confirmation_link(prereg, Confirmation.USER_REGISTRATION)
+    key = activation_url.split("/")[-1]
+    # Need to add test data to POST request as it doesn't originally contain the required parameters
+    modify_postdata(request, key=key, full_name=name, password="test", terms="true")
+
+    return accounts_register(request)
+
+
+@csrf_exempt
+def register_development_realm(request: HttpRequest) -> HttpResponse:
+    count = UserProfile.objects.count()
+    name = f"user-{count}"
+    email = f"{name}@zulip.com"
+    realm_name = f"realm-{count}"
+    realm_type = Realm.ORG_TYPES["business"]["id"]
+    realm_default_language = "en"
+    realm_subdomain = realm_name
+    prereg_realm = create_preregistration_realm(
+        email, realm_name, realm_subdomain, realm_type, realm_default_language
+    )
+    activation_url = create_confirmation_link(
+        prereg_realm, Confirmation.NEW_REALM_USER_REGISTRATION, no_associated_realm_object=True
+    )
+    key = activation_url.split("/")[-1]
+    # Need to add test data to POST request as it doesn't originally contain the required parameters
+    modify_postdata(
+        request,
+        key=key,
+        realm_name=realm_name,
+        realm_type=realm_type,
+        realm_default_language=realm_default_language,
+        full_name=name,
+        password="test",
+        realm_subdomain=realm_subdomain,
+        terms="true",
+        how_realm_creator_found_zulip="ad",
+        how_realm_creator_found_zulip_extra_context="test",
+    )
+
+    return accounts_register(request)
+
+
+@csrf_exempt
+def register_demo_development_realm(request: HttpRequest) -> HttpResponse:
+    user_profile = create_demo_helper(
+        request,
+        realm_type=Realm.ORG_TYPES["education"]["id"],
+        realm_default_language="en",
+        how_realm_creator_found_zulip="existing_user",
+        how_realm_creator_found_zulip_extra_context="test",
+        timezone="US/Pacific",
+    )
+
+    return redirect_and_log_into_subdomain(
+        ExternalAuthResult(user_profile=user_profile, data_dict={"is_realm_creation": True})
+    )

zerver/views/development/showroom.py

@@ -0,0 +1,63 @@
+import os
+
+from django.http import HttpRequest, HttpResponse
+from django.shortcuts import render
+
+background_colors = [
+    {"name": "Default background", "css_var": "--color-background"},
+    {"name": "Popover background", "css_var": "--color-background-popover-menu"},
+    {"name": "Modal background", "css_var": "--color-background-modal"},
+    {"name": "Compose background", "css_var": "--color-compose-box-background"},
+]
+
+
+def get_svg_filenames() -> list[str]:
+    icons_dir = os.path.join(os.path.dirname(__file__), "../../../web/icons")
+
+    # Get all .svg file names from the directory
+    svg_files = [f for f in os.listdir(icons_dir) if f.endswith(".svg")]
+
+    # Remove the .svg extension from the file names
+    icon_names = [os.path.splitext(f)[0] for f in svg_files]
+
+    # Sort the list alphabetically
+    return sorted(icon_names)
+
+
+def showroom_component_buttons(request: HttpRequest) -> HttpResponse:
+    context = {
+        "background_colors": background_colors,
+        "icons": get_svg_filenames(),
+        "page_is_showroom": True,
+        "showroom_component": "buttons",
+        "doc_root_title": "Button styles browser",
+        # We set isolated_page to avoid clutter from footer/header.
+        "isolated_page": True,
+    }
+    return render(request, "zerver/development/showroom/buttons.html", context)
+
+
+def showroom_component_banners(request: HttpRequest) -> HttpResponse:
+    context = {
+        "background_colors": background_colors,
+        "icons": get_svg_filenames(),
+        "page_is_showroom": True,
+        "showroom_component": "banners",
+        "doc_root_title": "Banner styles browser",
+        # We set isolated_page to avoid clutter from footer/header.
+        "isolated_page": True,
+    }
+    return render(request, "zerver/development/showroom/banners.html", context)
+
+
+def showroom_component_inputs(request: HttpRequest) -> HttpResponse:
+    context = {
+        "background_colors": background_colors,
+        "icons": get_svg_filenames(),
+        "page_is_showroom": True,
+        "showroom_component": "inputs",
+        "doc_root_title": "Input styles browser",
+        # We set isolated_page to avoid clutter from footer/header.
+        "isolated_page": True,
+    }
+    return render(request, "zerver/development/showroom/inputs.html", context)

zerver/views/devices.py

@@ -0,0 +1,24 @@
+from django.http import HttpRequest, HttpResponse
+from pydantic import Json
+
+from zerver.actions.devices import do_register_device, do_remove_device
+from zerver.lib.response import json_success
+from zerver.lib.typed_endpoint import typed_endpoint, typed_endpoint_without_parameters
+from zerver.models.users import UserProfile
+
+
+@typed_endpoint_without_parameters
+def register_device(request: HttpRequest, user_profile: UserProfile) -> HttpResponse:
+    device_id = do_register_device(user_profile)
+    return json_success(request, data={"device_id": device_id})
+
+
+@typed_endpoint
+def remove_device(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    device_id: Json[int],
+) -> HttpResponse:
+    do_remove_device(user_profile, device_id)
+    return json_success(request)

zerver/views/digest.py

@@ -0,0 +1,31 @@
+import time
+from datetime import timedelta
+
+from django.conf import settings
+from django.http import HttpRequest, HttpResponse
+from django.template.loader import render_to_string
+from django.utils.timezone import now as timezone_now
+
+from zerver.decorator import zulip_login_required
+from zerver.lib.digest import DIGEST_CUTOFF, get_digest_context
+from zerver.lib.send_email import get_inliner_instance
+
+
+@zulip_login_required
+def digest_page(request: HttpRequest) -> HttpResponse:
+    user_profile = request.user
+    assert user_profile.is_authenticated
+    cutoff = time.mktime((timezone_now() - timedelta(days=DIGEST_CUTOFF)).timetuple())
+
+    context = get_digest_context(user_profile, cutoff)
+    context.update(physical_address=settings.PHYSICAL_ADDRESS)
+
+    email_body_html = render_to_string("zerver/emails/digest.html", context, request=request)
+
+    inliner = get_inliner_instance()
+    inlined_body = inliner.inline(email_body_html)
+
+    context["inlined_digest_content"] = inlined_body
+    final_html = render_to_string("zerver/digest_base.html", context, request=request)
+
+    return HttpResponse(final_html)

zerver/views/documentation.py

@@ -0,0 +1,488 @@
+import os
+import random
+import re
+from collections import OrderedDict
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Any
+
+import werkzeug
+from django.conf import settings
+from django.http import HttpRequest, HttpResponse
+from django.template import loader
+from django.template.response import TemplateResponse
+from django.views.generic import TemplateView
+from lxml import html
+from lxml.etree import Element, SubElement, XPath, _Element
+from markupsafe import Markup
+from typing_extensions import override
+
+from zerver.context_processors import zulip_default_context
+from zerver.decorator import add_google_analytics_context
+from zerver.lib.html_to_text import get_content_description
+from zerver.lib.integrations import (
+    CATEGORIES,
+    INTEGRATIONS,
+    HubotIntegration,
+    IncomingWebhookIntegration,
+    Integration,
+    PythonAPIIntegration,
+    get_all_event_types_for_integration,
+)
+from zerver.lib.subdomains import get_subdomain
+from zerver.lib.templates import render_markdown_path
+from zerver.lib.typed_endpoint import PathOnly, typed_endpoint
+from zerver.models import Realm
+from zerver.openapi.openapi import get_endpoint_from_operationid, get_openapi_summary
+
+
+@dataclass
+class DocumentationArticle:
+    article_path: str
+    article_http_status: int
+    endpoint_path: str | None
+    endpoint_method: str | None
+
+
+def add_api_url_context(
+    context: dict[str, Any], request: HttpRequest, is_zilencer_endpoint: bool = False
+) -> None:
+    context.update(zulip_default_context(request))
+
+    if is_zilencer_endpoint:
+        context["api_url"] = (settings.ZULIP_SERVICES_URL or "https://push.zulipchat.com") + "/api"
+        return
+
+    subdomain = get_subdomain(request)
+    if subdomain != Realm.SUBDOMAIN_FOR_ROOT_DOMAIN or not settings.ROOT_DOMAIN_LANDING_PAGE:
+        display_subdomain = subdomain
+        html_settings_links = True
+    else:
+        display_subdomain = "your-org"
+        html_settings_links = False
+
+    display_host = Realm.host_for_subdomain(display_subdomain)
+    api_url_scheme_relative = display_host + "/api"
+    api_url = settings.EXTERNAL_URI_SCHEME + api_url_scheme_relative
+    zulip_url = settings.EXTERNAL_URI_SCHEME + display_host
+
+    context["display_subdomain"] = display_subdomain
+    context["display_host"] = display_host
+    context["external_url_scheme"] = settings.EXTERNAL_URI_SCHEME
+    context["api_url"] = api_url
+    context["api_url_scheme_relative"] = api_url_scheme_relative
+    context["zulip_url"] = zulip_url
+
+    context["html_settings_links"] = html_settings_links
+
+
+def add_canonical_link_context(context: dict[str, Any], request: HttpRequest) -> None:
+    if request.path in ["/api/", "/policies/", "/integrations/"]:
+        # Root doc pages have a trailing slash in the canonical URL.
+        canonical_path = request.path
+    else:
+        canonical_path = request.path.removesuffix("/")
+    context["REL_CANONICAL_LINK"] = f"https://zulip.com{canonical_path}"
+
+
+class ApiURLView(TemplateView):
+    @override
+    def get_context_data(self, **kwargs: Any) -> dict[str, str]:
+        context = super().get_context_data(**kwargs)
+        add_api_url_context(context, self.request)
+        return context
+
+
+sidebar_headings = XPath("//*[self::h1 or self::h2 or self::h3 or self::h4]")
+sidebar_links = XPath("//a[@href=$url]")
+
+
+class MarkdownDirectoryView(ApiURLView):
+    path_template = ""
+    policies_view = False
+    api_doc_view = False
+
+    def __init__(self, **kwargs: Any) -> None:
+        super().__init__(**kwargs)
+        self._post_render_callbacks: list[Callable[[HttpResponse], HttpResponse | None]] = []
+
+    def add_post_render_callback(
+        self, callback: Callable[[HttpResponse], HttpResponse | None]
+    ) -> None:
+        self._post_render_callbacks.append(callback)
+
+    def get_path(self, article: str) -> DocumentationArticle:
+        # We don't want to allow relative pathnames in `article`
+        # as they could introduce security vulnerabilities.
+        article = werkzeug.utils.secure_filename(article)
+
+        http_status = 200
+        if article == "":
+            article = "index"
+        elif article == "api-doc-template":
+            # This markdown template shouldn't be accessed directly.
+            article = "missing"
+            http_status = 404
+        elif len(article) > 100 or not re.match(r"^[0-9a-zA-Z_-]+$", article):
+            article = "missing"  # nocoverage
+            http_status = 404  # nocoverage
+
+        path = self.path_template % (article,)
+        endpoint_name = None
+        endpoint_method = None
+
+        if not self.path_template.startswith("/"):
+            # Relative paths only used for policies documentation
+            # when it is not configured or in the dev environment
+            assert self.policies_view
+
+            try:
+                loader.get_template(path)
+                return DocumentationArticle(
+                    article_path=path,
+                    article_http_status=http_status,
+                    endpoint_path=endpoint_name,
+                    endpoint_method=endpoint_method,
+                )
+            except loader.TemplateDoesNotExist:
+                return DocumentationArticle(
+                    article_path=self.path_template % ("missing",),
+                    article_http_status=404,
+                    endpoint_path=None,
+                    endpoint_method=None,
+                )
+
+        if not os.path.exists(path):
+            if self.api_doc_view:
+                try:
+                    # API endpoints documented in zerver/openapi/zulip.yaml
+                    endpoint_name, endpoint_method = get_endpoint_from_operationid(article)
+                    path = self.path_template % ("api-doc-template",)
+                except AssertionError:
+                    return DocumentationArticle(
+                        article_path=self.path_template % ("missing",),
+                        article_http_status=404,
+                        endpoint_path=None,
+                        endpoint_method=None,
+                    )
+            elif self.policies_view:
+                article = "missing"
+                http_status = 404
+                path = self.path_template % (article,)
+            else:
+                raise AssertionError("Invalid documentation view type")
+
+        return DocumentationArticle(
+            article_path=path,
+            article_http_status=http_status,
+            endpoint_path=endpoint_name,
+            endpoint_method=endpoint_method,
+        )
+
+    @override
+    def get_context_data(self, **kwargs: Any) -> dict[str, Any]:
+        article = kwargs["article"]
+        context: dict[str, Any] = super().get_context_data()
+
+        documentation_article = self.get_path(article)
+        context["article"] = documentation_article.article_path
+        not_index_page = not context["article"].endswith("/index.md")
+
+        if documentation_article.article_path.startswith("/") and os.path.exists(
+            documentation_article.article_path
+        ):
+            # Absolute path case
+            article_absolute_path = documentation_article.article_path
+        else:
+            # Relative path case
+            article_absolute_path = os.path.join(
+                settings.DEPLOY_ROOT, "templates", documentation_article.article_path
+            )
+
+        if self.policies_view:
+            context["page_is_policy_center"] = True
+            context["doc_root"] = "/policies/"
+            context["doc_root_title"] = "Terms and policies"
+            sidebar_article = self.get_path("sidebar_index")
+            if sidebar_article.article_http_status == 200:
+                sidebar_index = sidebar_article.article_path
+            else:
+                sidebar_index = None
+            title_base = "Zulip terms and policies"
+            # We don't add a rel-canonical link to self-hosted server policies docs.
+            if settings.CORPORATE_ENABLED:
+                add_canonical_link_context(context, self.request)
+        elif self.api_doc_view:
+            context["page_is_api_center"] = True
+            context["doc_root"] = "/api/"
+            context["doc_root_title"] = "API documentation"
+            sidebar_article = self.get_path("sidebar_index")
+            sidebar_index = sidebar_article.article_path
+            title_base = "Zulip API documentation"
+            add_canonical_link_context(context, self.request)
+        else:
+            raise AssertionError("Invalid documentation view type")
+
+        # The following is a somewhat hacky approach to extract titles from articles.
+        endpoint_name = None
+        endpoint_method = None
+        is_zilencer_endpoint = False
+        if os.path.exists(article_absolute_path):
+            with open(article_absolute_path) as article_file:
+                first_line = article_file.readlines()[0]
+            if self.api_doc_view and context["article"].endswith("api-doc-template.md"):
+                endpoint_name, endpoint_method = (
+                    documentation_article.endpoint_path,
+                    documentation_article.endpoint_method,
+                )
+                assert endpoint_name is not None
+                assert endpoint_method is not None
+                article_title = get_openapi_summary(endpoint_name, endpoint_method)
+                is_zilencer_endpoint = endpoint_name.startswith("/remotes/")
+            elif self.api_doc_view and "{generate_api_header(" in first_line:
+                api_operation = context["PAGE_METADATA_URL"].split("/api/")[1]
+                endpoint_name, endpoint_method = get_endpoint_from_operationid(api_operation)
+                article_title = get_openapi_summary(endpoint_name, endpoint_method)
+            else:
+                # Strip the header and then use the first line to get the article title
+                article_title = first_line.lstrip("#").strip()
+                endpoint_name = endpoint_method = None
+            if not_index_page:
+                context["PAGE_TITLE"] = f"{article_title} | {title_base}"
+            else:
+                context["PAGE_TITLE"] = title_base
+            placeholder_open_graph_description = (
+                f"REPLACEMENT_PAGE_DESCRIPTION_{int(2**24 * random.random())}"
+            )
+            context["PAGE_DESCRIPTION"] = placeholder_open_graph_description
+
+            def update_description(response: HttpResponse) -> None:
+                if placeholder_open_graph_description.encode() in response.content:
+                    first_paragraph_text = get_content_description(
+                        response.content, context["PAGE_METADATA_URL"]
+                    )
+                    response.content = response.content.replace(
+                        placeholder_open_graph_description.encode(),
+                        first_paragraph_text.encode(),
+                    )
+
+            self.add_post_render_callback(update_description)
+
+        # An "article" might require the api_url_context to be rendered
+        api_url_context: dict[str, Any] = {}
+        add_api_url_context(api_url_context, self.request, is_zilencer_endpoint)
+        api_url_context["run_content_validators"] = True
+        context["api_url_context"] = api_url_context
+        if endpoint_name and endpoint_method:
+            context["api_url_context"]["API_ENDPOINT_NAME"] = endpoint_name + ":" + endpoint_method
+
+        if sidebar_index is not None:
+            sidebar_html = render_markdown_path(sidebar_index)
+        else:
+            sidebar_html = ""
+        tree = html.fragment_fromstring(sidebar_html, create_parent=True)
+        if not context.get("page_is_policy_center", False):
+            home_h2 = Element("h2")
+            home_link = SubElement(home_h2, "a")
+            home_link.attrib["class"] = "no-underline"
+            home_link.attrib["href"] = context["doc_root"]
+            home_link.text = context["doc_root_title"] + " home"
+            tree.insert(0, home_h2)
+        url = context["doc_root"] + article
+        # Remove ID attributes from sidebar headings so they don't conflict with index page headings
+        headings = sidebar_headings(tree)
+        assert isinstance(headings, list)
+        for h in headings:
+            assert isinstance(h, _Element)
+            h.attrib.pop("id", "")
+        # Highlight current article link
+        links = sidebar_links(tree, url=url)
+        assert isinstance(links, list)
+        for a in links:
+            assert isinstance(a, _Element)
+            old_class = a.attrib.get("class", "")
+            assert isinstance(old_class, str)
+            a.attrib["class"] = old_class + " highlighted"
+        context["sidebar_html"] = Markup().join(
+            Markup(html.tostring(child, encoding="unicode")) for child in tree
+        )
+
+        add_google_analytics_context(context)
+        return context
+
+    @override
+    def get(
+        self, request: HttpRequest, *args: object, article: str = "", **kwargs: object
+    ) -> HttpResponse:
+        # Hack: It's hard to reinitialize urls.py from tests, and so
+        # we want to defer the use of settings.POLICIES_DIRECTORY to
+        # runtime.
+        if self.policies_view:
+            self.path_template = f"{settings.POLICIES_DIRECTORY}/%s.md"
+
+        documentation_article = self.get_path(article)
+        http_status = documentation_article.article_http_status
+        result = super().get(request, article=article)
+        assert isinstance(result, TemplateResponse)
+        for callback in self._post_render_callbacks:
+            result.add_post_render_callback(callback)
+        if http_status != 200:
+            result.status_code = http_status
+        return result
+
+
+def add_integrations_open_graph_context(context: dict[str, Any], request: HttpRequest) -> None:
+    path_name = request.path.rstrip("/").split("/")[-1]
+    description = (
+        "Zulip comes with over a hundred native integrations out of the box, "
+        "and integrates with Zapier and IFTTT to provide hundreds more. "
+        "Connect the apps you use every day to Zulip."
+    )
+
+    if path_name in INTEGRATIONS:
+        integration = INTEGRATIONS[path_name]
+        context["PAGE_TITLE"] = f"{integration.display_name} | Zulip integrations"
+        context["PAGE_DESCRIPTION"] = description
+
+    elif path_name in CATEGORIES:
+        category = CATEGORIES[path_name]
+        context["PAGE_TITLE"] = f"{category} | Zulip integrations"
+        context["PAGE_DESCRIPTION"] = description
+
+    elif path_name == "integrations":
+        context["PAGE_TITLE"] = "Zulip integrations"
+        context["PAGE_DESCRIPTION"] = description
+
+
+def build_integration_doc_html(integration: Integration, request: HttpRequest) -> str:
+    context: dict[str, Any] = {}
+    add_api_url_context(context, request)
+
+    context["integration_name"] = integration.name
+    context["integration_display_name"] = integration.display_name
+    if isinstance(integration, IncomingWebhookIntegration):
+        assert integration.url.startswith("api/")
+        context["integration_url"] = integration.url.removeprefix("api")
+        all_event_types = get_all_event_types_for_integration(integration)
+        if all_event_types is not None:
+            context["all_event_types"] = all_event_types
+    elif isinstance(integration, HubotIntegration):
+        context["hubot_docs_url"] = integration.hubot_docs_url
+    elif isinstance(integration, PythonAPIIntegration):
+        context["config_file_path"] = (
+            f"/usr/local/share/zulip/integrations/{integration.directory_name}/zulip_{integration.directory_name}_config.py"
+        )
+        context["integration_path"] = (
+            f"/usr/local/share/zulip/integrations/{integration.directory_name}"
+        )
+
+    return render_markdown_path(integration.doc, context, integration_doc=True)
+
+
+def add_base_integrations_context(request: HttpRequest) -> dict[str, Any]:
+    context: dict[str, Any] = {}
+    add_integrations_open_graph_context(context, request)
+    add_canonical_link_context(context, request)
+    add_google_analytics_context(context)
+    return context
+
+
+def get_visible_integrations_for_category(category_slug: str) -> list[Integration]:
+    enabled = sorted(
+        (
+            integration
+            for integration in INTEGRATIONS.values()
+            if integration.is_enabled_in_catalog() and not integration.legacy
+        ),
+        key=lambda integration: integration.name,
+    )
+    if category_slug == "all":
+        return enabled
+
+    category = CATEGORIES.get(category_slug)
+    return [integration for integration in enabled if category in integration.categories]
+
+
+def add_catalog_integrations_context(request: HttpRequest, category_slug: str) -> dict[str, Any]:
+    enabled_integrations_count = sum(v.is_enabled_in_catalog() for v in INTEGRATIONS.values())
+    # Subtract 1 so saying "Over X integrations" is correct. Then,
+    # round down to the nearest multiple of 10.
+    integrations_count_display = ((enabled_integrations_count - 1) // 10) * 10
+
+    context = add_base_integrations_context(request)
+    context.update(
+        {
+            "categories_dict": OrderedDict(sorted(CATEGORIES.items())),
+            "integrations_count_display": integrations_count_display,
+            "selected_category_slug": category_slug,
+            "visible_integrations": get_visible_integrations_for_category(category_slug),
+        }
+    )
+    return context
+
+
+def get_categories_for_integration(integration: Integration) -> list[tuple[str, str]]:
+    display_to_slug = {display: slug for slug, display in CATEGORIES.items()}
+    result = []
+    for display_name in integration.categories:
+        slug = display_to_slug.get(display_name)
+        assert slug is not None
+        result.append((slug, display_name))
+
+    return result
+
+
+def add_doc_integrations_context(
+    request: HttpRequest, integration: Integration, return_category_slug: str
+) -> dict[str, Any]:
+    context = add_base_integrations_context(request)
+    context.update(
+        {
+            "selected_integration": integration,
+            "integration_doc_html": build_integration_doc_html(integration, request),
+            "integration_categories": get_categories_for_integration(integration),
+            "return_category_slug": return_category_slug,
+        }
+    )
+    return context
+
+
+@typed_endpoint
+def integrations_catalog(
+    request: HttpRequest,
+    *,
+    category_slug: PathOnly[str],
+) -> HttpResponse:
+    if category_slug != "all" and category_slug not in CATEGORIES:
+        return TemplateResponse(request, "404.html", status=404)
+
+    return TemplateResponse(
+        request,
+        "zerver/integrations/catalog.html",
+        context=add_catalog_integrations_context(request, category_slug),
+        status=200,
+    )
+
+
+@typed_endpoint
+def integrations_doc(
+    request: HttpRequest,
+    *,
+    integration_name: PathOnly[str],
+) -> HttpResponse:
+    integration = INTEGRATIONS.get(integration_name)
+    if integration is None or not integration.is_enabled_in_catalog():
+        return TemplateResponse(request, "404.html", status=404)
+
+    return_category_slug = request.GET.get("category", "all")
+    category_slugs = [category[0] for category in get_categories_for_integration(integration)]
+    # If we have an invalid slug, back to list points to the root integrations page.
+    if return_category_slug != "all" and return_category_slug not in category_slugs:
+        return_category_slug = "all"
+
+    return TemplateResponse(
+        request,
+        "zerver/integrations/doc.html",
+        context=add_doc_integrations_context(request, integration, return_category_slug),
+        status=200,
+    )

zerver/views/drafts.py

@@ -0,0 +1,52 @@
+from django.http import HttpRequest, HttpResponse
+from pydantic import Json
+
+from zerver.lib.drafts import (
+    DraftData,
+    do_create_drafts,
+    do_delete_draft,
+    do_edit_draft,
+    draft_endpoint,
+)
+from zerver.lib.response import json_success
+from zerver.lib.typed_endpoint import PathOnly, typed_endpoint
+from zerver.models import Draft, UserProfile
+
+
+@draft_endpoint
+def fetch_drafts(request: HttpRequest, user_profile: UserProfile) -> HttpResponse:
+    user_drafts = Draft.objects.filter(user_profile=user_profile).order_by("last_edit_time")
+    draft_dicts = [draft.to_dict() for draft in user_drafts]
+    return json_success(request, data={"count": user_drafts.count(), "drafts": draft_dicts})
+
+
+@draft_endpoint
+@typed_endpoint
+def create_drafts(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    drafts: Json[list[DraftData]],
+) -> HttpResponse:
+    created_draft_objects = do_create_drafts(drafts, user_profile)
+    draft_ids = [draft_object.id for draft_object in created_draft_objects]
+    return json_success(request, data={"ids": draft_ids})
+
+
+@draft_endpoint
+@typed_endpoint
+def edit_draft(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    draft: Json[DraftData],
+    draft_id: PathOnly[int],
+) -> HttpResponse:
+    do_edit_draft(draft_id, draft, user_profile)
+    return json_success(request)
+
+
+@draft_endpoint
+def delete_draft(request: HttpRequest, user_profile: UserProfile, *, draft_id: int) -> HttpResponse:
+    do_delete_draft(draft_id, user_profile)
+    return json_success(request)

zerver/views/errors.py

@@ -0,0 +1,26 @@
+from django.conf import settings
+from django.http import HttpRequest, HttpResponse
+from django.shortcuts import render
+
+
+def config_error(
+    request: HttpRequest,
+    error_name: str,
+    *,
+    go_back_to_url: str | None = None,
+    go_back_to_url_name: str | None = None,
+) -> HttpResponse:
+    assert "/" not in error_name
+    context = {
+        "error_name": error_name,
+        "go_back_to_url": go_back_to_url or "/login/",
+        "go_back_to_url_name": go_back_to_url_name or "the login page",
+    }
+    if settings.DEVELOPMENT:
+        context["auth_settings_path"] = "zproject/dev-secrets.conf"
+        context["client_id_key_name"] = f"social_auth_{error_name}_key"
+    else:
+        context["auth_settings_path"] = "/etc/zulip/settings.py"
+        context["client_id_key_name"] = f"SOCIAL_AUTH_{error_name.upper()}_KEY"
+
+    return render(request, f"zerver/config_error/{error_name}.html", context=context, status=500)

zerver/views/events_register.py

@@ -0,0 +1,134 @@
+from typing import Annotated, Literal, TypeAlias
+
+from annotated_types import Len
+from django.conf import settings
+from django.contrib.auth.models import AnonymousUser
+from django.http import HttpRequest, HttpResponse
+from django.utils.translation import gettext as _
+from pydantic import Json, PositiveInt
+
+from zerver.context_processors import get_valid_realm_from_request
+from zerver.lib.compatibility import is_pronouns_field_type_supported
+from zerver.lib.events import DEFAULT_CLIENT_CAPABILITIES, ClientCapabilities, do_events_register
+from zerver.lib.exceptions import JsonableError, MissingAuthenticationError
+from zerver.lib.narrow_helpers import narrow_dataclasses_from_tuples
+from zerver.lib.request import RequestNotes
+from zerver.lib.response import json_success
+from zerver.lib.typed_endpoint import ApiParamConfig, typed_endpoint
+from zerver.models import Stream, UserProfile
+from zerver.views.streams import parse_include_subscribers
+
+
+def _default_all_public_streams(user_profile: UserProfile, all_public_streams: bool | None) -> bool:
+    if all_public_streams is not None:
+        return all_public_streams
+    else:
+        return user_profile.default_all_public_streams
+
+
+def _default_narrow(user_profile: UserProfile, narrow: list[list[str]]) -> list[list[str]]:
+    default_stream: Stream | None = user_profile.default_events_register_stream
+    if not narrow and default_stream is not None:
+        narrow = [["stream", default_stream.name]]
+    return narrow
+
+
+NarrowT: TypeAlias = list[Annotated[list[str], Len(min_length=2, max_length=2)]]
+
+
+@typed_endpoint
+def events_register_backend(
+    request: HttpRequest,
+    maybe_user_profile: UserProfile | AnonymousUser,
+    *,
+    all_public_streams: Json[bool] | None = None,
+    apply_markdown: Json[bool] = False,
+    client_capabilities: Json[ClientCapabilities] = DEFAULT_CLIENT_CAPABILITIES,
+    client_gravatar_raw: Annotated[Json[bool | None], ApiParamConfig("client_gravatar")] = None,
+    event_types: Json[list[str]] | None = None,
+    fetch_event_types: Json[list[str]] | None = None,
+    include_subscribers: Literal["true", "false", "partial"] = "false",
+    narrow: Json[NarrowT] | None = None,
+    presence_history_limit_days: Json[int] | None = None,
+    idle_queue_timeout: Json[PositiveInt | Literal["mobile"]] | None = None,
+    slim_presence: Json[bool] = False,
+) -> HttpResponse:
+    if narrow is None:
+        narrow = []
+    if client_gravatar_raw is None:
+        client_gravatar = maybe_user_profile.is_authenticated
+    else:
+        client_gravatar = client_gravatar_raw
+
+    parsed_include_subscribers = parse_include_subscribers(include_subscribers)
+
+    if maybe_user_profile.is_authenticated:
+        user_profile = maybe_user_profile
+        spectator_requested_language = None
+        assert isinstance(user_profile, UserProfile)
+        realm = user_profile.realm
+        include_streams = True
+
+        if all_public_streams and not user_profile.can_access_public_streams():
+            raise JsonableError(_("User not authorized for this query"))
+
+        all_public_streams = _default_all_public_streams(user_profile, all_public_streams)
+        narrow = _default_narrow(user_profile, narrow)
+    else:
+        user_profile = None
+        realm = get_valid_realm_from_request(request)
+        if not realm.allow_web_public_streams_access():
+            raise MissingAuthenticationError
+
+        # These parameters must be false for anonymous requests.
+        if client_gravatar:
+            raise JsonableError(
+                _("Invalid '{key}' parameter for anonymous request").format(key="client_gravatar")
+            )
+        if parsed_include_subscribers:
+            raise JsonableError(
+                _("Invalid '{key}' parameter for anonymous request").format(
+                    key="include_subscribers"
+                )
+            )
+
+        # Language set by spectator to be passed down to clients as user_settings.
+        spectator_requested_language = request.COOKIES.get(
+            settings.LANGUAGE_COOKIE_NAME, realm.default_language
+        )
+
+        all_public_streams = False
+        include_streams = False
+
+    client = RequestNotes.get_notes(request).client
+    assert client is not None
+
+    pronouns_field_type_supported = is_pronouns_field_type_supported(
+        request.headers.get("User-Agent")
+    )
+
+    # TODO: We eventually want to let callers pass in dictionaries over the wire,
+    #       but we will still need to support tuples for a long time.
+    modern_narrow = narrow_dataclasses_from_tuples(narrow)
+
+    ret = do_events_register(
+        user_profile,
+        realm,
+        client,
+        apply_markdown,
+        client_gravatar,
+        slim_presence,
+        None,
+        presence_history_limit_days,
+        event_types,
+        idle_queue_timeout,
+        all_public_streams,
+        narrow=modern_narrow,
+        include_subscribers=parsed_include_subscribers,
+        include_streams=include_streams,
+        client_capabilities=client_capabilities,
+        fetch_event_types=fetch_event_types,
+        spectator_requested_language=spectator_requested_language,
+        pronouns_field_type_supported=pronouns_field_type_supported,
+    )
+    return json_success(request, data=ret)

zerver/views/health.py

@@ -0,0 +1,67 @@
+from django.conf import settings
+from django.db.migrations.recorder import MigrationRecorder
+from django.http import HttpRequest, HttpResponse
+from django.utils.crypto import get_random_string
+from django.utils.translation import gettext as _
+from pika import BlockingConnection
+
+from zerver.lib.cache import cache_delete, cache_get, cache_set
+from zerver.lib.exceptions import ServerNotReadyError
+from zerver.lib.queue import get_queue_client
+from zerver.lib.redis_utils import get_redis_client
+from zerver.lib.response import json_success
+
+
+def check_database() -> None:
+    try:
+        if not MigrationRecorder.Migration.objects.exists():
+            raise ServerNotReadyError(_("Database is empty"))  # nocoverage
+    except ServerNotReadyError:  # nocoverage
+        raise
+    except Exception:  # nocoverage
+        raise ServerNotReadyError(_("Cannot query postgresql"))
+
+
+def check_rabbitmq() -> None:  # nocoverage
+    try:
+        conn = get_queue_client().connection
+        if conn is None:
+            raise ServerNotReadyError(_("Cannot connect to rabbitmq"))
+        assert isinstance(conn, BlockingConnection)
+        conn.process_data_events()
+    except ServerNotReadyError:
+        raise
+    except Exception:
+        raise ServerNotReadyError(_("Cannot query rabbitmq"))
+
+
+def check_redis() -> None:
+    try:
+        get_redis_client().ping()
+    except Exception:  # nocoverage
+        raise ServerNotReadyError(_("Cannot query redis"))
+
+
+def check_memcached() -> None:
+    try:
+        roundtrip_key = "health_check_" + get_random_string(32)
+        roundtrip_value = get_random_string(32)
+        cache_set(roundtrip_key, roundtrip_value)
+        got_value = cache_get(roundtrip_key)[0]
+        if got_value != roundtrip_value:
+            raise ServerNotReadyError(_("Cannot write to memcached"))  # nocoverage
+        cache_delete(roundtrip_key)
+    except ServerNotReadyError:  # nocoverage
+        raise
+    except Exception:  # nocoverage
+        raise ServerNotReadyError(_("Cannot query memcached"))
+
+
+def health(request: HttpRequest) -> HttpResponse:
+    check_database()
+    if settings.USING_RABBITMQ:  # nocoverage
+        check_rabbitmq()
+    check_redis()
+    check_memcached()
+
+    return json_success(request)

zerver/views/home.py

@@ -0,0 +1,302 @@
+import logging
+import secrets
+
+from django.conf import settings
+from django.http import HttpRequest, HttpResponse, HttpResponseRedirect
+from django.shortcuts import redirect, render
+from django.urls import reverse
+from django.utils.cache import patch_cache_control
+from django.utils.timezone import now as timezone_now
+
+from zerver.actions.user_settings import (
+    do_change_tos_version,
+    do_change_user_date_joined,
+    do_change_user_setting,
+)
+from zerver.actions.users import do_change_is_imported_stub
+from zerver.context_processors import get_realm_from_request, get_valid_realm_from_request
+from zerver.decorator import web_public_view, zulip_login_required
+from zerver.forms import ToSForm
+from zerver.lib.compatibility import is_banned_browser, is_outdated_desktop_app
+from zerver.lib.home import build_page_params_for_home_page_load, get_user_permission_info
+from zerver.lib.narrow_helpers import NeverNegatedNarrowTerm
+from zerver.lib.request import RequestNotes
+from zerver.lib.streams import access_stream_by_name
+from zerver.lib.subdomains import get_subdomain
+from zerver.models import Realm, RealmUserDefault, Stream, UserProfile
+
+
+def need_accept_tos(user_profile: UserProfile | None) -> bool:
+    if user_profile is None:
+        return False
+
+    if user_profile.tos_version == UserProfile.TOS_VERSION_BEFORE_FIRST_LOGIN:
+        return True
+
+    if settings.TERMS_OF_SERVICE_VERSION is None:
+        return False
+
+    return int(settings.TERMS_OF_SERVICE_VERSION.split(".")[0]) > user_profile.major_tos_version()
+
+
+@zulip_login_required
+def accounts_accept_terms(request: HttpRequest) -> HttpResponse:
+    assert request.user.is_authenticated
+
+    if request.method == "POST":
+        form = ToSForm(request.POST)
+        if form.is_valid():
+            first_time_login = (
+                request.user.tos_version == UserProfile.TOS_VERSION_BEFORE_FIRST_LOGIN
+            )
+            assert (
+                settings.TERMS_OF_SERVICE_VERSION is not None
+                or request.user.tos_version == UserProfile.TOS_VERSION_BEFORE_FIRST_LOGIN
+            )
+            do_change_tos_version(request.user, settings.TERMS_OF_SERVICE_VERSION)
+
+            email_address_visibility = form.cleaned_data["email_address_visibility"]
+            if (
+                email_address_visibility is not None
+                and email_address_visibility != request.user.email_address_visibility
+            ):
+                do_change_user_setting(
+                    request.user,
+                    "email_address_visibility",
+                    email_address_visibility,
+                    acting_user=request.user,
+                )
+
+            enable_marketing_emails = form.cleaned_data["enable_marketing_emails"]
+            if (
+                enable_marketing_emails is not None
+                and enable_marketing_emails != request.user.enable_marketing_emails
+            ):
+                do_change_user_setting(
+                    request.user,
+                    "enable_marketing_emails",
+                    enable_marketing_emails,
+                    acting_user=request.user,
+                )
+
+            if request.user.is_imported_stub:
+                do_change_is_imported_stub(request.user)
+
+            if first_time_login:
+                do_change_user_date_joined(request.user, timezone_now())
+
+            return redirect(home)
+    else:
+        form = ToSForm()
+
+    default_email_address_visibility = None
+    first_time_login = request.user.tos_version == UserProfile.TOS_VERSION_BEFORE_FIRST_LOGIN
+    if first_time_login:
+        realm_user_default = RealmUserDefault.objects.get(realm=request.user.realm)
+        default_email_address_visibility = realm_user_default.email_address_visibility
+
+    context = {
+        "form": form,
+        "email": request.user.delivery_email,
+        # Text displayed when updating TERMS_OF_SERVICE_VERSION.
+        "terms_of_service_message": settings.TERMS_OF_SERVICE_MESSAGE,
+        "terms_of_service_version": settings.TERMS_OF_SERVICE_VERSION,
+        # HTML template used when agreeing to terms of service the
+        # first time, e.g. after data import.
+        "first_time_terms_of_service_message_template": None,
+        "first_time_login": first_time_login,
+        "default_email_address_visibility": default_email_address_visibility,
+        "email_address_visibility_options_dict": UserProfile.EMAIL_ADDRESS_VISIBILITY_ID_TO_NAME_MAP,
+        "email_address_visibility_admins_only": UserProfile.EMAIL_ADDRESS_VISIBILITY_ADMINS,
+        "email_address_visibility_moderators": UserProfile.EMAIL_ADDRESS_VISIBILITY_MODERATORS,
+        "email_address_visibility_nobody": UserProfile.EMAIL_ADDRESS_VISIBILITY_NOBODY,
+    }
+
+    if (
+        request.user.tos_version == UserProfile.TOS_VERSION_BEFORE_FIRST_LOGIN
+        and settings.FIRST_TIME_TERMS_OF_SERVICE_TEMPLATE is not None
+    ):
+        context["first_time_terms_of_service_message_template"] = (
+            settings.FIRST_TIME_TERMS_OF_SERVICE_TEMPLATE
+        )
+
+    return render(
+        request,
+        "zerver/accounts_accept_terms.html",
+        context,
+    )
+
+
+def detect_narrowed_window(
+    request: HttpRequest, user_profile: UserProfile | None
+) -> tuple[list[NeverNegatedNarrowTerm], Stream | None, str | None]:
+    """This function implements Zulip's support for a mini Zulip window
+    that just handles messages from a single narrow"""
+    if user_profile is None:
+        return [], None, None
+
+    narrow: list[NeverNegatedNarrowTerm] = []
+    narrow_stream = None
+    narrow_topic_name = request.GET.get("topic")
+
+    if "stream" in request.GET:
+        try:
+            # TODO: We should support stream IDs and direct messages here as well.
+            narrow_stream_name = request.GET.get("stream")
+            assert narrow_stream_name is not None
+            (narrow_stream, _sub) = access_stream_by_name(user_profile, narrow_stream_name)
+            narrow = [NeverNegatedNarrowTerm(operator="stream", operand=narrow_stream.name)]
+        except Exception:
+            logging.warning("Invalid narrow requested, ignoring", extra=dict(request=request))
+        if narrow_stream is not None and narrow_topic_name is not None:
+            narrow.append(NeverNegatedNarrowTerm(operator="topic", operand=narrow_topic_name))
+    return narrow, narrow_stream, narrow_topic_name
+
+
+def update_last_reminder(user_profile: UserProfile | None) -> None:
+    """Reset our don't-spam-users-with-email counter since the
+    user has since logged in
+    """
+    if user_profile is None:
+        return
+
+    if user_profile.last_reminder is not None:  # nocoverage
+        # TODO: Look into the history of last_reminder; we may have
+        # eliminated that as a useful concept for non-bot users.
+        user_profile.last_reminder = None
+        user_profile.save(update_fields=["last_reminder"])
+
+
+def home(request: HttpRequest) -> HttpResponse:
+    subdomain = get_subdomain(request)
+
+    # If settings.ROOT_DOMAIN_LANDING_PAGE and this is the root
+    # domain, send the user the landing page.
+    if (
+        settings.ROOT_DOMAIN_LANDING_PAGE
+        and subdomain == Realm.SUBDOMAIN_FOR_ROOT_DOMAIN
+        and settings.CORPORATE_ENABLED
+    ):
+        from corporate.views.portico import hello_view
+
+        return hello_view(request)
+
+    if subdomain == settings.SOCIAL_AUTH_SUBDOMAIN:
+        return redirect(settings.ROOT_DOMAIN_URI)
+    elif subdomain == settings.SELF_HOSTING_MANAGEMENT_SUBDOMAIN:
+        return redirect(reverse("remote_billing_legacy_server_login"))
+    realm = get_realm_from_request(request)
+    if realm is None:
+        context = {
+            "current_url": request.get_host(),
+        }
+        return render(request, "zerver/invalid_realm.html", status=404, context=context)
+    if realm.allow_web_public_streams_access():
+        return web_public_view(home_real)(request)
+    return zulip_login_required(home_real)(request)
+
+
+def home_real(request: HttpRequest) -> HttpResponse:
+    # Before we do any real work, check if the app is banned.
+    client_user_agent = request.headers.get("User-Agent", "")
+    (insecure_desktop_app, banned_desktop_app, auto_update_broken) = is_outdated_desktop_app(
+        client_user_agent
+    )
+    if banned_desktop_app:
+        return render(
+            request,
+            "zerver/portico_error_pages/insecure_desktop_app.html",
+            context={
+                "auto_update_broken": auto_update_broken,
+            },
+        )
+    (unsupported_browser, browser_name) = is_banned_browser(client_user_agent)
+    if unsupported_browser:
+        return render(
+            request,
+            "zerver/portico_error_pages/unsupported_browser.html",
+            context={
+                "browser_name": browser_name,
+            },
+        )
+
+    # We need to modify the session object every two weeks or it will expire.
+    # This line makes reloading the page a sufficient action to keep the
+    # session alive.
+    request.session.modified = True
+
+    if request.user.is_authenticated:
+        user_profile = request.user
+        realm = user_profile.realm
+    else:
+        realm = get_valid_realm_from_request(request)
+        # We load the spectator experience.  We fall through to the shared code
+        # for loading the application, with user_profile=None encoding
+        # that we're a spectator, not a logged-in user.
+        user_profile = None
+
+    update_last_reminder(user_profile)
+
+    # If a user hasn't signed the current Terms of Service, send them there
+    if need_accept_tos(user_profile):
+        return accounts_accept_terms(request)
+
+    narrow, narrow_stream, narrow_topic_name = detect_narrowed_window(request, user_profile)
+
+    queue_id, page_params = build_page_params_for_home_page_load(
+        request=request,
+        user_profile=user_profile,
+        realm=realm,
+        insecure_desktop_app=insecure_desktop_app,
+        narrow=narrow,
+        narrow_stream=narrow_stream,
+        narrow_topic_name=narrow_topic_name,
+    )
+
+    log_data = RequestNotes.get_notes(request).log_data
+    assert log_data is not None
+    log_data["extra"] = f"[{queue_id}]"
+
+    csp_nonce = secrets.token_hex(24)
+
+    user_permission_info = get_user_permission_info(user_profile)
+    is_firefox_android = "Firefox" in client_user_agent and "Android" in client_user_agent
+
+    response = render(
+        request,
+        "zerver/app/index.html",
+        context={
+            "user_profile": user_profile,
+            "page_params": page_params,
+            "csp_nonce": csp_nonce,
+            "color_scheme": user_permission_info.color_scheme,
+            "enable_gravatar": settings.ENABLE_GRAVATAR,
+            "is_firefox_android": is_firefox_android,
+            "s3_avatar_public_url_prefix": settings.S3_AVATAR_PUBLIC_URL_PREFIX
+            if settings.LOCAL_UPLOADS_DIR is None
+            else "",
+            "has_web_public_streams": realm.web_public_streams_enabled(),
+        },
+    )
+    patch_cache_control(response, no_cache=True, no_store=True, must_revalidate=True)
+    return response
+
+
+@zulip_login_required
+def desktop_home(request: HttpRequest) -> HttpResponse:
+    return redirect(home)
+
+
+def doc_permalinks_view(request: HttpRequest, doc_id: str) -> HttpResponse:
+    DOC_PERMALINK_MAP: dict[str, str] = {
+        "usage-statistics": "https://zulip.readthedocs.io/en/stable/production/mobile-push-notifications.html#uploading-usage-statistics",
+        "basic-metadata": "https://zulip.readthedocs.io/en/stable/production/mobile-push-notifications.html#uploading-basic-metadata",
+        "why-service": "https://zulip.readthedocs.io/en/stable/production/mobile-push-notifications.html#why-a-push-notification-service-is-necessary",
+        "registration-transfer": "https://zulip.readthedocs.io/en/latest/production/mobile-push-notifications.html#moving-your-registration-to-a-new-server",
+    }
+
+    redirect_url = DOC_PERMALINK_MAP.get(doc_id)
+    if redirect_url is None:
+        return render(request, "404.html", status=404)
+
+    return HttpResponseRedirect(redirect_url)

zerver/views/invite.py

@@ -0,0 +1,316 @@
+import email.utils
+from typing import Annotated
+
+from django.conf import settings
+from django.db import transaction
+from django.http import HttpRequest, HttpResponse
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+from pydantic import Json, StringConstraints
+
+from confirmation import settings as confirmation_settings
+from zerver.actions.invites import (
+    do_create_multiuse_invite_link,
+    do_get_invites_controlled_by_user,
+    do_invite_users,
+    do_revoke_multi_use_invite,
+    do_revoke_user_invite,
+    do_send_user_invite_email,
+)
+from zerver.decorator import require_human_non_guest_user
+from zerver.lib.exceptions import InvitationError, JsonableError, OrganizationOwnerRequiredError
+from zerver.lib.response import json_success
+from zerver.lib.streams import access_stream_by_id, get_streams_to_which_user_cannot_add_subscribers
+from zerver.lib.typed_endpoint import ApiParamConfig, PathOnly, typed_endpoint
+from zerver.lib.typed_endpoint_validators import check_int_in_validator
+from zerver.lib.types import Invitee
+from zerver.lib.user_groups import UserGroupMembershipDetails, access_user_group_for_update
+from zerver.models import (
+    MultiuseInvite,
+    NamedUserGroup,
+    PreregistrationUser,
+    Realm,
+    Stream,
+    UserProfile,
+)
+
+# Convert INVITATION_LINK_VALIDITY_DAYS into minutes.
+# Because mypy fails to correctly infer the type of the validator, we want this constant
+# to be Optional[int] to avoid a mypy error when using it as the default value.
+# https://github.com/python/mypy/issues/13234
+INVITATION_LINK_VALIDITY_MINUTES: int | None = 24 * 60 * settings.INVITATION_LINK_VALIDITY_DAYS
+
+
+def check_role_based_permissions(
+    invited_as: int, user_profile: UserProfile, *, require_admin: bool
+) -> None:
+    if (
+        invited_as == PreregistrationUser.INVITE_AS["REALM_OWNER"]
+        and not user_profile.is_realm_owner
+    ):
+        raise OrganizationOwnerRequiredError
+
+    if require_admin and not user_profile.is_realm_admin:
+        raise JsonableError(_("Must be an organization administrator"))
+
+
+def access_invite_by_id(user_profile: UserProfile, invite_id: int) -> PreregistrationUser:
+    try:
+        prereg_user = PreregistrationUser.objects.get(id=invite_id)
+    except PreregistrationUser.DoesNotExist:
+        raise JsonableError(_("No such invitation"))
+
+    # Structurally, any invitation the user can actually access should
+    # have a referred_by set for the user who created it.
+    if prereg_user.referred_by is None or prereg_user.referred_by.realm != user_profile.realm:
+        raise JsonableError(_("No such invitation"))
+
+    if prereg_user.status != 0:
+        raise JsonableError(_("Invitation already used or deactivated."))
+
+    if prereg_user.referred_by_id != user_profile.id:
+        check_role_based_permissions(prereg_user.invited_as, user_profile, require_admin=True)
+    return prereg_user
+
+
+def access_multiuse_invite_by_id(user_profile: UserProfile, invite_id: int) -> MultiuseInvite:
+    try:
+        invite = MultiuseInvite.objects.get(id=invite_id)
+    except MultiuseInvite.DoesNotExist:
+        raise JsonableError(_("No such invitation"))
+
+    if invite.realm != user_profile.realm:
+        raise JsonableError(_("No such invitation"))
+
+    if invite.referred_by_id != user_profile.id:
+        check_role_based_permissions(invite.invited_as, user_profile, require_admin=True)
+
+    if invite.status == confirmation_settings.STATUS_REVOKED:
+        raise JsonableError(_("Invitation has already been revoked"))
+    return invite
+
+
+def access_streams_for_invite(stream_ids: list[int], user_profile: UserProfile) -> list[Stream]:
+    streams: list[Stream] = []
+
+    for stream_id in stream_ids:
+        try:
+            (stream, _sub) = access_stream_by_id(user_profile, stream_id)
+        except JsonableError:
+            raise JsonableError(
+                _("Invalid channel ID {channel_id}. No invites were sent.").format(
+                    channel_id=stream_id
+                )
+            )
+        streams.append(stream)
+
+    user_group_membership_details = UserGroupMembershipDetails(user_recursive_group_ids=None)
+    streams_to_which_user_cannot_add_subscribers = get_streams_to_which_user_cannot_add_subscribers(
+        streams,
+        user_profile,
+        allow_default_streams=True,
+        user_group_membership_details=user_group_membership_details,
+    )
+    if len(streams_to_which_user_cannot_add_subscribers) > 0:
+        raise JsonableError(_("You do not have permission to subscribe other users to channels."))
+
+    return streams
+
+
+def access_user_groups_for_invite(
+    group_ids: list[int] | None, user_profile: UserProfile
+) -> list[NamedUserGroup]:
+    user_groups: list[NamedUserGroup] = []
+    if group_ids:
+        with transaction.atomic(durable=True):
+            for group_id in group_ids:
+                user_group = access_user_group_for_update(
+                    group_id, user_profile, permission_setting="can_add_members_group"
+                )
+                user_groups.append(user_group)
+
+    return user_groups
+
+
+@require_human_non_guest_user
+@typed_endpoint
+def invite_users_backend(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    group_ids: Json[list[int]] | None = None,
+    include_realm_default_subscriptions: Json[bool] = False,
+    invite_as: Annotated[
+        Json[int],
+        check_int_in_validator(list(PreregistrationUser.INVITE_AS.values())),
+    ] = PreregistrationUser.INVITE_AS["MEMBER"],
+    invite_expires_in_minutes: Json[int | None] = INVITATION_LINK_VALIDITY_MINUTES,
+    invitee_emails_raw: Annotated[str, ApiParamConfig("invitee_emails")],
+    notify_referrer_on_join: Json[bool] = True,
+    stream_ids: Json[list[int]],
+    welcome_message_custom_text: Annotated[
+        str | None,
+        StringConstraints(
+            max_length=Realm.MAX_REALM_WELCOME_MESSAGE_CUSTOM_TEXT_LENGTH,
+        ),
+    ] = None,
+) -> HttpResponse:
+    if not user_profile.can_invite_users_by_email():
+        # Guest users case will not be handled here as it will
+        # be handled by the decorator above.
+        raise JsonableError(_("Insufficient permission"))
+
+    require_admin = invite_as in [
+        # Owners can only be invited by owners, checked by separate
+        # logic in check_role_based_permissions.
+        PreregistrationUser.INVITE_AS["REALM_OWNER"],
+        PreregistrationUser.INVITE_AS["REALM_ADMIN"],
+        PreregistrationUser.INVITE_AS["MODERATOR"],
+    ]
+    check_role_based_permissions(invite_as, user_profile, require_admin=require_admin)
+
+    if not invitee_emails_raw:
+        raise JsonableError(_("You must specify at least one email address."))
+
+    invitee_emails = get_invitees_set(invitee_emails_raw)
+
+    streams = access_streams_for_invite(stream_ids, user_profile)
+    user_groups = access_user_groups_for_invite(group_ids, user_profile)
+
+    if welcome_message_custom_text is not None and not user_profile.is_realm_admin:
+        raise JsonableError(_("Must be an organization administrator"))
+
+    skipped = do_invite_users(
+        user_profile,
+        invitee_emails,
+        streams,
+        notify_referrer_on_join,
+        user_groups,
+        invite_expires_in_minutes=invite_expires_in_minutes,
+        include_realm_default_subscriptions=include_realm_default_subscriptions,
+        invite_as=invite_as,
+        welcome_message_custom_text=welcome_message_custom_text,
+    )
+
+    if skipped:
+        raise InvitationError(
+            _(
+                "Some of those addresses are already using Zulip, "
+                "so we didn't send them an invitation. We did send "
+                "invitations to everyone else!"
+            ),
+            skipped,
+            sent_invitations=True,
+        )
+
+    return json_success(request)
+
+
+def get_invitee_emails_set(invitee_emails_raw: str) -> set[str]:
+    return {
+        email_addr
+        for name, email_addr in email.utils.getaddresses(
+            invitee_emails_raw.split("\n"), strict=False
+        )
+    } - {""}
+
+
+def get_invitees_set(invitee_emails_raw: str) -> set[Invitee]:
+    return {
+        Invitee(email=email, full_name=full_name)
+        for full_name, email in email.utils.getaddresses(
+            invitee_emails_raw.split("\n"), strict=False
+        )
+        if email
+    }
+
+
+@require_human_non_guest_user
+def get_user_invites(request: HttpRequest, user_profile: UserProfile) -> HttpResponse:
+    all_users = do_get_invites_controlled_by_user(user_profile)
+    return json_success(request, data={"invites": all_users})
+
+
+@require_human_non_guest_user
+@typed_endpoint
+def revoke_user_invite(
+    request: HttpRequest, user_profile: UserProfile, *, invite_id: PathOnly[int]
+) -> HttpResponse:
+    prereg_user = access_invite_by_id(user_profile, invite_id)
+    do_revoke_user_invite(prereg_user, acting_user=user_profile)
+    return json_success(request)
+
+
+@require_human_non_guest_user
+@typed_endpoint
+def revoke_multiuse_invite(
+    request: HttpRequest, user_profile: UserProfile, *, invite_id: PathOnly[int]
+) -> HttpResponse:
+    invite = access_multiuse_invite_by_id(user_profile, invite_id)
+    do_revoke_multi_use_invite(invite, acting_user=user_profile)
+    return json_success(request)
+
+
+@require_human_non_guest_user
+@typed_endpoint
+def resend_user_invite_email(
+    request: HttpRequest, user_profile: UserProfile, *, invite_id: PathOnly[int]
+) -> HttpResponse:
+    prereg_user = access_invite_by_id(user_profile, invite_id)
+    do_send_user_invite_email(prereg_user, event_time=timezone_now())
+    return json_success(request)
+
+
+@require_human_non_guest_user
+@typed_endpoint
+def generate_multiuse_invite_backend(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    group_ids: Json[list[int]] | None = None,
+    include_realm_default_subscriptions: Json[bool] = False,
+    invite_as: Annotated[
+        Json[int],
+        check_int_in_validator(list(PreregistrationUser.INVITE_AS.values())),
+    ] = PreregistrationUser.INVITE_AS["MEMBER"],
+    invite_expires_in_minutes: Json[int | None] = INVITATION_LINK_VALIDITY_MINUTES,
+    stream_ids: Json[list[int]] | None = None,
+    welcome_message_custom_text: Annotated[
+        str | None,
+        StringConstraints(
+            max_length=Realm.MAX_REALM_WELCOME_MESSAGE_CUSTOM_TEXT_LENGTH,
+        ),
+    ] = None,
+) -> HttpResponse:
+    if stream_ids is None:
+        stream_ids = []
+    if not user_profile.can_create_multiuse_invite_to_realm():
+        # Guest users case will not be handled here as it will
+        # be handled by the decorator above.
+        raise JsonableError(_("Insufficient permission"))
+
+    require_admin = invite_as in [
+        # Owners can only be invited by owners, checked by separate
+        # logic in check_role_based_permissions.
+        PreregistrationUser.INVITE_AS["REALM_OWNER"],
+        PreregistrationUser.INVITE_AS["REALM_ADMIN"],
+        PreregistrationUser.INVITE_AS["MODERATOR"],
+    ]
+    check_role_based_permissions(invite_as, user_profile, require_admin=require_admin)
+
+    streams = access_streams_for_invite(stream_ids, user_profile)
+    user_groups = access_user_groups_for_invite(group_ids, user_profile)
+
+    if welcome_message_custom_text is not None and not user_profile.is_realm_admin:
+        raise JsonableError(_("Must be an organization administrator"))
+
+    invite_link = do_create_multiuse_invite_link(
+        user_profile,
+        invite_as,
+        invite_expires_in_minutes,
+        include_realm_default_subscriptions,
+        streams,
+        user_groups,
+        welcome_message_custom_text,
+    )
+    return json_success(request, data={"invite_link": invite_link})

zerver/views/llms_txt.py

@@ -0,0 +1,197 @@
+import json
+from urllib.parse import urlencode
+
+from django.http import HttpRequest, HttpResponse
+
+from zerver.context_processors import get_valid_realm_from_request
+from zerver.lib.streams import get_web_public_streams_queryset
+
+
+def llms_txt(request: HttpRequest) -> HttpResponse:
+    """Serve /llms.txt so that LLMs can discover how to read web-public
+    channel messages via the Zulip API without authentication.
+
+    Returns 404 if the realm has no web-public streams.
+
+    See https://llmstxt.org/ for the llms.txt specification.
+    """
+    realm = get_valid_realm_from_request(request)
+
+    # web_public_streams_enabled() is a cheap in-memory check; skip the
+    # DB query entirely for realms that clearly have no web-public access.
+    if not realm.web_public_streams_enabled():
+        return HttpResponse(status=404)
+
+    streams = list(
+        get_web_public_streams_queryset(realm).order_by("name").values_list("id", "name")
+    )
+
+    # Return 404 if the feature is enabled at the realm level but no
+    # web-public channels actually exist yet.
+    if not streams:
+        return HttpResponse(status=404)
+
+    server_url = realm.url
+    example_channel_id = streams[0][0]
+    example_narrow = json.dumps(
+        [
+            {"operator": "channels", "operand": "web-public"},
+            {"operator": "channel", "operand": example_channel_id},
+            {"operator": "topic", "operand": "TOPIC_NAME"},
+        ],
+        separators=(",", ":"),
+    )
+    example_url = f"{server_url}/json/messages?" + urlencode(
+        {"anchor": "oldest", "num_before": 0, "num_after": 100, "narrow": example_narrow}
+    )
+    channel_list = "\n".join(f"- {name} ({channel_id})" for channel_id, name in streams)
+
+    content = f"""\
+# {realm.name}
+
+> {server_url} is a Zulip team chat server.
+> Zulip organizes messages into channels containing threads ("topics").
+>
+> Web-public channels can be read without logging in.
+
+## Reading web-public channel messages
+
+Use the Zulip REST API to fetch messages without authentication:
+
+    GET {server_url}/json/messages
+
+Required query parameters:
+
+- `anchor` — message ID or keyword (`oldest`, `newest`, `first_unread`)
+- `num_before` — number of messages before anchor (e.g. `0`)
+- `num_after` — number of messages after anchor (e.g. `100`)
+- `narrow` — JSON array of filter operators; URL-encode the entire
+  value in the query string. For example:
+  `[{{"operator":"channels","operand":"web-public"}},{{"operator":"channel","operand":CHANNEL_ID}},{{"operator":"topic","operand":"TOPIC_NAME"}}]`
+
+**Important**: the narrow must include `{{"operator":"channels","operand":"web-public"}}`
+to enable unauthenticated access.
+
+**Important**: Be kind to the Zulip server that you're interacting
+with. Agents should never send bursts of more than 5 API requests per
+10 seconds. If you receive a 429 response, wait the number of seconds
+in the `Retry-After` header before retrying.
+
+Batches of 100 messages are recommended. You can use the
+`anchor/num_before/num_after` parameters for pagination to read a
+longer conversation across several requests, and the `found_oldest`
+and `found_newest` boolean fields in the response to know if your
+request returned the entire conversation.
+
+## Example
+
+Fetch the 100 oldest messages from a topic:
+
+    GET {example_url}
+
+## Fetching messages in specific conversations / views
+
+Zulip conversations are referenced by URLs of these forms:
+
+    {server_url}/#narrow/channel/ID-NAME
+    {server_url}/#narrow/channel/ID-NAME/topic/TOPIC
+    {server_url}/#narrow/channel/ID-NAME/topic/TOPIC/with/MSG_ID
+    {server_url}/#narrow/channel/ID-NAME/topic/TOPIC/near/MSG_ID
+
+In the URL forms above, `ID-NAME` is the channel's numeric ID
+followed by its name (e.g., `113488-general`). When constructing
+API requests, use the numeric ID as the `channel` operand (e.g.,
+`{{"operator":"channel","operand":113488}}`), not the full `ID-NAME`
+string and not the channel name. Channel IDs are stable; channel
+names can be renamed.
+
+Never follow links to related conversations by fetching those URLs;
+translate them to the equivalent API request instead (see above). In a
+`#narrow/...` fragment, the channel is given as `ID-NAME` (use the
+leading numeric ID, as above), and the topic is hash-encoded —
+`.`-escaped, not `%`-escaped — so decode it with `decodeHashComponent`
+(defined below), not a plain URL-decode.
+
+(`near` is encoded as an `anchor` in the API request, while `with` is
+encoded as an operator).
+
+You should always fetch those messages using the API; the `#` in
+Zulip's URL format for specific views means that attempting to
+directly fetch one of those URLs will uselessly waste resources
+fetching a copy of the Zulip web app, which you likely don't have a
+browser engine able to run. The following code from
+web/src/internal_url.ts may be helpful for doing the encoding/decoding.
+
+```ts
+// ' and ! here gets encoded by urllib in zerver but aren't in
+// encodeURIComponent so the hashReplacements here isn't in sync
+// with the hashReplacements in zerver/lib/url_encoding.py.
+// We escape * to bypass server markdown double tag pattern.
+const hashReplacements = new Map([
+    ["%", "."],
+    ["!", ".21"],
+    ["'", ".27"],
+    ["(", ".28"],
+    [")", ".29"],
+    ["*", ".2A"],
+    [".", ".2E"],
+]);
+
+// Some browsers zealously URI-decode the contents of
+// window.location.hash.  So we hide our URI-encoding
+// by replacing % with . (like MediaWiki).
+export function encodeHashComponent(str: string): string {{
+    return encodeURIComponent(str).replaceAll(
+        /[%!'()*.]/g,
+        (matched) => hashReplacements.get(matched)!,
+    );
+}}
+
+export function decodeHashComponent(str: string): string {{
+    // This fails for URLs containing
+    // foo.foo or foo%foo due to our fault in special handling
+    // of such characters when encoding. This can also,
+    // fail independent of our fault.
+    // Here we let the calling code handle the exception.
+    return decodeURIComponent(str.replaceAll(".", "%"));
+}}
+```
+
+Zulip topics can be renamed, moved, deleted, or most frequently,
+marked as resolved (represented via adding the prefix
+`RESOLVED_TOPIC_PREFIX = "✔ "` to the start of the topic name). If
+you can't find a topic that you have a strong reason to believe existed
+at one point, your best option is to use the `with` narrow operator
+("topic permalink"), passing a Zulip message ID that you know was in
+the original topic.
+
+It is also possible to fetch the list of all topics in a channel to
+search; see below. That operation can be expensive, so use it
+sparingly.
+
+## Resources
+
+- Message fetch API: https://zulip.com/api/get-messages
+- List of topics in a channel API: https://zulip.com/api/get-stream-topics
+- Search operators: https://zulip.com/help/search-for-messages and https://zulip.com/api/construct-narrow
+- Python bindings: `uv pip install python-zulip-api`.
+- Zulip project: https://zulip.com
+- Zulip server source: https://github.com/zulip/zulip/
+- Official data export tool: https://zulip.com/help/export-your-organization
+- Zulip archive: https://github.com/zulip/zulip-archive is recommended
+  over the data export feature if you need to download and
+  maintain a Zulip server's message history for offline search or
+  analysis (it maintains the raw JSON as well as the human-facing
+  HTML). Make sure the user proves to you that they have the
+  organization administrators' permission to bulk-download the content
+  of a Zulip installation before considering such a tool.
+
+## Web-public channels on this server
+
+Each entry below is in the format `channel name (channel ID)`:
+
+{channel_list}
+
+"""
+
+    return HttpResponse(content, content_type="text/plain; charset=utf-8")

zerver/views/message_edit.py

@@ -0,0 +1,291 @@
+from datetime import timedelta
+from typing import Literal
+
+import orjson
+from django.contrib.auth.models import AnonymousUser
+from django.db import IntegrityError, transaction
+from django.http import HttpRequest, HttpResponse
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+from pydantic import Json, NonNegativeInt
+
+from zerver.actions.message_delete import do_delete_messages
+from zerver.actions.message_edit import check_update_message
+from zerver.context_processors import get_valid_realm_from_request
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.html_diff import highlight_html_differences
+from zerver.lib.message import (
+    access_message,
+    access_message_and_usermessage,
+    access_web_public_message,
+    messages_for_ids,
+    visible_edit_history_for_message,
+)
+from zerver.lib.request import RequestNotes
+from zerver.lib.response import json_success
+from zerver.lib.streams import can_delete_any_message_in_channel, can_delete_own_message_in_channel
+from zerver.lib.timestamp import datetime_to_timestamp
+from zerver.lib.topic import maybe_rename_empty_topic_to_general_chat
+from zerver.lib.typed_endpoint import OptionalTopic, PathOnly, typed_endpoint
+from zerver.lib.types import EditHistoryEvent, FormattedEditHistoryEvent
+from zerver.models import Message, UserProfile
+from zerver.models.realms import MessageEditHistoryVisibilityPolicyEnum
+from zerver.models.streams import Stream, get_stream_by_id_in_realm
+
+
+def fill_edit_history_entries(
+    raw_edit_history: list[EditHistoryEvent],
+    message: Message,
+    allow_empty_topic_name: bool,
+) -> list[FormattedEditHistoryEvent]:
+    """
+    This fills out the message edit history entries from the database
+    to have the current topic + content as of that time, plus data on
+    whatever changed. This makes it much simpler to do future
+    processing.
+    """
+    prev_content = message.content
+    prev_rendered_content = message.rendered_content
+    is_channel_message = message.is_channel_message
+    if is_channel_message:
+        prev_topic_name = maybe_rename_empty_topic_to_general_chat(
+            message.topic_name(), is_channel_message, allow_empty_topic_name
+        )
+    else:
+        prev_topic_name = ""
+
+    # Make sure that the latest entry in the history corresponds to the
+    # message's last edit time
+    if len(raw_edit_history) > 0:
+        assert message.last_edit_time is not None
+        assert datetime_to_timestamp(message.last_edit_time) == raw_edit_history[0]["timestamp"]
+
+    formatted_edit_history: list[FormattedEditHistoryEvent] = []
+    for edit_history_event in raw_edit_history:
+        formatted_entry: FormattedEditHistoryEvent = {
+            "content": prev_content,
+            "rendered_content": prev_rendered_content,
+            "timestamp": edit_history_event["timestamp"],
+            "topic": prev_topic_name,
+            "user_id": edit_history_event["user_id"],
+        }
+
+        if "prev_topic" in edit_history_event:
+            prev_topic_name = maybe_rename_empty_topic_to_general_chat(
+                edit_history_event["prev_topic"], is_channel_message, allow_empty_topic_name
+            )
+            formatted_entry["prev_topic"] = prev_topic_name
+
+        # Fill current values for content/rendered_content.
+        if "prev_content" in edit_history_event:
+            formatted_entry["prev_content"] = edit_history_event["prev_content"]
+            prev_content = formatted_entry["prev_content"]
+            formatted_entry["prev_rendered_content"] = edit_history_event["prev_rendered_content"]
+            prev_rendered_content = formatted_entry["prev_rendered_content"]
+            assert prev_rendered_content is not None
+            rendered_content = formatted_entry["rendered_content"]
+            assert rendered_content is not None
+            formatted_entry["content_html_diff"] = highlight_html_differences(
+                prev_rendered_content, rendered_content, message.id
+            )
+
+        if "prev_stream" in edit_history_event:
+            formatted_entry["prev_stream"] = edit_history_event["prev_stream"]
+            formatted_entry["stream"] = edit_history_event["stream"]
+
+        formatted_edit_history.append(formatted_entry)
+
+    initial_message_history: FormattedEditHistoryEvent = {
+        "content": prev_content,
+        "rendered_content": prev_rendered_content,
+        "timestamp": datetime_to_timestamp(message.date_sent),
+        "topic": prev_topic_name,
+        "user_id": message.sender_id,
+    }
+
+    formatted_edit_history.append(initial_message_history)
+
+    return formatted_edit_history
+
+
+@typed_endpoint
+def get_message_edit_history(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    allow_empty_topic_name: Json[bool] = False,
+    message_id: PathOnly[NonNegativeInt],
+) -> HttpResponse:
+    user_realm_message_edit_history_visibility_policy = (
+        user_profile.realm.message_edit_history_visibility_policy
+    )
+    if (
+        user_realm_message_edit_history_visibility_policy
+        == MessageEditHistoryVisibilityPolicyEnum.none.value
+    ):
+        raise JsonableError(_("Message edit history is disabled in this organization"))
+    message = access_message(user_profile, message_id, is_modifying_message=False)
+
+    # Extract the message edit history from the message
+    if message.edit_history is not None:
+        raw_edit_history = orjson.loads(message.edit_history)
+    else:
+        raw_edit_history = []
+
+    # Fill in all the extra data that will make it usable
+    message_edit_history = fill_edit_history_entries(
+        raw_edit_history, message, allow_empty_topic_name
+    )
+
+    visible_message_edit_history = visible_edit_history_for_message(
+        user_realm_message_edit_history_visibility_policy, message_edit_history
+    )
+
+    return json_success(
+        request, data={"message_history": list(reversed(visible_message_edit_history))}
+    )
+
+
+@typed_endpoint
+def update_message_backend(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    content: str | None = None,
+    message_id: PathOnly[NonNegativeInt],
+    prev_content_sha256: str | None = None,
+    propagate_mode: Literal["change_later", "change_one", "change_all"] = "change_one",
+    send_notification_to_new_thread: Json[bool] = True,
+    send_notification_to_old_thread: Json[bool] = False,
+    stream_id: Json[NonNegativeInt] | None = None,
+    topic_name: OptionalTopic = None,
+) -> HttpResponse:
+    updated_message_result = check_update_message(
+        user_profile,
+        message_id,
+        stream_id,
+        topic_name,
+        propagate_mode,
+        send_notification_to_old_thread,
+        send_notification_to_new_thread,
+        content,
+        prev_content_sha256,
+    )
+
+    # Include the number of messages changed in the logs
+    log_data = RequestNotes.get_notes(request).log_data
+    assert log_data is not None
+    log_data["extra"] = f"[{updated_message_result.changed_message_count}]"
+
+    return json_success(request, data={"detached_uploads": updated_message_result.detached_uploads})
+
+
+def validate_can_delete_message(user_profile: UserProfile, message: Message) -> None:
+    if user_profile.can_delete_any_message():
+        return
+
+    stream: Stream | None = None
+    if message.is_channel_message:
+        stream = get_stream_by_id_in_realm(message.recipient.type_id, user_profile.realm)
+        if can_delete_any_message_in_channel(user_profile, stream):
+            return
+
+    if message.sender != user_profile and message.sender.bot_owner_id != user_profile.id:
+        # Users can only delete messages sent by them or by their bots.
+        raise JsonableError(_("You don't have permission to delete this message"))
+
+    if not user_profile.can_delete_own_message():
+        if not message.is_channel_message:
+            raise JsonableError(_("You don't have permission to delete this message"))
+
+        assert stream is not None
+        # For channel messages, users are required to have either the
+        # channel-level permission or the organization-level permission to delete
+        # their own messages.
+        if not can_delete_own_message_in_channel(user_profile, stream):
+            raise JsonableError(_("You don't have permission to delete this message"))
+
+    deadline_seconds: int | None = user_profile.realm.message_content_delete_limit_seconds
+    if deadline_seconds is None:
+        # None means no time limit to delete message
+        return
+    if (timezone_now() - message.date_sent) > timedelta(seconds=deadline_seconds):
+        # User cannot delete message after deadline time of realm
+        raise JsonableError(_("The time limit for deleting this message has passed"))
+    return
+
+
+@transaction.atomic(durable=True)
+@typed_endpoint
+def delete_message_backend(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    message_id: PathOnly[NonNegativeInt],
+) -> HttpResponse:
+    # We lock the `Message` object to ensure that any transactions modifying the `Message` object
+    # concurrently are serialized properly with deleting the message; this prevents a deadlock
+    # that would otherwise happen because of the other transaction holding a lock on the `Message`
+    # row.
+    message = access_message(user_profile, message_id, lock_message=True, is_modifying_message=True)
+    validate_can_delete_message(user_profile, message)
+    try:
+        do_delete_messages(user_profile.realm, [message], acting_user=user_profile)
+    except (Message.DoesNotExist, IntegrityError):
+        raise JsonableError(_("Message already deleted"))
+    return json_success(request)
+
+
+@typed_endpoint
+def json_fetch_raw_message(
+    request: HttpRequest,
+    maybe_user_profile: UserProfile | AnonymousUser,
+    *,
+    allow_empty_topic_name: Json[bool] = False,
+    apply_markdown: Json[bool] = True,
+    message_id: PathOnly[NonNegativeInt],
+) -> HttpResponse:
+    if not maybe_user_profile.is_authenticated:
+        realm = get_valid_realm_from_request(request)
+        message = access_web_public_message(realm, message_id)
+        user_profile = None
+    else:
+        (message, user_message) = access_message_and_usermessage(
+            maybe_user_profile, message_id, is_modifying_message=False
+        )
+        user_profile = maybe_user_profile
+
+    flags = ["read"]
+    if not maybe_user_profile.is_authenticated:
+        message_edit_history_visibility_policy = realm.message_edit_history_visibility_policy
+    else:
+        if user_message:
+            flags = user_message.flags_list()
+        else:
+            flags = ["read", "historical"]
+        message_edit_history_visibility_policy = (
+            maybe_user_profile.realm.message_edit_history_visibility_policy
+        )
+
+    # Security note: It's important that we call this only with a
+    # message already fetched via `access_message` type methods,
+    # as we do above.
+    message_dict_list = messages_for_ids(
+        message_ids=[message.id],
+        user_message_flags={message_id: flags},
+        search_fields={},
+        apply_markdown=apply_markdown,
+        client_gravatar=True,
+        message_edit_history_visibility_policy=message_edit_history_visibility_policy,
+        allow_empty_topic_name=allow_empty_topic_name,
+        user_profile=user_profile,
+        realm=message.realm,
+    )
+    response = dict(
+        message=message_dict_list[0],
+        # raw_content is deprecated; we will need to wait until
+        # clients have been fully migrated to using the modern API
+        # before removing this, probably in 2023.
+        raw_content=message.content,
+    )
+    return json_success(request, response)

zerver/views/message_fetch.py

@@ -0,0 +1,386 @@
+from collections.abc import Iterable
+from typing import Annotated
+
+from django.conf import settings
+from django.contrib.auth.models import AnonymousUser
+from django.db import connection, transaction
+from django.db.models import F, Func, TextField
+from django.http import HttpRequest, HttpResponse
+from django.utils.translation import gettext as _
+from pydantic import Json, NonNegativeInt
+
+from zerver.context_processors import get_valid_realm_from_request
+from zerver.lib.exceptions import (
+    IncompatibleParametersError,
+    JsonableError,
+    MissingAuthenticationError,
+)
+from zerver.lib.message import get_first_visible_message_id, messages_for_ids
+from zerver.lib.narrow import (
+    NarrowParameter,
+    add_narrow_conditions,
+    clean_narrow_for_message_fetch,
+    fetch_messages,
+    get_base_query_for_search,
+    is_spectator_compatible,
+    is_web_public_narrow,
+    parse_anchor_value,
+)
+from zerver.lib.request import RequestNotes
+from zerver.lib.response import json_success
+from zerver.lib.topic import DB_TOPIC_NAME, MATCH_TOPIC
+from zerver.lib.typed_endpoint import ApiParamConfig, typed_endpoint
+from zerver.models import UserMessage, UserProfile
+
+MAX_MESSAGES_PER_FETCH = 5000
+
+
+def highlight_string(text: str, locs: Iterable[tuple[int, int]]) -> str:
+    highlight_start = '<span class="highlight">'
+    highlight_stop = "</span>"
+    pos = 0
+    result = ""
+    in_tag = False
+
+    for loc in locs:
+        (offset, length) = loc
+
+        prefix_start = pos
+        prefix_end = offset
+        match_start = offset
+        match_end = offset + length
+
+        prefix = text[prefix_start:prefix_end]
+        match = text[match_start:match_end]
+
+        for character in prefix + match:
+            if character == "<":
+                in_tag = True
+            elif character == ">":
+                in_tag = False
+        if in_tag:
+            result += prefix
+            result += match
+        else:
+            result += prefix
+            result += highlight_start
+            result += match
+            result += highlight_stop
+        pos = match_end
+
+    result += text[pos:]
+    return result
+
+
+def get_search_fields(
+    rendered_content: str,
+    escaped_topic_name: str,
+    content_matches: Iterable[tuple[int, int]],
+    topic_matches: Iterable[tuple[int, int]],
+) -> dict[str, str]:
+    return {
+        "match_content": highlight_string(rendered_content, content_matches),
+        MATCH_TOPIC: highlight_string(escaped_topic_name, topic_matches),
+    }
+
+
+def clean_narrow_for_web_public_api(
+    narrow: list[NarrowParameter] | None,
+) -> list[NarrowParameter] | None:
+    if narrow is None:
+        return None
+
+    # Remove {'operator': 'in', 'operand': 'home', 'negated': False} from narrow.
+    # This is to allow spectators to access all messages. The narrow should still pass
+    # is_web_public_narrow check after this change.
+    return [
+        term
+        for term in narrow
+        if not (term.operator == "in" and term.operand == "home" and not term.negated)
+    ]
+
+
+@typed_endpoint
+def get_messages_backend(
+    request: HttpRequest,
+    maybe_user_profile: UserProfile | AnonymousUser,
+    *,
+    allow_empty_topic_name: Json[bool] = False,
+    anchor_val: Annotated[str | None, ApiParamConfig("anchor")] = None,
+    anchor_date: str | None = None,
+    apply_markdown: Json[bool] = True,
+    client_gravatar: Json[bool] = True,
+    client_requested_message_ids: Annotated[
+        Json[list[NonNegativeInt] | None], ApiParamConfig("message_ids")
+    ] = None,
+    include_anchor: Json[bool] = True,
+    narrow: Json[list[NarrowParameter] | None] = None,
+    num_after: Json[NonNegativeInt] = 0,
+    num_before: Json[NonNegativeInt] = 0,
+    use_first_unread_anchor_val: Annotated[
+        Json[bool], ApiParamConfig("use_first_unread_anchor")
+    ] = False,
+) -> HttpResponse:
+    # User has to either provide message_ids or both num_before and num_after.
+    if (
+        num_before or num_after or anchor_val is not None or use_first_unread_anchor_val
+    ) and client_requested_message_ids is not None:
+        raise IncompatibleParametersError(
+            [
+                "num_before",
+                "num_after",
+                "anchor",
+                "message_ids",
+                "include_anchor",
+                "use_first_unread_anchor",
+            ]
+        )
+    elif client_requested_message_ids is not None:
+        include_anchor = False
+
+    anchor_info = None
+    if client_requested_message_ids is None:
+        anchor_info = parse_anchor_value(anchor_val, use_first_unread_anchor_val, anchor_date)
+
+    realm = get_valid_realm_from_request(request)
+    narrow = clean_narrow_for_message_fetch(narrow, realm, maybe_user_profile)
+
+    num_of_messages_requested = num_before + num_after
+    if client_requested_message_ids is not None:
+        num_of_messages_requested = len(client_requested_message_ids)
+
+    if num_of_messages_requested > MAX_MESSAGES_PER_FETCH:
+        raise JsonableError(
+            _("Too many messages requested (maximum {max_messages}).").format(
+                max_messages=MAX_MESSAGES_PER_FETCH,
+            )
+        )
+    if num_before > 0 and num_after > 0 and not include_anchor:
+        raise JsonableError(_("The anchor can only be excluded at an end of the range"))
+
+    if not maybe_user_profile.is_authenticated:
+        # If user is not authenticated, clients must include
+        # `streams:web-public` in their narrow query to indicate this
+        # is a web-public query.  This helps differentiate between
+        # cases of web-public queries (where we should return the
+        # web-public results only) and clients with buggy
+        # authentication code (where we should return an auth error).
+        #
+        # GetOldMessagesTest.test_unauthenticated_* tests ensure
+        # that we are not leaking any secure data (direct messages and
+        # non-web-public stream messages) via this path.
+        if not realm.allow_web_public_streams_access():
+            raise MissingAuthenticationError
+        narrow = clean_narrow_for_web_public_api(narrow)
+        if not is_web_public_narrow(narrow):
+            raise MissingAuthenticationError
+        assert narrow is not None
+        if not is_spectator_compatible(narrow):
+            raise MissingAuthenticationError
+
+        # We use None to indicate unauthenticated requests as it's more
+        # readable than using AnonymousUser, and the lack of Django
+        # stubs means that mypy can't check AnonymousUser well.
+        user_profile: UserProfile | None = None
+        is_web_public_query = True
+    else:
+        assert isinstance(maybe_user_profile, UserProfile)
+        user_profile = maybe_user_profile
+        assert user_profile is not None
+        is_web_public_query = False
+
+    assert realm is not None
+
+    if is_web_public_query:
+        # client_gravatar here is just the user-requested value. "finalize_payload" function
+        # is responsible for sending avatar_url based on each individual sender's
+        # email_address_visibility setting.
+        client_gravatar = False
+
+    if narrow is not None:
+        # Add some metadata to our logging data for narrows
+        verbose_operators = []
+        for term in narrow:
+            if term.operator == "is":
+                verbose_operators.append("is:" + term.operand)
+            else:
+                verbose_operators.append(term.operator)
+        log_data = RequestNotes.get_notes(request).log_data
+        assert log_data is not None
+        log_data["extra"] = "[{}]".format(",".join(verbose_operators))
+
+    with transaction.atomic(durable=True):
+        # We're about to perform a search, and then get results from
+        # it; this is done across multiple queries.  To prevent race
+        # conditions, we want the messages returned to be consistent
+        # with the version of the messages that was searched, to
+        # prevent changes which happened between them from leaking to
+        # clients who should not be able to see the new values, and
+        # when messages are deleted in between.  We set up
+        # repeatable-read isolation for this transaction, so that we
+        # prevent both phantom reads and non-repeatable reads.
+        #
+        # In a read-only repeatable-read transaction, it is not
+        # possible to encounter deadlocks or need retries due to
+        # serialization errors.
+        #
+        # You can only set the isolation level before any queries in
+        # the transaction, meaning it must be the top-most
+        # transaction, which durable=True establishes.  Except in
+        # tests, where durable=True is a lie, because there is an
+        # outer transaction for each test.  We thus skip this command
+        # in tests, since it would fail.
+        if not settings.TEST_SUITE:  # nocoverage
+            cursor = connection.cursor()
+            cursor.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY")
+
+        query_info = fetch_messages(
+            narrow=narrow,
+            user_profile=user_profile,
+            realm=realm,
+            is_web_public_query=is_web_public_query,
+            anchor_info=anchor_info,
+            include_anchor=include_anchor,
+            num_before=num_before,
+            num_after=num_after,
+            client_requested_message_ids=client_requested_message_ids,
+        )
+
+        anchor = query_info.anchor
+        include_history = query_info.include_history
+        is_search = query_info.is_search
+        rows = query_info.rows
+
+        # The following is a little messy, but ensures that the code paths
+        # are similar regardless of the value of include_history.  The
+        # 'user_messages' dictionary maps each message to the user's
+        # UserMessage object for that message, which we will attach to the
+        # rendered message dict before returning it.  We attempt to
+        # bulk-fetch rendered message dicts from remote cache using the
+        # 'messages' list.
+        result_message_ids: list[int] = []
+        user_message_flags: dict[int, list[str]] = {}
+        if is_web_public_query:
+            # For spectators, we treat all historical messages as read.
+            for row in rows:
+                message_id = row[0]
+                result_message_ids.append(message_id)
+                user_message_flags[message_id] = ["read"]
+        elif include_history:
+            assert user_profile is not None
+            result_message_ids = [row[0] for row in rows]
+
+            # TODO: This could be done with an outer join instead of two queries
+            um_rows = UserMessage.objects.filter(
+                user_profile=user_profile, message_id__in=result_message_ids
+            )
+            user_message_flags = {um.message_id: um.flags_list() for um in um_rows}
+
+            for message_id in result_message_ids:
+                if message_id not in user_message_flags:
+                    user_message_flags[message_id] = ["read", "historical"]
+        else:
+            for row in rows:
+                message_id = row[0]
+                flags = row[1]
+                user_message_flags[message_id] = UserMessage.flags_list_for_flags(flags)
+                result_message_ids.append(message_id)
+
+        search_fields: dict[int, dict[str, str]] = {}
+        if is_search:
+            for row in rows:
+                message_id = row[0]
+                (escaped_topic_name, rendered_content, content_matches, topic_matches) = row[-4:]
+                search_fields[message_id] = get_search_fields(
+                    rendered_content, escaped_topic_name, content_matches, topic_matches
+                )
+
+        message_list = messages_for_ids(
+            message_ids=result_message_ids,
+            user_message_flags=user_message_flags,
+            search_fields=search_fields,
+            apply_markdown=apply_markdown,
+            client_gravatar=client_gravatar,
+            allow_empty_topic_name=allow_empty_topic_name,
+            message_edit_history_visibility_policy=realm.message_edit_history_visibility_policy,
+            user_profile=user_profile,
+            realm=realm,
+        )
+
+    if client_requested_message_ids is not None:
+        ret = dict(
+            messages=message_list,
+            result="success",
+            msg="",
+            history_limited=query_info.history_limited,
+            found_anchor=False,
+            found_oldest=False,
+            found_newest=False,
+        )
+    else:
+        ret = dict(
+            messages=message_list,
+            result="success",
+            msg="",
+            found_anchor=query_info.found_anchor,
+            found_oldest=query_info.found_oldest,
+            found_newest=query_info.found_newest,
+            history_limited=query_info.history_limited,
+            anchor=anchor,
+        )
+
+    return json_success(request, data=ret)
+
+
+@typed_endpoint
+def messages_in_narrow_backend(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    msg_ids: Json[list[int]],
+    narrow: Json[list[NarrowParameter]],
+) -> HttpResponse:
+    first_visible_message_id = get_first_visible_message_id(user_profile.realm)
+    msg_ids = [message_id for message_id in msg_ids if message_id >= first_visible_message_id]
+    # This query is limited to messages the user has access to because they
+    # actually received them, as reflected in `zerver_usermessage`.
+    query = get_base_query_for_search(user_profile.realm_id, user_profile, need_user_message=True)
+    query = query.filter(id__in=msg_ids)
+
+    cleaned_narrow = clean_narrow_for_message_fetch(narrow, user_profile.realm, user_profile)
+    query, is_search, _is_dm_narrow = add_narrow_conditions(
+        user_profile=user_profile,
+        query=query,
+        narrow=cleaned_narrow,
+        is_web_public_query=False,
+        realm=user_profile.realm,
+    )
+
+    if not is_search:
+        # `add_narrow_conditions` adds the following columns only if narrow has search operands.
+        query = query.annotate(
+            escaped_topic_name=Func(
+                F(DB_TOPIC_NAME), function="escape_html", output_field=TextField()
+            ),
+        )
+
+    search_fields = {}
+    for row in query.values(
+        "id",
+        "escaped_topic_name",
+        "rendered_content",
+        *["content_matches", "topic_matches"] if is_search else [],
+    ):
+        message_id = row["id"]
+        escaped_topic_name: str = row["escaped_topic_name"]
+        rendered_content: str = row["rendered_content"]
+        content_matches = row.get("content_matches", [])
+        topic_matches = row.get("topic_matches", [])
+        search_fields[str(message_id)] = get_search_fields(
+            rendered_content,
+            escaped_topic_name,
+            content_matches,
+            topic_matches,
+        )
+
+    return json_success(request, data={"messages": search_fields})

zerver/views/message_flags.py

@@ -0,0 +1,193 @@
+from typing import Annotated
+
+from django.http import HttpRequest, HttpResponse
+from django.utils.translation import gettext as _
+from pydantic import Json, NonNegativeInt
+
+from zerver.actions.message_flags import (
+    do_mark_all_as_read,
+    do_mark_stream_messages_as_read,
+    do_update_message_flags,
+)
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.narrow import (
+    NarrowParameter,
+    clean_narrow_for_message_fetch,
+    fetch_messages,
+    parse_anchor_value,
+)
+from zerver.lib.request import RequestNotes
+from zerver.lib.response import json_success
+from zerver.lib.streams import access_stream_by_id
+from zerver.lib.topic import maybe_rename_general_chat_to_empty_topic, user_message_exists_for_topic
+from zerver.lib.typed_endpoint import (
+    ApiParamConfig,
+    typed_endpoint,
+    typed_endpoint_without_parameters,
+)
+from zerver.models import UserActivity, UserProfile
+
+
+def get_latest_update_message_flag_activity(user_profile: UserProfile) -> UserActivity | None:
+    return (
+        UserActivity.objects.filter(
+            user_profile=user_profile,
+            query__in=["update_message_flags", "update_message_flags_for_narrow"],
+        )
+        .order_by("last_visit")
+        .last()
+    )
+
+
+# NOTE: If this function name is changed, add the new name to the
+# query in get_latest_update_message_flag_activity
+@typed_endpoint
+def update_message_flags(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    flag: str,
+    messages: Json[list[int]],
+    operation: Annotated[str, ApiParamConfig("op")],
+) -> HttpResponse:
+    request_notes = RequestNotes.get_notes(request)
+    assert request_notes.log_data is not None
+
+    (count, ignored_because_not_subscribed_channels) = do_update_message_flags(
+        user_profile, operation, flag, messages
+    )
+
+    target_count_str = str(len(messages))
+    log_data_str = f"[{operation} {flag}/{target_count_str}] actually {count}"
+    request_notes.log_data["extra"] = log_data_str
+
+    return json_success(
+        request,
+        data={
+            "messages": messages,  # Useless, but included for backwards compatibility.
+            "ignored_because_not_subscribed_channels": ignored_because_not_subscribed_channels,
+        },
+    )
+
+
+MAX_MESSAGES_PER_UPDATE = 5000
+
+
+# NOTE: If this function name is changed, add the new name to the
+# query in get_latest_update_message_flag_activity
+@typed_endpoint
+def update_message_flags_for_narrow(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    anchor_val: Annotated[str, ApiParamConfig("anchor")],
+    flag: str,
+    include_anchor: Json[bool] = True,
+    narrow: Json[list[NarrowParameter] | None],
+    num_after: Json[NonNegativeInt],
+    num_before: Json[NonNegativeInt],
+    operation: Annotated[str, ApiParamConfig("op")],
+) -> HttpResponse:
+    anchor_info = parse_anchor_value(anchor_val, use_first_unread_anchor=False)
+
+    if num_before > 0 and num_after > 0 and not include_anchor:
+        raise JsonableError(_("The anchor can only be excluded at an end of the range"))
+
+    # Clamp such that num_before + num_after <= MAX_MESSAGES_PER_UPDATE.
+    num_before = min(
+        num_before, max(MAX_MESSAGES_PER_UPDATE - num_after, MAX_MESSAGES_PER_UPDATE // 2)
+    )
+    num_after = min(num_after, MAX_MESSAGES_PER_UPDATE - num_before)
+
+    narrow = clean_narrow_for_message_fetch(narrow, user_profile.realm, user_profile)
+
+    query_info = fetch_messages(
+        narrow=narrow,
+        user_profile=user_profile,
+        realm=user_profile.realm,
+        is_web_public_query=False,
+        anchor_info=anchor_info,
+        include_anchor=include_anchor,
+        num_before=num_before,
+        num_after=num_after,
+    )
+
+    messages = [row[0] for row in query_info.rows]
+    (updated_count, ignored_because_not_subscribed_channels) = do_update_message_flags(
+        user_profile, operation, flag, messages
+    )
+
+    return json_success(
+        request,
+        data={
+            "processed_count": len(messages),
+            "updated_count": updated_count,
+            "first_processed_id": messages[0] if messages else None,
+            "last_processed_id": messages[-1] if messages else None,
+            "found_oldest": query_info.found_oldest,
+            "found_newest": query_info.found_newest,
+            "ignored_because_not_subscribed_channels": ignored_because_not_subscribed_channels,
+        },
+    )
+
+
+@typed_endpoint_without_parameters
+def mark_all_as_read(request: HttpRequest, user_profile: UserProfile) -> HttpResponse:
+    request_notes = RequestNotes.get_notes(request)
+    count = do_mark_all_as_read(user_profile, timeout=50)
+    if count is None:
+        return json_success(request, data={"complete": False})
+
+    log_data_str = f"[{count} updated]"
+    assert request_notes.log_data is not None
+    request_notes.log_data["extra"] = log_data_str
+
+    return json_success(request, data={"complete": True})
+
+
+@typed_endpoint
+def mark_stream_as_read(
+    request: HttpRequest, user_profile: UserProfile, *, stream_id: Json[int]
+) -> HttpResponse:
+    stream, _sub = access_stream_by_id(user_profile, stream_id)
+    assert stream.recipient_id is not None
+    count = do_mark_stream_messages_as_read(user_profile, stream.recipient_id)
+
+    log_data_str = f"[{count} updated]"
+    log_data = RequestNotes.get_notes(request).log_data
+    assert log_data is not None
+    log_data["extra"] = log_data_str
+
+    return json_success(request)
+
+
+@typed_endpoint
+def mark_topic_as_read(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    topic_name: str,
+    stream_id: Json[int],
+) -> HttpResponse:
+    stream, _sub = access_stream_by_id(user_profile, stream_id)
+    assert stream.recipient_id is not None
+
+    if topic_name:
+        topic_name = maybe_rename_general_chat_to_empty_topic(topic_name)
+        topic_exists = user_message_exists_for_topic(
+            user_profile=user_profile,
+            recipient_id=stream.recipient_id,
+            topic_name=topic_name,
+        )
+
+        if not topic_exists:
+            raise JsonableError(_("No such topic '{topic}'").format(topic=topic_name))
+
+    count = do_mark_stream_messages_as_read(user_profile, stream.recipient_id, topic_name)
+
+    log_data_str = f"[{count} updated]"
+    log_data = RequestNotes.get_notes(request).log_data
+    assert log_data is not None
+    log_data["extra"] = log_data_str
+
+    return json_success(request)

zerver/views/message_report.py

@@ -0,0 +1,49 @@
+from typing import Annotated
+
+from django.http import HttpRequest, HttpResponse
+from django.utils.translation import gettext as _
+from django.utils.translation import override as override_language
+from pydantic import StringConstraints
+
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.message import access_message
+from zerver.lib.message_report import send_message_report, send_report_confirmation_dm
+from zerver.lib.response import json_success
+from zerver.lib.typed_endpoint import typed_endpoint
+from zerver.lib.typed_endpoint_validators import check_string_in_validator
+from zerver.models import Realm, UserProfile
+
+
+@typed_endpoint
+def report_message_backend(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    message_id: int,
+    *,
+    description: Annotated[
+        str, StringConstraints(max_length=Realm.MAX_REPORT_MESSAGE_EXPLANATION_LENGTH)
+    ] = "",
+    report_type: Annotated[str, check_string_in_validator(Realm.REPORT_MESSAGE_REASONS)],
+) -> HttpResponse:
+    if report_type == "other" and description == "":
+        raise JsonableError(_("An explanation is required."))
+
+    if user_profile.realm.moderation_request_channel is None:
+        raise JsonableError(_("Message reporting is not enabled in this organization."))
+
+    reported_message = access_message(user_profile, message_id, is_modifying_message=False)
+    with override_language(user_profile.realm.default_language):
+        message_report_id = send_message_report(
+            user_profile,
+            user_profile.realm,
+            reported_message,
+            report_type,
+            description,
+        )
+
+    if message_report_id is None:
+        raise JsonableError(_("Failed to send the message report."))
+
+    send_report_confirmation_dm(user_profile, user_profile.realm, reported_message, report_type)
+
+    return json_success(request)

zerver/views/message_send.py

@@ -0,0 +1,280 @@
+from collections.abc import Iterable, Sequence
+from dataclasses import asdict
+from email.headerregistry import Address
+from typing import Annotated, Literal, TypedDict, cast
+
+from django.conf import settings
+from django.core import validators
+from django.core.exceptions import ValidationError
+from django.http import HttpRequest, HttpResponse
+from django.utils.translation import gettext as _
+from pydantic import Json, StringConstraints
+
+from zerver.actions.message_send import (
+    check_send_message,
+    compute_irc_user_fullname,
+    compute_jabber_user_fullname,
+    create_mirror_user_if_needed,
+    extract_private_recipients,
+    extract_stream_indicator,
+)
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.markdown import render_message_markdown
+from zerver.lib.request import RequestNotes
+from zerver.lib.response import json_success
+from zerver.lib.typed_endpoint import (
+    DOCUMENTATION_PENDING,
+    ApiParamConfig,
+    OptionalTopic,
+    typed_endpoint,
+)
+from zerver.lib.zcommand import process_zcommands
+from zerver.models import Client, Message, RealmDomain, UserProfile
+from zerver.models.users import get_user_including_cross_realm
+
+
+class SendMessageResponseData(TypedDict, total=False):
+    id: int
+    message_url: str
+    message_link: str
+    automatic_new_visibility_policy: int
+
+
+class InvalidMirrorInputError(Exception):
+    pass
+
+
+def create_mirrored_message_users(
+    client: Client,
+    user_profile: UserProfile,
+    recipients: Iterable[str],
+    sender: str,
+    recipient_type_name: str,
+) -> UserProfile:
+    sender_email = sender.strip().lower()
+    referenced_users = {sender_email}
+    if recipient_type_name == "private":
+        referenced_users.update(email.lower() for email in recipients)
+
+    if client.name == "irc_mirror":
+        user_check = same_realm_irc_user
+        fullname_function = compute_irc_user_fullname
+    elif client.name in ("jabber_mirror", "JabberMirror"):
+        user_check = same_realm_jabber_user
+        fullname_function = compute_jabber_user_fullname
+    else:
+        raise InvalidMirrorInputError("Unrecognized mirroring client")
+
+    for email in referenced_users:
+        # Check that all referenced users are in our realm:
+        if not user_check(user_profile, email):
+            raise InvalidMirrorInputError("At least one user cannot be mirrored")
+
+    # Create users for the referenced users, if needed.
+    for email in referenced_users:
+        create_mirror_user_if_needed(user_profile.realm, email, fullname_function)
+
+    sender_user_profile = get_user_including_cross_realm(sender_email, user_profile.realm)
+    return sender_user_profile
+
+
+def same_realm_irc_user(user_profile: UserProfile, email: str) -> bool:
+    # Check whether the target email address is an IRC user in the
+    # same realm as user_profile, i.e. if the domain were example.com,
+    # the IRC user would need to be username@irc.example.com
+    try:
+        validators.validate_email(email)
+    except ValidationError:
+        return False
+
+    domain = Address(addr_spec=email).domain.lower()
+    domain = domain.removeprefix("irc.")
+
+    # Assumes allow_subdomains=False for all RealmDomain's corresponding to
+    # these realms.
+    return RealmDomain.objects.filter(realm=user_profile.realm, domain=domain).exists()
+
+
+def same_realm_jabber_user(user_profile: UserProfile, email: str) -> bool:
+    try:
+        validators.validate_email(email)
+    except ValidationError:
+        return False
+
+    # If your Jabber users have a different email domain than the
+    # Zulip users, this is where you would do any translation.
+    domain = Address(addr_spec=email).domain.lower()
+
+    # Assumes allow_subdomains=False for all RealmDomain's corresponding to
+    # these realms.
+    return RealmDomain.objects.filter(realm=user_profile.realm, domain=domain).exists()
+
+
+@typed_endpoint
+def send_message_backend(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    forged_str: Annotated[
+        str | None, ApiParamConfig("forged", documentation_status=DOCUMENTATION_PENDING)
+    ] = None,
+    local_id: str | None = None,
+    message_content: Annotated[str, ApiParamConfig("content")],
+    queue_id: str | None = None,
+    read_by_sender: Json[bool] | None = None,
+    req_sender: Annotated[
+        str | None, ApiParamConfig("sender", documentation_status=DOCUMENTATION_PENDING)
+    ] = None,
+    req_to: Annotated[str | None, ApiParamConfig("to")] = None,
+    req_type: Annotated[Literal["direct", "private", "stream", "channel"], ApiParamConfig("type")],
+    time: Annotated[
+        Json[float] | None, ApiParamConfig("time", documentation_status=DOCUMENTATION_PENDING)
+    ] = None,
+    topic_name: OptionalTopic = None,
+    widget_content: Annotated[
+        str | None, ApiParamConfig("widget_content", documentation_status=DOCUMENTATION_PENDING)
+    ] = None,
+) -> HttpResponse:
+    recipient_type_name = req_type
+    if recipient_type_name == "direct":
+        # For now, use "private" from Message.API_RECIPIENT_TYPES.
+        # TODO: Use "direct" here, as well as in events and
+        # message (created, schdeduled, drafts) objects/dicts.
+        recipient_type_name = "private"
+    elif recipient_type_name == "channel":
+        # For now, use "stream" from Message.API_RECIPIENT_TYPES.
+        # TODO: Use "channel" here, as well as in events and
+        # message (created, schdeduled, drafts) objects/dicts.
+        recipient_type_name = "stream"
+
+    # If to is None, then we default to an
+    # empty list of recipients.
+    message_to: Sequence[int] | Sequence[str] = []
+
+    if req_to is not None:
+        if recipient_type_name == "stream":
+            stream_indicator = extract_stream_indicator(req_to)
+
+            # For legacy reasons check_send_message expects
+            # a list of streams, instead of a single stream.
+            #
+            # Also, mypy can't detect that a single-item
+            # list populated from a Union[int, str] is actually
+            # a Union[Sequence[int], Sequence[str]].
+            if isinstance(stream_indicator, int):
+                message_to = [stream_indicator]
+            else:
+                message_to = [stream_indicator]
+        else:
+            message_to = extract_private_recipients(req_to)
+
+    # Temporary hack: We're transitioning `forged` from accepting
+    # `yes` to accepting `true` like all of our normal booleans.
+    forged = forged_str is not None and forged_str in ["yes", "true"]
+
+    client = RequestNotes.get_notes(request).client
+    assert client is not None
+    can_forge_sender = user_profile.can_forge_sender
+    if forged and not can_forge_sender:
+        raise JsonableError(_("User not authorized for this query"))
+
+    realm = user_profile.realm
+
+    if client.name in ["irc_mirror", "jabber_mirror", "JabberMirror"]:
+        # Here's how security works for mirroring:
+        #
+        # For direct messages, the message must be (1) both sent and
+        # received exclusively by users in your realm, and (2)
+        # received by the forwarding user.
+        #
+        # For stream messages, the message must be (1) being forwarded
+        # by an API superuser for your realm and (2) being sent to a
+        # mirrored stream.
+        #
+        # The most important security checks are in
+        # `create_mirrored_message_users` below, which checks the
+        # same-realm constraint.
+        if req_sender is None:
+            raise JsonableError(_("Missing sender"))
+        if recipient_type_name != "private" and not can_forge_sender:
+            raise JsonableError(_("User not authorized for this query"))
+
+        # For now, mirroring only works with recipient emails, not for
+        # recipient user IDs.
+        if not all(isinstance(to_item, str) for to_item in message_to):
+            raise JsonableError(_("Mirroring not allowed with recipient user IDs"))
+
+        # We need this manual cast so that mypy doesn't complain about
+        # create_mirrored_message_users not being able to accept a Sequence[int]
+        # type parameter.
+        message_to = cast(Sequence[str], message_to)
+
+        try:
+            mirror_sender = create_mirrored_message_users(
+                client, user_profile, message_to, req_sender, recipient_type_name
+            )
+        except InvalidMirrorInputError:
+            raise JsonableError(_("Invalid mirrored message"))
+
+        sender = mirror_sender
+    else:
+        if req_sender is not None:
+            raise JsonableError(_("Invalid mirrored message"))
+        sender = user_profile
+
+    if read_by_sender is None:
+        # Legacy default: a message you sent from a non-API client is
+        # automatically marked as read for yourself.
+        read_by_sender = client.default_read_by_sender()
+
+    data: SendMessageResponseData = {}
+    sent_message_result = check_send_message(
+        sender,
+        client,
+        recipient_type_name,
+        message_to,
+        topic_name,
+        message_content,
+        forged=forged,
+        forged_timestamp=time,
+        forwarder_user_profile=user_profile,
+        realm=realm,
+        local_id=local_id,
+        sender_queue_id=queue_id,
+        widget_content=widget_content,
+        read_by_sender=read_by_sender,
+    )
+    data["id"] = sent_message_result.message_id
+    data["message_url"] = sent_message_result.message_url
+    data["message_link"] = sent_message_result.message_link
+    if sent_message_result.automatic_new_visibility_policy:
+        data["automatic_new_visibility_policy"] = (
+            sent_message_result.automatic_new_visibility_policy
+        )
+    return json_success(request, data=data)
+
+
+@typed_endpoint
+def zcommand_backend(
+    request: HttpRequest, user_profile: UserProfile, *, command: str
+) -> HttpResponse:
+    return json_success(request, data=asdict(process_zcommands(command, user_profile)))
+
+
+@typed_endpoint
+def render_message_backend(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    content: Annotated[str, StringConstraints(max_length=settings.MAX_MESSAGE_LENGTH)],
+) -> HttpResponse:
+    message = Message()
+    message.sender = user_profile
+    message.realm = user_profile.realm
+    message.content = content
+    client = RequestNotes.get_notes(request).client
+    assert client is not None
+    message.sending_client = client
+
+    rendering_result = render_message_markdown(message, content, realm=user_profile.realm)
+    return json_success(request, data={"rendered": rendering_result.rendered_content})

zerver/views/message_summary.py

@@ -0,0 +1,39 @@
+from django.conf import settings
+from django.http import HttpRequest, HttpResponse
+from django.utils.translation import gettext as _
+from pydantic import Json
+
+from analytics.lib.counts import COUNT_STATS
+from zerver.actions.message_summary import do_summarize_narrow
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.narrow import NarrowParameter
+from zerver.lib.response import json_success
+from zerver.lib.typed_endpoint import typed_endpoint
+from zerver.models import UserProfile
+
+
+@typed_endpoint
+def get_messages_summary(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    narrow: Json[list[NarrowParameter] | None] = None,
+) -> HttpResponse:
+    if settings.TOPIC_SUMMARIZATION_MODEL is None:  # nocoverage
+        raise JsonableError(_("AI features are not enabled on this server."))
+
+    if not user_profile.can_summarize_topics():
+        raise JsonableError(_("Insufficient permission"))
+
+    if settings.MAX_PER_USER_MONTHLY_AI_COST is not None:
+        used_credits = COUNT_STATS["ai_credit_usage::day"].current_month_accumulated_count_for_user(
+            user_profile
+        )
+        if used_credits >= settings.MAX_PER_USER_MONTHLY_AI_COST * 1000000000:
+            raise JsonableError(_("Reached monthly limit for AI credits."))
+
+    summary = do_summarize_narrow(user_profile, narrow)
+    if summary is None:  # nocoverage
+        raise JsonableError(_("No messages in conversation to summarize"))
+
+    return json_success(request, {"summary": summary})

zerver/views/muted_users.py

@@ -0,0 +1,51 @@
+from django.db import IntegrityError
+from django.http import HttpRequest, HttpResponse
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+
+from zerver.actions.muted_users import do_mute_user, do_unmute_user
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.muted_users import get_mute_object
+from zerver.lib.response import json_success
+from zerver.lib.users import access_user_by_id_including_cross_realm
+from zerver.models import UserProfile
+
+
+def mute_user(request: HttpRequest, user_profile: UserProfile, muted_user_id: int) -> HttpResponse:
+    if user_profile.id == muted_user_id:
+        raise JsonableError(_("Cannot mute self"))
+
+    # Arguably, access_used_by_id is not quite the right check; in the
+    # corner case of a limited guest trying to mute a deactivated user
+    # who they no longer have access to because the user was
+    # deactivated... it might from a policy perspective be OK to allow
+    # that operation even though this API will reject it.
+    #
+    # But it's quite possibly something nobody will try to do, so we
+    # just reuse the existing shared code path.
+    muted_user = access_user_by_id_including_cross_realm(
+        user_profile, muted_user_id, allow_bots=True, allow_deactivated=True, for_admin=False
+    )
+    date_muted = timezone_now()
+
+    try:
+        do_mute_user(user_profile, muted_user, date_muted)
+    except IntegrityError:
+        raise JsonableError(_("User already muted"))
+
+    return json_success(request)
+
+
+def unmute_user(
+    request: HttpRequest, user_profile: UserProfile, muted_user_id: int
+) -> HttpResponse:
+    muted_user = access_user_by_id_including_cross_realm(
+        user_profile, muted_user_id, allow_bots=True, allow_deactivated=True, for_admin=False
+    )
+    mute_object = get_mute_object(user_profile, muted_user)
+
+    if mute_object is None:
+        raise JsonableError(_("User is not muted"))
+
+    do_unmute_user(mute_object)
+    return json_success(request)

zerver/views/navigation_views.py

@@ -0,0 +1,117 @@
+from typing import Annotated
+
+from django.http import HttpRequest, HttpResponse
+from django.utils.translation import gettext as _
+from pydantic import Json, StringConstraints
+
+from zerver.actions.navigation_views import (
+    do_add_navigation_view,
+    do_remove_navigation_view,
+    do_update_navigation_view,
+)
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.navigation_views import get_navigation_view, get_navigation_views_for_user
+from zerver.lib.response import json_success
+from zerver.lib.typed_endpoint import typed_endpoint
+from zerver.models import NavigationView, UserProfile
+
+BUILT_IN_VIEW_FRAGMENTS = [
+    "inbox",
+    "recent",
+    "feed",
+    "drafts",
+    # Reactions view
+    "narrow/has/reaction/sender/me",
+    # Mentions view
+    "narrow/is/mentioned",
+    # Starred messages view
+    "narrow/is/starred",
+    "scheduled",
+    "reminders",
+]
+
+
+def get_navigation_views(
+    request: HttpRequest,
+    user_profile: UserProfile,
+) -> HttpResponse:
+    """
+    Fetch navigation views for the specified user.
+    """
+    navigation_views = get_navigation_views_for_user(user_profile)
+    return json_success(request, data={"navigation_views": navigation_views})
+
+
+@typed_endpoint
+def add_navigation_view(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    fragment: Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)],
+    is_pinned: Json[bool],
+    name: Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)] | None = None,
+) -> HttpResponse:
+    """
+    Add a new navigation view (or settings for a default view) for the user.
+    """
+    if fragment in BUILT_IN_VIEW_FRAGMENTS:
+        if name is not None:
+            raise JsonableError(_("Built-in views cannot have a custom name."))
+    else:
+        if not name:
+            raise JsonableError(_("Custom views must have a valid name."))
+
+    if NavigationView.objects.filter(user=user_profile, fragment=fragment).exists():
+        raise JsonableError(_("Navigation view already exists."))
+    if name is not None and NavigationView.objects.filter(user=user_profile, name=name).exists():
+        raise JsonableError(_("Navigation view already exists."))
+
+    do_add_navigation_view(
+        user_profile,
+        fragment,
+        is_pinned,
+        name,
+    )
+    return json_success(request)
+
+
+@typed_endpoint
+def update_navigation_view(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    fragment: Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)],
+    *,
+    is_pinned: Json[bool] | None = None,
+    name: Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)] | None = None,
+) -> HttpResponse:
+    """
+    Update an existing navigation view for the user.
+    """
+    if fragment in BUILT_IN_VIEW_FRAGMENTS and name is not None:
+        raise JsonableError(_("Built-in views cannot have a custom name."))
+    if name is not None and NavigationView.objects.filter(user=user_profile, name=name).exists():
+        raise JsonableError(_("Navigation view already exists."))
+
+    navigation_view = get_navigation_view(user_profile, fragment)
+
+    do_update_navigation_view(
+        user_profile,
+        navigation_view,
+        is_pinned,
+        name,
+    )
+    return json_success(request)
+
+
+def remove_navigation_view(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    fragment: Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)],
+) -> HttpResponse:
+    """
+    Remove a navigation view for the user.
+    """
+    navigation_view = get_navigation_view(user_profile, fragment)
+    do_remove_navigation_view(user_profile, navigation_view)
+    return json_success(request)

zerver/views/onboarding_steps.py

@@ -0,0 +1,61 @@
+from datetime import timedelta
+
+from django.conf import settings
+from django.http import HttpRequest, HttpResponse
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+from pydantic import Json
+
+from zerver.actions.onboarding_steps import do_mark_onboarding_step_as_read
+from zerver.actions.scheduled_messages import check_schedule_message
+from zerver.decorator import human_users_only
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.onboarding_steps import ALL_ONBOARDING_STEPS
+from zerver.lib.response import json_success
+from zerver.lib.typed_endpoint import typed_endpoint
+from zerver.models import UserProfile
+from zerver.models.clients import get_client
+from zerver.models.users import get_system_bot
+
+
+@human_users_only
+@typed_endpoint
+def mark_onboarding_step_as_read(
+    request: HttpRequest,
+    user: UserProfile,
+    *,
+    onboarding_step: str,
+    schedule_navigation_tour_video_reminder_delay: Json[int] | None = None,
+) -> HttpResponse:
+    if not any(step.name == onboarding_step for step in ALL_ONBOARDING_STEPS):
+        raise JsonableError(
+            _("Unknown onboarding_step: {onboarding_step}").format(onboarding_step=onboarding_step)
+        )
+
+    if schedule_navigation_tour_video_reminder_delay is not None:
+        assert onboarding_step == "navigation_tour_video"
+
+        realm = user.realm
+        sender = get_system_bot(settings.WELCOME_BOT, realm.id)
+        client = get_client("Internal")
+        message_content = _("""
+You asked to watch the [Welcome to Zulip video]({navigation_tour_video_url}) later. Is this a good time?
+""").format(navigation_tour_video_url=settings.NAVIGATION_TOUR_VIDEO_URL)
+        deliver_at = timezone_now() + timedelta(
+            seconds=schedule_navigation_tour_video_reminder_delay
+        )
+
+        check_schedule_message(
+            sender,
+            client,
+            "private",
+            [user.id],
+            None,
+            message_content,
+            deliver_at,
+            realm,
+            skip_events=True,
+        )
+
+    do_mark_onboarding_step_as_read(user, onboarding_step)
+    return json_success(request)

zerver/views/presence.py

@@ -0,0 +1,227 @@
+import time
+from typing import Annotated, Any
+
+from django.conf import settings
+from django.http import HttpRequest, HttpResponse
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+from pydantic import Json, StringConstraints
+
+from zerver.actions.presence import update_user_presence
+from zerver.actions.user_status import do_update_user_status
+from zerver.decorator import human_users_only
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.presence import get_presence_dicts_for_rows, get_presence_response
+from zerver.lib.request import RequestNotes
+from zerver.lib.response import json_success
+from zerver.lib.timestamp import datetime_to_timestamp
+from zerver.lib.typed_endpoint import (
+    ApiParamConfig,
+    PathOnly,
+    typed_endpoint,
+    typed_endpoint_without_parameters,
+)
+from zerver.lib.user_status import check_update_user_status, get_user_status
+from zerver.lib.users import access_user_by_id, check_can_access_user
+from zerver.models import UserPresence, UserProfile
+from zerver.models.users import get_active_user, get_active_user_profile_by_id_in_realm
+
+
+@typed_endpoint_without_parameters
+def get_presence_backend(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    user_id_or_email: PathOnly[str],
+) -> HttpResponse:
+    try:
+        try:
+            user_id = int(user_id_or_email)
+            target = get_active_user_profile_by_id_in_realm(user_id, user_profile.realm)
+        except ValueError:
+            email = user_id_or_email
+            target = get_active_user(email, user_profile.realm)
+    except UserProfile.DoesNotExist:
+        raise JsonableError(_("No such user"))
+
+    # Check bot_type here, rather than is_bot, because that matches
+    # authenticated_json_view's check of .is_incoming_webhook; the
+    # narrow user cache can hence be optimized to only have the
+    # nullable bot_type, as long as this check matches.
+    if target.bot_type is not None:
+        raise JsonableError(_("Presence is not supported for bot users."))
+
+    if settings.CAN_ACCESS_ALL_USERS_GROUP_LIMITS_PRESENCE and not check_can_access_user(
+        target, user_profile
+    ):
+        raise JsonableError(_("Insufficient permission"))
+
+    server_timestamp = time.time()
+    # Other callers of get_presence_dicts_for_rows also fetch
+    # user_profile__enable_offline_push_notifications, but it is no
+    # longer read by the helper (the legacy `pushable` field is
+    # hardcoded to False), so we omit it here.
+    presence_rows = list(
+        UserPresence.objects.filter(user_profile_id=target.id).values(
+            "last_active_time",
+            "last_connected_time",
+            "user_profile__email",
+            "user_profile_id",
+            "user_profile__date_joined",
+        )
+    )
+    if not presence_rows:
+        raise JsonableError(
+            _("No presence data for {user_id_or_email}").format(user_id_or_email=user_id_or_email)
+        )
+    modern_presence = get_presence_dicts_for_rows(presence_rows, slim_presence=True)[str(target.id)]
+    legacy_presence = get_presence_dicts_for_rows(presence_rows, slim_presence=False)[target.email]
+
+    # The legacy API reports "offline" in the aggregated dict when the
+    # user's latest presence update is older than OFFLINE_THRESHOLD_SECS.
+    # The modern format leaves that computation to the client.
+    aggregated_info = legacy_presence["aggregated"]
+    aggr_status_duration = datetime_to_timestamp(timezone_now()) - aggregated_info["timestamp"]
+    if aggr_status_duration > settings.OFFLINE_THRESHOLD_SECS:
+        aggregated_info["status"] = "offline"
+    for val in legacy_presence.values():
+        val.pop("client", None)
+        val.pop("pushable", None)
+
+    presence = {**modern_presence, **legacy_presence}
+    return json_success(request, data={"presence": presence, "server_timestamp": server_timestamp})
+
+
+def get_status_backend(
+    request: HttpRequest, user_profile: UserProfile, user_id: int
+) -> HttpResponse:
+    target_user = access_user_by_id(user_profile, user_id, allow_bots=False, for_admin=False)
+    return json_success(request, data={"status": get_user_status(target_user)})
+
+
+@human_users_only
+@typed_endpoint
+def update_user_status_backend(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    away: Json[bool] | None = None,
+    emoji_code: str | None = None,
+    emoji_name: str | None = None,
+    # TODO: emoji_type is the more appropriate name for this parameter, but changing
+    # that requires nontrivial work on the API documentation, since it's not clear
+    # that the reactions endpoint would prefer such a change.
+    emoji_type: Annotated[str | None, ApiParamConfig("reaction_type")] = None,
+    status_text: Annotated[
+        str | None, StringConstraints(strip_whitespace=True, max_length=60)
+    ] = None,
+) -> HttpResponse:
+    user_status = check_update_user_status(
+        user_profile.realm,
+        away=away,
+        status_text=status_text,
+        emoji_name=emoji_name,
+        emoji_code=emoji_code,
+        emoji_type=emoji_type,
+    )
+
+    client = RequestNotes.get_notes(request).client
+    assert client is not None
+    do_update_user_status(
+        user_profile=user_profile,
+        away=away,
+        status_text=user_status.status_text,
+        client_id=client.id,
+        emoji_name=user_status.emoji_name,
+        emoji_code=user_status.emoji_code,
+        reaction_type=user_status.reaction_type,
+    )
+
+    return json_success(request)
+
+
+@typed_endpoint
+def update_user_status_admin(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    user_id: PathOnly[Json[int]],
+    status_text: Annotated[
+        str | None, StringConstraints(strip_whitespace=True, max_length=60)
+    ] = None,
+    emoji_name: str | None = None,
+    emoji_code: str | None = None,
+    emoji_type: Annotated[str | None, ApiParamConfig("reaction_type")] = None,
+) -> HttpResponse:
+    target_user = access_user_by_id(user_profile, user_id, allow_bots=False, for_admin=True)
+
+    user_status = check_update_user_status(
+        target_user.realm,
+        status_text=status_text,
+        emoji_name=emoji_name,
+        emoji_code=emoji_code,
+        emoji_type=emoji_type,
+    )
+
+    client = RequestNotes.get_notes(request).client
+    assert client is not None
+    do_update_user_status(
+        user_profile=target_user,
+        away=None,
+        status_text=user_status.status_text,
+        client_id=client.id,
+        emoji_name=user_status.emoji_name,
+        emoji_code=user_status.emoji_code,
+        reaction_type=user_status.reaction_type,
+    )
+
+    return json_success(request)
+
+
+@human_users_only
+@typed_endpoint
+def update_active_status_backend(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    history_limit_days: Json[int] | None = None,
+    last_update_id: Json[int] | None = None,
+    new_user_input: Json[bool] = False,
+    ping_only: Json[bool] = False,
+    slim_presence: Json[bool] = False,
+    status: str,
+) -> HttpResponse:
+    if last_update_id is not None:
+        # This param being submitted by the client, means they want to use
+        # the modern API.
+        slim_presence = True
+
+    status_val = UserPresence.status_from_string(status)
+    if status_val is None:
+        raise JsonableError(_("Invalid status: {status}").format(status=status))
+
+    client = RequestNotes.get_notes(request).client
+    assert client is not None
+    update_user_presence(user_profile, client, timezone_now(), status_val, new_user_input)
+
+    if ping_only:
+        ret: dict[str, Any] = {}
+    else:
+        ret = get_presence_response(
+            user_profile,
+            slim_presence,
+            last_update_id_fetched_by_client=last_update_id,
+            history_limit_days=history_limit_days,
+        )
+
+    return json_success(request, data=ret)
+
+
+def get_statuses_for_realm(request: HttpRequest, user_profile: UserProfile) -> HttpResponse:
+    # This isn't used by the web app; it's available for API use by
+    # bots and other clients.  We may want to add slim_presence
+    # support for it (or just migrate its API wholesale) later.
+    data = get_presence_response(user_profile, slim_presence=False)
+
+    # We're not interested in the last_update_id field in this context.
+    data.pop("presence_last_update_id", None)
+    return json_success(request, data=data)

zerver/views/push_notifications.py

@@ -0,0 +1,430 @@
+from typing import Annotated
+
+import orjson
+from django.conf import settings
+from django.db import transaction
+from django.http import HttpRequest, HttpResponse, HttpResponseRedirect
+from django.shortcuts import render
+from django.urls import reverse
+from django.utils.crypto import constant_time_compare
+from django.utils.translation import gettext as _
+from pydantic import Json
+
+from zerver.actions.push_notifications import (
+    do_register_push_device,
+    do_rotate_push_key,
+    do_rotate_token,
+)
+from zerver.decorator import human_users_only, zulip_login_required
+from zerver.lib import redis_utils
+from zerver.lib.devices import b64decode_token_id_base64, check_device_id
+from zerver.lib.exceptions import (
+    ErrorCode,
+    JsonableError,
+    MissingRemoteRealmError,
+    PushServiceNotConfiguredError,
+    RemoteRealmServerMismatchError,
+    ResourceNotFoundError,
+)
+from zerver.lib.push_notifications import (
+    InvalidPushDeviceTokenError,
+    NoActivePushDeviceError,
+    add_push_device_token,
+    remove_push_device_token,
+    send_e2ee_test_push_notification,
+    send_test_push_notification,
+    uses_notification_bouncer,
+    validate_token,
+)
+from zerver.lib.push_registration import RegisterPushDeviceToBouncerQueueItem, check_push_key
+from zerver.lib.queue import mobile_notifications_queue_name, queue_event_on_commit
+from zerver.lib.remote_server import (
+    SELF_HOSTING_REGISTRATION_TAKEOVER_CHALLENGE_TOKEN_REDIS_KEY,
+    UserDataForRemoteBilling,
+    get_realms_info_for_push_bouncer,
+    send_server_data_to_push_bouncer,
+    send_to_push_bouncer,
+)
+from zerver.lib.response import json_success
+from zerver.lib.typed_endpoint import (
+    ApiParamConfig,
+    ApnsAppId,
+    typed_endpoint,
+    typed_endpoint_without_parameters,
+)
+from zerver.lib.typed_endpoint_validators import check_string_in_validator, uint32_validator
+from zerver.models import Device, PushDeviceToken, UserProfile
+from zerver.views.errors import config_error
+
+redis_client = redis_utils.get_redis_client()
+
+
+@human_users_only
+@typed_endpoint
+def add_apns_device_token(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    appid: ApnsAppId,
+    token: str,
+) -> HttpResponse:
+    validate_token(token, PushDeviceToken.APNS)
+    add_push_device_token(user_profile, token, PushDeviceToken.APNS, ios_app_id=appid)
+    return json_success(request)
+
+
+@human_users_only
+@typed_endpoint
+def add_android_reg_id(
+    request: HttpRequest, user_profile: UserProfile, *, token: str
+) -> HttpResponse:
+    validate_token(token, PushDeviceToken.FCM)
+    add_push_device_token(user_profile, token, PushDeviceToken.FCM)
+    return json_success(request)
+
+
+@human_users_only
+@typed_endpoint
+def remove_apns_device_token(
+    request: HttpRequest, user_profile: UserProfile, *, token: str
+) -> HttpResponse:
+    validate_token(token, PushDeviceToken.APNS)
+    remove_push_device_token(user_profile, token, PushDeviceToken.APNS)
+    return json_success(request)
+
+
+@human_users_only
+@typed_endpoint
+def remove_android_reg_id(
+    request: HttpRequest, user_profile: UserProfile, *, token: str
+) -> HttpResponse:
+    validate_token(token, PushDeviceToken.FCM)
+    remove_push_device_token(user_profile, token, PushDeviceToken.FCM)
+    return json_success(request)
+
+
+@human_users_only
+@typed_endpoint
+def send_test_push_notification_api(
+    request: HttpRequest, user_profile: UserProfile, *, token: str | None = None
+) -> HttpResponse:
+    # If a token is specified in the request, the test notification is supposed to be sent
+    # to that device. If no token is provided, the test notification should be sent to
+    # all devices registered for the user.
+    if token is not None:
+        try:
+            devices = [PushDeviceToken.objects.get(token=token, user=user_profile)]
+        except PushDeviceToken.DoesNotExist:
+            raise InvalidPushDeviceTokenError
+    else:
+        devices = list(PushDeviceToken.objects.filter(user=user_profile))
+
+    send_test_push_notification(user_profile, devices)
+
+    return json_success(request)
+
+
+@human_users_only
+@typed_endpoint
+def send_e2ee_test_push_notification_api(
+    request: HttpRequest, user_profile: UserProfile, *, device_id: Json[int] | None = None
+) -> HttpResponse:
+    # We skip devices with `push_token_id` set to `null` as they are
+    # not yet registered with the bouncer to send mobile push notifications.
+    if device_id is not None:
+        push_devices = Device.objects.filter(
+            id=device_id, user=user_profile, push_token_id__isnull=False
+        )
+    else:
+        # Uses 'zerver_device_user_push_token_id_idx' index.
+        push_devices = Device.objects.filter(user=user_profile, push_token_id__isnull=False)
+
+    if len(push_devices) == 0:
+        raise NoActivePushDeviceError
+
+    send_e2ee_test_push_notification(user_profile, push_devices)
+
+    return json_success(request)
+
+
+class CannotManageRemoteBillingError(JsonableError):
+    def __init__(self) -> None:
+        super().__init__(_("You do not have permission to manage plans and billing."))
+
+
+def self_hosting_auth_view_common(
+    request: HttpRequest, user_profile: UserProfile, next_page: str | None = None
+) -> str:
+    if not user_profile.has_billing_access:
+        # We may want to replace this with an html error page at some point,
+        # but this endpoint shouldn't be accessible via the UI to an unauthorized
+        # user_profile - and they need to directly enter the URL in their browser. So a json
+        # error may be sufficient.
+        raise CannotManageRemoteBillingError
+
+    if not uses_notification_bouncer():
+        if settings.CORPORATE_ENABLED:
+            # This endpoint makes no sense on zulipchat.com, so just 404.
+            raise ResourceNotFoundError(_("Server doesn't use the push notification service"))
+        else:
+            return reverse(self_hosting_auth_not_configured)
+
+    realm_info = get_realms_info_for_push_bouncer(user_profile.realm_id)[0]
+
+    user_info = UserDataForRemoteBilling(
+        uuid=user_profile.uuid,
+        email=user_profile.delivery_email,
+        full_name=user_profile.full_name,
+    )
+
+    post_data = {
+        "user": user_info.model_dump_json(),
+        "realm": realm_info.model_dump_json(),
+        # The uri_scheme is necessary for the bouncer to know the correct URL
+        # to redirect the user to for re-authing in case the session expires.
+        # Otherwise, the bouncer would know only the realm.host but be missing
+        # the knowledge of whether to use http or https.
+        "uri_scheme": settings.EXTERNAL_URI_SCHEME,
+    }
+    if next_page is not None:
+        post_data["next_page"] = next_page
+
+    try:
+        result = send_to_push_bouncer("POST", "server/billing", post_data)
+    except MissingRemoteRealmError:
+        # Upload realm info and re-try. It should work now.
+        send_server_data_to_push_bouncer(consider_usage_statistics=False)
+        result = send_to_push_bouncer("POST", "server/billing", post_data)
+
+    if result["result"] != "success":  # nocoverage
+        raise JsonableError(_("Error returned by the bouncer: {result}").format(result=result))
+
+    redirect_url = result["billing_access_url"]
+    assert isinstance(redirect_url, str)
+    return redirect_url
+
+
+@zulip_login_required
+@typed_endpoint
+def self_hosting_auth_redirect_endpoint(
+    request: HttpRequest,
+    *,
+    next_page: str | None = None,
+) -> HttpResponse:
+    """
+    This endpoint is used by the web app running in the browser. We serve HTML
+    error pages, and in case of success a simple redirect to the remote billing
+    access link received from the bouncer.
+    """
+
+    user = request.user
+    assert user.is_authenticated
+    assert isinstance(user, UserProfile)
+
+    try:
+        redirect_url = self_hosting_auth_view_common(request, user, next_page)
+    except ResourceNotFoundError:
+        return render(request, "404.html", status=404)
+    except RemoteRealmServerMismatchError:
+        return render(
+            request,
+            "zerver/portico_error_pages/remote_realm_server_mismatch_error.html",
+            status=403,
+        )
+
+    return HttpResponseRedirect(redirect_url)
+
+
+@typed_endpoint
+def self_hosting_auth_json_endpoint(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    next_page: str | None = None,
+) -> HttpResponse:
+    """
+    This endpoint is used by the desktop application. It makes an API request here,
+    expecting a JSON response with either the billing access link, or appropriate
+    error information.
+    """
+
+    redirect_url = self_hosting_auth_view_common(request, user_profile, next_page)
+
+    return json_success(request, data={"billing_access_url": redirect_url})
+
+
+@zulip_login_required
+def self_hosting_auth_not_configured(request: HttpRequest) -> HttpResponse:
+    # Use the same access model as the main endpoints for consistency
+    # and to not have to worry about this endpoint leaking some kind of
+    # sensitive configuration information in the future.
+    user = request.user
+    assert user.is_authenticated
+    assert isinstance(user, UserProfile)
+    if not user.has_billing_access:
+        raise CannotManageRemoteBillingError
+
+    if settings.CORPORATE_ENABLED or uses_notification_bouncer():
+        # This error page should only be available if the config error
+        # is actually real.
+        return render(request, "404.html", status=404)
+
+    return config_error(
+        request,
+        "remote_billing_bouncer_not_configured",
+        go_back_to_url="/",
+        go_back_to_url_name="the app",
+    )
+
+
+class VerificationSecretNotPreparedError(JsonableError):
+    code = ErrorCode.REMOTE_SERVER_VERIFICATION_SECRET_NOT_PREPARED
+
+    def __init__(self) -> None:
+        super().__init__(_("Verification secret not prepared"))
+
+
+@typed_endpoint_without_parameters
+def self_hosting_registration_transfer_challenge_verify(
+    request: HttpRequest, access_token: str
+) -> HttpResponse:
+    json_data = redis_client.get(
+        redis_utils.REDIS_KEY_PREFIX + SELF_HOSTING_REGISTRATION_TAKEOVER_CHALLENGE_TOKEN_REDIS_KEY
+    )
+    if json_data is None:
+        raise VerificationSecretNotPreparedError
+
+    data = orjson.loads(json_data)
+    if not constant_time_compare(data["access_token"], access_token):
+        # Without knowing the access_token, the client gets the same error
+        # as if we're not serving the verification secret at all.
+        raise VerificationSecretNotPreparedError
+
+    verification_secret = data["verification_secret"]
+
+    return json_success(request, data={"verification_secret": verification_secret})
+
+
+@human_users_only
+@typed_endpoint
+def register_push_device(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    # `device_id` to configure for push notifications.
+    device_id: Json[int],
+    token_kind: Annotated[
+        str | None, check_string_in_validator(Device.PushTokenKind.values)
+    ] = None,
+    # Key that the client is requesting be used for encrypting
+    # push notifications for delivery to it.
+    # Consists of a 1-byte prefix identifying the symmetric
+    # cryptosystem in use, followed by the secret key.
+    push_key: str | None = None,
+    # ID which client uses to reference `push_key`.
+    push_key_id: Annotated[Json[int] | None, uint32_validator()] = None,
+    # Key that the client claims was used to encrypt
+    # `encrypted_push_registration`.
+    bouncer_public_key: str | None = None,
+    # Registration data encrypted by mobile client for bouncer.
+    encrypted_push_registration: str | None = None,
+    # ID which client uses to reference token provided by FCM/APNs.
+    token_id_base64: Annotated[str | None, ApiParamConfig("token_id")] = None,
+) -> HttpResponse:
+    if not (settings.ZILENCER_ENABLED or uses_notification_bouncer()):
+        raise PushServiceNotConfiguredError
+
+    device = check_device_id(device_id, user_profile.id)
+
+    push_key_fields_present = push_key is not None and push_key_id is not None
+    token_fields_present = (
+        bouncer_public_key is not None
+        and encrypted_push_registration is not None
+        and token_id_base64 is not None
+        and token_kind is not None
+    )
+
+    if not (push_key_fields_present or token_fields_present):
+        raise JsonableError(
+            _(
+                "Missing parameters: must provide either all push key fields, all token fields, or both."
+            )
+        )
+
+    if push_key_fields_present and not token_fields_present:
+        if device.push_key_id is None:
+            raise JsonableError(_("No push registration exists to rotate key for."))
+
+        assert push_key is not None and push_key_id is not None
+        push_key_bytes = check_push_key(push_key)
+        do_rotate_push_key(user_profile, device, push_key_bytes, push_key_id)
+        return json_success(request)
+
+    # A registration request already in progress, don't enqueue another one
+    # to bouncer to avoid bugs due to race.
+    if device.push_registration_error_code is None and device.pending_push_token_id is not None:
+        raise JsonableError(_("A registration for the device already in progress."))
+
+    with transaction.atomic(durable=True):
+        assert token_id_base64 is not None
+        assert token_kind is not None
+        assert bouncer_public_key is not None
+        assert encrypted_push_registration is not None
+
+        token_id_int = b64decode_token_id_base64(token_id_base64)
+        if token_fields_present and not push_key_fields_present:
+            if device.push_key_id is None:
+                raise JsonableError(_("No push registration exists to rotate token for."))
+
+            if device.push_registration_error_code is None and device.push_token_id == token_id_int:
+                return json_success(request)
+
+            do_rotate_token(user_profile, device, token_id_int, token_id_base64)
+
+        # All fields present
+        if push_key_fields_present and token_fields_present:
+            assert push_key is not None
+            assert push_key_id is not None
+            push_key_bytes = check_push_key(push_key)
+
+            # Idempotent
+            if (
+                device.push_registration_error_code is None
+                and device.push_key_id == push_key_id
+                and device.push_key is not None
+                and bytes(device.push_key) == push_key_bytes
+                and device.push_token_id == token_id_int
+                and device.push_token_kind == token_kind
+            ):
+                return json_success(request)
+
+            do_register_push_device(
+                user_profile,
+                device,
+                token_kind=token_kind,
+                push_key_bytes=push_key_bytes,
+                push_key_id=push_key_id,
+                token_id_int=token_id_int,
+                token_id_base64=token_id_base64,
+            )
+
+        # We use a queue worker to make the request to the bouncer
+        # to complete the registration, so that any transient failures
+        # can be managed between the two servers, without the mobile
+        # device and its often-irregular network access in the picture.
+        queue_item: RegisterPushDeviceToBouncerQueueItem = {
+            "user_profile_id": user_profile.id,
+            "device_id": device_id,
+            "bouncer_public_key": bouncer_public_key,
+            "encrypted_push_registration": encrypted_push_registration,
+            "token_id_base64": token_id_base64,
+        }
+        queue_event_on_commit(
+            mobile_notifications_queue_name(user_profile.id),
+            {
+                "type": "register_push_device_to_bouncer",
+                "payload": queue_item,
+            },
+        )
+
+    return json_success(request)

zerver/views/reactions.py

@@ -0,0 +1,77 @@
+from django.db import transaction
+from django.http import HttpRequest, HttpResponse
+from django.utils.translation import gettext as _
+
+from zerver.actions.reactions import check_add_reaction, do_remove_reaction
+from zerver.lib.emoji import get_emoji_data
+from zerver.lib.exceptions import JsonableError, ReactionDoesNotExistError
+from zerver.lib.message import access_message
+from zerver.lib.response import json_success
+from zerver.lib.typed_endpoint import typed_endpoint
+from zerver.models import Reaction, UserProfile
+
+
+# transaction.atomic is required since we use FOR UPDATE queries in access_message
+@transaction.atomic(durable=True)
+@typed_endpoint
+def add_reaction(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    message_id: int,
+    *,
+    emoji_code: str | None = None,
+    emoji_name: str,
+    reaction_type: str | None = None,
+) -> HttpResponse:
+    check_add_reaction(user_profile, message_id, emoji_name, emoji_code, reaction_type)
+
+    return json_success(request)
+
+
+# transaction.atomic is required since we use FOR UPDATE queries in access_message
+@transaction.atomic(durable=True)
+@typed_endpoint
+def remove_reaction(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    message_id: int,
+    *,
+    emoji_code: str | None = None,
+    emoji_name: str | None = None,
+    reaction_type: str = "unicode_emoji",
+) -> HttpResponse:
+    message = access_message(user_profile, message_id, lock_message=True, is_modifying_message=True)
+
+    if emoji_code is None:
+        if emoji_name is None:
+            raise JsonableError(
+                _("At least one of the following arguments must be present: emoji_name, emoji_code")
+            )
+        # A correct full Zulip client implementation should always
+        # pass an emoji_code, because of the corner cases discussed in
+        # the long block comments elsewhere in this file.  However, to
+        # make it easy for simple API clients to use the reactions API
+        # without needing the mapping between emoji names and codes,
+        # we allow instead passing the emoji_name and looking up the
+        # corresponding code using the current data.
+        emoji_code = get_emoji_data(message.realm_id, emoji_name).emoji_code
+
+    if not Reaction.objects.filter(
+        user_profile=user_profile,
+        message=message,
+        emoji_code=emoji_code,
+        reaction_type=reaction_type,
+    ).exists():
+        raise ReactionDoesNotExistError
+
+    # Unlike adding reactions, while deleting a reaction, we don't
+    # check whether the provided (emoji_type, emoji_code) pair is
+    # valid in this realm.  Since there's a row in the database, we
+    # know it was valid when the user added their reaction in the
+    # first place, so it is safe to just remove the reaction if it
+    # exists.  And the (reaction_type, emoji_code) pair may no longer be
+    # valid in legitimate situations (e.g. if a realm emoji was
+    # deactivated by an administrator in the meantime).
+    do_remove_reaction(user_profile, message, emoji_code, reaction_type)
+
+    return json_success(request)

zerver/views/read_receipts.py

@@ -0,0 +1,87 @@
+from django.http.request import HttpRequest
+from django.http.response import HttpResponse
+from django.utils.translation import gettext as _
+from pydantic import NonNegativeInt
+
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.message import access_message
+from zerver.lib.response import json_success
+from zerver.lib.typed_endpoint import PathOnly, typed_endpoint
+from zerver.models import UserMessage, UserProfile
+
+
+@typed_endpoint
+def read_receipts(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    message_id: PathOnly[NonNegativeInt],
+) -> HttpResponse:
+    message = access_message(user_profile, message_id, is_modifying_message=False)
+
+    if not user_profile.realm.enable_read_receipts:
+        raise JsonableError(_("Read receipts are disabled in this organization."))
+
+    # This query implements a few decisions:
+    # * Most importantly, this is where we enforce the
+    #   send_read_receipts privacy setting.
+    #
+    # * The message sender is never included, since presumably they
+    #   read the message before sending it, and showing the sender as
+    #   having read their own message is likely to be confusing.
+    #
+    # * Users who have muted the current user are not included, since
+    #   the current user could infer that they have been muted by
+    #   said users by noting that the muters immediately read every
+    #   message that the current user sends to mutually subscribed
+    #   streams.
+    #
+    # * Users muted by the current user are also not included, as this
+    #   is consistent with other aspects of how muting works.
+    #
+    # * Deactivated users are excluded. While in theory someone
+    #   could be interested in the information, not including them
+    #   is a cleaner policy, and usually read receipts are only of
+    #   interest shortly after a message was sent.
+    #
+    # * We do not filter on the historical flag. This means that a user
+    #   who stars a public stream message that they did not originally
+    #   receive will appear in read receipts for that message.
+    #
+    # * Marking a message as unread causes it to appear as not read
+    #   via read receipts, as well. This is consistent with the fact
+    #   that users can view a message without leaving it marked as
+    #   read in other ways (desktop/email/push notifications).
+    #
+    # * Bots are included. Most bots never mark any messages as read,
+    #   but one could imagine having them to do so via the API to
+    #   communicate useful information. For example, the `read` flag
+    #   could be used by a bot to track which messages have been
+    #   bridged to another chat system or otherwise processed
+    #   successfully by the bot, and users might find it useful to be
+    #   able to inspect that in the UI. If this behavior is not
+    #   desired for a bot, it can be disabled using the
+    #   send_read_receipts privacy setting.
+    #
+    # Note that we do not attempt to present how many users received a
+    # message but have NOT marked the message as read. There are
+    # tricky corner cases involved in doing so, such as the
+    # `historical` flag for public stream messages; but the most
+    # important one is how to handle users who read a message and then
+    # later unsubscribed from a stream.
+    user_ids = (
+        UserMessage.objects.filter(
+            message_id=message.id,
+            user_profile__is_active=True,
+            user_profile__send_read_receipts=True,
+        )
+        .exclude(user_profile_id=message.sender_id)
+        .exclude(user_profile__muter__muted_user_id=user_profile.id)
+        .exclude(user_profile__muted__user_profile_id=user_profile.id)
+        .extra(  # noqa: S610
+            where=[UserMessage.where_read()],
+        )
+        .values_list("user_profile_id", flat=True)
+    )
+
+    return json_success(request, {"user_ids": list(user_ids)})

zerver/views/realm.py

@@ -0,0 +1,836 @@
+from collections.abc import Mapping
+from typing import Annotated, Any, Literal
+
+from django.conf import settings
+from django.core.exceptions import ValidationError
+from django.db import transaction
+from django.http import HttpRequest, HttpResponse
+from django.shortcuts import render
+from django.urls import reverse
+from django.utils.translation import gettext as _
+from django.views.decorators.http import require_safe
+from pydantic import Json, NonNegativeInt, StringConstraints
+from pydantic.functional_validators import AfterValidator
+
+from confirmation.models import Confirmation, ConfirmationKeyError, get_object_from_key
+from zerver.actions.create_realm import do_change_realm_subdomain
+from zerver.actions.realm_settings import (
+    do_change_realm_org_type,
+    do_change_realm_permission_group_setting,
+    do_deactivate_realm,
+    do_reactivate_realm,
+    do_set_realm_authentication_methods,
+    do_set_realm_moderation_request_channel,
+    do_set_realm_new_stream_announcements_stream,
+    do_set_realm_property,
+    do_set_realm_signup_announcements_stream,
+    do_set_realm_user_default_setting,
+    do_set_realm_zulip_update_announcements_stream,
+    parse_and_set_setting_value_if_required,
+    validate_authentication_methods_dict_from_api,
+)
+from zerver.decorator import require_post, require_realm_admin, require_realm_owner
+from zerver.forms import check_subdomain_available as check_subdomain
+from zerver.lib.demo_organizations import (
+    check_demo_organization_has_set_email,
+    demo_organization_owner_email_exists,
+    get_demo_organization_deadline_days_remaining,
+)
+from zerver.lib.exceptions import JsonableError, OrganizationOwnerRequiredError
+from zerver.lib.i18n import get_available_language_codes
+from zerver.lib.response import json_success
+from zerver.lib.retention import parse_message_retention_days
+from zerver.lib.streams import access_stream_by_id
+from zerver.lib.typed_endpoint import ApiParamConfig, typed_endpoint
+from zerver.lib.typed_endpoint_validators import (
+    check_int_in_validator,
+    check_string_in_validator,
+    parse_enum_from_string_value,
+)
+from zerver.lib.user_groups import (
+    GroupSettingChangeRequest,
+    access_user_group_for_setting,
+    get_group_setting_value_for_api,
+    get_system_user_group_by_name,
+    parse_group_setting_value,
+    validate_can_manage_all_groups,
+    validate_group_setting_value_change,
+)
+from zerver.lib.validator import check_capped_url, check_string
+from zerver.lib.workplace_users import (
+    realm_eligible_for_non_workplace_pricing,
+    realm_on_discounted_cloud_plan,
+    validate_workplace_users_group,
+)
+from zerver.models import Realm, RealmReactivationStatus, RealmUserDefault, UserProfile
+from zerver.models.groups import SystemGroups
+from zerver.models.realms import (
+    DigestWeekdayEnum,
+    MessageEditHistoryVisibilityPolicyEnum,
+    OrgTypeEnum,
+    RealmMediaPreviewSizeEnum,
+    RealmTopicsPolicyEnum,
+)
+from zerver.models.users import ResolvedTopicNoticeAutoReadPolicyEnum
+from zerver.views.user_settings import check_settings_values
+
+
+def parse_jitsi_server_url(value: str, special_values_map: Mapping[str, str | None]) -> str | None:
+    if value in special_values_map:
+        return special_values_map[value]
+
+    return value
+
+
+JITSI_SERVER_URL_MAX_LENGTH = 200
+
+
+def check_jitsi_url(value: str) -> str:
+    var_name = "jitsi_server_url"
+    value = check_string(var_name, value)
+
+    if value in list(Realm.JITSI_SERVER_SPECIAL_VALUES_MAP.keys()):
+        return value
+
+    validator = check_capped_url(JITSI_SERVER_URL_MAX_LENGTH)
+    try:
+        return validator(var_name, value)
+    except ValidationError:
+        raise JsonableError(_("{var_name} is not an allowed_type").format(var_name=var_name))
+
+
+DEFAULT_AVATAR_SOURCES = [key for key, _ in Realm.AVATAR_SOURCES]
+
+
+@require_realm_admin
+@typed_endpoint
+def update_realm(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    allow_message_editing: Json[bool] | None = None,
+    authentication_methods: Json[dict[str, Any]] | None = None,
+    avatar_changes_disabled: Json[bool] | None = None,
+    can_access_all_users_group: Json[GroupSettingChangeRequest] | None = None,
+    can_add_custom_emoji_group: Json[GroupSettingChangeRequest] | None = None,
+    can_add_subscribers_group: Json[GroupSettingChangeRequest] | None = None,
+    can_create_bots_group: Json[GroupSettingChangeRequest] | None = None,
+    can_create_groups: Json[GroupSettingChangeRequest] | None = None,
+    can_create_private_channel_group: Json[GroupSettingChangeRequest] | None = None,
+    can_create_public_channel_group: Json[GroupSettingChangeRequest] | None = None,
+    can_create_web_public_channel_group: Json[GroupSettingChangeRequest] | None = None,
+    can_create_write_only_bots_group: Json[GroupSettingChangeRequest] | None = None,
+    can_delete_any_message_group: Json[GroupSettingChangeRequest] | None = None,
+    can_delete_own_message_group: Json[GroupSettingChangeRequest] | None = None,
+    can_invite_users_group: Json[GroupSettingChangeRequest] | None = None,
+    can_manage_all_groups: Json[GroupSettingChangeRequest] | None = None,
+    can_manage_billing_group: Json[GroupSettingChangeRequest] | None = None,
+    can_mention_many_users_group: Json[GroupSettingChangeRequest] | None = None,
+    can_move_messages_between_channels_group: Json[GroupSettingChangeRequest] | None = None,
+    can_move_messages_between_topics_group: Json[GroupSettingChangeRequest] | None = None,
+    can_resolve_topics_group: Json[GroupSettingChangeRequest] | None = None,
+    can_set_delete_message_policy_group: Json[GroupSettingChangeRequest] | None = None,
+    can_set_topics_policy_group: Json[GroupSettingChangeRequest] | None = None,
+    can_summarize_topics_group: Json[GroupSettingChangeRequest] | None = None,
+    create_multiuse_invite_group: Json[GroupSettingChangeRequest] | None = None,
+    default_avatar_source: Annotated[str, check_string_in_validator(DEFAULT_AVATAR_SOURCES)]
+    | None = None,
+    default_code_block_language: str | None = None,
+    default_language: str | None = None,
+    description: Annotated[
+        str | None, StringConstraints(max_length=Realm.MAX_REALM_DESCRIPTION_LENGTH)
+    ] = None,
+    digest_emails_enabled: Json[bool] | None = None,
+    digest_weekday: Json[DigestWeekdayEnum] | None = None,
+    direct_message_initiator_group: Json[GroupSettingChangeRequest] | None = None,
+    direct_message_permission_group: Json[GroupSettingChangeRequest] | None = None,
+    disallow_disposable_email_addresses: Json[bool] | None = None,
+    email_changes_disabled: Json[bool] | None = None,
+    emails_restricted_to_domains: Json[bool] | None = None,
+    enable_guest_user_dm_warning: Json[bool] | None = None,
+    enable_guest_user_indicator: Json[bool] | None = None,
+    enable_read_receipts: Json[bool] | None = None,
+    enable_spectator_access: Json[bool] | None = None,
+    gif_rating_policy: Json[int] | None = None,
+    media_preview_size: Json[RealmMediaPreviewSizeEnum] | None = None,
+    inline_image_preview: Json[bool] | None = None,
+    inline_url_embed_preview: Json[bool] | None = None,
+    invite_required: Json[bool] | None = None,
+    jitsi_server_url_raw: Annotated[
+        Json[str] | None,
+        AfterValidator(check_jitsi_url),
+        ApiParamConfig("jitsi_server_url"),
+    ] = None,
+    message_content_allowed_in_email_notifications: Json[bool] | None = None,
+    message_content_delete_limit_seconds_raw: Annotated[
+        Json[int | str] | None,
+        ApiParamConfig("message_content_delete_limit_seconds"),
+    ] = None,
+    message_content_edit_limit_seconds_raw: Annotated[
+        Json[int | str] | None, ApiParamConfig("message_content_edit_limit_seconds")
+    ] = None,
+    message_edit_history_visibility_policy: Annotated[
+        str | None,
+        AfterValidator(
+            lambda val: parse_enum_from_string_value(
+                val,
+                "message_edit_history_visibility_policy",
+                MessageEditHistoryVisibilityPolicyEnum,
+            )
+        ),
+    ] = None,
+    message_retention_days_raw: Annotated[
+        Json[int | str] | None, ApiParamConfig("message_retention_days")
+    ] = None,
+    moderation_request_channel_id: Json[int] | None = None,
+    move_messages_between_streams_limit_seconds_raw: Annotated[
+        Json[int | str] | None,
+        ApiParamConfig("move_messages_between_streams_limit_seconds"),
+    ] = None,
+    move_messages_within_stream_limit_seconds_raw: Annotated[
+        Json[int | str] | None,
+        ApiParamConfig("move_messages_within_stream_limit_seconds"),
+    ] = None,
+    name: Annotated[str | None, StringConstraints(max_length=Realm.MAX_REALM_NAME_LENGTH)] = None,
+    name_changes_disabled: Json[bool] | None = None,
+    new_stream_announcements_stream_id: Json[int] | None = None,
+    org_type: Json[OrgTypeEnum] | None = None,
+    require_e2ee_push_notifications: Json[bool] | None = None,
+    require_unique_names: Json[bool] | None = None,
+    send_channel_events_messages: Json[bool] | None = None,
+    send_welcome_emails: Json[bool] | None = None,
+    signup_announcements_stream_id: Json[int] | None = None,
+    string_id: Annotated[
+        str | None, StringConstraints(max_length=Realm.MAX_REALM_SUBDOMAIN_LENGTH)
+    ] = None,
+    topics_policy: Annotated[
+        str | None,
+        AfterValidator(
+            lambda val: parse_enum_from_string_value(
+                val,
+                "topics_policy",
+                RealmTopicsPolicyEnum,
+            )
+        ),
+    ] = None,
+    video_chat_provider: Json[int] | None = None,
+    waiting_period_threshold: Json[NonNegativeInt] | None = None,
+    want_advertise_in_communities_directory: Json[bool] | None = None,
+    zulip_update_announcements_stream_id: Json[int] | None = None,
+    # Note: push_notifications_enabled and push_notifications_enabled_end_timestamp
+    # are not offered here as it is maintained by the server, not via the API.
+    welcome_message_custom_text: Annotated[
+        str | None,
+        StringConstraints(
+            max_length=Realm.MAX_REALM_WELCOME_MESSAGE_CUSTOM_TEXT_LENGTH,
+        ),
+    ] = None,
+    workplace_users_group: Json[GroupSettingChangeRequest] | None = None,
+) -> HttpResponse:
+    # Realm object is being refetched here to make sure that we
+    # do not use stale object from cache which can happen when a
+    # previous request tried updating multiple settings in a single
+    # request.
+    #
+    # TODO: Change the cache flushing strategy to make sure cache
+    # does not contain stale objects.
+    realm = Realm.objects.get(id=user_profile.realm_id)
+
+    # Additional validation/error checking beyond types go here, so
+    # the entire request can succeed or fail atomically.
+    if default_language is not None and default_language not in get_available_language_codes():
+        raise JsonableError(_("Invalid language '{language}'").format(language=default_language))
+    if authentication_methods is not None:
+        if not user_profile.is_realm_owner:
+            raise OrganizationOwnerRequiredError
+
+        validate_authentication_methods_dict_from_api(realm, authentication_methods)
+        if True not in authentication_methods.values():
+            raise JsonableError(_("At least one authentication method must be enabled."))
+
+    if video_chat_provider is not None and video_chat_provider not in {
+        p["id"] for p in realm.get_enabled_video_chat_providers().values()
+    }:
+        raise JsonableError(
+            _("Invalid video_chat_provider {video_chat_provider}").format(
+                video_chat_provider=video_chat_provider
+            )
+        )
+    if gif_rating_policy is not None and gif_rating_policy not in {
+        p["id"] for p in Realm.GIF_RATING_POLICY_OPTIONS.values()
+    }:
+        raise JsonableError(
+            _("Invalid gif_rating_policy {gif_rating_policy}").format(
+                gif_rating_policy=gif_rating_policy
+            )
+        )
+
+    message_retention_days: int | None = None
+    if message_retention_days_raw is not None:
+        if not user_profile.is_realm_owner:
+            raise OrganizationOwnerRequiredError
+        realm.ensure_not_on_limited_plan()
+        message_retention_days = parse_message_retention_days(  # used by locals() below
+            message_retention_days_raw, Realm.MESSAGE_RETENTION_SPECIAL_VALUES_MAP
+        )
+
+    if can_create_groups is not None:
+        realm.ensure_not_on_limited_plan()
+
+    if (
+        invite_required is not None
+        or create_multiuse_invite_group is not None
+        or can_create_groups is not None
+        or can_invite_users_group is not None
+        or can_manage_all_groups is not None
+        or can_manage_billing_group is not None
+    ) and not user_profile.is_realm_owner:
+        raise OrganizationOwnerRequiredError
+
+    if (
+        emails_restricted_to_domains is not None or disallow_disposable_email_addresses is not None
+    ) and not user_profile.is_realm_owner:
+        raise OrganizationOwnerRequiredError
+
+    if waiting_period_threshold is not None and not user_profile.is_realm_owner:
+        raise OrganizationOwnerRequiredError
+
+    if realm.demo_organization_scheduled_deletion_date is not None and invite_required is not None:
+        check_demo_organization_has_set_email(realm)
+
+    if enable_spectator_access:
+        realm.ensure_not_on_limited_plan()
+
+    if can_access_all_users_group is not None:
+        realm.can_enable_restricted_user_access_for_guests()
+
+    data: dict[str, Any] = {}
+
+    message_content_delete_limit_seconds: int | None = None
+    if message_content_delete_limit_seconds_raw is not None:
+        (
+            message_content_delete_limit_seconds,
+            setting_value_changed,
+        ) = parse_and_set_setting_value_if_required(
+            realm,
+            "message_content_delete_limit_seconds",
+            message_content_delete_limit_seconds_raw,
+            acting_user=user_profile,
+        )
+
+        if setting_value_changed:
+            data["message_content_delete_limit_seconds"] = message_content_delete_limit_seconds
+
+    message_content_edit_limit_seconds: int | None = None
+    if message_content_edit_limit_seconds_raw is not None:
+        (
+            message_content_edit_limit_seconds,
+            setting_value_changed,
+        ) = parse_and_set_setting_value_if_required(
+            realm,
+            "message_content_edit_limit_seconds",
+            message_content_edit_limit_seconds_raw,
+            acting_user=user_profile,
+        )
+
+        if setting_value_changed:
+            data["message_content_edit_limit_seconds"] = message_content_edit_limit_seconds
+
+    move_messages_within_stream_limit_seconds: int | None = None
+    if move_messages_within_stream_limit_seconds_raw is not None:
+        (
+            move_messages_within_stream_limit_seconds,
+            setting_value_changed,
+        ) = parse_and_set_setting_value_if_required(
+            realm,
+            "move_messages_within_stream_limit_seconds",
+            move_messages_within_stream_limit_seconds_raw,
+            acting_user=user_profile,
+        )
+
+        if setting_value_changed:
+            data["move_messages_within_stream_limit_seconds"] = (
+                move_messages_within_stream_limit_seconds
+            )
+
+    move_messages_between_streams_limit_seconds: int | None = None
+    if move_messages_between_streams_limit_seconds_raw is not None:
+        (
+            move_messages_between_streams_limit_seconds,
+            setting_value_changed,
+        ) = parse_and_set_setting_value_if_required(
+            realm,
+            "move_messages_between_streams_limit_seconds",
+            move_messages_between_streams_limit_seconds_raw,
+            acting_user=user_profile,
+        )
+
+        if setting_value_changed:
+            data["move_messages_between_streams_limit_seconds"] = (
+                move_messages_between_streams_limit_seconds
+            )
+
+    jitsi_server_url: str | None = None
+    if jitsi_server_url_raw is not None:
+        jitsi_server_url = parse_jitsi_server_url(
+            jitsi_server_url_raw,
+            Realm.JITSI_SERVER_SPECIAL_VALUES_MAP,
+        )
+
+        # We handle the "None" case separately here because
+        # in the loop below, do_set_realm_property is called only when
+        # the setting value is not "None". For values other than "None",
+        # the loop itself sets the value of 'jitsi_server_url' by
+        # calling do_set_realm_property.
+        if jitsi_server_url is None and realm.jitsi_server_url is not None:
+            do_set_realm_property(
+                realm,
+                "jitsi_server_url",
+                jitsi_server_url,
+                acting_user=user_profile,
+            )
+
+            data["jitsi_server_url"] = jitsi_server_url
+
+    if workplace_users_group is not None:
+        # Remove this when the feature is ready for production.
+        assert settings.DEVELOPMENT
+
+        if not realm_eligible_for_non_workplace_pricing(realm):
+            raise JsonableError(
+                _("Organization is not eligible for discounted pricing for non workplace users.")
+            )
+
+        if realm_on_discounted_cloud_plan(realm):
+            raise JsonableError(
+                _(
+                    "Discounted pricing for workplace users cannot be enabled with current discounted plan."
+                )
+            )
+
+    # The user of `locals()` here is a bit of a code smell, but it's
+    # restricted to the elements present in realm.property_types.
+    #
+    # TODO: It should be possible to deduplicate this function up
+    # further by some more advanced usage of the
+    # `typed_endpoint` extraction.
+    req_vars = {}
+    req_group_setting_vars = {}
+
+    for k, v in locals().items():
+        if k in realm.property_types:
+            req_vars[k] = v
+
+        for setting_name in Realm.REALM_PERMISSION_GROUP_SETTINGS:
+            if k == setting_name:
+                req_group_setting_vars[k] = v
+
+    for k, v in req_vars.items():
+        if v is not None and getattr(realm, k) != v:
+            do_set_realm_property(realm, k, v, acting_user=user_profile)
+            if isinstance(v, str):
+                data[k] = "updated"
+            else:
+                data[k] = v
+
+    nobody_group = get_system_user_group_by_name(SystemGroups.NOBODY, user_profile.realm_id)
+    for setting_name, permission_configuration in Realm.REALM_PERMISSION_GROUP_SETTINGS.items():
+        expected_current_setting_value = None
+        assert setting_name in req_group_setting_vars
+        if req_group_setting_vars[setting_name] is None:
+            continue
+
+        setting_value = req_group_setting_vars[setting_name]
+        new_setting_value = parse_group_setting_value(setting_value.new, nobody_group)
+
+        if setting_value.old is not None:
+            expected_current_setting_value = parse_group_setting_value(
+                setting_value.old, nobody_group
+            )
+
+        current_value = getattr(realm, setting_name)
+        current_setting_api_value = get_group_setting_value_for_api(current_value)
+
+        if validate_group_setting_value_change(
+            current_setting_api_value, new_setting_value, expected_current_setting_value
+        ):
+            with transaction.atomic(durable=True):
+                user_group = access_user_group_for_setting(
+                    new_setting_value,
+                    user_profile,
+                    setting_name=setting_name,
+                    permission_configuration=permission_configuration,
+                    current_setting_value=current_value,
+                )
+
+                if setting_name == "workplace_users_group":
+                    validate_workplace_users_group(new_setting_value, realm)
+
+                if setting_name == "can_manage_all_groups":
+                    validate_can_manage_all_groups(new_setting_value, realm)
+
+                do_change_realm_permission_group_setting(
+                    realm,
+                    setting_name,
+                    user_group,
+                    old_setting_api_value=current_setting_api_value,
+                    acting_user=user_profile,
+                )
+            data[setting_name] = new_setting_value
+
+    # The following realm properties do not fit the pattern above
+    # authentication_methods is not supported by the do_set_realm_property
+    # framework because it's tracked through the RealmAuthenticationMethod table.
+    if authentication_methods is not None and (
+        realm.authentication_methods_dict() != authentication_methods
+    ):
+        do_set_realm_authentication_methods(realm, authentication_methods, acting_user=user_profile)
+        data["authentication_methods"] = authentication_methods
+
+    # Channel-valued settings are not yet fully supported by the
+    # property_types framework, and thus have explicit blocks here.
+    if moderation_request_channel_id is not None and (
+        realm.moderation_request_channel is None
+        or realm.moderation_request_channel.id != moderation_request_channel_id
+    ):
+        new_moderation_request_channel_id = None
+        if moderation_request_channel_id >= 0:
+            (new_moderation_request_channel_id, sub) = access_stream_by_id(
+                user_profile, moderation_request_channel_id, require_content_access=False
+            )
+        do_set_realm_moderation_request_channel(
+            realm,
+            new_moderation_request_channel_id,
+            moderation_request_channel_id,
+            acting_user=user_profile,
+        )
+        data["moderation_request_channel_id"] = moderation_request_channel_id
+
+    if new_stream_announcements_stream_id is not None and (
+        realm.new_stream_announcements_stream is None
+        or (realm.new_stream_announcements_stream.id != new_stream_announcements_stream_id)
+    ):
+        new_stream_announcements_stream_new = None
+        if new_stream_announcements_stream_id >= 0:
+            (new_stream_announcements_stream_new, sub) = access_stream_by_id(
+                user_profile,
+                new_stream_announcements_stream_id,
+                require_content_access=False,
+            )
+        do_set_realm_new_stream_announcements_stream(
+            realm,
+            new_stream_announcements_stream_new,
+            new_stream_announcements_stream_id,
+            acting_user=user_profile,
+        )
+        data["new_stream_announcements_stream_id"] = new_stream_announcements_stream_id
+
+    if signup_announcements_stream_id is not None and (
+        realm.signup_announcements_stream is None
+        or realm.signup_announcements_stream.id != signup_announcements_stream_id
+    ):
+        new_signup_announcements_stream = None
+        if signup_announcements_stream_id >= 0:
+            (new_signup_announcements_stream, sub) = access_stream_by_id(
+                user_profile, signup_announcements_stream_id, require_content_access=False
+            )
+        do_set_realm_signup_announcements_stream(
+            realm,
+            new_signup_announcements_stream,
+            signup_announcements_stream_id,
+            acting_user=user_profile,
+        )
+        data["signup_announcements_stream_id"] = signup_announcements_stream_id
+
+    if zulip_update_announcements_stream_id is not None and (
+        realm.zulip_update_announcements_stream is None
+        or realm.zulip_update_announcements_stream.id != zulip_update_announcements_stream_id
+    ):
+        new_zulip_update_announcements_stream = None
+        if zulip_update_announcements_stream_id >= 0:
+            (new_zulip_update_announcements_stream, sub) = access_stream_by_id(
+                user_profile,
+                zulip_update_announcements_stream_id,
+                require_content_access=False,
+            )
+        do_set_realm_zulip_update_announcements_stream(
+            realm,
+            new_zulip_update_announcements_stream,
+            zulip_update_announcements_stream_id,
+            acting_user=user_profile,
+        )
+        data["zulip_update_announcements_stream_id"] = zulip_update_announcements_stream_id
+
+    if string_id is not None:
+        if not user_profile.is_realm_owner:
+            raise OrganizationOwnerRequiredError
+        if realm.demo_organization_scheduled_deletion_date is None:
+            raise JsonableError(_("Must be a demo organization."))
+        check_demo_organization_has_set_email(realm)
+        try:
+            check_subdomain(string_id)
+        except ValidationError as err:
+            raise JsonableError(str(err.message))
+
+        do_change_realm_subdomain(realm, string_id, acting_user=user_profile)
+        data["realm_uri"] = realm.url
+        data["realm_url"] = realm.url
+
+    if org_type is not None:
+        do_change_realm_org_type(realm, org_type, acting_user=user_profile)
+        data["org_type"] = org_type
+
+    return json_success(request, data)
+
+
+@require_realm_owner
+@typed_endpoint
+def deactivate_realm(
+    request: HttpRequest, user: UserProfile, *, deletion_delay_days: Json[int | None] = None
+) -> HttpResponse:
+    realm = user.realm
+
+    # Demo organizations have different conditions for deactivation by
+    # the organization owner than permanent organizations do.
+    if realm.demo_organization_scheduled_deletion_date is not None:
+        # We require that demo organization data be deleted, and if the
+        # demo organization owner has not configured an email address,
+        # then it must be deleted immediately.
+        owner_email_configured = demo_organization_owner_email_exists(realm)
+        if deletion_delay_days is None or (not owner_email_configured and deletion_delay_days != 0):
+            raise JsonableError(_("Invalid data deletion time for demo organization."))
+
+        # If the demo organization owner has configured an email address,
+        # then the data must be deleted before the demo organization's
+        # scheduled deletion date.
+        days_before_scheduled_deletion = get_demo_organization_deadline_days_remaining(realm)
+        if days_before_scheduled_deletion < deletion_delay_days:
+            raise JsonableError(_("Invalid data deletion time for demo organization."))
+
+        do_deactivate_realm(
+            realm,
+            acting_user=user,
+            deactivation_reason="owner_request",
+            email_owners=owner_email_configured,
+            deletion_delay_days=deletion_delay_days,
+        )
+        return json_success(request)
+
+    if settings.MAX_DEACTIVATED_REALM_DELETION_DAYS is not None and (
+        deletion_delay_days is None
+        or deletion_delay_days > settings.MAX_DEACTIVATED_REALM_DELETION_DAYS
+    ):
+        raise JsonableError(
+            _("Data deletion time must be at most {max_allowed_days} days in the future.").format(
+                max_allowed_days=settings.MAX_DEACTIVATED_REALM_DELETION_DAYS,
+            )
+        )
+
+    if (
+        settings.MIN_DEACTIVATED_REALM_DELETION_DAYS is not None
+        and deletion_delay_days is not None
+        and deletion_delay_days < settings.MIN_DEACTIVATED_REALM_DELETION_DAYS
+    ):
+        raise JsonableError(
+            _("Data deletion time must be at least {min_allowed_days} days in the future.").format(
+                min_allowed_days=settings.MIN_DEACTIVATED_REALM_DELETION_DAYS,
+            )
+        )
+
+    do_deactivate_realm(
+        realm,
+        acting_user=user,
+        deactivation_reason="owner_request",
+        email_owners=True,
+        deletion_delay_days=deletion_delay_days,
+    )
+    return json_success(request)
+
+
+@require_safe
+def check_subdomain_available(request: HttpRequest, subdomain: str) -> HttpResponse:
+    try:
+        check_subdomain(subdomain)
+        return json_success(request, data={"msg": "available"})
+    except ValidationError as e:
+        return json_success(request, data={"msg": e.message})
+
+
+def realm_reactivation_get(request: HttpRequest, confirmation_key: str) -> HttpResponse:
+    try:
+        get_object_from_key(
+            confirmation_key, [Confirmation.REALM_REACTIVATION], mark_object_used=False
+        )
+    except ConfirmationKeyError:  # nocoverage
+        return render(request, "zerver/realm_reactivation_link_error.html", status=404)
+
+    return render(
+        request,
+        "confirmation/redirect_to_post.html",
+        context={
+            "target_url": reverse("realm_reactivation"),
+            "key": confirmation_key,
+        },
+    )
+
+
+@require_post
+@typed_endpoint
+def realm_reactivation(request: HttpRequest, *, key: str) -> HttpResponse:
+    try:
+        obj = get_object_from_key(key, [Confirmation.REALM_REACTIVATION], mark_object_used=True)
+    except ConfirmationKeyError:
+        return render(request, "zerver/realm_reactivation_link_error.html", status=404)
+
+    assert isinstance(obj, RealmReactivationStatus)
+    realm = obj.realm
+
+    do_reactivate_realm(realm)
+
+    context = {"realm": realm}
+    return render(request, "zerver/realm_reactivation.html", context)
+
+
+emojiset_choices = {emojiset["key"] for emojiset in RealmUserDefault.emojiset_choices()}
+
+
+@require_realm_admin
+@typed_endpoint
+def update_realm_user_settings_defaults(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    automatically_follow_topics_policy: Json[
+        Annotated[
+            int,
+            check_int_in_validator(UserProfile.AUTOMATICALLY_CHANGE_VISIBILITY_POLICY_CHOICES),
+        ]
+    ]
+    | None = None,
+    automatically_follow_topics_where_mentioned: Json[bool] | None = None,
+    automatically_unmute_topics_in_muted_streams_policy: Json[
+        Annotated[
+            int,
+            check_int_in_validator(UserProfile.AUTOMATICALLY_CHANGE_VISIBILITY_POLICY_CHOICES),
+        ]
+    ]
+    | None = None,
+    color_scheme: Json[Annotated[int, check_int_in_validator(UserProfile.COLOR_SCHEME_CHOICES)]]
+    | None = None,
+    demote_inactive_streams: Json[
+        Annotated[int, check_int_in_validator(UserProfile.DEMOTE_STREAMS_CHOICES)]
+    ]
+    | None = None,
+    desktop_icon_count_display: Json[
+        Annotated[int, check_int_in_validator(UserProfile.DESKTOP_ICON_COUNT_DISPLAY_CHOICES)]
+    ]
+    | None = None,
+    display_emoji_reaction_users: Json[bool] | None = None,
+    email_address_visibility: Json[
+        Annotated[int, check_int_in_validator(UserProfile.EMAIL_ADDRESS_VISIBILITY_TYPES)]
+    ]
+    | None = None,
+    email_notifications_batching_period_seconds: Json[int] | None = None,
+    emojiset: Annotated[str, check_string_in_validator(emojiset_choices)] | None = None,
+    enable_desktop_notifications: Json[bool] | None = None,
+    enable_digest_emails: Json[bool] | None = None,
+    enable_drafts_synchronization: Json[bool] | None = None,
+    enable_followed_topic_audible_notifications: Json[bool] | None = None,
+    enable_followed_topic_desktop_notifications: Json[bool] | None = None,
+    enable_followed_topic_email_notifications: Json[bool] | None = None,
+    enable_followed_topic_push_notifications: Json[bool] | None = None,
+    enable_followed_topic_wildcard_mentions_notify: Json[bool] | None = None,
+    enable_offline_email_notifications: Json[bool] | None = None,
+    enable_offline_push_notifications: Json[bool] | None = None,
+    enable_online_push_notifications: Json[bool] | None = None,
+    enable_sounds: Json[bool] | None = None,
+    enable_stream_audible_notifications: Json[bool] | None = None,
+    enable_stream_desktop_notifications: Json[bool] | None = None,
+    enable_stream_email_notifications: Json[bool] | None = None,
+    enable_stream_push_notifications: Json[bool] | None = None,
+    # enable_login_emails is not included here, because we don't want
+    # security-related settings to be controlled by organization administrators.
+    # enable_marketing_emails is not included here, since we don't at
+    # present allow organizations to customize this. (The user's selection
+    # in the signup form takes precedence over RealmUserDefault).
+    #
+    # We may want to change this model in the future, since some SSO signups
+    # do not offer an opportunity to prompt the user at all during signup.
+    enter_sends: Json[bool] | None = None,
+    fluid_layout_width: Json[bool] | None = None,
+    hide_ai_features: Json[bool] | None = None,
+    high_contrast_mode: Json[bool] | None = None,
+    left_side_userlist: Json[bool] | None = None,
+    message_content_in_email_notifications: Json[bool] | None = None,
+    notification_sound: str | None = None,
+    pm_content_in_desktop_notifications: Json[bool] | None = None,
+    presence_enabled: Json[bool] | None = None,
+    realm_name_in_email_notifications_policy: Json[
+        Annotated[
+            int,
+            check_int_in_validator(UserProfile.REALM_NAME_IN_EMAIL_NOTIFICATIONS_POLICY_CHOICES),
+        ]
+    ]
+    | None = None,
+    receives_typing_notifications: Json[bool] | None = None,
+    resolved_topic_notice_auto_read_policy: Annotated[
+        str | None,
+        AfterValidator(
+            lambda val: parse_enum_from_string_value(
+                val,
+                "resolved_topic_notice_auto_read_policy",
+                ResolvedTopicNoticeAutoReadPolicyEnum,
+            )
+        ),
+    ] = None,
+    send_private_typing_notifications: Json[bool] | None = None,
+    send_read_receipts: Json[bool] | None = None,
+    send_stream_typing_notifications: Json[bool] | None = None,
+    starred_message_counts: Json[bool] | None = None,
+    translate_emoticons: Json[bool] | None = None,
+    twenty_four_hour_time: Json[bool] | None = None,
+    user_list_style: Json[
+        Annotated[int, check_int_in_validator(UserProfile.USER_LIST_STYLE_CHOICES)]
+    ]
+    | None = None,
+    web_animate_image_previews: Literal["always", "on_hover", "never"] | None = None,
+    web_channel_default_view: Json[
+        Annotated[int, check_int_in_validator(UserProfile.WEB_CHANNEL_DEFAULT_VIEW_CHOICES)]
+    ]
+    | None = None,
+    web_escape_navigates_to_home_view: Json[bool] | None = None,
+    web_font_size_px: Json[int] | None = None,
+    web_home_view: Literal["recent", "inbox", "all_messages"] | None = None,
+    web_inbox_show_channel_folders: Json[bool] | None = None,
+    web_left_sidebar_show_channel_folders: Json[bool] | None = None,
+    web_left_sidebar_unreads_count_summary: Json[bool] | None = None,
+    web_line_height_percent: Json[int] | None = None,
+    web_mark_read_on_scroll_policy: Json[
+        Annotated[
+            int,
+            check_int_in_validator(UserProfile.WEB_MARK_READ_ON_SCROLL_POLICY_CHOICES),
+        ]
+    ]
+    | None = None,
+    web_navigate_to_sent_message: Json[bool] | None = None,
+    web_stream_unreads_count_display_policy: Json[
+        Annotated[
+            int,
+            check_int_in_validator(UserProfile.WEB_STREAM_UNREADS_COUNT_DISPLAY_POLICY_CHOICES),
+        ]
+    ]
+    | None = None,
+    web_suggest_update_timezone: Json[bool] | None = None,
+    wildcard_mentions_notify: Json[bool] | None = None,
+) -> HttpResponse:
+    if notification_sound is not None or email_notifications_batching_period_seconds is not None:
+        check_settings_values(notification_sound, email_notifications_batching_period_seconds)
+
+    realm_user_default = RealmUserDefault.objects.get(realm=user_profile.realm)
+
+    request_settings = {k: v for k, v in locals().items() if k in RealmUserDefault.property_types}
+    for k, v in request_settings.items():
+        if v is not None and getattr(realm_user_default, k) != v:
+            do_set_realm_user_default_setting(realm_user_default, k, v, acting_user=user_profile)
+
+    return json_success(request)

zerver/views/realm_domains.py

@@ -0,0 +1,76 @@
+from django.core.exceptions import ValidationError
+from django.http import HttpRequest, HttpResponse
+from django.utils.translation import gettext as _
+from pydantic import Json
+
+from zerver.actions.realm_domains import (
+    do_add_realm_domain,
+    do_change_realm_domain,
+    do_remove_realm_domain,
+)
+from zerver.decorator import require_realm_owner
+from zerver.lib.domains import validate_domain
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.response import json_success
+from zerver.lib.typed_endpoint import PathOnly, typed_endpoint
+from zerver.models import RealmDomain, UserProfile
+from zerver.models.realms import get_realm_domains
+
+
+def list_realm_domains(request: HttpRequest, user_profile: UserProfile) -> HttpResponse:
+    domains = get_realm_domains(user_profile.realm)
+    return json_success(request, data={"domains": domains})
+
+
+@require_realm_owner
+@typed_endpoint
+def create_realm_domain(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    allow_subdomains: Json[bool],
+    domain: str,
+) -> HttpResponse:
+    domain = domain.strip().lower()
+    try:
+        validate_domain(domain)
+    except ValidationError as e:
+        raise JsonableError(_("Invalid domain: {error}").format(error=e.messages[0]))
+    if RealmDomain.objects.filter(realm=user_profile.realm, domain=domain).exists():
+        raise JsonableError(
+            _("The domain {domain} is already a part of your organization.").format(domain=domain)
+        )
+    realm_domain = do_add_realm_domain(
+        user_profile.realm, domain, allow_subdomains, acting_user=user_profile
+    )
+    return json_success(request, data={"new_domain": [realm_domain.id, realm_domain.domain]})
+
+
+@require_realm_owner
+@typed_endpoint
+def patch_realm_domain(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    allow_subdomains: Json[bool],
+    domain: PathOnly[str],
+) -> HttpResponse:
+    try:
+        realm_domain = RealmDomain.objects.get(realm=user_profile.realm, domain=domain)
+        do_change_realm_domain(realm_domain, allow_subdomains, acting_user=user_profile)
+    except RealmDomain.DoesNotExist:
+        raise JsonableError(_("No entry found for domain {domain}.").format(domain=domain))
+    return json_success(request)
+
+
+@require_realm_owner
+@typed_endpoint
+def delete_realm_domain(
+    request: HttpRequest, user_profile: UserProfile, *, domain: PathOnly[str]
+) -> HttpResponse:
+    try:
+        realm_domain = RealmDomain.objects.get(realm=user_profile.realm, domain=domain)
+        do_remove_realm_domain(realm_domain, acting_user=user_profile)
+    except RealmDomain.DoesNotExist:
+        raise JsonableError(_("No entry found for domain {domain}.").format(domain=domain))
+    return json_success(request)

zerver/views/realm_emoji.py

@@ -0,0 +1,69 @@
+from django.conf import settings
+from django.core.files.uploadedfile import UploadedFile
+from django.http import HttpRequest, HttpResponse
+from django.utils.translation import gettext as _
+
+from zerver.actions.realm_emoji import check_add_realm_emoji, do_remove_realm_emoji
+from zerver.decorator import require_human_non_guest_user
+from zerver.lib.emoji import check_remove_custom_emoji, check_valid_emoji_name, name_to_codepoint
+from zerver.lib.exceptions import JsonableError, ResourceNotFoundError
+from zerver.lib.response import json_success
+from zerver.lib.typed_endpoint import PathOnly, typed_endpoint
+from zerver.lib.upload import get_file_info
+from zerver.models import RealmEmoji, UserProfile
+from zerver.models.realm_emoji import get_all_custom_emoji_for_realm
+
+
+def list_emoji(request: HttpRequest, user_profile: UserProfile) -> HttpResponse:
+    # We don't do any checks here because the list of realm
+    # emoji is public.
+    return json_success(
+        request, data=dict(emoji=get_all_custom_emoji_for_realm(user_profile.realm_id))
+    )
+
+
+@require_human_non_guest_user
+@typed_endpoint
+def upload_emoji(
+    request: HttpRequest, user_profile: UserProfile, *, emoji_name: PathOnly[str]
+) -> HttpResponse:
+    emoji_name = emoji_name.strip().replace(" ", "_")
+    valid_built_in_emoji = name_to_codepoint.keys()
+    check_valid_emoji_name(emoji_name)
+
+    if not user_profile.can_add_custom_emoji():
+        raise JsonableError(_("Insufficient permission"))
+
+    if RealmEmoji.objects.filter(
+        realm=user_profile.realm, name=emoji_name, deactivated=False
+    ).exists():
+        raise JsonableError(_("A custom emoji with this name already exists."))
+    if len(request.FILES) != 1:
+        raise JsonableError(_("You must upload exactly one file."))
+    if emoji_name in valid_built_in_emoji and not user_profile.is_realm_admin:
+        raise JsonableError(_("Only administrators can override default emoji."))
+    [emoji_file] = request.FILES.values()
+    assert isinstance(emoji_file, UploadedFile)
+    assert emoji_file.size is not None
+    if emoji_file.size > settings.MAX_EMOJI_FILE_SIZE_MIB * 1024 * 1024:
+        raise JsonableError(
+            _("Uploaded file is larger than the allowed limit of {max_size} MiB").format(
+                max_size=settings.MAX_EMOJI_FILE_SIZE_MIB,
+            )
+        )
+
+    _filename, content_type = get_file_info(emoji_file)
+    check_add_realm_emoji(user_profile.realm, emoji_name, user_profile, emoji_file, content_type)
+    return json_success(request)
+
+
+def delete_emoji(request: HttpRequest, user_profile: UserProfile, emoji_name: str) -> HttpResponse:
+    if not RealmEmoji.objects.filter(
+        realm=user_profile.realm, name=emoji_name, deactivated=False
+    ).exists():
+        raise ResourceNotFoundError(
+            _("Emoji '{emoji_name}' does not exist").format(emoji_name=emoji_name)
+        )
+    check_remove_custom_emoji(user_profile, emoji_name)
+    do_remove_realm_emoji(user_profile.realm, emoji_name, acting_user=user_profile)
+    return json_success(request)

zerver/views/realm_export.py

@@ -0,0 +1,159 @@
+from datetime import timedelta
+from typing import Annotated
+
+from django.db import transaction
+from django.http import HttpRequest, HttpResponse
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+
+from analytics.models import RealmCount
+from zerver.actions.realm_export import do_delete_realm_export, notify_realm_export
+from zerver.decorator import require_realm_admin
+from zerver.lib.exceptions import JsonableError, OrganizationOwnerRequiredError
+from zerver.lib.export import get_realm_exports_serialized
+from zerver.lib.queue import queue_event_on_commit
+from zerver.lib.response import json_success
+from zerver.lib.send_email import FromAddress
+from zerver.lib.typed_endpoint import typed_endpoint
+from zerver.lib.typed_endpoint_validators import check_string_in_validator
+from zerver.models import RealmExport, UserProfile
+from zerver.models.realms import DEFAULT_REALM_EXPORT_TYPE_SLUG, RealmExportSlug
+
+
+@transaction.atomic(durable=True)
+@require_realm_admin
+@typed_endpoint
+def export_realm(
+    request: HttpRequest,
+    user: UserProfile,
+    *,
+    export_type: Annotated[
+        RealmExportSlug,
+        check_string_in_validator(RealmExport.EXPORT_TYPES.keys()),
+    ] = DEFAULT_REALM_EXPORT_TYPE_SLUG,
+) -> HttpResponse:
+    realm = user.realm
+    EXPORT_LIMIT = 5
+    export_type_value = RealmExport.EXPORT_TYPES[export_type]
+
+    if (
+        export_type_value == RealmExport.EXPORT_FULL_WITHOUT_CONSENT
+        and not realm.owner_full_content_access
+    ):
+        raise JsonableError(
+            _("Exports of all public and private data are not enabled for this organization.")
+        )
+
+    if (
+        export_type_value == RealmExport.EXPORT_FULL_WITHOUT_CONSENT
+        and realm.owner_full_content_access
+        and not user.is_realm_owner
+    ):
+        raise OrganizationOwnerRequiredError
+
+    # Exporting organizations with a huge amount of history can
+    # potentially consume a lot of disk or otherwise have accidental
+    # DoS risk; for that reason, we require large exports to be done
+    # manually on the command line.
+    #
+    # It's very possible that higher limits would be completely safe.
+    MAX_MESSAGE_HISTORY = 250000
+    MAX_UPLOAD_QUOTA = 20 * 1024 * 1024 * 1024
+
+    # Filter based upon the number of events that have occurred in the delta
+    # If we are at the limit, the incoming request is rejected
+    event_time_delta = timezone_now() - timedelta(days=7)
+    limit_check = RealmExport.objects.filter(
+        realm=realm, date_requested__gte=event_time_delta
+    ).count()
+    if limit_check >= EXPORT_LIMIT:
+        raise JsonableError(_("Exceeded rate limit."))
+
+    # The RealmCount analytics table lets us efficiently get an estimate
+    # for the number of messages in an organization. It won't match the
+    # actual number of messages in the export, because this measures the
+    # number of messages that went to DMs / Group DMs / public or private
+    # channels at the time they were sent.
+    # Thus, messages that were deleted or moved between channels and
+    # private messages for which the users didn't consent for export will be
+    # treated differently for this check vs. in the export code.
+    realm_count_query = RealmCount.objects.filter(
+        realm=realm, property="messages_sent:message_type:day"
+    )
+    if export_type_value == RealmExport.EXPORT_PUBLIC:
+        realm_count_query = realm_count_query.filter(subgroup="public_stream")
+    exportable_messages_estimate = sum(realm_count.value for realm_count in realm_count_query)
+
+    if (
+        exportable_messages_estimate > MAX_MESSAGE_HISTORY
+        or user.realm.currently_used_upload_space_bytes() > MAX_UPLOAD_QUOTA
+    ):
+        raise JsonableError(
+            _(
+                "The export you requested is too large for automatic processing. Please request a manual export by contacting {email}."
+            ).format(
+                email=FromAddress.SUPPORT,
+            )
+        )
+
+    row = RealmExport.objects.create(
+        realm=realm,
+        type=export_type_value,
+        acting_user=user,
+        status=RealmExport.REQUESTED,
+        date_requested=timezone_now(),
+    )
+
+    # Allow for UI updates on a pending export
+    notify_realm_export(realm)
+
+    # Using the deferred_work queue processor to avoid
+    # killing the process after 60s
+    event = {
+        "type": "realm_export",
+        "user_profile_id": user.id,
+        "realm_export_id": row.id,
+    }
+    queue_event_on_commit("deferred_work", event)
+    return json_success(request, data={"id": row.id})
+
+
+@require_realm_admin
+def get_realm_exports(request: HttpRequest, user: UserProfile) -> HttpResponse:
+    realm_exports = get_realm_exports_serialized(user.realm)
+    return json_success(request, data={"exports": realm_exports})
+
+
+@require_realm_admin
+def delete_realm_export(request: HttpRequest, user: UserProfile, export_id: int) -> HttpResponse:
+    try:
+        export_row = RealmExport.objects.get(realm_id=user.realm_id, id=export_id)
+    except RealmExport.DoesNotExist:
+        raise JsonableError(_("Invalid data export ID"))
+
+    if export_row.status in [RealmExport.DELETED, RealmExport.EXPORT_FROM_PRIOR_SERVER]:
+        raise JsonableError(_("Export already deleted"))
+    if export_row.status == RealmExport.FAILED:
+        raise JsonableError(_("Export failed, nothing to delete"))
+    if export_row.status in [RealmExport.REQUESTED, RealmExport.STARTED]:
+        raise JsonableError(_("Export still in progress"))
+    do_delete_realm_export(export_row, user)
+    return json_success(request)
+
+
+@require_realm_admin
+def get_users_export_consents(request: HttpRequest, user: UserProfile) -> HttpResponse:
+    rows = UserProfile.objects.filter(realm=user.realm, is_active=True, is_bot=False).values(
+        "id",
+        "allow_private_data_export",
+        "email_address_visibility",
+    )
+    export_consents = [
+        {
+            "user_id": row["id"],
+            "consented": row["allow_private_data_export"],
+            "email_address_visibility": row["email_address_visibility"],
+        }
+        for row in rows
+    ]
+    return json_success(request, data={"export_consents": export_consents})

zerver/views/realm_icon.py

@@ -0,0 +1,67 @@
+from django.conf import settings
+from django.core.files.uploadedfile import UploadedFile
+from django.http import HttpRequest, HttpResponse
+from django.shortcuts import redirect
+from django.utils.translation import gettext as _
+
+from zerver.actions.realm_icon import do_change_icon_source
+from zerver.decorator import require_realm_admin
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.realm_icon import realm_icon_url
+from zerver.lib.response import json_success
+from zerver.lib.upload import get_file_info, upload_icon_image
+from zerver.lib.url_encoding import append_url_query_string
+from zerver.models import UserProfile
+
+
+@require_realm_admin
+def upload_icon(request: HttpRequest, user_profile: UserProfile) -> HttpResponse:
+    if len(request.FILES) != 1:
+        raise JsonableError(_("You must upload exactly one icon."))
+
+    [icon_file] = request.FILES.values()
+    assert isinstance(icon_file, UploadedFile)
+    assert icon_file.size is not None
+    if icon_file.size > settings.MAX_ICON_FILE_SIZE_MIB * 1024 * 1024:
+        raise JsonableError(
+            _("Uploaded file is larger than the allowed limit of {max_size} MiB").format(
+                max_size=settings.MAX_ICON_FILE_SIZE_MIB,
+            )
+        )
+    _filename, content_type = get_file_info(icon_file)
+    upload_icon_image(icon_file, user_profile, content_type=content_type)
+    do_change_icon_source(
+        user_profile.realm, user_profile.realm.ICON_UPLOADED, acting_user=user_profile
+    )
+    icon_url = realm_icon_url(user_profile.realm)
+
+    json_result = dict(
+        icon_url=icon_url,
+    )
+    return json_success(request, data=json_result)
+
+
+@require_realm_admin
+def delete_icon_backend(request: HttpRequest, user_profile: UserProfile) -> HttpResponse:
+    # We don't actually delete the icon because it might still
+    # be needed if the URL was cached and it is rewritten
+    # in any case after next update.
+    do_change_icon_source(
+        user_profile.realm, user_profile.realm.ICON_FROM_GRAVATAR, acting_user=user_profile
+    )
+    gravatar_url = realm_icon_url(user_profile.realm)
+    json_result = dict(
+        icon_url=gravatar_url,
+    )
+    return json_success(request, data=json_result)
+
+
+def get_icon_backend(request: HttpRequest, user_profile: UserProfile) -> HttpResponse:
+    url = realm_icon_url(user_profile.realm)
+
+    # We can rely on the URL already having query parameters. Because
+    # our templates depend on being able to use the ampersand to
+    # add query parameters to our url, get_icon_url does '?version=version_number'
+    # hacks to prevent us from having to jump through decode/encode hoops.
+    url = append_url_query_string(url, request.META["QUERY_STRING"])
+    return redirect(url)

zerver/views/realm_linkifiers.py

@@ -0,0 +1,104 @@
+from django.core.exceptions import ValidationError
+from django.http import HttpRequest, HttpResponse
+from django.utils.translation import gettext as _
+from pydantic import Json
+
+from zerver.actions.realm_linkifiers import (
+    check_reorder_linkifiers,
+    do_add_linkifier,
+    do_remove_linkifier,
+    do_update_linkifier,
+)
+from zerver.decorator import require_realm_admin
+from zerver.lib.exceptions import JsonableError, ValidationFailureError
+from zerver.lib.response import json_success
+from zerver.lib.typed_endpoint import PathOnly, typed_endpoint
+from zerver.models import RealmFilter, UserProfile
+from zerver.models.linkifiers import linkifiers_for_realm
+
+
+# Custom realm linkifiers
+def list_linkifiers(request: HttpRequest, user_profile: UserProfile) -> HttpResponse:
+    linkifiers = linkifiers_for_realm(user_profile.realm_id)
+    return json_success(request, data={"linkifiers": linkifiers})
+
+
+@require_realm_admin
+@typed_endpoint
+def create_linkifier(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    pattern: str,
+    url_template: str,
+    example_input: str | None = None,
+    reverse_template: str | None = None,
+    alternative_url_templates: Json[list[str]] | None = None,
+) -> HttpResponse:
+    try:
+        linkifier_id = do_add_linkifier(
+            realm=user_profile.realm,
+            pattern=pattern,
+            url_template=url_template,
+            example_input=example_input,
+            reverse_template=reverse_template,
+            alternative_url_templates=alternative_url_templates,
+            acting_user=user_profile,
+        )
+        return json_success(request, data={"id": linkifier_id})
+    except ValidationError as e:
+        raise ValidationFailureError(e)
+
+
+@require_realm_admin
+def delete_linkifier(
+    request: HttpRequest, user_profile: UserProfile, filter_id: int
+) -> HttpResponse:
+    try:
+        do_remove_linkifier(realm=user_profile.realm, id=filter_id, acting_user=None)
+    except RealmFilter.DoesNotExist:
+        raise JsonableError(_("Linkifier not found."))
+    return json_success(request)
+
+
+@require_realm_admin
+@typed_endpoint
+def update_linkifier(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    filter_id: PathOnly[int],
+    pattern: str,
+    url_template: str,
+    example_input: str | None = None,
+    reverse_template: str | None = None,
+    alternative_url_templates: Json[list[str]] | None = None,
+) -> HttpResponse:
+    try:
+        do_update_linkifier(
+            realm=user_profile.realm,
+            id=filter_id,
+            pattern=pattern,
+            url_template=url_template,
+            example_input=example_input,
+            reverse_template=reverse_template,
+            alternative_url_templates=alternative_url_templates,
+            acting_user=user_profile,
+        )
+        return json_success(request)
+    except RealmFilter.DoesNotExist:
+        raise JsonableError(_("Linkifier not found."))
+    except ValidationError as e:
+        raise ValidationFailureError(e)
+
+
+@require_realm_admin
+@typed_endpoint
+def reorder_linkifiers(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    ordered_linkifier_ids: Json[list[int]],
+) -> HttpResponse:
+    check_reorder_linkifiers(user_profile.realm, ordered_linkifier_ids, acting_user=user_profile)
+    return json_success(request)

zerver/views/realm_logo.py

@@ -0,0 +1,70 @@
+from django.conf import settings
+from django.core.files.uploadedfile import UploadedFile
+from django.http import HttpRequest, HttpResponse
+from django.shortcuts import redirect
+from django.utils.translation import gettext as _
+from pydantic import Json
+
+from zerver.actions.realm_logo import do_change_logo_source
+from zerver.decorator import require_realm_admin
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.realm_logo import get_realm_logo_url
+from zerver.lib.response import json_success
+from zerver.lib.typed_endpoint import typed_endpoint
+from zerver.lib.upload import get_file_info, upload_logo_image
+from zerver.lib.url_encoding import append_url_query_string
+from zerver.models import UserProfile
+
+
+@require_realm_admin
+@typed_endpoint
+def upload_logo(
+    request: HttpRequest, user_profile: UserProfile, *, night: Json[bool]
+) -> HttpResponse:
+    user_profile.realm.ensure_not_on_limited_plan()
+
+    if len(request.FILES) != 1:
+        raise JsonableError(_("You must upload exactly one logo."))
+    [logo_file] = request.FILES.values()
+    assert isinstance(logo_file, UploadedFile)
+    assert logo_file.size is not None
+    if logo_file.size > settings.MAX_LOGO_FILE_SIZE_MIB * 1024 * 1024:
+        raise JsonableError(
+            _("Uploaded file is larger than the allowed limit of {max_size} MiB").format(
+                max_size=settings.MAX_LOGO_FILE_SIZE_MIB,
+            )
+        )
+    _filename, content_type = get_file_info(logo_file)
+    upload_logo_image(logo_file, user_profile, night, content_type=content_type)
+    do_change_logo_source(
+        user_profile.realm, user_profile.realm.LOGO_UPLOADED, night, acting_user=user_profile
+    )
+    return json_success(request)
+
+
+@require_realm_admin
+@typed_endpoint
+def delete_logo_backend(
+    request: HttpRequest, user_profile: UserProfile, *, night: Json[bool]
+) -> HttpResponse:
+    # We don't actually delete the logo because it might still
+    # be needed if the URL was cached and it is rewritten
+    # in any case after next update.
+    do_change_logo_source(
+        user_profile.realm, user_profile.realm.LOGO_DEFAULT, night, acting_user=user_profile
+    )
+    return json_success(request)
+
+
+@typed_endpoint
+def get_logo_backend(
+    request: HttpRequest, user_profile: UserProfile, *, night: Json[bool]
+) -> HttpResponse:
+    url = get_realm_logo_url(user_profile.realm, night)
+
+    # We can rely on the URL already having query parameters. Because
+    # our templates depend on being able to use the ampersand to
+    # add query parameters to our url, get_logo_url does '?version=version_number'
+    # hacks to prevent us from having to jump through decode/encode hoops.
+    url = append_url_query_string(url, request.META["QUERY_STRING"])
+    return redirect(url)

zerver/views/realm_playgrounds.py

@@ -0,0 +1,77 @@
+import re
+from typing import Annotated
+
+from django.http import HttpRequest, HttpResponse
+from django.utils.translation import gettext as _
+from pydantic import AfterValidator
+
+from zerver.actions.realm_playgrounds import check_add_realm_playground, do_remove_realm_playground
+from zerver.actions.realm_settings import do_set_realm_property
+from zerver.decorator import require_realm_admin
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.response import json_success
+from zerver.lib.typed_endpoint import PathOnly, typed_endpoint
+from zerver.lib.validator import check_capped_string
+from zerver.models import Realm, RealmPlayground, UserProfile
+from zerver.models.realm_playgrounds import PLAYGROUND_LANGUAGE_REGEX
+
+
+def check_pygments_language(var_name: str, val: object) -> str:
+    s = check_capped_string(RealmPlayground.MAX_PYGMENTS_LANGUAGE_LENGTH)(var_name, val)
+    # We don't want to restrict the language here to be only from the list of valid
+    # Pygments languages. Keeping it open would allow us to hook up a "playground"
+    # for custom "languages" that aren't known to Pygments. We use a similar strategy
+    # even in our fenced_code Markdown processor.
+    if not re.match(rf"^{PLAYGROUND_LANGUAGE_REGEX}$", s):
+        for char in s:
+            if not re.match(rf"^{PLAYGROUND_LANGUAGE_REGEX}$", char):
+                raise JsonableError(
+                    _("Invalid character in language: {character}").format(character=char)
+                )
+    if s in RealmPlayground.RESTRICTED_KEYWORDS:
+        raise JsonableError(_("Language '{language}' is not allowed.").format(language=s))
+    return s
+
+
+def access_playground_by_id(realm: Realm, playground_id: int) -> RealmPlayground:
+    try:
+        realm_playground = RealmPlayground.objects.get(id=playground_id, realm=realm)
+    except RealmPlayground.DoesNotExist:
+        raise JsonableError(_("Invalid playground"))
+    return realm_playground
+
+
+@require_realm_admin
+@typed_endpoint
+def add_realm_playground(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    name: str,
+    pygments_language: Annotated[
+        str, AfterValidator(lambda x: check_pygments_language("pygments_language", x))
+    ],
+    url_template: str,
+) -> HttpResponse:
+    playground_id = check_add_realm_playground(
+        realm=user_profile.realm,
+        acting_user=user_profile,
+        name=name.strip(),
+        pygments_language=pygments_language.strip(),
+        url_template=url_template.strip(),
+    )
+    return json_success(request, data={"id": playground_id})
+
+
+@require_realm_admin
+@typed_endpoint
+def delete_realm_playground(
+    request: HttpRequest, user_profile: UserProfile, *, playground_id: PathOnly[int]
+) -> HttpResponse:
+    realm_playground = access_playground_by_id(user_profile.realm, playground_id)
+    if user_profile.realm.default_code_block_language == realm_playground.pygments_language:
+        do_set_realm_property(
+            user_profile.realm, "default_code_block_language", "", acting_user=user_profile
+        )
+    do_remove_realm_playground(user_profile.realm, realm_playground, acting_user=user_profile)
+    return json_success(request)

zerver/views/reminders.py

@@ -0,0 +1,50 @@
+from django.http import HttpRequest, HttpResponse
+from django.utils.timezone import now as timezone_now
+from pydantic import Json, NonNegativeInt
+
+from zerver.actions.reminders import do_delete_reminder, schedule_reminder_for_message
+from zerver.lib.exceptions import DeliveryTimeNotInFutureError
+from zerver.lib.reminders import access_reminder
+from zerver.lib.request import RequestNotes
+from zerver.lib.response import json_success
+from zerver.lib.timestamp import timestamp_to_datetime
+from zerver.lib.typed_endpoint import PathOnly, typed_endpoint
+from zerver.models import UserProfile
+
+
+@typed_endpoint
+def create_reminders_message_backend(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    message_id: Json[int],
+    scheduled_delivery_timestamp: Json[int],
+    note: str | None = None,
+) -> HttpResponse:
+    deliver_at = timestamp_to_datetime(scheduled_delivery_timestamp)
+    if deliver_at <= timezone_now():
+        raise DeliveryTimeNotInFutureError
+
+    client = RequestNotes.get_notes(request).client
+    assert client is not None
+
+    reminder_id = schedule_reminder_for_message(
+        user_profile,
+        client,
+        message_id,
+        deliver_at,
+        note=note or "",
+    )
+    return json_success(request, data={"reminder_id": reminder_id})
+
+
+@typed_endpoint
+def delete_reminder(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    reminder_id: PathOnly[NonNegativeInt],
+) -> HttpResponse:
+    reminder = access_reminder(user_profile, reminder_id)
+    do_delete_reminder(user_profile, reminder)
+    return json_success(request)

zerver/views/report.py

@@ -0,0 +1,39 @@
+# System documented in https://zulip.readthedocs.io/en/latest/subsystems/logging.html
+import logging
+
+from django.http import HttpRequest, HttpResponse
+from django.views.decorators.csrf import csrf_exempt
+from django.views.decorators.http import require_POST
+
+from zerver.lib.response import json_success
+from zerver.lib.typed_endpoint import JsonBodyPayload, typed_endpoint
+from zerver.lib.validator import WildValue, check_string
+
+
+@csrf_exempt
+@require_POST
+@typed_endpoint
+def report_csp_violations(
+    request: HttpRequest, *, csp_report: JsonBodyPayload[WildValue]
+) -> HttpResponse:
+    def get_attr(csp_report_attr: str) -> str:
+        return csp_report.get(csp_report_attr, "").tame(check_string)
+
+    logging.warning(
+        "CSP violation in document('%s'). "
+        "blocked URI('%s'), original policy('%s'), "
+        "violated directive('%s'), effective directive('%s'), "
+        "disposition('%s'), referrer('%s'), "
+        "status code('%s'), script sample('%s')",
+        get_attr("document-uri"),
+        get_attr("blocked-uri"),
+        get_attr("original-policy"),
+        get_attr("violated-directive"),
+        get_attr("effective-directive"),
+        get_attr("disposition"),
+        get_attr("referrer"),
+        get_attr("status-code"),
+        get_attr("script-sample"),
+    )
+
+    return json_success(request)

zerver/views/saved_snippets.py

@@ -0,0 +1,84 @@
+from typing import Annotated
+
+from django.conf import settings
+from django.http import HttpRequest, HttpResponse
+from pydantic import StringConstraints
+
+from zerver.actions.saved_snippets import (
+    do_create_saved_snippet,
+    do_delete_saved_snippet,
+    do_edit_saved_snippet,
+    do_get_saved_snippets,
+)
+from zerver.lib.response import json_success
+from zerver.lib.typed_endpoint import PathOnly, typed_endpoint
+from zerver.models import SavedSnippet, UserProfile
+
+
+def get_saved_snippets(
+    request: HttpRequest,
+    user_profile: UserProfile,
+) -> HttpResponse:
+    return json_success(request, data={"saved_snippets": do_get_saved_snippets(user_profile)})
+
+
+@typed_endpoint
+def create_saved_snippet(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    content: Annotated[
+        str,
+        StringConstraints(
+            min_length=1, max_length=settings.MAX_MESSAGE_LENGTH, strip_whitespace=True
+        ),
+    ],
+    title: Annotated[
+        str,
+        StringConstraints(
+            min_length=1, max_length=SavedSnippet.MAX_TITLE_LENGTH, strip_whitespace=True
+        ),
+    ],
+) -> HttpResponse:
+    title = title.strip()
+    content = content.strip()
+    saved_snippet = do_create_saved_snippet(title, content, user_profile)
+    return json_success(request, data={"saved_snippet_id": saved_snippet.id})
+
+
+@typed_endpoint
+def edit_saved_snippet(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    content: Annotated[
+        str | None,
+        StringConstraints(
+            min_length=1, max_length=settings.MAX_MESSAGE_LENGTH, strip_whitespace=True
+        ),
+    ] = None,
+    saved_snippet_id: PathOnly[int],
+    title: Annotated[
+        str | None,
+        StringConstraints(
+            min_length=1, max_length=SavedSnippet.MAX_TITLE_LENGTH, strip_whitespace=True
+        ),
+    ] = None,
+) -> HttpResponse:
+    if title is None and content is None:
+        # No changes are requested; exit early to avoid sending a
+        # spurious event to clients.
+        return json_success(request)
+
+    do_edit_saved_snippet(saved_snippet_id, title, content, user_profile)
+    return json_success(request)
+
+
+def delete_saved_snippet(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    saved_snippet_id: int,
+) -> HttpResponse:
+    do_delete_saved_snippet(saved_snippet_id, user_profile)
+    return json_success(request)

zerver/views/scheduled_messages.py

@@ -0,0 +1,185 @@
+from typing import Annotated
+
+from django.http import HttpRequest, HttpResponse
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+from pydantic import Json, NonNegativeInt
+
+from zerver.actions.scheduled_messages import (
+    check_schedule_message,
+    delete_scheduled_message,
+    edit_scheduled_message,
+)
+from zerver.lib.exceptions import DeliveryTimeNotInFutureError, JsonableError
+from zerver.lib.recipient_parsing import extract_direct_message_recipient_ids, extract_stream_id
+from zerver.lib.request import RequestNotes
+from zerver.lib.response import json_success
+from zerver.lib.scheduled_messages import (
+    get_undelivered_reminders,
+    get_undelivered_scheduled_messages,
+)
+from zerver.lib.timestamp import timestamp_to_datetime
+from zerver.lib.typed_endpoint import (
+    ApiParamConfig,
+    OptionalTopic,
+    PathOnly,
+    typed_endpoint,
+    typed_endpoint_without_parameters,
+)
+from zerver.lib.typed_endpoint_validators import check_string_in_validator
+from zerver.models import Message, UserProfile
+
+
+@typed_endpoint_without_parameters
+def fetch_scheduled_messages(request: HttpRequest, user_profile: UserProfile) -> HttpResponse:
+    return json_success(
+        request, data={"scheduled_messages": get_undelivered_scheduled_messages(user_profile)}
+    )
+
+
+@typed_endpoint_without_parameters
+def fetch_reminders(request: HttpRequest, user_profile: UserProfile) -> HttpResponse:
+    return json_success(request, data={"reminders": get_undelivered_reminders(user_profile)})
+
+
+@typed_endpoint
+def delete_scheduled_messages(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    scheduled_message_id: PathOnly[NonNegativeInt],
+) -> HttpResponse:
+    delete_scheduled_message(user_profile, scheduled_message_id)
+    return json_success(request)
+
+
+@typed_endpoint
+def update_scheduled_message_backend(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    message_content: Annotated[str | None, ApiParamConfig("content")] = None,
+    req_type: Annotated[
+        Annotated[str, check_string_in_validator(Message.API_RECIPIENT_TYPES)] | None,
+        ApiParamConfig("type"),
+    ] = None,
+    scheduled_delivery_timestamp: Json[int] | None = None,
+    scheduled_message_id: PathOnly[NonNegativeInt],
+    to: Json[int | list[int]] | None = None,
+    topic_name: OptionalTopic = None,
+) -> HttpResponse:
+    if (
+        req_type is None
+        and to is None
+        and topic_name is None
+        and message_content is None
+        and scheduled_delivery_timestamp is None
+    ):
+        raise JsonableError(_("Nothing to change"))
+
+    recipient_type_name = None
+    if req_type:
+        if to is None:
+            raise JsonableError(_("Recipient required when updating type of scheduled message."))
+        else:
+            recipient_type_name = req_type
+
+    if recipient_type_name is not None and recipient_type_name == "channel":
+        # For now, use "stream" from Message.API_RECIPIENT_TYPES.
+        # TODO: Use "channel" here, as well as in events and
+        # message (created, schdeduled, drafts) objects/dicts.
+        recipient_type_name = "stream"
+
+    if recipient_type_name is not None and recipient_type_name == "stream" and topic_name is None:
+        raise JsonableError(_("Topic required when updating scheduled message type to channel."))
+
+    if recipient_type_name is not None and recipient_type_name == "direct":
+        # For now, use "private" from Message.API_RECIPIENT_TYPES.
+        # TODO: Use "direct" here, as well as in events and
+        # scheduled message objects/dicts.
+        recipient_type_name = "private"
+
+    message_to = None
+    if to is not None:
+        # Because the recipient_type_name may not be updated/changed,
+        # we extract these updated recipient IDs in edit_scheduled_message.
+        message_to = to
+
+    deliver_at = None
+    if scheduled_delivery_timestamp is not None:
+        deliver_at = timestamp_to_datetime(scheduled_delivery_timestamp)
+        if deliver_at <= timezone_now():
+            raise DeliveryTimeNotInFutureError
+
+    sender = user_profile
+    client = RequestNotes.get_notes(request).client
+    assert client is not None
+
+    edit_scheduled_message(
+        sender,
+        client,
+        scheduled_message_id,
+        recipient_type_name,
+        message_to,
+        topic_name,
+        message_content,
+        deliver_at,
+        realm=user_profile.realm,
+    )
+
+    return json_success(request)
+
+
+@typed_endpoint
+def create_scheduled_message_backend(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    message_content: Annotated[str, ApiParamConfig("content")],
+    read_by_sender: Json[bool] | None = None,
+    req_to: Annotated[Json[int | list[int]], ApiParamConfig("to")],
+    req_type: Annotated[
+        Annotated[str, check_string_in_validator(Message.API_RECIPIENT_TYPES)],
+        ApiParamConfig("type"),
+    ],
+    scheduled_delivery_timestamp: Json[int],
+    topic_name: OptionalTopic = None,
+) -> HttpResponse:
+    recipient_type_name = req_type
+    if recipient_type_name == "direct":
+        # For now, use "private" from Message.API_RECIPIENT_TYPES.
+        # TODO: Use "direct" here, as well as in events and
+        # scheduled message objects/dicts.
+        recipient_type_name = "private"
+    elif recipient_type_name == "channel":
+        # For now, use "stream" from Message.API_RECIPIENT_TYPES.
+        # TODO: Use "channel" here, as well as in events and
+        # message (created, schdeduled, drafts) objects/dicts.
+        recipient_type_name = "stream"
+
+    deliver_at = timestamp_to_datetime(scheduled_delivery_timestamp)
+    if deliver_at <= timezone_now():
+        raise DeliveryTimeNotInFutureError
+
+    sender = user_profile
+    client = RequestNotes.get_notes(request).client
+    assert client is not None
+
+    if recipient_type_name == "stream":
+        stream_id = extract_stream_id(req_to)
+        message_to = [stream_id]
+    else:
+        message_to = extract_direct_message_recipient_ids(req_to)
+
+    scheduled_message_id = check_schedule_message(
+        sender,
+        client,
+        recipient_type_name,
+        message_to,
+        topic_name,
+        message_content,
+        deliver_at,
+        realm=user_profile.realm,
+        read_by_sender=read_by_sender,
+    )
+    return json_success(request, data={"scheduled_message_id": scheduled_message_id})

zerver/views/sentry.py

@@ -0,0 +1,124 @@
+import logging
+from contextlib import suppress
+from urllib.parse import urlsplit
+
+import orjson
+from django.conf import settings
+from django.http import HttpRequest, HttpResponse
+from django.utils.translation import gettext as _
+from django.views.decorators.csrf import csrf_exempt
+from pybreaker import CircuitBreaker, CircuitBreakerError
+from requests.exceptions import HTTPError, ProxyError, RequestException, Timeout
+from sentry_sdk.integrations.logging import ignore_logger
+
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.outgoing_http import OutgoingSession
+from zerver.lib.validator import check_url, to_wild_value
+
+# In order to not overload Sentry if it's having a bad day, we tell
+# Sentry to ignore exceptions that we have when talking to Sentry.
+logger = logging.getLogger(__name__)
+ignore_logger(logger.name)
+
+
+class SentryTunnelSession(OutgoingSession):
+    def __init__(self) -> None:
+        super().__init__(role="sentry_tunnel", timeout=1)
+
+
+@csrf_exempt
+def sentry_tunnel(
+    request: HttpRequest,
+) -> HttpResponse:
+    try:
+        envelope_header_line, envelope_items = request.body.split(b"\n", 1)
+        envelope_header = to_wild_value("envelope_header", envelope_header_line.decode("utf-8"))
+        dsn = urlsplit(envelope_header["dsn"].tame(check_url))
+    except Exception:
+        raise JsonableError(_("Invalid request format"))
+
+    if dsn.geturl() != settings.SENTRY_FRONTEND_DSN:
+        raise JsonableError(_("Invalid DSN"))
+
+    assert dsn.hostname
+    project_id = dsn.path.strip("/")
+    url = dsn._replace(netloc=dsn.hostname, path=f"/api/{project_id}/envelope/").geturl()
+
+    # Adjust the payload to explicitly contain the IP address of the
+    # user we see.  If left blank, Sentry will assume the IP it
+    # received the request from, which is Zulip's, which can make
+    # debugging more complicated.
+    updated_body = request.body
+    # If we fail to update the body for any reason, leave it as-is; it
+    # is better to misreport the IP than to drop the report entirely.
+    with suppress(Exception):
+        # This parses the Sentry ingestion format, known as an
+        # Envelope.  See https://develop.sentry.dev/sdk/envelopes/ for
+        # spec.
+        parts = [envelope_header_line, b"\n"]
+        while envelope_items != b"":
+            item_header_line, rest = envelope_items.split(b"\n", 1)
+            parts.append(item_header_line)
+            parts.append(b"\n")
+            item_header = orjson.loads(item_header_line.decode("utf-8"))
+            length = item_header.get("length")
+            if length is None:
+                item_body, envelope_items = [*rest.split(b"\n", 1), b""][:2]
+            else:
+                item_body, envelope_items = rest[0:length], rest[length:]
+            if item_header.get("type") in ("transaction", "event"):
+                # Event schema:
+                # https://develop.sentry.dev/sdk/event-payloads/#core-interfaces
+                # https://develop.sentry.dev/sdk/event-payloads/user/
+                #
+                # Transaction schema:
+                # https://develop.sentry.dev/sdk/event-payloads/transaction/#anatomy
+                # Note that "Transactions are Events enriched with Span data."
+                payload_data = orjson.loads(item_body)
+                if "user" in payload_data:
+                    payload_data["user"]["ip_address"] = request.META.get("REMOTE_ADDR")
+                    item_body = orjson.dumps(payload_data)
+            parts.append(item_body)
+            if length is None:
+                parts.append(b"\n")
+        updated_body = b"".join(parts)
+
+    try:
+        sentry_request(url, updated_body)
+    except CircuitBreakerError:
+        logger.warning("Dropped a client exception due to circuit-breaking")
+    except RequestException:
+        # This logger has been configured, above, to not report to Sentry
+        logger.exception("Error while reporting to Sentry")
+    return HttpResponse(status=200)
+
+
+# Circuit-break and temporarily stop trying to report to
+# Sentry if it keeps timing out.  We include ProxyError in
+# here because we are likely making our requests through
+# Smokescreen as a CONNECT proxy, so failures from Smokescreen
+# failing to connect at the TCP level will report as
+# ProxyErrors.
+def open_circuit_for(exc_value: Exception) -> bool:
+    if isinstance(exc_value, ProxyError | Timeout):
+        return True
+    if isinstance(exc_value, HTTPError):
+        response = exc_value.response
+        if response.status_code == 429 or response.status_code >= 500:
+            return True
+    return False
+
+
+# Open the circuit after 2 failures, and leave it open for 30s.
+@CircuitBreaker(
+    fail_max=2,
+    reset_timeout=30,
+    name="Sentry tunnel",
+    exclude=(open_circuit_for,),
+)
+def sentry_request(url: str, data: bytes) -> None:
+    SentryTunnelSession().post(
+        url=url,
+        data=data,
+        headers={"Content-Type": "application/x-sentry-envelope"},
+    ).raise_for_status()

zerver/views/storage.py

@@ -0,0 +1,64 @@
+from django.http import HttpRequest, HttpResponse
+from pydantic import Json
+
+from zerver.decorator import require_bot_user
+from zerver.lib.bot_storage import (
+    StateError,
+    get_bot_storage,
+    get_keys_in_bot_storage,
+    remove_bot_storage,
+    set_bot_storage,
+)
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.response import json_success
+from zerver.lib.typed_endpoint import typed_endpoint
+from zerver.models import UserProfile
+
+
+@require_bot_user
+@typed_endpoint
+def update_storage(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    storage: Json[dict[str, str]],
+) -> HttpResponse:
+    try:
+        set_bot_storage(user_profile, list(storage.items()))
+    except StateError as e:  # nocoverage
+        raise JsonableError(str(e))
+    return json_success(request)
+
+
+@require_bot_user
+@typed_endpoint
+def get_storage(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    keys: Json[list[str] | None] = None,
+) -> HttpResponse:
+    if keys is None:
+        keys = get_keys_in_bot_storage(user_profile)
+    try:
+        storage = {key: get_bot_storage(user_profile, key) for key in keys}
+    except StateError as e:
+        raise JsonableError(str(e))
+    return json_success(request, data={"storage": storage})
+
+
+@require_bot_user
+@typed_endpoint
+def remove_storage(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    keys: Json[list[str] | None] = None,
+) -> HttpResponse:
+    if keys is None:
+        keys = get_keys_in_bot_storage(user_profile)
+    try:
+        remove_bot_storage(user_profile, keys)
+    except StateError as e:
+        raise JsonableError(str(e))
+    return json_success(request)

zerver/views/submessage.py

@@ -0,0 +1,65 @@
+import orjson
+from django.core.exceptions import ValidationError
+from django.db import transaction
+from django.http import HttpRequest, HttpResponse
+from django.utils.translation import gettext as _
+from pydantic import Json
+
+from zerver.actions.submessage import do_add_submessage, verify_submessage_sender
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.message import access_message
+from zerver.lib.response import json_success
+from zerver.lib.typed_endpoint import typed_endpoint
+from zerver.lib.validator import validate_poll_data, validate_todo_data
+from zerver.lib.widget import get_widget_type
+from zerver.models import UserProfile
+
+
+# transaction.atomic is required since we use FOR UPDATE queries in access_message.
+@transaction.atomic(durable=True)
+@typed_endpoint
+def process_submessage(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    content: str,
+    message_id: Json[int],
+    msg_type: str,
+) -> HttpResponse:
+    message = access_message(user_profile, message_id, lock_message=True, is_modifying_message=True)
+
+    verify_submessage_sender(
+        message_id=message.id,
+        message_sender_id=message.sender_id,
+        submessage_sender_id=user_profile.id,
+    )
+
+    try:
+        widget_data = orjson.loads(content)
+    except orjson.JSONDecodeError:
+        raise JsonableError(_("Invalid json for submessage"))
+
+    widget_type = get_widget_type(message_id=message.id)
+
+    is_widget_author = message.sender_id == user_profile.id
+
+    if widget_type == "poll":
+        try:
+            validate_poll_data(poll_data=widget_data, is_widget_author=is_widget_author)
+        except ValidationError as error:
+            raise JsonableError(error.message)
+
+    if widget_type == "todo":
+        try:
+            validate_todo_data(todo_data=widget_data, is_widget_author=is_widget_author)
+        except ValidationError as error:
+            raise JsonableError(error.message)
+
+    do_add_submessage(
+        realm=user_profile.realm,
+        sender_id=user_profile.id,
+        message_id=message.id,
+        msg_type=msg_type,
+        content=content,
+    )
+    return json_success(request)

zerver/views/thumbnail.py

@@ -0,0 +1,70 @@
+import re
+
+from django.contrib.auth.models import AnonymousUser
+from django.http import HttpRequest, HttpResponseBase, HttpResponseForbidden
+from django.utils.translation import gettext as _
+
+from zerver.lib.attachments import validate_attachment_request
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.response import json_success
+from zerver.lib.thumbnail import missing_thumbnails
+from zerver.lib.typed_endpoint import PathOnly, typed_endpoint
+from zerver.models import ImageAttachment, UserProfile
+from zerver.views.upload import serve_file
+
+
+@typed_endpoint
+def backend_serve_thumbnail(
+    request: HttpRequest,
+    maybe_user_profile: UserProfile | AnonymousUser,
+    *,
+    size: str,
+    url: str,
+) -> HttpResponseBase:
+    # This URL used to be passed arbitrary URLs, and pass them through
+    # Camo; we no longer support doing so, and instead return a 403.
+    #
+    # Modern thumbnailing uses URLs of the style
+    # `/user_uploads/thumbnail/.../300x200.webp`; this endpoint is
+    # kept for backward compatibility, and for future extension for
+    # thumbnailing external URLs.
+    upload_path_parts = re.match(r"user_uploads/(\d+)/(.*)", url)
+    if not upload_path_parts:
+        return HttpResponseForbidden()
+
+    realm_id_str = upload_path_parts[1]
+    path_id = upload_path_parts[2]
+
+    # We do not have ImageAttachment rows for historical uploads, so
+    # we cannot serve a "new" thumbnail for these requests; serve the
+    # full-size file.
+    return serve_file(request, maybe_user_profile, realm_id_str, path_id)
+
+
+@typed_endpoint
+def check_thumbnail_status(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    realm_id_str: PathOnly[str],
+    filename: PathOnly[str],
+) -> HttpResponseBase:
+    path_id = f"{realm_id_str}/{filename}"
+
+    if not validate_attachment_request(user_profile, path_id)[0]:
+        # Technically, there are 2 cases here:
+        # * The user put something in their compose box that has the form
+        # of an upload URL, but does not actually correspond to a previously
+        # uploaded file.  validate_attachment_request will return None.
+        # * The user is polling thumbnail status for a file they don't have
+        # permission to access.  validate_attachment_request will return False.
+        raise JsonableError(_("Invalid attachment"))
+
+    try:
+        image_attachment = ImageAttachment.objects.get(path_id=path_id)
+    except ImageAttachment.DoesNotExist:
+        raise JsonableError(_("Invalid attachment"))
+
+    needed_thumbnails = missing_thumbnails(image_attachment)
+
+    return json_success(request, data={"has_thumbnail": len(needed_thumbnails) == 0})

zerver/views/tusd.py

@@ -0,0 +1,354 @@
+from typing import Annotated, Any
+
+import orjson
+from django.conf import settings
+from django.contrib.auth.models import AnonymousUser
+from django.db import transaction
+from django.http import HttpRequest, HttpResponse, HttpResponseNotFound
+from django.utils.http import content_disposition_header
+from django.utils.translation import gettext as _
+from django.views.decorators.csrf import csrf_exempt
+from pydantic import BaseModel, ConfigDict, Field
+from pydantic.alias_generators import to_pascal
+
+from confirmation.models import Confirmation, ConfirmationKeyError, get_object_from_key
+from zerver.decorator import get_basic_credentials, validate_api_key
+from zerver.lib.exceptions import AccessDeniedError, JsonableError
+from zerver.lib.mime_types import INLINE_MIME_TYPES, bare_content_type, guess_type
+from zerver.lib.rate_limiter import is_local_addr
+from zerver.lib.typed_endpoint import JsonBodyPayload, typed_endpoint
+from zerver.lib.upload import (
+    RealmUploadQuotaError,
+    attachment_source,
+    check_upload_within_quota,
+    create_attachment,
+    delete_message_attachment,
+    generate_message_upload_path,
+    get_upload_backend,
+    maybe_add_charset,
+    needs_charset_detection,
+    sanitize_name,
+)
+from zerver.models import ArchivedAttachment, Attachment, PreregistrationRealm, Realm, UserProfile
+
+
+# See https://tus.github.io/tusd/advanced-topics/hooks/ for the spec
+# for these.
+class TusUpload(BaseModel):
+    model_config = ConfigDict(populate_by_name=True, alias_generator=to_pascal)
+    id: Annotated[str, Field(alias="ID")]
+    size: int | None
+    size_is_deferred: bool
+    offset: int
+    meta_data: dict[str, str]
+    is_partial: bool
+    is_final: bool
+    partial_uploads: list[str] | None
+    storage: dict[str, str] | None
+
+
+class TusHTTPRequest(BaseModel):
+    model_config = ConfigDict(populate_by_name=True, alias_generator=to_pascal)
+    method: str
+    uri: Annotated[str, Field(alias="URI")]
+    remote_addr: str
+    header: dict[str, list[str]]
+
+
+class TusEvent(BaseModel):
+    model_config = ConfigDict(populate_by_name=True, alias_generator=to_pascal)
+    upload: TusUpload
+    http_request: Annotated[TusHTTPRequest, Field(alias="HTTPRequest")]
+
+
+class TusHook(BaseModel):
+    model_config = ConfigDict(populate_by_name=True, alias_generator=to_pascal)
+    type: str
+    event: TusEvent
+
+
+# Note that we do not raise JsonableError in these views
+# because our client is not a consumer of the Zulip API -- it's tusd,
+# which has its own ideas of what error responses look like.
+def tusd_json_response(data: dict[str, Any]) -> HttpResponse:
+    return HttpResponse(
+        content=orjson.dumps(data, option=orjson.OPT_APPEND_NEWLINE),
+        content_type="application/json",
+        status=200,
+    )
+
+
+def reject_upload(message: str, status_code: int) -> HttpResponse:
+    # Due to https://github.com/transloadit/uppy/issues/5460, uppy
+    # will retry responses with a statuscode of exactly 400, so we
+    # return 4xx status codes which are more specific to trigger an
+    # immediate rejection.
+    return tusd_json_response(
+        {
+            "HttpResponse": {
+                "StatusCode": status_code,
+                "Body": orjson.dumps({"message": message}).decode(),
+                "Header": {
+                    "Content-Type": "application/json",
+                },
+            },
+            "RejectUpload": True,
+        }
+    )
+
+
+def handle_upload_pre_create_hook(
+    request: HttpRequest, user_profile: UserProfile, data: TusUpload
+) -> HttpResponse:
+    if data.size_is_deferred or data.size is None:
+        return reject_upload("SizeIsDeferred is not supported", 411)
+
+    max_file_upload_size_mebibytes = user_profile.realm.get_max_file_upload_size_mebibytes()
+    if data.size > max_file_upload_size_mebibytes * 1024 * 1024:
+        if user_profile.realm.plan_type != Realm.PLAN_TYPE_SELF_HOSTED:
+            return reject_upload(
+                _(
+                    "File is larger than the maximum upload size ({max_size} MiB) allowed by your organization's plan."
+                ).format(
+                    max_size=max_file_upload_size_mebibytes,
+                ),
+                413,
+            )
+        else:
+            return reject_upload(
+                _(
+                    "File is larger than this server's configured maximum upload size ({max_size} MiB)."
+                ).format(
+                    max_size=max_file_upload_size_mebibytes,
+                ),
+                413,
+            )
+
+    try:
+        check_upload_within_quota(user_profile.realm, data.size)
+    except RealmUploadQuotaError as e:
+        return reject_upload(str(e), 413)
+
+    # Determine the path_id to store it at
+    file_name = sanitize_name(data.meta_data.get("filename", ""), strict=True)
+    path_id = generate_message_upload_path(str(user_profile.realm_id), file_name)
+    return tusd_json_response({"ChangeFileInfo": {"ID": path_id}})
+
+
+def handle_upload_pre_finish_hook(
+    request: HttpRequest, user_profile: UserProfile, data: TusUpload
+) -> HttpResponse:
+    # With an S3 backend, the filename we passed in pre_create's
+    # data.id has a randomly-generated "multipart-id" appended with a
+    # `+`.  Our path_ids cannot contain `+`, so we strip any suffix
+    # starting with `+`.
+    path_id = data.id.partition("+")[0]
+
+    tus_metadata = data.meta_data
+    filename = tus_metadata.get("filename", "")
+
+    # We want to store as the filename a version that clients are
+    # likely to be able to accept via "Save as..."
+    if filename in {"", ".", ".."}:
+        filename = "uploaded-file"
+
+    content_type = tus_metadata.get("filetype")
+    if not content_type:
+        content_type = guess_type(filename)[0]
+        if content_type is None:
+            content_type = "application/octet-stream"
+
+    # Only open the object at all when charset detection actually needs to
+    # inspect its bytes. With the S3 backend, opening it starts a
+    # GetObject read that we'd otherwise hold open across the self-copy
+    # below. AWS S3 doesn't mind, but S3-compatible backends that lock per
+    # object (e.g. MinIO) hold a read lock for the life of an open
+    # GetObject, so the copy would block until that idle reader is torn
+    # down -- tens of seconds later.
+    if needs_charset_detection(content_type):
+        content_type = maybe_add_charset(content_type, attachment_source(path_id))
+
+    if settings.LOCAL_UPLOADS_DIR is None:
+        # We "copy" the file to itself to update the Content-Type,
+        # Content-Disposition, and storage class of the data.  This
+        # parallels the work from upload_content_to_s3 in
+        # zerver.lib.uploads.s3
+        s3_metadata = {
+            "user_profile_id": str(user_profile.id),
+            "realm_id": str(user_profile.realm_id),
+        }
+
+        is_attachment = bare_content_type(content_type) not in INLINE_MIME_TYPES
+        content_disposition = content_disposition_header(is_attachment, filename) or "inline"
+
+        from zerver.lib.upload.s3 import S3UploadBackend
+
+        upload_backend = get_upload_backend()
+        assert isinstance(upload_backend, S3UploadBackend)
+        key = upload_backend.uploads_bucket.Object(path_id)
+        key.copy_from(
+            ContentType=content_type,
+            ContentDisposition=content_disposition,
+            CopySource={"Bucket": settings.S3_AUTH_UPLOADS_BUCKET, "Key": path_id},
+            Metadata=s3_metadata,
+            MetadataDirective="REPLACE",
+            StorageClass=settings.S3_UPLOADS_STORAGE_CLASS,
+        )
+
+        # https://tus.github.io/tusd/storage-backends/overview/#storage-format
+        # tusd also creates a .info file next to the upload, which
+        # must be preserved for HEAD requests (to check for upload
+        # state) to work.  These files are inaccessible via Zulip, and
+        # small enough to not pose any notable storage use; but we
+        # should store them with the right StorageClass.
+        if settings.S3_UPLOADS_STORAGE_CLASS != "STANDARD":
+            info_key = upload_backend.uploads_bucket.Object(path_id + ".info")
+            info_key.copy_from(
+                CopySource={"Bucket": settings.S3_AUTH_UPLOADS_BUCKET, "Key": path_id + ".info"},
+                MetadataDirective="COPY",
+                StorageClass=settings.S3_UPLOADS_STORAGE_CLASS,
+            )
+
+    # Open a fresh source for create_attachment/maybe_thumbnail to
+    # read from, now that the self-copy above (if any) has finished.
+    file_data = attachment_source(path_id)
+
+    with transaction.atomic(durable=True):
+        create_attachment(
+            filename,
+            path_id,
+            content_type,
+            file_data,
+            user_profile,
+            user_profile.realm,
+        )
+
+    path = "/user_uploads/" + path_id
+    return tusd_json_response(
+        {
+            "HttpResponse": {
+                "StatusCode": 200,
+                "Body": orjson.dumps({"url": path, "filename": filename}).decode(),
+                "Header": {
+                    "Content-Type": "application/json",
+                },
+            },
+        }
+    )
+
+
+def handle_upload_pre_terminate_hook(
+    request: HttpRequest, user_profile: UserProfile, data: TusUpload
+) -> HttpResponse:
+    path_id = data.id.partition("+")[0]
+
+    if (
+        Attachment.objects.filter(path_id=path_id).exists()
+        or ArchivedAttachment.objects.filter(path_id=path_id).exists()
+    ):
+        # Once we have it in our Attachments table (i.e. the
+        # pre-upload-finished hook has run), it is ours to manage and
+        # we no longer accept terminations.
+        return tusd_json_response({"RejectTermination": True})
+    return tusd_json_response({})
+
+
+def authenticate_user(request: HttpRequest) -> UserProfile | AnonymousUser:
+    # This acts like the authenticated_rest_api_view wrapper, while
+    # allowing fallback to session-based request.user
+    if "Authorization" in request.headers:
+        try:
+            role, api_key = get_basic_credentials(request)
+            return validate_api_key(
+                request,
+                role,
+                api_key,
+            )
+
+        except JsonableError:
+            pass
+
+    # If that failed, fall back to session auth
+    return request.user
+
+
+def handle_preregistration_pre_create_hook(
+    request: HttpRequest, preregistration_realm: PreregistrationRealm, data: TusUpload
+) -> HttpResponse:
+    if data.size_is_deferred or data.size is None:
+        return reject_upload("SizeIsDeferred is not supported", 411)
+
+    if settings.MAX_WEB_DATA_IMPORT_SIZE_MB is not None:
+        max_upload_size = settings.MAX_WEB_DATA_IMPORT_SIZE_MB * 1024 * 1024  # 1G
+        if data.size > max_upload_size:
+            return reject_upload(
+                _(
+                    "Uploaded file exceeds the maximum file size for imports ({max_file_size} MiB)."
+                ).format(max_file_size=settings.MAX_WEB_DATA_IMPORT_SIZE_MB),
+                413,
+            )
+
+    filename = f"import/{preregistration_realm.id}/slack.zip"
+
+    # Delete any existing upload, so tusd doesn't declare that there's nothing
+    # to do. This also has the nice benefit of deleting the previous upload.
+    delete_message_attachment(filename)
+
+    return tusd_json_response({"ChangeFileInfo": {"ID": filename}})
+
+
+def handle_preregistration_pre_finish_hook(
+    request: HttpRequest, preregistration_realm: PreregistrationRealm, data: TusUpload
+) -> HttpResponse:
+    # Save the filename to display the uploaded file to user. We need to store it in
+    # the database so that is available even after a refresh.
+    filename = data.meta_data["filename"]
+    preregistration_realm.data_import_metadata["uploaded_import_file_name"] = filename
+    preregistration_realm.save(update_fields=["data_import_metadata"])
+    return tusd_json_response({})
+
+
+@csrf_exempt
+@typed_endpoint
+def handle_tusd_hook(
+    request: HttpRequest,
+    *,
+    payload: JsonBodyPayload[TusHook],
+) -> HttpResponse:
+    # Make sure this came from localhost
+    if not is_local_addr(request.META["REMOTE_ADDR"]):
+        raise AccessDeniedError
+
+    hook_name = payload.type
+    maybe_user = authenticate_user(request)
+    if maybe_user.is_authenticated:
+        # Authenticated requests are file upload requests
+        if hook_name == "pre-create":
+            return handle_upload_pre_create_hook(request, maybe_user, payload.event.upload)
+        elif hook_name == "pre-finish":
+            return handle_upload_pre_finish_hook(request, maybe_user, payload.event.upload)
+        elif hook_name == "pre-terminate":
+            return handle_upload_pre_terminate_hook(request, maybe_user, payload.event.upload)
+        else:
+            return HttpResponseNotFound()
+
+    # Check if unauthenticated requests are for realm creation
+    key = payload.event.upload.meta_data.get("key")
+    if key is None:
+        return reject_upload("Unauthenticated upload", 401)
+    try:
+        prereg_object = get_object_from_key(
+            key, [Confirmation.NEW_REALM_USER_REGISTRATION], mark_object_used=False
+        )
+    except ConfirmationKeyError:
+        return reject_upload("Unauthenticated upload", 401)
+
+    assert isinstance(prereg_object, PreregistrationRealm)
+    assert prereg_object.created_realm is None
+
+    if hook_name == "pre-create":
+        return handle_preregistration_pre_create_hook(request, prereg_object, payload.event.upload)
+    elif hook_name == "pre-finish":
+        return handle_preregistration_pre_finish_hook(request, prereg_object, payload.event.upload)
+    else:  # nocoverage
+        return HttpResponseNotFound()

zerver/views/typing.py

@@ -0,0 +1,114 @@
+from typing import Annotated, Literal
+
+from django.http import HttpRequest, HttpResponse
+from django.utils.translation import gettext as _
+from pydantic import Json, NonNegativeInt
+
+from zerver.actions.message_edit import validate_user_can_edit_message
+from zerver.actions.typing import (
+    check_send_typing_notification,
+    do_send_direct_message_edit_typing_notification,
+    do_send_stream_message_edit_typing_notification,
+    do_send_stream_typing_notification,
+)
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.message import access_message
+from zerver.lib.response import json_success
+from zerver.lib.streams import access_stream_by_id, access_stream_for_send_message
+from zerver.lib.topic import (
+    maybe_rename_general_chat_to_empty_topic,
+    maybe_rename_no_topic_to_empty_topic,
+)
+from zerver.lib.typed_endpoint import ApiParamConfig, OptionalTopic, PathOnly, typed_endpoint
+from zerver.models import Recipient, UserProfile
+from zerver.models.recipients import get_direct_message_group_user_ids
+
+
+@typed_endpoint
+def send_notification_backend(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    notification_to: Annotated[Json[list[int] | None], ApiParamConfig("to")] = None,
+    operator: Annotated[Literal["start", "stop"], ApiParamConfig("op")],
+    req_type: Annotated[Literal["direct", "stream", "channel"], ApiParamConfig("type")] = "direct",
+    stream_id: Json[int | None] = None,
+    topic: OptionalTopic = None,
+) -> HttpResponse:
+    recipient_type_name = req_type
+    if recipient_type_name == "channel":
+        # For now, use "stream" from Message.API_RECIPIENT_TYPES.
+        # TODO: Use "channel" here, as well as in events and
+        # message (created, schdeduled, drafts) objects/dicts.
+        recipient_type_name = "stream"
+
+    if recipient_type_name == "stream":
+        if stream_id is None:
+            raise JsonableError(_("Missing '{var_name}' argument").format(var_name="stream_id"))
+
+        if topic is None:
+            raise JsonableError(_("Missing topic"))
+
+        if not user_profile.send_stream_typing_notifications:
+            raise JsonableError(_("User has disabled typing notifications for channel messages"))
+
+        # Verify that the user has access to the stream and has
+        # permission to send messages to it.
+        stream = access_stream_by_id(user_profile, stream_id)[0]
+        access_stream_for_send_message(user_profile, stream, forwarder_user_profile=None)
+        topic = maybe_rename_general_chat_to_empty_topic(topic)
+        topic = maybe_rename_no_topic_to_empty_topic(topic)
+        do_send_stream_typing_notification(user_profile, operator, stream, topic)
+    else:
+        if notification_to is None:
+            raise JsonableError(_("Missing 'to' argument"))
+
+        user_ids = notification_to
+        to_length = len(user_ids)
+        if to_length == 0:
+            raise JsonableError(_("Empty 'to' list"))
+
+        if not user_profile.send_private_typing_notifications:
+            raise JsonableError(_("User has disabled typing notifications for direct messages"))
+
+        check_send_typing_notification(user_profile, user_ids, operator)
+
+    return json_success(request)
+
+
+@typed_endpoint
+def send_message_edit_notification_backend(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    message_id: PathOnly[NonNegativeInt],
+    operator: Annotated[Literal["start", "stop"], ApiParamConfig("op")],
+) -> HttpResponse:
+    # Technically, this endpoint doesn't modify the message, but we're
+    # attempting to send a typing notification that we're editing the
+    # message.
+    message = access_message(user_profile, message_id, is_modifying_message=True)
+    validate_user_can_edit_message(user_profile, message, edit_limit_buffer=0)
+    recipient = message.recipient
+
+    if recipient.type == Recipient.STREAM:
+        if not user_profile.send_stream_typing_notifications:
+            raise JsonableError(_("User has disabled typing notifications for channel messages"))
+
+        channel_id = recipient.type_id
+        topic = message.topic_name()
+        do_send_stream_message_edit_typing_notification(
+            user_profile, channel_id, message_id, operator, topic
+        )
+
+    else:
+        if not user_profile.send_private_typing_notifications:
+            raise JsonableError(_("User has disabled typing notifications for direct messages"))
+
+        recipient_ids = list(get_direct_message_group_user_ids(recipient))
+
+        do_send_direct_message_edit_typing_notification(
+            user_profile, recipient_ids, message_id, operator
+        )
+
+    return json_success(request)

zerver/views/unsubscribe.py

@@ -0,0 +1,80 @@
+from collections.abc import Callable
+
+from django.http import HttpRequest, HttpResponse
+from django.shortcuts import render
+from django.views.decorators.csrf import csrf_exempt
+
+from confirmation.models import Confirmation, ConfirmationKeyError, get_object_from_key
+from zerver.actions.user_settings import do_change_user_setting
+from zerver.context_processors import common_context
+from zerver.lib.send_email import clear_scheduled_emails
+from zerver.models import ScheduledEmail, UserProfile
+
+
+def process_unsubscribe(
+    request: HttpRequest,
+    confirmation_key: str,
+    subscription_type: str,
+    unsubscribe_function: Callable[[UserProfile], None],
+) -> HttpResponse:
+    try:
+        user_profile = get_object_from_key(
+            confirmation_key, [Confirmation.UNSUBSCRIBE], mark_object_used=False
+        )
+    except ConfirmationKeyError:
+        return render(request, "zerver/unsubscribe_link_error.html")
+
+    assert isinstance(user_profile, UserProfile)
+    unsubscribe_function(user_profile)
+    context = common_context(user_profile)
+    context.update(subscription_type=subscription_type)
+    return render(request, "zerver/unsubscribe_success.html", context=context)
+
+
+# Email unsubscribe functions. All have the function signature
+# processor(user_profile).
+
+
+def do_missedmessage_unsubscribe(user_profile: UserProfile) -> None:
+    do_change_user_setting(
+        user_profile, "enable_offline_email_notifications", False, acting_user=user_profile
+    )
+
+
+def do_welcome_unsubscribe(user_profile: UserProfile) -> None:
+    clear_scheduled_emails([user_profile.id], ScheduledEmail.WELCOME)
+
+
+def do_digest_unsubscribe(user_profile: UserProfile) -> None:
+    do_change_user_setting(user_profile, "enable_digest_emails", False, acting_user=user_profile)
+
+
+def do_login_unsubscribe(user_profile: UserProfile) -> None:
+    do_change_user_setting(user_profile, "enable_login_emails", False, acting_user=user_profile)
+
+
+def do_marketing_unsubscribe(user_profile: UserProfile) -> None:
+    do_change_user_setting(user_profile, "enable_marketing_emails", False, acting_user=user_profile)
+
+
+# The keys are part of the URL for the unsubscribe link and must be valid
+# without encoding.
+# The values are a tuple of (display name, unsubscribe function), where the
+# display name is what we call this class of email in user-visible text.
+email_unsubscribers = {
+    "missed_messages": ("missed messages", do_missedmessage_unsubscribe),
+    "welcome": ("welcome", do_welcome_unsubscribe),
+    "digest": ("digest", do_digest_unsubscribe),
+    "login": ("login", do_login_unsubscribe),
+    "marketing": ("marketing", do_marketing_unsubscribe),
+}
+
+
+# Login NOT required. These are for one-click unsubscribes.
+@csrf_exempt
+def email_unsubscribe(request: HttpRequest, email_type: str, confirmation_key: str) -> HttpResponse:
+    if email_type in email_unsubscribers:
+        display_name, unsubscribe_function = email_unsubscribers[email_type]
+        return process_unsubscribe(request, confirmation_key, display_name, unsubscribe_function)
+
+    return render(request, "zerver/unsubscribe_link_error.html")

zerver/views/upload.py

@@ -0,0 +1,493 @@
+import base64
+import binascii
+import os
+from datetime import timedelta
+from urllib.parse import quote, urlsplit
+
+from django.conf import settings
+from django.contrib.auth.models import AnonymousUser
+from django.core.files.uploadedfile import UploadedFile
+from django.core.signing import BadSignature, TimestampSigner
+from django.db import transaction
+from django.http import (
+    FileResponse,
+    HttpRequest,
+    HttpResponse,
+    HttpResponseBase,
+    HttpResponseForbidden,
+    HttpResponseNotFound,
+)
+from django.shortcuts import redirect
+from django.urls import reverse
+from django.utils.cache import patch_cache_control, patch_vary_headers
+from django.utils.http import content_disposition_header
+from django.utils.translation import gettext as _
+
+from zerver.context_processors import get_valid_realm_from_request
+from zerver.decorator import zulip_redirect_to_login
+from zerver.lib.attachments import validate_attachment_request
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.mime_types import INLINE_MIME_TYPES, bare_content_type, guess_type
+from zerver.lib.response import json_success
+from zerver.lib.storage import static_path
+from zerver.lib.thumbnail import (
+    THUMBNAIL_OUTPUT_FORMATS,
+    BaseThumbnailFormat,
+    StoredThumbnailFormat,
+    get_image_thumbnail_path,
+)
+from zerver.lib.upload import (
+    attachment_source,
+    check_upload_within_quota,
+    get_public_upload_root_url,
+    maybe_add_charset,
+    needs_charset_detection,
+    upload_message_attachment_from_request,
+)
+from zerver.lib.upload.local import assert_is_local_storage_path
+from zerver.lib.upload.s3 import get_signed_upload_url
+from zerver.models import Attachment, ImageAttachment, Realm, UserProfile
+from zerver.worker.thumbnail import ensure_thumbnails
+
+
+def patch_disposition_header(response: HttpResponse, filename: str, is_attachment: bool) -> None:
+    content_disposition = content_disposition_header(is_attachment, filename)
+
+    if content_disposition is not None:
+        response.headers["Content-Disposition"] = content_disposition
+
+
+def internal_nginx_redirect(internal_path: str, content_type: str | None = None) -> HttpResponse:
+    # The following headers from this initial response are
+    # _preserved_, if present, and sent unmodified to the client;
+    # all other headers are overridden by the redirected URL:
+    #  - Content-Type
+    #  - Content-Disposition
+    #  - Accept-Ranges
+    #  - Set-Cookie
+    #  - Cache-Control
+    #  - Expires
+    # As such, we default to unsetting the Content-type header to
+    # allow nginx to set it from the static file; the caller can set
+    # Content-Disposition and Cache-Control on this response as they
+    # desire, and the client will see those values.  In some cases
+    # (local files) we do wish to control the Content-Type, so also
+    # support setting it explicitly.
+    response = HttpResponse(content_type=content_type)
+    response["X-Accel-Redirect"] = internal_path
+    if content_type is None:
+        del response["Content-Type"]
+    return response
+
+
+def serve_s3(
+    request: HttpRequest,
+    path_id: str,
+    attachment: Attachment,
+    *,
+    filename: str | None = None,
+    force_download: bool = False,
+) -> HttpResponse:
+    if filename is None:
+        filename = attachment.file_name
+
+    # Save going to S3 for 0-byte files -- in part for performance, to
+    # save a round-trip to S3, but also because S3 can erroneously
+    # return HTTP 416 errors when a `Range: 0-5242879` header is sent
+    # (as nginx does by default with the `slice` directive) for a
+    # 0-byte file.
+    if attachment.size == 0:
+        content_type = (
+            attachment.content_type or guess_type(filename)[0] or "application/octet-stream"
+        )
+        if needs_charset_detection(content_type):
+            content_type = maybe_add_charset(content_type, attachment_source(path_id))
+        download = force_download or bare_content_type(content_type) not in INLINE_MIME_TYPES
+
+        response = HttpResponse("", content_type=content_type)
+        response.headers["Content-Length"] = "0"
+        patch_cache_control(response, private=True, immutable=True)
+        patch_disposition_header(response, filename, download)
+        return response
+
+    url = get_signed_upload_url(path_id, filename, force_download=force_download)
+    assert url.startswith("https://")
+
+    if settings.DEVELOPMENT:
+        # In development, we do not have the nginx server to offload
+        # the response to; serve a redirect to the short-lived S3 URL.
+        # This means the content cannot be cached by the browser, but
+        # this is acceptable in development.
+        return redirect(url)
+
+    # We over-escape the path, to work around it being impossible to
+    # get the _unescaped_ new internal request URL in nginx.
+    parsed_url = urlsplit(url)
+    assert parsed_url.hostname is not None
+    assert parsed_url.path is not None
+    assert parsed_url.query is not None
+    escaped_path_parts = parsed_url.hostname + quote(parsed_url.path) + "?" + parsed_url.query
+    response = internal_nginx_redirect("/internal/s3/" + escaped_path_parts)
+
+    # It is important that S3 generate both the Content-Type and
+    # Content-Disposition headers; when the file was uploaded, we
+    # stored the browser-provided value for the former, and set
+    # Content-Disposition according to if that was safe.  As such,
+    # only S3 knows if a given attachment is safe to inline; we only
+    # override Content-Disposition to "attachment", and do so by
+    # telling S3 that is what we want in the signed URL.
+    patch_cache_control(response, private=True, immutable=True)
+    return response
+
+
+def serve_local(
+    request: HttpRequest,
+    path_id: str,
+    filename: str,
+    force_download: bool = False,
+    content_type: str | None = None,
+) -> HttpResponseBase:
+    assert settings.LOCAL_FILES_DIR is not None
+    local_path = os.path.join(settings.LOCAL_FILES_DIR, path_id)
+    assert_is_local_storage_path("files", local_path)
+    if not os.path.isfile(local_path):
+        return HttpResponseNotFound("<p>File not found</p>")
+
+    if content_type is None:
+        content_type = guess_type(filename)[0] or "application/octet-stream"
+
+    if needs_charset_detection(content_type):
+        content_type = maybe_add_charset(content_type, attachment_source(path_id))
+    download = force_download or bare_content_type(content_type) not in INLINE_MIME_TYPES
+
+    if settings.DEVELOPMENT:
+        # In development, we do not have the nginx server to offload
+        # the response to; serve it directly ourselves.  FileResponse
+        # handles setting Content-Type, Content-Disposition, etc.
+        response: HttpResponseBase = FileResponse(
+            open(local_path, "rb"),  # noqa: SIM115
+            as_attachment=download,
+            filename=filename,
+            content_type=content_type,
+        )
+        patch_cache_control(response, private=True, immutable=True)
+        return response
+
+    # For local responses, we are in charge of generating both
+    # Content-Type and Content-Disposition headers; unlike with S3
+    # storage, the Content-Type is not stored with the file in any
+    # way, so Django makes the determination of it, and thus as well
+    # if that type is safe to have a Content-Disposition of "inline".
+    # nginx respects the values we send.
+    response = internal_nginx_redirect(
+        quote(f"/internal/local/uploads/{path_id}"), content_type=content_type
+    )
+    patch_disposition_header(response, filename, download)
+    patch_cache_control(response, private=True, immutable=True)
+    return response
+
+
+def serve_file_download_backend(
+    request: HttpRequest,
+    maybe_user_profile: UserProfile | AnonymousUser,
+    realm_id_str: str,
+    filename: str,
+) -> HttpResponseBase:
+    return serve_file(
+        request, maybe_user_profile, realm_id_str, filename, url_only=False, force_download=True
+    )
+
+
+def serve_file_backend(
+    request: HttpRequest,
+    maybe_user_profile: UserProfile | AnonymousUser,
+    realm_id_str: str,
+    filename: str,
+    thumbnail_format: str | None = None,
+) -> HttpResponseBase:
+    return serve_file(
+        request,
+        maybe_user_profile,
+        realm_id_str,
+        filename,
+        url_only=False,
+        thumbnail_format=thumbnail_format,
+    )
+
+
+def serve_file_url_backend(
+    request: HttpRequest, user_profile: UserProfile, realm_id_str: str, filename: str
+) -> HttpResponseBase:
+    """
+    We should return a signed, short-lived URL
+    that the client can use for native mobile download, rather than serving a redirect.
+    """
+
+    return serve_file(request, user_profile, realm_id_str, filename, url_only=True)
+
+
+def closest_thumbnail_format(
+    requested_format: BaseThumbnailFormat,
+    request: HttpRequest,
+    rendered_formats: list[StoredThumbnailFormat],
+) -> StoredThumbnailFormat:
+    # Serve a "close" format -- preferring animated which
+    # matches, followed by the format they requested, or one
+    # their browser supports, in the size closest to what they
+    # requested, with the minimum bytes.
+    def grade_format(
+        possible_format: StoredThumbnailFormat,
+    ) -> tuple[bool, bool, float, int, int]:
+        return (
+            possible_format.animated != requested_format.animated,
+            possible_format.extension != requested_format.extension,
+            0.0
+            if (accepted_type := request.accepted_type(possible_format.content_type)) is None
+            else -accepted_type.quality,
+            abs(requested_format.max_width - possible_format.max_width),
+            possible_format.byte_size,
+        )
+
+    return min(rendered_formats, key=grade_format)
+
+
+def serve_file(
+    request: HttpRequest,
+    maybe_user_profile: UserProfile | AnonymousUser,
+    realm_id_str: str,
+    filename: str,
+    thumbnail_format: str | None = None,
+    url_only: bool = False,
+    force_download: bool = False,
+) -> HttpResponseBase:
+    path_id = f"{realm_id_str}/{filename}"
+    realm = get_valid_realm_from_request(request)
+    is_authorized, attachment = validate_attachment_request(maybe_user_profile, path_id, realm)
+
+    def serve_image_error(status: int, image_path: str) -> HttpResponseBase:
+        # We cannot use X-Accel-Redirect to offload the serving of
+        # this image to nginx, because it does not preserve the status
+        # code of this response, nor the Vary: header.
+        return FileResponse(open(static_path(image_path), "rb"), status=status)
+
+    if attachment is None:
+        if request.get_preferred_type(["text/html", "image/png"]) == "image/png":
+            response = serve_image_error(404, "images/errors/image-not-exist.png")
+        else:
+            response = HttpResponseNotFound(
+                _("<p>This file does not exist or has been deleted.</p>")
+            )
+        patch_vary_headers(response, ("Accept",))
+        return response
+    if not is_authorized:
+        if request.get_preferred_type(["text/html", "image/png"]) == "image/png":
+            response = serve_image_error(403, "images/errors/image-no-auth.png")
+        elif isinstance(maybe_user_profile, AnonymousUser):
+            response = zulip_redirect_to_login(request)
+        else:
+            response = HttpResponseForbidden(_("<p>You are not authorized to view this file.</p>"))
+        patch_vary_headers(response, ("Accept",))
+        return response
+    if url_only:
+        url = generate_unauthed_file_access_url(path_id)
+        return json_success(request, data=dict(url=url))
+
+    if thumbnail_format is not None:
+        # Check if this is something that we thumbnail at all
+        try:
+            image_attachment = ImageAttachment.objects.get(path_id=path_id)
+        except ImageAttachment.DoesNotExist:
+            return serve_image_error(404, "images/errors/image-not-exist.png")
+
+        # Validate that this is a potential thumbnail format
+        requested_format = BaseThumbnailFormat.from_string(thumbnail_format)
+        if requested_format is None:
+            return serve_image_error(404, "images/errors/image-not-exist.png")
+
+        rendered_formats = [StoredThumbnailFormat(**f) for f in image_attachment.thumbnail_metadata]
+
+        # We never generate animated versions if the input was still,
+        # so filter those out
+        if image_attachment.frames == 1:
+            potential_output_formats = [
+                thumbnail_format
+                for thumbnail_format in THUMBNAIL_OUTPUT_FORMATS
+                if not thumbnail_format.animated
+            ]
+        else:
+            potential_output_formats = list(THUMBNAIL_OUTPUT_FORMATS)
+        if requested_format not in potential_output_formats:
+            if rendered_formats == []:
+                # We haven't rendered anything, and they requested
+                # something we don't support.
+                return serve_image_error(404, "images/errors/image-not-exist.png")
+            elif requested_format in rendered_formats:
+                # Not a _current_ format, but we did render it at the
+                # time, so fine to serve.  We also end up here for
+                # TRANSCODED_IMAGE_FORMAT requests, which are not in
+                # the default THUMBNAIL_OUTPUT_FORMATS, but may exist
+                # for some images types not in INLINE_MIME_TYPES.
+                pass
+            else:
+                # Find something "close enough".  This will not be a
+                # common occurrence -- the client has out of date
+                # information about which formats are supported, and
+                # the thumbnails were generated with an even earlier
+                # set, or the client is just guessing a format and
+                # hoping.
+                requested_format = closest_thumbnail_format(
+                    requested_format, request, rendered_formats
+                )
+        elif requested_format not in rendered_formats:
+            # They requested a valid format, but one we've not
+            # rendered yet.  Take a lock on the row, then render every
+            # missing format, synchronously.  The lock prevents us
+            # from doing double-work if the background worker is
+            # currently processing the row.
+            with transaction.atomic(savepoint=False):
+                ensure_thumbnails(
+                    # ensure_thumbnails can in rare cases end up deleting the ImageAttachment row, if thumbnailing
+                    # fails. Since a DELETE can happen, we want a full FOR UPDATE lock.
+                    ImageAttachment.objects.select_for_update(no_key=False).get(
+                        id=image_attachment.id
+                    ),
+                )
+
+        # Update the path that we are fetching to be the thumbnail
+        path_id = get_image_thumbnail_path(image_attachment, requested_format)
+        served_filename = str(requested_format)
+        content_type: str | None = None  # Guess from filename
+    else:
+        served_filename = attachment.file_name
+        content_type = attachment.content_type
+
+    if settings.LOCAL_UPLOADS_DIR is not None:
+        return serve_local(
+            request,
+            path_id,
+            filename=served_filename,
+            force_download=force_download,
+            content_type=content_type,
+        )
+    else:
+        return serve_s3(
+            request,
+            path_id,
+            attachment,
+            filename=served_filename,
+            force_download=force_download,
+        )
+
+
+USER_UPLOADS_ACCESS_TOKEN_SALT = "user_uploads_"
+
+
+def generate_unauthed_file_access_url(path_id: str) -> str:
+    signed_data = TimestampSigner(salt=USER_UPLOADS_ACCESS_TOKEN_SALT).sign(path_id)
+    token = base64.b16encode(signed_data.encode()).decode()
+
+    filename = path_id.rsplit("/", 1)[-1]
+    return reverse("file_unauthed_from_token", args=[token, filename])
+
+
+def get_file_path_id_from_token(token: str) -> str | None:
+    signer = TimestampSigner(salt=USER_UPLOADS_ACCESS_TOKEN_SALT)
+    try:
+        signed_data = base64.b16decode(token).decode()
+        path_id = signer.unsign(
+            signed_data, max_age=timedelta(seconds=settings.SIGNED_ACCESS_TOKEN_VALIDITY_IN_SECONDS)
+        )
+    except (BadSignature, binascii.Error):
+        return None
+
+    return path_id
+
+
+def serve_file_unauthed_from_token(
+    request: HttpRequest, token: str, filename: str
+) -> HttpResponseBase:
+    path_id = get_file_path_id_from_token(token)
+    if path_id is None:
+        raise JsonableError(_("Invalid token"))
+    if path_id.split("/")[-1] != filename:
+        raise JsonableError(_("Invalid filename"))
+    try:
+        attachment = Attachment.objects.get(path_id=path_id)
+    except Attachment.DoesNotExist:
+        raise JsonableError(_("Invalid token"))
+
+    if settings.LOCAL_UPLOADS_DIR is not None:
+        return serve_local(
+            request,
+            path_id,
+            filename=attachment.file_name,
+            content_type=attachment.content_type,
+        )
+    else:
+        return serve_s3(request, path_id, attachment)
+
+
+def serve_local_avatar_unauthed(request: HttpRequest, path: str) -> HttpResponseBase:
+    """Serves avatar images off disk, via nginx (or directly in dev), with no auth.
+
+    This is done unauthed because these need to be accessed from HTML
+    emails, where the client does not have any auth.  We rely on the
+    URL being generated using the AVATAR_SALT secret.
+
+    """
+    if settings.LOCAL_AVATARS_DIR is None:
+        # We do not expect clients to hit this URL when using the S3
+        # backend; however, there is no reason to not serve the
+        # redirect to S3 where the content lives.
+        url = get_public_upload_root_url() + path
+        return redirect(url, permanent=True)
+
+    local_path = os.path.join(settings.LOCAL_AVATARS_DIR, path)
+    assert_is_local_storage_path("avatars", local_path)
+    if not os.path.isfile(local_path):
+        return HttpResponseNotFound("<p>File not found</p>")
+
+    if settings.DEVELOPMENT:
+        response: HttpResponseBase = FileResponse(open(local_path, "rb"))  # noqa: SIM115
+    else:
+        response = internal_nginx_redirect(quote(f"/internal/local/user_avatars/{path}"))
+
+    patch_cache_control(response, max_age=31536000, public=True, immutable=True)
+    return response
+
+
+def upload_file_backend(request: HttpRequest, user_profile: UserProfile) -> HttpResponse:
+    if len(request.FILES) == 0:
+        raise JsonableError(_("You must specify a file to upload"))
+    if len(request.FILES) != 1:
+        raise JsonableError(_("You may only upload one file at a time"))
+
+    [user_file] = request.FILES.values()
+    assert isinstance(user_file, UploadedFile)
+    file_size = user_file.size
+    assert file_size is not None
+    max_file_upload_size_mebibytes = user_profile.realm.get_max_file_upload_size_mebibytes()
+    if file_size > max_file_upload_size_mebibytes * 1024 * 1024:
+        if user_profile.realm.plan_type != Realm.PLAN_TYPE_SELF_HOSTED:
+            raise JsonableError(
+                _(
+                    "File is larger than the maximum upload size ({max_size} MiB) allowed by your organization's plan."
+                ).format(
+                    max_size=max_file_upload_size_mebibytes,
+                )
+            )
+        else:
+            raise JsonableError(
+                _(
+                    "File is larger than this server's configured maximum upload size ({max_size} MiB)."
+                ).format(
+                    max_size=max_file_upload_size_mebibytes,
+                )
+            )
+    check_upload_within_quota(user_profile.realm, file_size)
+
+    url, filename = upload_message_attachment_from_request(user_file, user_profile)
+
+    # TODO/compatibility: uri is a deprecated alias for url that can
+    # be removed once there are no longer clients relying on it.
+    return json_success(request, data={"uri": url, "url": url, "filename": filename})

zerver/views/user_groups.py

@@ -0,0 +1,628 @@
+from django.conf import settings
+from django.db import transaction
+from django.http import HttpRequest, HttpResponse
+from django.utils.translation import gettext as _
+from django.utils.translation import override as override_language
+from pydantic import Json
+
+from zerver.actions.message_send import do_send_messages, internal_prep_private_message
+from zerver.actions.user_groups import (
+    add_subgroups_to_user_group,
+    bulk_add_members_to_user_groups,
+    bulk_remove_members_from_user_groups,
+    check_add_user_group,
+    do_change_user_group_permission_setting,
+    do_deactivate_user_group,
+    do_reactivate_user_group,
+    do_update_user_group_description,
+    do_update_user_group_name,
+    remove_subgroups_from_user_group,
+)
+from zerver.decorator import require_non_guest_user, require_user_group_create_permission
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.mention import MentionBackend, silent_mention_syntax_for_user
+from zerver.lib.response import json_success
+from zerver.lib.typed_endpoint import PathOnly, typed_endpoint
+from zerver.lib.types import UserGroupMembersData
+from zerver.lib.user_groups import (
+    GroupSettingChangeRequest,
+    access_user_group_for_deactivation,
+    access_user_group_for_setting,
+    access_user_group_for_update,
+    access_user_group_to_read_membership,
+    check_group_membership_management_permissions_with_admins_only,
+    check_user_group_name,
+    get_direct_memberships_of_users,
+    get_group_setting_value_for_api,
+    get_role_based_system_groups_dict,
+    get_subgroup_ids,
+    get_system_user_group_by_name,
+    get_user_group_direct_member_ids,
+    get_user_group_member_ids,
+    is_user_in_group,
+    lock_subgroups_with_respect_to_supergroup,
+    parse_group_setting_value,
+    user_groups_in_realm_serialized,
+    validate_group_membership_management_setting,
+    validate_group_setting_value_change,
+)
+from zerver.lib.users import access_user_by_id, user_ids_to_users
+from zerver.models import NamedUserGroup, UserProfile
+from zerver.models.groups import SystemGroups
+from zerver.models.users import get_system_bot
+from zerver.views.streams import compose_views
+
+
+@transaction.atomic(durable=True)
+@require_user_group_create_permission
+@typed_endpoint
+def add_user_group(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    can_add_members_group: Json[int | UserGroupMembersData] | None = None,
+    can_join_group: Json[int | UserGroupMembersData] | None = None,
+    can_leave_group: Json[int | UserGroupMembersData] | None = None,
+    can_manage_group: Json[int | UserGroupMembersData] | None = None,
+    can_mention_group: Json[int | UserGroupMembersData] | None = None,
+    can_remove_members_group: Json[int | UserGroupMembersData] | None = None,
+    description: str,
+    members: Json[list[int]],
+    name: str,
+    subgroups: Json[list[int]] | None = None,
+) -> HttpResponse:
+    user_profile.realm.ensure_not_on_limited_plan()
+    user_profiles = user_ids_to_users(
+        members, user_profile.realm, allow_deactivated=False, allow_bots=True
+    )
+    name = check_user_group_name(name)
+
+    group_settings_map = {}
+    request_settings_dict = locals()
+    nobody_group = get_system_user_group_by_name(SystemGroups.NOBODY, user_profile.realm_id)
+    for setting_name, permission_config in NamedUserGroup.GROUP_PERMISSION_SETTINGS.items():
+        if setting_name not in request_settings_dict:  # nocoverage
+            continue
+
+        if request_settings_dict[setting_name] is not None:
+            setting_value = parse_group_setting_value(
+                request_settings_dict[setting_name], nobody_group
+            )
+            setting_value_group = access_user_group_for_setting(
+                setting_value,
+                user_profile,
+                setting_name=setting_name,
+                permission_configuration=permission_config,
+            )
+            group_settings_map[setting_name] = setting_value_group
+
+    user_group = check_add_user_group(
+        user_profile.realm,
+        name,
+        user_profiles,
+        description,
+        group_settings_map=group_settings_map,
+        acting_user=user_profile,
+    )
+
+    if subgroups is not None and len(subgroups) != 0:
+        with lock_subgroups_with_respect_to_supergroup(
+            subgroups, user_group.id, user_profile, permission_setting=None, creating_group=True
+        ) as context:
+            add_subgroups_to_user_group(
+                context.supergroup, context.direct_subgroups, acting_user=user_profile
+            )
+    return json_success(request, data={"group_id": user_group.id})
+
+
+@require_non_guest_user
+@typed_endpoint
+def get_user_groups(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    include_deactivated_groups: Json[bool] = False,
+) -> HttpResponse:
+    user_groups = user_groups_in_realm_serialized(
+        user_profile.realm, include_deactivated_groups=include_deactivated_groups
+    ).api_groups
+    return json_success(request, data={"user_groups": user_groups})
+
+
+@transaction.atomic(durable=True)
+@require_non_guest_user
+@typed_endpoint
+def edit_user_group(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    can_add_members_group: Json[GroupSettingChangeRequest] | None = None,
+    can_join_group: Json[GroupSettingChangeRequest] | None = None,
+    can_leave_group: Json[GroupSettingChangeRequest] | None = None,
+    can_manage_group: Json[GroupSettingChangeRequest] | None = None,
+    can_mention_group: Json[GroupSettingChangeRequest] | None = None,
+    can_remove_members_group: Json[GroupSettingChangeRequest] | None = None,
+    deactivated: Json[bool] | None = None,
+    description: str | None = None,
+    name: str | None = None,
+    user_group_id: PathOnly[int],
+) -> HttpResponse:
+    if (
+        name is None
+        and description is None
+        and can_add_members_group is None
+        and can_join_group is None
+        and can_leave_group is None
+        and can_manage_group is None
+        and can_mention_group is None
+        and can_remove_members_group is None
+        and deactivated is None
+    ):
+        raise JsonableError(_("No new data supplied"))
+
+    user_group = access_user_group_for_update(
+        user_group_id, user_profile, permission_setting="can_manage_group", allow_deactivated=True
+    )
+
+    if name is not None and name != user_group.name:
+        name = check_user_group_name(name)
+        do_update_user_group_name(user_group, name, acting_user=user_profile)
+
+    if description is not None and description != user_group.description:
+        do_update_user_group_description(user_group, description, acting_user=user_profile)
+
+    if deactivated is not None and not deactivated and user_group.deactivated:
+        do_reactivate_user_group(user_group, acting_user=user_profile)
+
+    request_settings_dict = locals()
+    system_groups_name_dict = get_role_based_system_groups_dict(user_profile.realm)
+    nobody_group = system_groups_name_dict[SystemGroups.NOBODY]
+    for setting_name, permission_config in NamedUserGroup.GROUP_PERMISSION_SETTINGS.items():
+        if setting_name not in request_settings_dict:  # nocoverage
+            continue
+
+        if request_settings_dict[setting_name] is None:
+            continue
+
+        setting_value = request_settings_dict[setting_name]
+        new_setting_value = parse_group_setting_value(setting_value.new, nobody_group)
+
+        expected_current_setting_value = None
+        if setting_value.old is not None:
+            expected_current_setting_value = parse_group_setting_value(
+                setting_value.old, nobody_group
+            )
+
+        current_value = getattr(user_group, setting_name)
+        current_setting_api_value = get_group_setting_value_for_api(current_value)
+        if validate_group_setting_value_change(
+            current_setting_api_value, new_setting_value, expected_current_setting_value
+        ):
+            setting_value_group = access_user_group_for_setting(
+                new_setting_value,
+                user_profile,
+                setting_name=setting_name,
+                permission_configuration=permission_config,
+                current_setting_value=current_value,
+            )
+
+            if setting_name in NamedUserGroup.MEMBERSHIP_MANAGEMENT_SETTINGS:
+                validate_group_membership_management_setting(
+                    user_group,
+                    setting_name,
+                    new_setting_value,
+                    user_profile.realm,
+                    system_groups_name_dict,
+                )
+
+            do_change_user_group_permission_setting(
+                user_group,
+                setting_name,
+                setting_value_group,
+                old_setting_api_value=current_setting_api_value,
+                acting_user=user_profile,
+            )
+
+    return json_success(request)
+
+
+@typed_endpoint
+@transaction.atomic(durable=True)
+def deactivate_user_group(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    user_group_id: PathOnly[Json[int]],
+) -> HttpResponse:
+    user_group = access_user_group_for_deactivation(user_group_id, user_profile)
+    do_deactivate_user_group(user_group, acting_user=user_profile)
+    return json_success(request)
+
+
+@require_non_guest_user
+@typed_endpoint
+@transaction.atomic(durable=True)
+def update_user_group_backend(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    add: Json[list[int]] | None = None,
+    add_subgroups: Json[list[int]] | None = None,
+    delete: Json[list[int]] | None = None,
+    delete_subgroups: Json[list[int]] | None = None,
+    user_group_id: PathOnly[Json[int]],
+) -> HttpResponse:
+    if not add and not delete and not add_subgroups and not delete_subgroups:
+        raise JsonableError(
+            _(
+                'Nothing to do. Specify at least one of "add", "delete", "add_subgroups" or "delete_subgroups".'
+            )
+        )
+
+    thunks = []
+    if add:
+        thunks.append(
+            lambda: add_members_to_group_backend(
+                request, user_profile, user_group_id=user_group_id, members=add
+            )
+        )
+    if delete:
+        thunks.append(
+            lambda: remove_members_from_group_backend(
+                request, user_profile, user_group_id=user_group_id, members=delete
+            )
+        )
+
+    if add_subgroups:
+        thunks.append(
+            lambda: add_subgroups_to_group_backend(
+                request, user_profile, user_group_id=user_group_id, subgroup_ids=add_subgroups
+            )
+        )
+
+    if delete_subgroups:
+        thunks.append(
+            lambda: remove_subgroups_from_group_backend(
+                request, user_profile, user_group_id=user_group_id, subgroup_ids=delete_subgroups
+            )
+        )
+
+    data = compose_views(thunks)
+
+    return json_success(request, data)
+
+
+def notify_for_user_group_subscription_changes(
+    acting_user: UserProfile,
+    recipient_users: list[UserProfile],
+    user_group: NamedUserGroup,
+    *,
+    send_subscription_message: bool = False,
+    send_unsubscription_message: bool = False,
+) -> None:
+    realm = acting_user.realm
+    mention_backend = MentionBackend(realm.id)
+
+    notifications = []
+    notification_bot = get_system_bot(settings.NOTIFICATION_BOT, realm.id)
+    for recipient_user in recipient_users:
+        if recipient_user.id == acting_user.id:
+            # Don't send notification message if you subscribed/unsubscribed yourself.
+            continue
+        if recipient_user.is_bot:
+            # Don't send notification message to bots.
+            continue
+
+        assert recipient_user.is_active
+
+        with override_language(recipient_user.default_language):
+            if send_subscription_message:
+                message = _("{user_full_name} added you to the group {group_name}.").format(
+                    user_full_name=silent_mention_syntax_for_user(acting_user),
+                    group_name=f"@_*{user_group.name}*",
+                )
+            if send_unsubscription_message:
+                message = _("{user_full_name} removed you from the group {group_name}.").format(
+                    user_full_name=silent_mention_syntax_for_user(acting_user),
+                    group_name=f"@_*{user_group.name}*",
+                )
+
+        notifications.append(
+            internal_prep_private_message(
+                sender=notification_bot,
+                recipient_user=recipient_user,
+                content=message,
+                mention_backend=mention_backend,
+                acting_user=acting_user,
+            )
+        )
+
+    if len(notifications) > 0:
+        do_send_messages(notifications)
+
+
+def add_members_to_group_backend(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    user_group_id: int,
+    members: list[int],
+) -> HttpResponse:
+    if len(members) == 1 and user_profile.id == members[0]:
+        try:
+            user_group = access_user_group_for_update(
+                user_group_id,
+                user_profile,
+                permission_setting="can_join_group",
+                allow_deactivated=True,
+            )
+        except JsonableError:
+            # User can still join the group if user has permission to add
+            # anyone in the group.
+            user_group = access_user_group_for_update(
+                user_group_id,
+                user_profile,
+                permission_setting="can_add_members_group",
+                allow_deactivated=True,
+            )
+    else:
+        user_group = access_user_group_for_update(
+            user_group_id,
+            user_profile,
+            permission_setting="can_add_members_group",
+            allow_deactivated=True,
+        )
+
+    member_users = user_ids_to_users(
+        members, user_profile.realm, allow_deactivated=False, allow_bots=True
+    )
+    existing_member_ids = set(
+        get_direct_memberships_of_users(user_group.usergroup_ptr, member_users)
+    )
+
+    for member_user in member_users:
+        if member_user.id in existing_member_ids:
+            raise JsonableError(
+                _("User {user_id} is already a member of this group").format(
+                    user_id=member_user.id,
+                )
+            )
+
+    member_user_ids = [member_user.id for member_user in member_users]
+    bulk_add_members_to_user_groups([user_group], member_user_ids, acting_user=user_profile)
+    notify_for_user_group_subscription_changes(
+        acting_user=user_profile,
+        recipient_users=member_users,
+        user_group=user_group,
+        send_subscription_message=True,
+    )
+    return json_success(request)
+
+
+def remove_members_from_group_backend(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    user_group_id: int,
+    members: list[int],
+) -> HttpResponse:
+    user_profiles = user_ids_to_users(
+        members, user_profile.realm, allow_deactivated=False, allow_bots=True
+    )
+    if len(members) == 1 and user_profile.id == members[0]:
+        try:
+            user_group = access_user_group_for_update(
+                user_group_id,
+                user_profile,
+                permission_setting="can_leave_group",
+                allow_deactivated=True,
+            )
+        except JsonableError:
+            # User can still leave the group if user has permission to remove
+            # anyone from the group.
+            user_group = access_user_group_for_update(
+                user_group_id,
+                user_profile,
+                permission_setting="can_remove_members_group",
+                allow_deactivated=True,
+            )
+    else:
+        user_group = access_user_group_for_update(
+            user_group_id,
+            user_profile,
+            permission_setting="can_remove_members_group",
+            allow_deactivated=True,
+        )
+
+    group_member_ids = get_user_group_direct_member_ids(user_group)
+    for member in members:
+        if member not in group_member_ids:
+            raise JsonableError(
+                _("There is no member '{user_id}' in this user group").format(user_id=member)
+            )
+
+    user_profile_ids = [user.id for user in user_profiles]
+    bulk_remove_members_from_user_groups([user_group], user_profile_ids, acting_user=user_profile)
+    notify_for_user_group_subscription_changes(
+        acting_user=user_profile,
+        recipient_users=user_profiles,
+        user_group=user_group,
+        send_unsubscription_message=True,
+    )
+    return json_success(request)
+
+
+def add_subgroups_to_group_backend(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    user_group_id: int,
+    subgroup_ids: list[int],
+) -> HttpResponse:
+    with lock_subgroups_with_respect_to_supergroup(
+        subgroup_ids, user_group_id, user_profile, permission_setting="can_add_members_group"
+    ) as context:
+        existing_direct_subgroup_ids = context.supergroup.direct_subgroups.all().values_list(
+            "id", flat=True
+        )
+        for group in context.direct_subgroups:
+            if group.id in existing_direct_subgroup_ids:
+                raise JsonableError(
+                    _("User group {group_id} is already a subgroup of this group.").format(
+                        group_id=group.id
+                    )
+                )
+
+        recursive_subgroup_ids = {
+            recursive_subgroup.id for recursive_subgroup in context.recursive_subgroups
+        }
+        if user_group_id in recursive_subgroup_ids:
+            raise JsonableError(
+                _(
+                    "User group {user_group_id} is already a subgroup of one of the passed subgroups."
+                ).format(user_group_id=user_group_id)
+            )
+
+        realm = user_profile.realm
+        if user_group_id == realm.workplace_users_group_id or user_group_id in get_subgroup_ids(
+            realm.workplace_users_group
+        ):
+            # If the user group being updated is used for workplace_users_group, we need
+            # to make sure that only admins have permission to update members for all its
+            # recursive subgroups.
+            non_system_subgroups = [
+                group for group in context.recursive_subgroups if not group.is_system_group
+            ]
+            system_groups_name_dict = get_role_based_system_groups_dict(realm)
+            if not check_group_membership_management_permissions_with_admins_only(
+                non_system_subgroups, realm, system_groups_name_dict
+            ):
+                raise JsonableError(
+                    _(
+                        "Subgroups of a group used for 'workplace_users_group' must allow only organization administrators to manage their membership."
+                    )
+                )
+
+        add_subgroups_to_user_group(
+            context.supergroup, context.direct_subgroups, acting_user=user_profile
+        )
+    return json_success(request)
+
+
+def remove_subgroups_from_group_backend(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    user_group_id: int,
+    subgroup_ids: list[int],
+) -> HttpResponse:
+    with lock_subgroups_with_respect_to_supergroup(
+        subgroup_ids, user_group_id, user_profile, permission_setting="can_manage_group"
+    ) as context:
+        # While the recursive subgroups in the context are not used, it is important that
+        # we acquire a lock for these rows while updating the subgroups to acquire the locks
+        # in a consistent order for subgroup membership changes.
+        existing_direct_subgroup_ids = context.supergroup.direct_subgroups.all().values_list(
+            "id", flat=True
+        )
+        for group in context.direct_subgroups:
+            if group.id not in existing_direct_subgroup_ids:
+                raise JsonableError(
+                    _("User group {group_id} is not a subgroup of this group.").format(
+                        group_id=group.id
+                    )
+                )
+
+        remove_subgroups_from_user_group(
+            context.supergroup, context.direct_subgroups, acting_user=user_profile
+        )
+
+    return json_success(request)
+
+
+@require_non_guest_user
+@typed_endpoint
+@transaction.atomic(durable=True)
+def update_subgroups_of_user_group(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    add: Json[list[int]] | None = None,
+    delete: Json[list[int]] | None = None,
+    user_group_id: PathOnly[Json[int]],
+) -> HttpResponse:
+    if not add and not delete:
+        raise JsonableError(_('Nothing to do. Specify at least one of "add" or "delete".'))
+
+    thunks = []
+    if add:
+        thunks.append(
+            lambda: add_subgroups_to_group_backend(
+                request, user_profile, user_group_id=user_group_id, subgroup_ids=add
+            )
+        )
+    if delete:
+        thunks.append(
+            lambda: remove_subgroups_from_group_backend(
+                request, user_profile, user_group_id=user_group_id, subgroup_ids=delete
+            )
+        )
+
+    data = compose_views(thunks)
+
+    return json_success(request, data)
+
+
+@require_non_guest_user
+@typed_endpoint
+def get_is_user_group_member(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    direct_member_only: Json[bool] = False,
+    user_group_id: PathOnly[Json[int]],
+    user_id: PathOnly[Json[int]],
+) -> HttpResponse:
+    user_group = access_user_group_to_read_membership(user_group_id, user_profile.realm)
+    target_user = access_user_by_id(user_profile, user_id, allow_bots=True, for_admin=False)
+
+    return json_success(
+        request,
+        data={
+            "is_user_group_member": is_user_in_group(
+                user_group.id, target_user, direct_member_only=direct_member_only
+            )
+        },
+    )
+
+
+@require_non_guest_user
+@typed_endpoint
+def get_user_group_members(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    direct_member_only: Json[bool] = False,
+    user_group_id: PathOnly[Json[int]],
+) -> HttpResponse:
+    user_group = access_user_group_to_read_membership(user_group_id, user_profile.realm)
+
+    return json_success(
+        request,
+        data={
+            "members": get_user_group_member_ids(user_group, direct_member_only=direct_member_only)
+        },
+    )
+
+
+@require_non_guest_user
+@typed_endpoint
+def get_subgroups_of_user_group(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    direct_subgroup_only: Json[bool] = False,
+    user_group_id: PathOnly[Json[int]],
+) -> HttpResponse:
+    user_group = access_user_group_to_read_membership(user_group_id, user_profile.realm)
+
+    return json_success(
+        request,
+        data={"subgroups": get_subgroup_ids(user_group, direct_subgroup_only=direct_subgroup_only)},
+    )

zerver/views/user_settings.py

@@ -0,0 +1,599 @@
+from email.headerregistry import Address
+from typing import Annotated, Any
+
+from django.conf import settings
+from django.contrib.auth import authenticate, update_session_auth_hash
+from django.contrib.auth.tokens import default_token_generator
+from django.core.exceptions import ValidationError
+from django.core.files.uploadedfile import UploadedFile
+from django.db import transaction
+from django.db.models import F
+from django.http import HttpRequest, HttpResponse, HttpResponseRedirect
+from django.shortcuts import render
+from django.urls import reverse
+from django.utils.html import escape
+from django.utils.safestring import SafeString
+from django.utils.translation import gettext as _
+from django.utils.translation import gettext_lazy
+from pydantic import BaseModel, Json, model_validator
+from pydantic.functional_validators import AfterValidator
+
+from confirmation.models import (
+    Confirmation,
+    ConfirmationKeyError,
+    get_object_from_key,
+    render_confirmation_key_error,
+)
+from zerver.actions.user_settings import (
+    bulk_change_user_setting,
+    check_change_full_name,
+    do_change_avatar_fields,
+    do_change_password,
+    do_change_user_delivery_email,
+    do_change_user_setting,
+    do_regenerate_api_key,
+    do_start_email_change_process,
+    set_avatar_to_default,
+)
+from zerver.actions.users import generate_password_reset_url
+from zerver.decorator import human_users_only, require_post
+from zerver.lib.avatar import avatar_url
+from zerver.lib.email_notifications import enqueue_welcome_emails
+from zerver.lib.email_validation import (
+    get_realm_email_validator,
+    validate_email_is_valid,
+    validate_email_not_already_in_realm,
+)
+from zerver.lib.exceptions import (
+    JsonableError,
+    OrganizationAdministratorRequiredError,
+    RateLimitedError,
+    UserDeactivatedError,
+)
+from zerver.lib.i18n import get_available_language_codes
+from zerver.lib.rate_limiter import (
+    RateLimitedUser,
+    readable_expiry_string_for_plaintext,
+    should_rate_limit,
+)
+from zerver.lib.response import json_success
+from zerver.lib.send_email import FromAddress, send_email
+from zerver.lib.sounds import get_available_notification_sounds
+from zerver.lib.typed_endpoint import typed_endpoint, typed_endpoint_without_parameters
+from zerver.lib.typed_endpoint_validators import (
+    check_int_in_validator,
+    check_string_in_validator,
+    parse_enum_from_string_value,
+    timezone_validator,
+)
+from zerver.lib.upload import get_file_info, upload_avatar_image
+from zerver.lib.user_groups import (
+    get_recursive_group_members_union_for_groups,
+    user_group_ids_to_user_groups,
+)
+from zerver.lib.users import user_ids_to_users
+from zerver.models import EmailChangeStatus, RealmAuditLog, UserBaseSettings, UserProfile
+from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.models.realms import avatar_changes_disabled, name_changes_disabled
+from zerver.models.users import ResolvedTopicNoticeAutoReadPolicyEnum
+from zerver.views.auth import redirect_to_deactivation_notice
+from zproject.backends import check_password_strength, email_belongs_to_ldap
+
+AVATAR_CHANGES_DISABLED_ERROR = gettext_lazy("Avatar changes are disabled in this organization.")
+
+
+def validate_email_change_request(user_profile: UserProfile, new_email: str) -> None:
+    if not user_profile.is_active:
+        raise UserDeactivatedError
+
+    if user_profile.realm.email_changes_disabled and not user_profile.is_realm_admin:
+        raise JsonableError(_("Email address changes are disabled in this organization."))
+
+    error = validate_email_is_valid(
+        new_email,
+        get_realm_email_validator(user_profile.realm),
+    )
+    if error:
+        raise JsonableError(error)
+
+    try:
+        validate_email_not_already_in_realm(
+            user_profile.realm,
+            new_email,
+            allow_inactive_mirror_dummies=False,
+            verbose=False,
+        )
+    except ValidationError as e:
+        raise JsonableError(e.message)
+
+
+def confirm_email_change_get(request: HttpRequest, confirmation_key: str) -> HttpResponse:
+    try:
+        get_object_from_key(confirmation_key, [Confirmation.EMAIL_CHANGE], mark_object_used=False)
+    except ConfirmationKeyError as exception:  # nocoverage
+        return render_confirmation_key_error(request, exception)
+
+    return render(
+        request,
+        "confirmation/redirect_to_post.html",
+        context={
+            "target_url": reverse("confirm_email_change"),
+            "key": confirmation_key,
+        },
+    )
+
+
+@require_post
+@typed_endpoint
+def confirm_email_change(request: HttpRequest, *, key: str) -> HttpResponse:
+    with transaction.atomic(durable=True):
+        try:
+            email_change_object = get_object_from_key(
+                key, [Confirmation.EMAIL_CHANGE], mark_object_used=True
+            )
+        except ConfirmationKeyError as exception:
+            return render_confirmation_key_error(request, exception)
+
+        assert isinstance(email_change_object, EmailChangeStatus)
+        new_email = email_change_object.new_email
+        old_email = email_change_object.old_email
+
+        user_profile = UserProfile.objects.select_for_update(no_key=True).get(
+            id=email_change_object.user_profile_id
+        )
+
+        if user_profile.delivery_email != old_email:
+            # This is not expected to be possible, since we deactivate
+            # any previous email changes when we create a new one, but
+            # double-check.
+            return render_confirmation_key_error(
+                request, ConfirmationKeyError(ConfirmationKeyError.EXPIRED)
+            )  # nocoverage
+
+        if user_profile.realm.deactivated:
+            return redirect_to_deactivation_notice()
+        try:
+            validate_email_change_request(user_profile, new_email)
+        except UserDeactivatedError:
+            context = {"realm_url": user_profile.realm.url}
+            return render(
+                request,
+                "zerver/portico_error_pages/user_deactivated.html",
+                context=context,
+                status=401,
+            )
+        do_change_user_delivery_email(user_profile, new_email, acting_user=user_profile)
+
+    user_profile = UserProfile.objects.get(id=email_change_object.user_profile_id)
+    context = {"realm_name": user_profile.realm.name, "new_email": new_email}
+    language = user_profile.default_language
+
+    if old_email == "":
+        # The assertions here are to help document the only circumstance under which
+        # this condition should be possible.
+        assert (
+            user_profile.realm.demo_organization_scheduled_deletion_date is not None
+            and user_profile.is_realm_owner
+        )
+        # Schedule onboarding emails for demo organization owner now that we have an
+        # email address for their account.
+        enqueue_welcome_emails(user_profile, demo_organization_creator=True)
+        # Because demo organizations are created without setting an email and password
+        # we want to redirect to setting a password after configuring and confirming
+        # an email for the owner's account.
+        reset_password_url = generate_password_reset_url(user_profile, default_token_generator)
+        return HttpResponseRedirect(reset_password_url)
+
+    send_email(
+        "zerver/emails/notify_change_in_email",
+        to_emails=[old_email],
+        from_name=FromAddress.security_email_from_name(user_profile=user_profile),
+        from_address=FromAddress.SUPPORT,
+        language=language,
+        context=context,
+        realm=user_profile.realm,
+    )
+    old_email_address = Address(addr_spec=old_email)
+    new_email_address = Address(addr_spec=new_email)
+    ctx = {
+        "new_email_html_tag": SafeString(
+            f'<a href="mailto:{escape(new_email)}">{escape(new_email_address.username)}@<wbr>{escape(new_email_address.domain)}</wbr></a>'
+        ),
+        "old_email_html_tag": SafeString(
+            f'<a href="mailto:{escape(old_email)}">{escape(old_email_address.username)}@<wbr>{escape(old_email_address.domain)}</wbr></a>'
+        ),
+    }
+    return render(request, "confirmation/confirm_email_change.html", context=ctx)
+
+
+emojiset_choices = {emojiset["key"] for emojiset in UserProfile.emojiset_choices()}
+web_home_view_options = ["recent", "inbox", "all_messages"]
+web_animate_image_previews_options = ["always", "on_hover", "never"]
+
+
+def check_settings_values(
+    notification_sound: str | None,
+    email_notifications_batching_period_seconds: int | None,
+    default_language: str | None = None,
+) -> None:
+    # We can't use typed_endpoint for this widget because
+    # get_available_language_codes requires provisioning to be
+    # complete.
+    if default_language is not None and default_language not in get_available_language_codes():
+        raise JsonableError(_("Invalid default_language"))
+
+    if (
+        notification_sound is not None
+        and notification_sound not in get_available_notification_sounds()
+        and notification_sound != "none"
+    ):
+        raise JsonableError(
+            _("Invalid notification sound '{notification_sound}'").format(
+                notification_sound=notification_sound
+            )
+        )
+
+    if email_notifications_batching_period_seconds is not None and (
+        email_notifications_batching_period_seconds <= 0
+        or email_notifications_batching_period_seconds > 7 * 24 * 60 * 60
+    ):
+        # We set a limit of one week for the batching period
+        raise JsonableError(
+            _("Invalid email batching period: {seconds} seconds").format(
+                seconds=email_notifications_batching_period_seconds
+            )
+        )
+
+
+class TargetUsersData(BaseModel):
+    user_ids: list[int] = []
+    group_ids: list[int] = []
+    skip_if_already_edited: bool
+
+    @model_validator(mode="after")
+    def validate_terms(self) -> "TargetUsersData":
+        if not self.user_ids and not self.group_ids:
+            raise JsonableError(_("Either user_ids or group_ids must be provided."))
+        return self
+
+
+@human_users_only
+@typed_endpoint
+def json_change_settings(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    allow_private_data_export: Json[bool] | None = None,
+    automatically_follow_topics_policy: Annotated[
+        Json[int],
+        check_int_in_validator(UserProfile.AUTOMATICALLY_CHANGE_VISIBILITY_POLICY_CHOICES),
+    ]
+    | None = None,
+    automatically_follow_topics_where_mentioned: Json[bool] | None = None,
+    automatically_unmute_topics_in_muted_streams_policy: Annotated[
+        Json[int],
+        check_int_in_validator(UserProfile.AUTOMATICALLY_CHANGE_VISIBILITY_POLICY_CHOICES),
+    ]
+    | None = None,
+    color_scheme: Annotated[Json[int], check_int_in_validator(UserProfile.COLOR_SCHEME_CHOICES)]
+    | None = None,
+    default_language: str | None = None,
+    demote_inactive_streams: Annotated[
+        Json[int], check_int_in_validator(UserProfile.DEMOTE_STREAMS_CHOICES)
+    ]
+    | None = None,
+    desktop_icon_count_display: Annotated[
+        Json[int], check_int_in_validator(UserProfile.DESKTOP_ICON_COUNT_DISPLAY_CHOICES)
+    ]
+    | None = None,
+    display_emoji_reaction_users: Json[bool] | None = None,
+    email: str | None = None,
+    email_address_visibility: Annotated[
+        Json[int], check_int_in_validator(UserProfile.EMAIL_ADDRESS_VISIBILITY_TYPES)
+    ]
+    | None = None,
+    email_notifications_batching_period_seconds: Json[int] | None = None,
+    emojiset: Annotated[str, check_string_in_validator(emojiset_choices)] | None = None,
+    enable_desktop_notifications: Json[bool] | None = None,
+    enable_digest_emails: Json[bool] | None = None,
+    enable_drafts_synchronization: Json[bool] | None = None,
+    enable_followed_topic_audible_notifications: Json[bool] | None = None,
+    enable_followed_topic_desktop_notifications: Json[bool] | None = None,
+    enable_followed_topic_email_notifications: Json[bool] | None = None,
+    enable_followed_topic_push_notifications: Json[bool] | None = None,
+    enable_followed_topic_wildcard_mentions_notify: Json[bool] | None = None,
+    enable_login_emails: Json[bool] | None = None,
+    enable_marketing_emails: Json[bool] | None = None,
+    enable_offline_email_notifications: Json[bool] | None = None,
+    enable_offline_push_notifications: Json[bool] | None = None,
+    enable_online_push_notifications: Json[bool] | None = None,
+    enable_sounds: Json[bool] | None = None,
+    enable_stream_audible_notifications: Json[bool] | None = None,
+    enable_stream_desktop_notifications: Json[bool] | None = None,
+    enable_stream_email_notifications: Json[bool] | None = None,
+    enable_stream_push_notifications: Json[bool] | None = None,
+    enter_sends: Json[bool] | None = None,
+    fluid_layout_width: Json[bool] | None = None,
+    full_name: str | None = None,
+    high_contrast_mode: Json[bool] | None = None,
+    hide_ai_features: Json[bool] | None = None,
+    left_side_userlist: Json[bool] | None = None,
+    message_content_in_email_notifications: Json[bool] | None = None,
+    new_password: str | None = None,
+    notification_sound: str | None = None,
+    old_password: str | None = None,
+    pm_content_in_desktop_notifications: Json[bool] | None = None,
+    presence_enabled: Json[bool] | None = None,
+    realm_name_in_email_notifications_policy: Annotated[
+        Json[int],
+        check_int_in_validator(UserProfile.REALM_NAME_IN_EMAIL_NOTIFICATIONS_POLICY_CHOICES),
+    ]
+    | None = None,
+    receives_typing_notifications: Json[bool] | None = None,
+    resolved_topic_notice_auto_read_policy: Annotated[
+        str | None,
+        AfterValidator(
+            lambda val: parse_enum_from_string_value(
+                val,
+                "resolved_topic_notice_auto_read_policy",
+                ResolvedTopicNoticeAutoReadPolicyEnum,
+            )
+        ),
+    ] = None,
+    send_private_typing_notifications: Json[bool] | None = None,
+    send_read_receipts: Json[bool] | None = None,
+    send_stream_typing_notifications: Json[bool] | None = None,
+    starred_message_counts: Json[bool] | None = None,
+    target_users: Json[TargetUsersData] | None = None,
+    timezone: Annotated[str, timezone_validator()] | None = None,
+    translate_emoticons: Json[bool] | None = None,
+    twenty_four_hour_time: Json[bool] | None = None,
+    user_list_style: Annotated[
+        Json[int], check_int_in_validator(UserProfile.USER_LIST_STYLE_CHOICES)
+    ]
+    | None = None,
+    web_animate_image_previews: Annotated[
+        str, check_string_in_validator(web_animate_image_previews_options)
+    ]
+    | None = None,
+    web_channel_default_view: Annotated[
+        Json[int], check_int_in_validator(UserProfile.WEB_CHANNEL_DEFAULT_VIEW_CHOICES)
+    ]
+    | None = None,
+    web_escape_navigates_to_home_view: Json[bool] | None = None,
+    web_font_size_px: Json[int] | None = None,
+    web_home_view: Annotated[str, check_string_in_validator(web_home_view_options)] | None = None,
+    web_inbox_show_channel_folders: Json[bool] | None = None,
+    web_left_sidebar_show_channel_folders: Json[bool] | None = None,
+    web_left_sidebar_unreads_count_summary: Json[bool] | None = None,
+    web_line_height_percent: Json[int] | None = None,
+    web_mark_read_on_scroll_policy: Annotated[
+        Json[int], check_int_in_validator(UserProfile.WEB_MARK_READ_ON_SCROLL_POLICY_CHOICES)
+    ]
+    | None = None,
+    web_navigate_to_sent_message: Json[bool] | None = None,
+    web_stream_unreads_count_display_policy: Annotated[
+        Json[int],
+        check_int_in_validator(UserProfile.WEB_STREAM_UNREADS_COUNT_DISPLAY_POLICY_CHOICES),
+    ]
+    | None = None,
+    web_suggest_update_timezone: Json[bool] | None = None,
+    wildcard_mentions_notify: Json[bool] | None = None,
+) -> HttpResponse:
+    # UserProfile object is being refetched here to make sure that we
+    # do not use stale object from cache which can happen when a
+    # previous request tried updating multiple settings in a single
+    # request.
+    #
+    # TODO: Change the cache flushing strategy to make sure cache
+    # does not contain stale objects.
+    user_profile = UserProfile.objects.get(id=user_profile.id)
+
+    # Loop over user_profile.property_types
+    request_settings = {
+        setting_name: requested_value
+        for setting_name, requested_value in locals().items()
+        if setting_name in user_profile.property_types and requested_value is not None
+    }
+
+    if target_users is not None:
+        if not user_profile.is_realm_admin:
+            raise OrganizationAdministratorRequiredError
+
+        # "email" and "password" are not user settings but users still update them
+        # using this endpoint.
+        if email is not None or new_password is not None or timezone is not None:
+            raise JsonableError(_("Cannot change this setting for other users."))
+
+        for setting_name in request_settings:
+            if setting_name in UserBaseSettings.SECURITY_SENSITIVE_USER_SETTINGS:
+                raise JsonableError(_("Cannot change this setting for other users."))
+
+        user_ids = target_users.user_ids
+        group_member_ids = set()
+
+        if target_users.group_ids:
+            valid_groups = user_group_ids_to_user_groups(target_users.group_ids, user_profile.realm)
+            valid_group_ids = [group.id for group in valid_groups]
+            group_member_ids = set(
+                get_recursive_group_members_union_for_groups(valid_group_ids).values_list(
+                    "id", flat=True
+                )
+            )
+
+        all_user_ids = list(set(user_ids) | group_member_ids)
+
+        # User settings do not apply to bots, so exclude them from the query even if they
+        # are in a member of a targeted group or are included in user_ids list. Deactivated
+        # users are included, so that the right thing happens if they are later reactivated.
+        users = user_ids_to_users(
+            all_user_ids, user_profile.realm, allow_deactivated=True, allow_bots=False
+        )
+
+        for setting_name, requested_value in request_settings.items():
+            if target_users.skip_if_already_edited:
+                already_edited_user_ids = set(
+                    RealmAuditLog.objects.filter(
+                        modified_user__in=users,
+                        event_type=AuditLogEventType.USER_SETTING_CHANGED,
+                        extra_data__property=setting_name,
+                        acting_user=F("modified_user"),
+                    ).values_list("modified_user_id", flat=True)
+                )
+
+                eligible_users = [user for user in users if user.id not in already_edited_user_ids]
+            else:
+                eligible_users = users
+
+            users_to_update = [
+                user for user in eligible_users if getattr(user, setting_name) != requested_value
+            ]
+
+            if users_to_update:
+                bulk_change_user_setting(
+                    user_profile.realm,
+                    users_to_update,
+                    setting_name,
+                    requested_value,
+                    acting_user=user_profile,
+                )
+
+        return json_success(request, data={})
+
+    if (
+        default_language is not None
+        or notification_sound is not None
+        or email_notifications_batching_period_seconds is not None
+    ):
+        check_settings_values(
+            notification_sound, email_notifications_batching_period_seconds, default_language
+        )
+
+    if new_password is not None:
+        return_data: dict[str, Any] = {}
+        if email_belongs_to_ldap(user_profile.realm, user_profile.delivery_email):
+            raise JsonableError(_("Your Zulip password is managed in LDAP"))
+
+        try:
+            if not authenticate(
+                request,
+                username=user_profile.delivery_email,
+                password=old_password,
+                realm=user_profile.realm,
+                return_data=return_data,
+            ):
+                raise JsonableError(_("Wrong password!"))
+        except RateLimitedError as e:
+            assert e.secs_to_freedom is not None
+            secs_to_freedom = int(e.secs_to_freedom)
+            retry_after_string = readable_expiry_string_for_plaintext(secs_to_freedom)
+            raise JsonableError(
+                _("You're making too many attempts! Try again in {retry_after_string}.").format(
+                    retry_after_string=retry_after_string
+                ),
+            )
+
+        if not check_password_strength(new_password):
+            raise JsonableError(_("New password is too weak!"))
+
+        do_change_password(user_profile, new_password)
+        # Password changes invalidates sessions, see
+        # https://docs.djangoproject.com/en/5.0/topics/auth/default/#session-invalidation-on-password-change
+        # for details. To avoid this logging the user out of their own
+        # session (which would provide a confusing UX at best), we
+        # update the session hash here.
+        update_session_auth_hash(request, user_profile)
+        # We also save the session to the DB immediately to mitigate
+        # race conditions. In theory, there is still a race condition
+        # and to completely avoid it we will have to use some kind of
+        # mutex lock in `django.contrib.auth.get_user` where session
+        # is verified. To make that lock work we will have to control
+        # the AuthenticationMiddleware which is currently controlled
+        # by Django,
+        request.session.save()
+
+    result: dict[str, Any] = {}
+
+    if email is not None:
+        new_email = email.strip()
+        if user_profile.delivery_email != new_email:
+            validate_email_change_request(user_profile, new_email)
+
+            if should_rate_limit(request):
+                ratelimited, time_until_free = RateLimitedUser(
+                    user_profile, domain="email_change_by_user"
+                ).rate_limit()
+                if ratelimited:
+                    raise RateLimitedError(time_until_free)
+
+            do_start_email_change_process(user_profile, new_email)
+
+    if full_name is not None and user_profile.full_name != full_name:
+        if name_changes_disabled(user_profile.realm) and not user_profile.is_realm_admin:
+            # Failingly silently is fine -- they can't do it through the UI, so
+            # they'd have to be trying to break the rules.
+            pass
+        else:
+            # Note that check_change_full_name strips the passed name automatically
+            check_change_full_name(user_profile, full_name, user_profile)
+
+    for k, v in request_settings.items():
+        if getattr(user_profile, k) != v:
+            do_change_user_setting(user_profile, k, v, acting_user=user_profile)
+
+    if timezone is not None and user_profile.timezone != timezone:
+        do_change_user_setting(user_profile, "timezone", timezone, acting_user=user_profile)
+
+    return json_success(request, data=result)
+
+
+def set_avatar_backend(request: HttpRequest, user_profile: UserProfile) -> HttpResponse:
+    if len(request.FILES) != 1:
+        raise JsonableError(_("You must upload exactly one avatar."))
+
+    if avatar_changes_disabled(user_profile.realm) and not user_profile.is_realm_admin:
+        raise JsonableError(str(AVATAR_CHANGES_DISABLED_ERROR))
+
+    [user_file] = request.FILES.values()
+    assert isinstance(user_file, UploadedFile)
+    assert user_file.size is not None
+    if user_file.size > settings.MAX_AVATAR_FILE_SIZE_MIB * 1024 * 1024:
+        raise JsonableError(
+            _("Uploaded file is larger than the allowed limit of {max_size} MiB").format(
+                max_size=settings.MAX_AVATAR_FILE_SIZE_MIB,
+            )
+        )
+    _filename, content_type = get_file_info(user_file)
+    upload_avatar_image(user_file, user_profile, content_type=content_type)
+    do_change_avatar_fields(user_profile, UserProfile.AVATAR_FROM_USER, acting_user=user_profile)
+    user_avatar_url = avatar_url(user_profile)
+
+    json_result = dict(
+        avatar_url=user_avatar_url,
+    )
+    return json_success(request, data=json_result)
+
+
+def delete_avatar_backend(request: HttpRequest, user_profile: UserProfile) -> HttpResponse:
+    if avatar_changes_disabled(user_profile.realm) and not user_profile.is_realm_admin:
+        raise JsonableError(str(AVATAR_CHANGES_DISABLED_ERROR))
+
+    if user_profile.avatar_source == UserProfile.AVATAR_FROM_USER:
+        set_avatar_to_default(user_profile, acting_user=user_profile)
+
+    default_avatar_url = avatar_url(user_profile)
+    json_result = dict(
+        avatar_url=default_avatar_url,
+    )
+    return json_success(request, data=json_result)
+
+
+# We don't use @human_users_only here, because there are use cases for
+# a bot regenerating its own API key.
+@typed_endpoint_without_parameters
+def regenerate_api_key(request: HttpRequest, user_profile: UserProfile) -> HttpResponse:
+    new_api_key = do_regenerate_api_key(user_profile, user_profile)
+    json_result = dict(
+        api_key=new_api_key,
+    )
+    return json_success(request, data=json_result)

zerver/views/user_topics.py

@@ -0,0 +1,115 @@
+from datetime import datetime
+from typing import Annotated, Literal
+
+from django.http import HttpRequest, HttpResponse
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+from pydantic import Json, StringConstraints
+
+from zerver.actions.user_topics import do_set_user_topic_visibility_policy
+from zerver.lib.response import json_success
+from zerver.lib.streams import (
+    access_stream_by_id,
+    access_stream_by_name,
+    access_stream_to_remove_visibility_policy_by_id,
+    access_stream_to_remove_visibility_policy_by_name,
+    check_for_exactly_one_stream_arg,
+)
+from zerver.lib.typed_endpoint import typed_endpoint
+from zerver.lib.typed_endpoint_validators import check_int_in_validator
+from zerver.models import UserProfile, UserTopic
+from zerver.models.constants import MAX_TOPIC_NAME_LENGTH
+
+
+def mute_topic(
+    user_profile: UserProfile,
+    stream_id: int | None,
+    stream_name: str | None,
+    topic_name: str,
+    date_muted: datetime,
+) -> None:
+    if stream_name is not None:
+        (stream, _sub) = access_stream_by_name(user_profile, stream_name)
+    else:
+        assert stream_id is not None
+        (stream, _sub) = access_stream_by_id(user_profile, stream_id)
+
+    do_set_user_topic_visibility_policy(
+        user_profile,
+        stream,
+        topic_name,
+        visibility_policy=UserTopic.VisibilityPolicy.MUTED,
+        last_updated=date_muted,
+    )
+
+
+def unmute_topic(
+    user_profile: UserProfile,
+    stream_id: int | None,
+    stream_name: str | None,
+    topic_name: str,
+) -> None:
+    error = _("Topic is not muted")
+
+    if stream_name is not None:
+        stream = access_stream_to_remove_visibility_policy_by_name(user_profile, stream_name, error)
+    else:
+        assert stream_id is not None
+        stream = access_stream_to_remove_visibility_policy_by_id(user_profile, stream_id, error)
+
+    do_set_user_topic_visibility_policy(
+        user_profile, stream, topic_name, visibility_policy=UserTopic.VisibilityPolicy.INHERIT
+    )
+
+
+@typed_endpoint
+def update_muted_topic(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    op: Literal["add", "remove"],
+    topic: Annotated[str, StringConstraints(max_length=MAX_TOPIC_NAME_LENGTH)],
+    stream: str | None = None,
+    stream_id: Json[int] | None = None,
+) -> HttpResponse:
+    check_for_exactly_one_stream_arg(stream_id=stream_id, stream=stream)
+
+    if op == "add":
+        mute_topic(
+            user_profile=user_profile,
+            stream_id=stream_id,
+            stream_name=stream,
+            topic_name=topic,
+            date_muted=timezone_now(),
+        )
+    elif op == "remove":
+        unmute_topic(
+            user_profile=user_profile,
+            stream_id=stream_id,
+            stream_name=stream,
+            topic_name=topic,
+        )
+    return json_success(request)
+
+
+@typed_endpoint
+def update_user_topic(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    stream_id: Json[int],
+    topic: Annotated[str, StringConstraints(max_length=MAX_TOPIC_NAME_LENGTH)],
+    visibility_policy: Json[
+        Annotated[int, check_int_in_validator(UserTopic.VisibilityPolicy.values)]
+    ],
+) -> HttpResponse:
+    if visibility_policy == UserTopic.VisibilityPolicy.INHERIT:
+        error = _("Invalid channel ID")
+        stream = access_stream_to_remove_visibility_policy_by_id(user_profile, stream_id, error)
+    else:
+        (stream, _sub) = access_stream_by_id(user_profile, stream_id)
+
+    do_set_user_topic_visibility_policy(
+        user_profile, stream, topic, visibility_policy=visibility_policy
+    )
+    return json_success(request)

zerver/views/video_calls.py

@@ -0,0 +1,765 @@
+import datetime
+import hashlib
+import json
+import logging
+import random
+from abc import ABC, abstractmethod
+from base64 import b64encode
+from typing import Any, Literal
+from urllib.parse import quote, urlencode, urljoin, urlsplit
+
+import requests
+from defusedxml import ElementTree
+from django.conf import settings
+from django.core.signing import Signer
+from django.http import HttpRequest, HttpResponse
+from django.middleware import csrf
+from django.shortcuts import redirect, render
+from django.utils.crypto import constant_time_compare, salted_hmac
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+from django.views.decorators.cache import never_cache
+from django.views.decorators.csrf import csrf_exempt
+from django.views.decorators.http import require_POST
+from oauthlib.oauth2 import OAuth2Error
+from pydantic import Json
+from requests import Response
+from requests_oauthlib import OAuth2Session
+from typing_extensions import TypedDict, override
+
+from zerver.actions.video_calls import do_set_video_call_provider_token
+from zerver.decorator import zulip_login_required
+from zerver.lib.cache import (
+    cache_with_key,
+    flush_zoom_server_access_token_cache,
+    zoom_server_access_token_cache_key,
+)
+from zerver.lib.exceptions import ErrorCode, JsonableError
+from zerver.lib.message import truncate_content
+from zerver.lib.outgoing_http import OutgoingSession
+from zerver.lib.partial import partial
+from zerver.lib.pysa import mark_sanitized
+from zerver.lib.response import json_success
+from zerver.lib.subdomains import get_subdomain
+from zerver.lib.typed_endpoint import typed_endpoint, typed_endpoint_without_parameters
+from zerver.lib.url_encoding import append_url_query_string
+from zerver.lib.utils import assert_is_not_none
+from zerver.models import UserProfile
+from zerver.models.realms import get_realm
+
+
+class VideoCallSession(OutgoingSession):
+    def __init__(self) -> None:
+        super().__init__(role="video_calls", timeout=5)
+
+
+class InvalidVideoCallProviderTokenError(JsonableError):
+    code = ErrorCode.INVALID_VIDEO_CALL_PROVIDER_TOKEN
+
+    def __init__(self, provider_name: str) -> None:
+        super().__init__(
+            _("Invalid {provider_name} access token").format(provider_name=provider_name)
+        )
+
+
+class CreateVideoCallFailedError(JsonableError):
+    def __init__(self, provider_name: str) -> None:
+        super().__init__(
+            _("Failed to create {provider_name} call").format(provider_name=provider_name)
+        )
+
+
+class VideoCallProviderNotConfiguredError(JsonableError):
+    def __init__(self, provider_name: str) -> None:
+        super().__init__(
+            _("{provider_name} credentials have not been configured").format(
+                provider_name=provider_name
+            )
+        )
+
+
+class VideoCallProviderCredentialsError(JsonableError):
+    def __init__(self, provider_name: str) -> None:
+        super().__init__(
+            _("Invalid {provider_name} credentials").format(provider_name=provider_name)
+        )
+
+
+class VideoCallServerConnectionError(JsonableError):
+    def __init__(self, provider_name: str, reason: str) -> None:
+        super().__init__(
+            _("Error connecting to the {provider_name} server: {reason}").format(
+                provider_name=provider_name, reason=reason
+            )
+        )
+
+
+class VideoCallServerAuthError(JsonableError):
+    def __init__(self, provider_name: str) -> None:
+        super().__init__(
+            _("Error authenticating to the {provider_name} server").format(
+                provider_name=provider_name
+            )
+        )
+
+
+class VideoCallServerStatusError(JsonableError):
+    def __init__(self, provider_name: str, status: str | None) -> None:
+        super().__init__(
+            _("{provider_name} server returned an unexpected error: {status}").format(
+                provider_name=provider_name, status=status
+            )
+        )
+
+
+class VideoCallProviderSessionIdError(JsonableError):
+    def __init__(self, provider_name: str) -> None:
+        super().__init__(
+            _("Invalid {provider_name} session identifier").format(provider_name=provider_name)
+        )
+
+
+class UnknownZoomUserError(JsonableError):
+    code = ErrorCode.UNKNOWN_ZOOM_USER
+
+    def __init__(self) -> None:
+        super().__init__(_("Unknown Zoom user email"))
+
+
+class ConstructorGroupsService:
+    def __init__(self) -> None:
+        if (
+            (url := settings.CONSTRUCTOR_GROUPS_URL) is None
+            or (access_key := settings.CONSTRUCTOR_GROUPS_ACCESS_KEY) is None
+            or (secret_key := settings.CONSTRUCTOR_GROUPS_SECRET_KEY) is None
+        ):
+            raise VideoCallProviderNotConfiguredError("Constructor Groups")
+
+        self.access_key = access_key
+        self.secret_key = secret_key
+        self.base_url = url.rstrip("/")
+
+    def _make_authenticated_request(
+        self, method: str, endpoint: str, data: dict[str, Any] | None = None
+    ) -> dict[str, Any]:
+        """Make authenticated request to Constructor Groups XAPI"""
+        url = f"{self.base_url}{endpoint}"
+
+        # Construct authentication token: ACCESS_KEY|SHA256(ACCESS_KEY|SECRET_KEY)
+        combined_string = f"{self.access_key}|{self.secret_key}"
+        hash_hex = hashlib.sha256(combined_string.encode("utf-8")).hexdigest()
+        auth_token = f"{self.access_key}|{hash_hex}"
+
+        headers = {
+            "Authorization": f"Bearer {auth_token}",
+            "Content-Type": "application/json",
+        }
+
+        try:
+            session = VideoCallSession()
+            if method.upper() == "POST":
+                response = session.post(url, headers=headers, json=data or {})
+            else:
+                raise ValueError(f"Unsupported HTTP method: {method}")
+
+            response.raise_for_status()
+            return response.json()
+        except requests.HTTPError as e:
+            logging.exception(
+                "Constructor Groups API request failed with status %s", e.response.status_code
+            )
+            raise CreateVideoCallFailedError("Constructor Groups")
+        except Exception:
+            logging.exception("Constructor Groups API request failed")
+            raise CreateVideoCallFailedError("Constructor Groups")
+
+    def get_or_create_default_room(
+        self, creator_email: str, name: str, fallback_name: str
+    ) -> dict[str, Any]:
+        data = {
+            "creator_email": creator_email,
+            "name": name,
+            "fallback_name": fallback_name,
+        }
+
+        return self._make_authenticated_request("POST", "/room/default", data)
+
+
+class OAuthVideoCallProvider(ABC):
+    provider_name: str = NotImplemented
+    client_id: str | None = NotImplemented
+    client_secret: str | None = NotImplemented
+    authorization_scope: str | None = NotImplemented
+    authorization_url: str = NotImplemented
+    token_url: str = NotImplemented
+    auto_refresh_url: str = NotImplemented
+    create_meeting_url: str = NotImplemented
+    token_key_name: str = NotImplemented
+
+    def get_token(self, user: UserProfile) -> object | None:
+        return user.third_party_api_state.get(self.token_key_name)
+
+    def update_token(self, user: UserProfile, token: dict[str, object] | None) -> None:
+        do_set_video_call_provider_token(user, self.token_key_name, token)
+
+    @abstractmethod
+    def get_meeting_details(self, request: HttpRequest, response: Response) -> HttpResponse:
+        pass
+
+    def __get_session(self, user: UserProfile) -> OAuth2Session:
+        if self.client_id is None or self.client_secret is None:
+            raise VideoCallProviderNotConfiguredError(self.provider_name)
+
+        return OAuth2Session(
+            self.client_id,
+            scope=self.authorization_scope,
+            redirect_uri=urljoin(
+                settings.ROOT_DOMAIN_URI, f"/calls/{self.provider_name.lower()}/complete"
+            ),
+            auto_refresh_url=self.auto_refresh_url,
+            auto_refresh_kwargs={
+                "client_id": self.client_id,
+                "client_secret": self.client_secret,
+            },
+            token=self.get_token(user),
+            token_updater=partial(self.update_token, user),
+        )
+
+    def __get_sid(self, request: HttpRequest) -> str:
+        # This is used to prevent CSRF attacks on the OAuth
+        # authentication flow.  We want this value to be unpredictable and
+        # tied to the session, but we don’t want to expose the main CSRF
+        # token directly to the server.
+
+        csrf.get_token(request)
+        # Use 'mark_sanitized' to cause Pysa to ignore the flow of user controlled
+        # data out of this function. 'request.META' is indeed user controlled, but
+        # post-HMAC output is no longer meaningfully controllable.
+        return mark_sanitized(
+            ""
+            if getattr(request, "_dont_enforce_csrf_checks", False)
+            else salted_hmac(
+                f"Zulip {self.provider_name.capitalize()} sid", request.META["CSRF_COOKIE"]
+            ).hexdigest()
+        )
+
+    def register_user(self, request: HttpRequest, **kwargs: Any) -> HttpResponse:
+        assert isinstance(request.user, UserProfile)
+        oauth = self.__get_session(request.user)
+        authorization_url, _state = oauth.authorization_url(
+            self.authorization_url,
+            state=json.dumps(
+                {"realm": get_subdomain(request), "sid": self.__get_sid(request)},
+            ),
+            **kwargs,
+        )
+        return redirect(authorization_url)
+
+    def complete_user(
+        self, request: HttpRequest, sid: str, code: str, **kwargs: Any
+    ) -> HttpResponse:
+        if not constant_time_compare(sid, self.__get_sid(request)):
+            raise VideoCallProviderSessionIdError(self.provider_name)
+        assert isinstance(request.user, UserProfile)
+        oauth = self.__get_session(request.user)
+        try:
+            token = oauth.fetch_token(
+                self.token_url, code=code, client_secret=self.client_secret, **kwargs
+            )
+        except OAuth2Error:
+            raise VideoCallProviderCredentialsError(self.provider_name)
+
+        self.update_token(request.user, token)
+        return render(request, "zerver/close_window.html")
+
+    def oauth_post(
+        self,
+        user: UserProfile,
+        url: str,
+        *,
+        json: object = None,
+        **kwargs: Any,
+    ) -> Response:
+        oauth = self.__get_session(user)
+        if not oauth.authorized:
+            raise InvalidVideoCallProviderTokenError(self.provider_name)
+
+        try:
+            response = oauth.post(url, json=json, **kwargs)
+        except OAuth2Error:
+            self.update_token(user, None)
+            raise InvalidVideoCallProviderTokenError(self.provider_name)
+
+        if response.status_code == 401:
+            self.update_token(user, None)
+            raise InvalidVideoCallProviderTokenError(self.provider_name)
+
+        return response
+
+    def make_video_call(
+        self, request: HttpRequest, user: UserProfile, payload: object = {}, **kwargs: Any
+    ) -> HttpResponse:
+        response = self.oauth_post(user, self.create_meeting_url, json=payload, **kwargs)
+        if not response.ok:
+            raise CreateVideoCallFailedError(self.provider_name)
+
+        return self.get_meeting_details(request, response)
+
+
+class WebexOAuthProvider(OAuthVideoCallProvider):
+    provider_name = "Webex"
+    token_key_name = "webex"
+
+    def __init__(self) -> None:
+        self.client_id = settings.VIDEO_WEBEX_CLIENT_ID
+        self.client_secret = settings.VIDEO_WEBEX_CLIENT_SECRET
+        self.authorization_url = urljoin(settings.VIDEO_WEBEX_API_URL, "authorize")
+        self.token_url = urljoin(settings.VIDEO_WEBEX_API_URL, "access_token")
+        self.auto_refresh_url = urljoin(settings.VIDEO_WEBEX_API_URL, "access_token")
+        self.create_meeting_url = urljoin(settings.VIDEO_WEBEX_API_URL, "meetings")
+        self.create_room_url = urljoin(settings.VIDEO_WEBEX_API_URL, "rooms")
+        # We need spark:all to create meetings with an associated roomId, which are
+        # ad-hoc meetings in our case: https://developer.webex.com/meeting/docs/meetings.
+        self.authorization_scope = "meeting:schedules_read meeting:schedules_write spark:all"
+
+    @override
+    def get_meeting_details(self, request: HttpRequest, response: Response) -> HttpResponse:
+        return json_success(request, data={"url": response.json()["webLink"]})
+
+    # For ad-hoc meetings, we need to create a public room as a prerequisite,
+    # where a public room is a room that lets anyone in the Webex organization
+    # join meetings associated with it. Note that the public room feature is
+    # restricted to paid Webex organizations.
+    def maybe_generate_public_room_id(self, user: UserProfile) -> str | None:
+        create_room_payload = {
+            "title": "Webex Zulip Meeting",
+            "isPublic": True,
+            "description": "A Webex meeting created via the Zulip client.",
+        }
+        response = self.oauth_post(user, self.create_room_url, json=create_room_payload)
+
+        # This is probably a free organization, because the Webex API sends a 403
+        # response when trying to create a public room with user credentials that
+        # belong to a free organization with the message: "Spaces belonging to a
+        # free org cannot be made public". For details, see error docs at
+        # https://developer.webex.com/messaging/docs/api/v1/rooms/create-a-room.
+        if response.status_code == 403:
+            return None
+        elif not response.ok:
+            raise JsonableError(
+                _("Failed to create {provider_name} public room.").format(
+                    provider_name=self.provider_name
+                )
+            )
+        return response.json()["id"]
+
+
+class ZoomGeneralOAuthProvider(OAuthVideoCallProvider):
+    provider_name = "Zoom"
+    authorization_scope = None
+    token_key_name = "zoom"
+
+    def __init__(self) -> None:
+        self.client_id = settings.VIDEO_ZOOM_CLIENT_ID
+        self.client_secret = settings.VIDEO_ZOOM_CLIENT_SECRET
+        self.authorization_url = urljoin(settings.VIDEO_ZOOM_OAUTH_URL, "/oauth/authorize")
+        self.token_url = urljoin(settings.VIDEO_ZOOM_OAUTH_URL, "/oauth/token")
+        self.auto_refresh_url = urljoin(settings.VIDEO_ZOOM_OAUTH_URL, "/oauth/token")
+        self.create_meeting_url = urljoin(settings.VIDEO_ZOOM_API_URL, "/v2/users/me/meetings")
+
+    @override
+    def get_meeting_details(self, request: HttpRequest, response: Response) -> HttpResponse:
+        return json_success(request, data={"url": response.json()["join_url"]})
+
+
+@zulip_login_required
+@never_cache
+def register_zoom_user(request: HttpRequest) -> HttpResponse:
+    return ZoomGeneralOAuthProvider().register_user(request=request)
+
+
+@zulip_login_required
+@never_cache
+def register_webex_user(request: HttpRequest) -> HttpResponse:
+    return WebexOAuthProvider().register_user(request=request)
+
+
+class StateDictRealm(TypedDict):
+    realm: str
+    sid: str
+
+
+class StateDict(TypedDict):
+    sid: str
+
+
+class ZoomVideoSettings(TypedDict):
+    host_video: bool
+    participant_video: bool
+
+
+class ZoomPayload(TypedDict):
+    settings: ZoomVideoSettings
+    default_password: bool
+
+
+class WebexPersonalRoomMeetingPayload(TypedDict):
+    title: str
+    start: str
+    end: str
+    scheduledType: Literal["personalRoomMeeting"]
+
+
+class WebexAdhocMeetingPayload(TypedDict):
+    title: str
+    adhoc: Literal[True]
+    roomId: str
+
+
+@never_cache
+@zulip_login_required
+@typed_endpoint
+def complete_zoom_user(
+    request: HttpRequest,
+    *,
+    code: str,
+    state: Json[StateDictRealm],
+) -> HttpResponse:
+    if get_subdomain(request) != state["realm"]:
+        return redirect(urljoin(get_realm(state["realm"]).url, request.get_full_path()))
+    return ZoomGeneralOAuthProvider().complete_user(request, code=code, sid=state["sid"])
+
+
+@never_cache
+@zulip_login_required
+@typed_endpoint
+def complete_webex_user(
+    request: HttpRequest,
+    *,
+    code: str,
+    state: Json[StateDictRealm],
+) -> HttpResponse:
+    if get_subdomain(request) != state["realm"]:
+        return redirect(urljoin(get_realm(state["realm"]).url, request.get_full_path()))
+    return WebexOAuthProvider().complete_user(request, code=code, sid=state["sid"])
+
+
+@cache_with_key(zoom_server_access_token_cache_key, timeout=3600 - 240)
+def get_zoom_server_to_server_access_token(account_id: str) -> str:
+    if settings.VIDEO_ZOOM_CLIENT_ID is None:
+        raise VideoCallProviderNotConfiguredError("Zoom")
+
+    client_id = settings.VIDEO_ZOOM_CLIENT_ID.encode("utf-8")
+    client_secret = str(settings.VIDEO_ZOOM_CLIENT_SECRET).encode("utf-8")
+
+    url = urljoin(settings.VIDEO_ZOOM_OAUTH_URL, "/oauth/token")
+    data = {"grant_type": "account_credentials", "account_id": account_id}
+
+    client_information = client_id + b":" + client_secret
+    encoded_client = b64encode(client_information).decode("ascii")
+    headers = {"Host": urlsplit(url).hostname, "Authorization": f"Basic {encoded_client}"}
+
+    response = VideoCallSession().post(url, data, headers=headers)
+    if not response.ok:
+        # {reason: 'Bad request', error: 'invalid_request'} for invalid account ID
+        # {'reason': 'Invalid client_id or client_secret', 'error': 'invalid_client'}
+        raise VideoCallProviderCredentialsError("Zoom")
+    return response.json()["access_token"]
+
+
+def get_zoom_server_to_server_call(
+    user: UserProfile, access_token: str, payload: ZoomPayload
+) -> str:
+    email = user.delivery_email
+    url = f"{settings.VIDEO_ZOOM_API_URL}/v2/users/{email}/meetings"
+    headers = {"Authorization": f"Bearer {access_token}", "Content-Type": "application/json"}
+    response = VideoCallSession().post(url, json=payload, headers=headers)
+    if not response.ok:
+        response_dict = response.json()
+        zoom_api_error_code = response_dict["code"]
+        if zoom_api_error_code == 1001:
+            # {code: 1001, message: "User does not exist: {email}"}
+            raise UnknownZoomUserError
+        if zoom_api_error_code == 124:
+            # For the error responses below, we flush any
+            # cached access token for the Zoom account.
+            # {code: 124, message: "Invalid access token"}
+            # {code: 124, message: "Access token is expired"}
+            account_id = str(settings.VIDEO_ZOOM_SERVER_TO_SERVER_ACCOUNT_ID)
+
+            # We are managing expiry ourselves, so this shouldn't
+            # happen. Log an error, and flush the access token from
+            # the cache, so that future requests should proceed.
+            logging.error(
+                "Unexpected Zoom error 124: %s",
+                response_dict.get("message", str(response_dict)),
+            )
+            flush_zoom_server_access_token_cache(account_id)
+        raise CreateVideoCallFailedError("Zoom")
+    return response.json()["join_url"]
+
+
+def make_server_authenticated_zoom_video_call(
+    request: HttpRequest,
+    user: UserProfile,
+    *,
+    payload: ZoomPayload,
+) -> HttpResponse:
+    account_id = str(settings.VIDEO_ZOOM_SERVER_TO_SERVER_ACCOUNT_ID)
+    access_token = get_zoom_server_to_server_access_token(account_id)
+    url = get_zoom_server_to_server_call(user, access_token, payload)
+    return json_success(request, data={"url": url})
+
+
+@typed_endpoint
+def make_zoom_video_call(
+    request: HttpRequest,
+    user: UserProfile,
+    *,
+    is_video_call: Json[bool] = True,
+) -> HttpResponse:
+    # The meeting host has the ability to configure both their own and
+    # participants' default video on/off state for the meeting. That's
+    # why when creating a meeting, configure the video on/off default
+    # according to the desired call type. Each Zoom user can still have
+    # their own personal setting to not start video by default.
+    video_settings = ZoomVideoSettings(host_video=is_video_call, participant_video=is_video_call)
+    payload = ZoomPayload(
+        settings=video_settings,
+        # Generate a default password depending on the user settings. This will
+        # result in the password being appended to the returned Join URL.
+        #
+        # If we don't request a password to be set, the waiting room will be
+        # forcibly enabled in Zoom organizations that require some kind of
+        # authentication for all meetings.
+        default_password=True,
+    )
+    if settings.VIDEO_ZOOM_SERVER_TO_SERVER_ACCOUNT_ID is not None:
+        return make_server_authenticated_zoom_video_call(request, user, payload=payload)
+    return ZoomGeneralOAuthProvider().make_video_call(request=request, user=user, payload=payload)
+
+
+@typed_endpoint_without_parameters
+def make_webex_video_call(request: HttpRequest, user: UserProfile) -> HttpResponse:
+    room_id = WebexOAuthProvider().maybe_generate_public_room_id(user)
+    payload: WebexAdhocMeetingPayload | WebexPersonalRoomMeetingPayload
+
+    # Quoting from https://developer.webex.com/meeting/docs/api/v1/meetings/create-a-meeting:
+    # An ad-hoc meeting is a non-recurring instant meeting for the target room
+    # which is supposed to be started immediately after being created for a
+    # quick collaboration.
+
+    # Public room IDs can only be generated if the user belongs to a paid Webex
+    # organization. They are necessary to generate ad-hoc Webex meetings which
+    # can be joined by anyone in the Webex organization. We fall back to
+    # generating a personal room meeting link if the user belongs to a free
+    # Webex organization.
+    if room_id is not None:
+        payload = WebexAdhocMeetingPayload(adhoc=True, roomId=room_id, title="Webex Ad-hoc meeting")
+    else:
+        start_time = timezone_now()
+        end_time = start_time + datetime.timedelta(minutes=40)
+        payload = WebexPersonalRoomMeetingPayload(
+            scheduledType="personalRoomMeeting",
+            start=start_time.isoformat(timespec="seconds"),
+            end=end_time.isoformat(timespec="seconds"),
+            title="Webex Personal Room Meeting",
+        )
+
+    return WebexOAuthProvider().make_video_call(request, user, payload=payload)
+
+
+@csrf_exempt
+@require_POST
+@typed_endpoint_without_parameters
+def deauthorize_zoom_user(request: HttpRequest) -> HttpResponse:
+    return json_success(request)
+
+
+@typed_endpoint
+def get_bigbluebutton_url(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    meeting_name: str,
+    voice_only: Json[bool] = False,
+) -> HttpResponse:
+    # https://docs.bigbluebutton.org/dev/api.html#create for reference on the API calls
+    # https://docs.bigbluebutton.org/dev/api.html#usage for reference for checksum
+    id = "zulip-" + str(random.randint(100000000000, 999999999999))
+
+    # We sign our data here to ensure a Zulip user cannot tamper with
+    # the join link to gain access to other meetings that are on the
+    # same bigbluebutton server.
+    signed = Signer().sign_object(
+        {
+            "meeting_id": id,
+            "name": meeting_name,
+            "lock_settings_disable_cam": voice_only,
+            "moderator": request.user.id,
+        }
+    )
+    url = append_url_query_string("/calls/bigbluebutton/join", "bigbluebutton=" + signed)
+    return json_success(request, {"url": url})
+
+
+# We use zulip_login_required here mainly to get access to the user's
+# full name from Zulip to prepopulate the user's name in the
+# BigBlueButton meeting.  Since the meeting's details are encoded in
+# the link the user is clicking, there is no validation tying this
+# meeting to the Zulip organization it was created in.
+@zulip_login_required
+@never_cache
+@typed_endpoint
+def join_bigbluebutton(request: HttpRequest, *, bigbluebutton: str) -> HttpResponse:
+    assert request.user.is_authenticated
+
+    if settings.BIG_BLUE_BUTTON_URL is None or settings.BIG_BLUE_BUTTON_SECRET is None:
+        raise VideoCallProviderNotConfiguredError("BigBlueButton")
+
+    try:
+        bigbluebutton_data = Signer().unsign_object(bigbluebutton)
+    except Exception:
+        raise JsonableError(_("Invalid signature."))
+
+    create_params = urlencode(
+        {
+            "meetingID": bigbluebutton_data["meeting_id"],
+            "name": bigbluebutton_data["name"],
+            "lockSettingsDisableCam": bigbluebutton_data["lock_settings_disable_cam"],
+        },
+        quote_via=quote,
+    )
+
+    checksum = hashlib.sha256(
+        ("create" + create_params + settings.BIG_BLUE_BUTTON_SECRET).encode()
+    ).hexdigest()
+
+    try:
+        response = VideoCallSession().get(
+            append_url_query_string(settings.BIG_BLUE_BUTTON_URL + "api/create", create_params)
+            + "&checksum="
+            + checksum
+        )
+        response.raise_for_status()
+    except requests.RequestException as e:
+        if e.response is not None:
+            reason = f"HTTP {response.status_code}: {response.text:.200}"
+        else:
+            reason = str(e)
+        raise VideoCallServerConnectionError("BigBlueButton", reason=reason)
+
+    payload = ElementTree.fromstring(response.text)
+    if assert_is_not_none(payload.find("messageKey")).text == "checksumError":
+        raise VideoCallServerAuthError("BigBlueButton")
+
+    status = assert_is_not_none(payload.find("returncode")).text
+    if status != "SUCCESS":
+        raise VideoCallServerStatusError("BigBlueButton", status=status)
+
+    join_params = urlencode(
+        {
+            "meetingID": bigbluebutton_data["meeting_id"],
+            # We use the moderator role only for the user who created the
+            # meeting, the attendee role for everyone else, so that only
+            # the user who created the meeting can convert a voice-only
+            # call to a video call.
+            "role": "MODERATOR" if bigbluebutton_data["moderator"] == request.user.id else "VIEWER",
+            "fullName": request.user.full_name,
+            # https://docs.bigbluebutton.org/dev/api.html#create
+            # The createTime option is used to have the user redirected to a link
+            # that is only valid for this meeting.
+            #
+            # Even if the same link in Zulip is used again, a new
+            # createTime parameter will be created, as the meeting on
+            # the BigBlueButton server has to be recreated. (after a
+            # few minutes)
+            "createTime": assert_is_not_none(payload.find("createTime")).text,
+        },
+        quote_via=quote,
+    )
+
+    checksum = hashlib.sha256(
+        ("join" + join_params + settings.BIG_BLUE_BUTTON_SECRET).encode()
+    ).hexdigest()
+    redirect_url_base = append_url_query_string(
+        settings.BIG_BLUE_BUTTON_URL + "api/join", join_params
+    )
+    return redirect(append_url_query_string(redirect_url_base, "checksum=" + checksum))
+
+
+@typed_endpoint_without_parameters
+def make_constructor_groups_video_call(
+    request: HttpRequest,
+    user_profile: UserProfile,
+) -> HttpResponse:
+    service = ConstructorGroupsService()
+    room_name = _("{full_name}'s Zulip room").format(full_name=user_profile.full_name)
+
+    room_data = service.get_or_create_default_room(
+        creator_email=user_profile.delivery_email,
+        name=room_name,
+        fallback_name=f"{room_name} ({user_profile.realm_id}-{user_profile.id})",
+    )
+
+    room_url = room_data.get("url", "")
+    if not room_url:
+        logging.error("Constructor Groups API returned room without URL: %s", room_data)
+        raise CreateVideoCallFailedError("Constructor Groups")
+
+    return json_success(request, {"url": room_url})
+
+
+# Nextcloud Talk API limits room names to 255 characters.
+MAX_NEXTCLOUD_TALK_ROOM_NAME_LENGTH = 255
+
+
+@typed_endpoint
+def create_nextcloud_talk_url(
+    request: HttpRequest, user: UserProfile, *, room_name: str
+) -> HttpResponse:
+    if (
+        settings.NEXTCLOUD_SERVER is None
+        or settings.NEXTCLOUD_TALK_USERNAME is None
+        or settings.NEXTCLOUD_TALK_PASSWORD is None
+    ):
+        raise VideoCallProviderNotConfiguredError("Nextcloud Talk")
+
+    room_name = truncate_content(room_name, MAX_NEXTCLOUD_TALK_ROOM_NAME_LENGTH, "...")
+    # https://nextcloud-talk.readthedocs.io/en/stable/conversation/#creating-a-new-conversation
+    api_url = urljoin(settings.NEXTCLOUD_SERVER, "/ocs/v2.php/apps/spreed/api/v4/room")
+
+    payload = {
+        # Create a PUBLIC conversation (roomType=3) which allows guest access
+        # https://nextcloud-talk.readthedocs.io/en/latest/constants/#conversation-types
+        "roomType": 3,
+        "roomName": room_name,
+    }
+    username = str(settings.NEXTCLOUD_TALK_USERNAME)
+    password = str(settings.NEXTCLOUD_TALK_PASSWORD)
+    credentials = f"{username}:{password}".encode()
+    encoded_credentials = b64encode(credentials).decode("ascii")
+
+    headers = {
+        "OCS-APIRequest": "true",
+        "Accept": "application/json",
+        "Authorization": f"Basic {encoded_credentials}",
+    }
+
+    try:
+        response = VideoCallSession().post(api_url, json=payload, headers=headers, timeout=10)
+        response.raise_for_status()
+    except requests.RequestException as e:
+        if e.response is not None:
+            reason = f"HTTP {response.status_code}: {response.text:.200}"
+        else:
+            reason = str(e)
+        raise VideoCallServerConnectionError("Nextcloud Talk", reason=reason)
+    try:
+        data = response.json()
+        token = data["ocs"]["data"]["token"]
+    except (KeyError, ValueError):
+        raise CreateVideoCallFailedError("Nextcloud Talk")
+
+    call_url = urljoin(settings.NEXTCLOUD_SERVER, f"/index.php/call/{token}")
+    return json_success(request, data={"url": call_url})

zerver/views/welcome_bot_custom_message.py

@@ -0,0 +1,44 @@
+from typing import Annotated
+
+from django.conf import settings
+from django.http import HttpRequest, HttpResponse
+from django.utils.translation import gettext as _
+from pydantic import StringConstraints
+
+from zerver.actions.message_send import internal_send_private_message
+from zerver.decorator import require_realm_admin
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.onboarding import get_custom_welcome_message_string
+from zerver.lib.response import json_success
+from zerver.lib.typed_endpoint import typed_endpoint
+from zerver.models.realms import Realm
+from zerver.models.users import UserProfile, get_system_bot
+
+
+@require_realm_admin
+@typed_endpoint
+def send_test_welcome_bot_custom_message(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    welcome_message_custom_text: Annotated[
+        str,
+        StringConstraints(
+            max_length=Realm.MAX_REALM_WELCOME_MESSAGE_CUSTOM_TEXT_LENGTH,
+        ),
+    ],
+) -> HttpResponse:
+    if len(welcome_message_custom_text) == 0:
+        raise JsonableError(_("Message must not be empty"))
+
+    welcome_bot_custom_message_string = get_custom_welcome_message_string(
+        user_profile.realm, welcome_message_custom_text
+    )
+    message_id = internal_send_private_message(
+        get_system_bot(settings.WELCOME_BOT, user_profile.realm_id),
+        user_profile,
+        welcome_bot_custom_message_string,
+        disable_external_notifications=True,
+    )
+    assert message_id is not None
+    return json_success(request, data={"message_id": message_id})