snapshot of https://github.com/zulip/zulip.git @ 48c69911c7dd4e03b20552eb3bd77ad8151c1357 (2026-09-15)

zerver/lib/upload/__init__.py

@@ -0,0 +1,693 @@
+import io
+import logging
+import os
+import re
+import unicodedata
+from collections.abc import Callable, Iterator
+from contextlib import contextmanager
+from datetime import datetime
+from email.message import EmailMessage
+from typing import IO, Any
+from urllib.parse import unquote, urljoin
+
+import chardet
+import pyvips
+from django.conf import settings
+from django.core.files.uploadedfile import UploadedFile
+from django.db import transaction
+from django.utils.translation import gettext as _
+
+from zerver.lib.avatar_hash import user_avatar_base_path_from_ids, user_avatar_path
+from zerver.lib.exceptions import ErrorCode, JsonableError
+from zerver.lib.mime_types import INLINE_MIME_TYPES, bare_content_type, guess_type
+from zerver.lib.outgoing_http import OutgoingSession
+from zerver.lib.thumbnail import (
+    MAX_EMOJI_GIF_FILE_SIZE_BYTES,
+    MEDIUM_AVATAR_SIZE,
+    THUMBNAIL_ACCEPT_IMAGE_TYPES,
+    BadImageError,
+    maybe_thumbnail,
+    resize_avatar,
+    resize_emoji,
+)
+from zerver.lib.upload.base import StreamingSourceWithSize, ZulipUploadBackend
+from zerver.models import (
+    ArchivedAttachment,
+    Attachment,
+    ImageAttachment,
+    Message,
+    Realm,
+    RealmEmoji,
+    ScheduledMessage,
+    UserProfile,
+)
+from zerver.models.users import is_cross_realm_bot_email
+
+
+class RealmUploadQuotaError(JsonableError):
+    code = ErrorCode.REALM_UPLOAD_QUOTA
+
+
+def check_upload_within_quota(realm: Realm, uploaded_file_size: int) -> None:
+    upload_quota = realm.upload_quota_bytes()
+    if upload_quota is None:
+        return
+    used_space = realm.currently_used_upload_space_bytes()
+    if (used_space + uploaded_file_size) > upload_quota:
+        raise RealmUploadQuotaError(_("Upload would exceed your organization's upload quota."))
+
+
+def needs_charset_detection(content_type: str) -> bool:
+    # We only add a charset if it doesn't already have one, and is a
+    # text type which we serve inline; currently, this is only text/plain.
+    fake_msg = EmailMessage()
+    fake_msg["content-type"] = content_type
+    return (
+        fake_msg.get_content_maintype() == "text"
+        and fake_msg.get_content_type() in INLINE_MIME_TYPES
+        and fake_msg.get_content_charset() is None
+    )
+
+
+def maybe_add_charset(content_type: str, file_data: bytes | StreamingSourceWithSize) -> str:
+    # Callers must gate on needs_charset_detection(content_type) first,
+    # so that they can avoid opening file_data at all when it won't be
+    # inspected -- with the S3 backend, opening a source starts an open
+    # GetObject read immediately, rather than lazily on first read.
+    assert needs_charset_detection(content_type)
+    fake_msg = EmailMessage()
+    fake_msg["content-type"] = content_type
+
+    early_abort = False
+    if isinstance(file_data, bytes):
+        detected = chardet.detect(file_data)
+    else:
+        chunk_size = 4096
+        reader = file_data.reader()
+        try:
+            detector = chardet.UniversalDetector()
+            total_read = 0
+            while True:
+                data = reader.read(chunk_size)
+                detector.feed(data)
+                if detector.done or len(data) < chunk_size:
+                    break
+                total_read += chunk_size
+                if total_read >= 32 * 1024:
+                    # If there's no BOM and no high bytes, the detector
+                    # never says "done" before EOF -- we bail out
+                    # arbitrarily at 32k.
+                    early_abort = True
+                    break
+            detector.close()
+            detected = detector.result
+        finally:
+            reader.close()
+    if early_abort and detected["confidence"] == 1.0 and detected["encoding"] == "ascii":
+        # An early abort which didn't see high-byte characters is not
+        # a confident "ASCII", as they may come later in the file; we
+        # would prefer to leave off the charset rather than be wrong.
+        pass
+    elif detected["confidence"] >= 0.90 and detected["encoding"]:
+        fake_msg.set_param("charset", detected["encoding"], replace=True)
+    elif detected["confidence"] >= 0.73 and detected["encoding"] == "ISO-8859-1":
+        # Older versions of chardet capped ISO-8859-1 confidence to at
+        # most 73%.  Newer versions do not, but are still somewhat
+        # reluctant to guess ISO-8859-1 confidently for short files.
+        fake_msg.set_param("charset", detected["encoding"], replace=True)
+    elif detected["confidence"] >= 0.66 and detected["encoding"] == "utf-8":
+        # UTF-8 is far and wide the most common current encoding,
+        # so we set a much lower threshold if that's the best guess.
+        # https://en.wikipedia.org/wiki/Popularity_of_text_encodings
+        fake_msg.set_param("charset", detected["encoding"], replace=True)
+    return str(fake_msg["content-type"])
+
+
+def create_attachment(
+    file_name: str,
+    path_id: str,
+    content_type: str,
+    file_data: bytes | StreamingSourceWithSize,
+    user_profile: UserProfile,
+    realm: Realm,
+) -> None:
+    assert (user_profile.realm_id == realm.id) or is_cross_realm_bot_email(
+        user_profile.delivery_email
+    )
+    if isinstance(file_data, bytes):
+        file_size = len(file_data)
+        file_vips_data: bytes | pyvips.Source = file_data
+    else:
+        file_size = file_data.size
+        file_vips_data = file_data.vips_source
+
+    attachment = Attachment.objects.create(
+        file_name=file_name,
+        path_id=path_id,
+        owner=user_profile,
+        realm=realm,
+        size=file_size,
+        content_type=content_type,
+    )
+    maybe_thumbnail(file_vips_data, content_type, path_id, realm.id)
+    from zerver.actions.uploads import notify_attachment_update
+
+    notify_attachment_update(user_profile, "add", attachment.to_dict())
+
+
+def get_file_info(user_file: UploadedFile[bytes]) -> tuple[str, str]:
+    uploaded_file_name = user_file.name
+    assert uploaded_file_name is not None
+
+    # It appears Django's UploadedFile.content_type defaults to an empty string,
+    # even though the value is documented as `str | None`. So we check for both.
+    content_type = user_file.content_type
+    if content_type is None or content_type == "":
+        guessed_type = guess_type(uploaded_file_name)[0]
+        if guessed_type is not None:
+            content_type = guessed_type
+        else:
+            # Fallback to application/octet-stream if unable to determine a
+            # different content-type from the filename.
+            content_type = "application/octet-stream"
+
+    fake_msg = EmailMessage()
+    extras = {}
+    if user_file.content_type_extra:
+        extras = {k: v.decode() if v else None for k, v in user_file.content_type_extra.items()}
+    fake_msg.add_header("content-type", content_type, **extras)
+    content_type = str(fake_msg["content-type"])
+
+    uploaded_file_name = unquote(uploaded_file_name)
+
+    return uploaded_file_name, content_type
+
+
+# Common and wrappers
+_upload_backend: ZulipUploadBackend | None = None
+
+
+def get_upload_backend() -> ZulipUploadBackend:
+    global _upload_backend
+    if _upload_backend is None:
+        if settings.LOCAL_UPLOADS_DIR is not None:
+            from zerver.lib.upload.local import LocalUploadBackend
+
+            _upload_backend = LocalUploadBackend()
+        else:  # nocoverage
+            from zerver.lib.upload.s3 import S3UploadBackend
+
+            _upload_backend = S3UploadBackend()
+    return _upload_backend
+
+
+# Message attachment uploads
+
+
+def get_public_upload_root_url() -> str:
+    return get_upload_backend().get_public_upload_root_url()
+
+
+def generate_message_upload_path(realm_id: str, uploaded_file_name: str) -> str:
+    return get_upload_backend().generate_message_upload_path(realm_id, uploaded_file_name)
+
+
+def store_message_attachment(
+    path_id: str,
+    filename: str,
+    content_type: str,
+    file_data: bytes,
+    user_profile: UserProfile | None,
+    target_realm: Realm | None,
+) -> None:
+    get_upload_backend().store_message_attachment(
+        path_id, filename, content_type, file_data, user_profile, target_realm
+    )
+
+
+# Most Linux filesystems limit each component of a path to 255 bytes,
+# and we store uploaded files with their sanitized name as the last
+# component of the path.
+MAX_FILE_NAME_LENGTH = 255
+
+
+def truncate_file_name(file_name: str, max_length: int) -> str:
+    """Truncates a file name to at most max_length bytes, preserving its
+    extension if that leaves room for the rest of the name.
+    """
+    if len(file_name.encode()) <= max_length:
+        return file_name
+
+    stem, extension = os.path.splitext(file_name)
+    if len(extension.encode()) >= max_length:
+        # The extension alone is too long to preserve, so truncate the
+        # name as a whole instead.
+        stem, extension = file_name, ""
+
+    # Slicing the encoded name can cut a multi-byte character in half;
+    # errors="ignore" discards such a partial character.
+    stem_length = max_length - len(extension.encode())
+    return stem.encode()[:stem_length].decode(errors="ignore") + extension
+
+
+def sanitize_name(value: str, *, strict: bool = False) -> str:
+    """Sanitizes a value to be safe to store in a Linux filesystem, in
+    S3, and in a URL.  So Unicode is allowed, but not special
+    characters other than ".", "-", and "_".
+
+    In "strict" mode, it does not allow Unicode, allowing only ASCII
+    [A-Za-z0-9_] as word characters.  This is for the benefit of tusd,
+    which is not Unicode-aware.
+
+    This implementation is based on django.utils.text.slugify; it is
+    modified by:
+    * adding '.' to the list of allowed characters.
+    * preserving the case of the value.
+    * not stripping trailing dashes and underscores.
+    * limiting the length of the result.
+
+    """
+    if strict:
+        value = re.sub(r"[^A-Za-z0-9_ .-]", "", value).strip()
+    else:
+        value = unicodedata.normalize("NFKC", value)
+        value = re.sub(r"[^\w\s.-]", "", value).strip()
+    value = re.sub(r"[-\s]+", "-", value)
+
+    # NFKC normalization can make the value considerably longer -- a
+    # single character can normalize to dozens of them -- so this has
+    # to happen after normalizing, not to the caller's value.
+    value = truncate_file_name(value, MAX_FILE_NAME_LENGTH)
+
+    # Django's MultiPartParser never returns files named this, but we
+    # could get them after removing spaces; change the name to a safer
+    # value.
+    if value in {"", ".", ".."}:
+        return "uploaded-file"
+    return value
+
+
+def upload_message_attachment(
+    uploaded_file_name: str,
+    content_type: str,
+    file_data: bytes,
+    user_profile: UserProfile,
+    target_realm: Realm | None = None,
+) -> tuple[str, str]:
+    if target_realm is None:
+        target_realm = user_profile.realm
+    path_id = get_upload_backend().generate_message_upload_path(
+        str(target_realm.id), sanitize_name(uploaded_file_name)
+    )
+    if needs_charset_detection(content_type):
+        content_type = maybe_add_charset(content_type, file_data)
+
+    # NULL bytes are the one thing we can't store in the original
+    # filename column, due to PostgreSQL limitations
+    uploaded_file_name = re.sub(r"\x00", "", uploaded_file_name)
+
+    with transaction.atomic(durable=True):
+        get_upload_backend().store_message_attachment(
+            path_id,
+            uploaded_file_name,
+            content_type,
+            file_data,
+            user_profile,
+            target_realm,
+        )
+        create_attachment(
+            uploaded_file_name,
+            path_id,
+            content_type,
+            file_data,
+            user_profile,
+            target_realm,
+        )
+    return f"/user_uploads/{path_id}", uploaded_file_name
+
+
+def claim_attachment(
+    path_id: str,
+    message: Message | ScheduledMessage,
+    is_message_realm_public: bool,
+    is_message_web_public: bool = False,
+) -> Attachment:
+    attachment = Attachment.objects.get(path_id=path_id)
+    if isinstance(message, ScheduledMessage):
+        attachment.scheduled_messages.add(message)
+        # Setting the is_web_public and is_realm_public flags would be incorrect
+        # in the scheduled message case - since the attachment becomes such only
+        # when the message is actually posted.
+        return attachment
+
+    assert isinstance(message, Message)
+    attachment.messages.add(message)
+    attachment.is_web_public = attachment.is_web_public or is_message_web_public
+    attachment.is_realm_public = attachment.is_realm_public or is_message_realm_public
+    attachment.save()
+    return attachment
+
+
+def upload_message_attachment_from_request(
+    user_file: UploadedFile[bytes], user_profile: UserProfile
+) -> tuple[str, str]:
+    uploaded_file_name, content_type = get_file_info(user_file)
+    return upload_message_attachment(
+        uploaded_file_name, content_type, user_file.read(), user_profile
+    )
+
+
+def attachment_source(path_id: str) -> StreamingSourceWithSize:
+    return get_upload_backend().attachment_source(path_id)
+
+
+def save_attachment_contents(path_id: str, filehandle: IO[bytes]) -> None:
+    get_upload_backend().save_attachment_contents(path_id, filehandle)
+
+
+def delete_message_attachment(path_id: str, *, raw_path: bool = False) -> None:
+    get_upload_backend().delete_message_attachment_from_storage(path_id)
+
+
+@contextmanager
+def delete_message_attachments(
+    *,
+    raw_paths: bool = False,
+    delete_from: tuple[type[ImageAttachment | Attachment | ArchivedAttachment], ...] = (),
+) -> Iterator[Callable[[str], None]]:
+    if delete_from == ():
+        flush_path_ids: Callable[[list[str]], None] | None = None
+    else:
+
+        def delete_from_database(path_ids: list[str]) -> None:
+            for db_class in delete_from:
+                db_class._default_manager.filter(path_id__in=path_ids).delete()
+
+        flush_path_ids = delete_from_database
+    with get_upload_backend().delete_message_attachments_from_storage(
+        raw_paths=raw_paths, flush=flush_path_ids
+    ) as delete_one:
+        yield delete_one
+
+
+def all_message_attachments(
+    *, include_thumbnails: bool = False, prefix: str = ""
+) -> Iterator[tuple[str, datetime]]:
+    return get_upload_backend().all_message_attachments(include_thumbnails, prefix)
+
+
+# Avatar image uploads
+
+
+def get_avatar_url(hash_key: str, medium: bool = False) -> str:
+    return get_upload_backend().get_avatar_url(hash_key, medium)
+
+
+def get_avatar_path(hash_key: str, medium: bool = False) -> str:
+    return get_upload_backend().get_avatar_path(hash_key, medium)
+
+
+def write_avatar_images(
+    file_path: str,
+    user_profile: UserProfile,
+    image_data: bytes,
+    *,
+    content_type: str | None,
+    backend: ZulipUploadBackend | None = None,
+    future: bool = True,
+) -> None:
+    if backend is None:
+        backend = get_upload_backend()
+    backend.store_single_avatar_image(
+        file_path + ".original",
+        user_profile=user_profile,
+        image_data=image_data,
+        content_type=content_type,
+        future=future,
+    )
+
+    backend.store_single_avatar_image(
+        backend.get_avatar_path(file_path, medium=False),
+        user_profile=user_profile,
+        image_data=resize_avatar(image_data),
+        content_type="image/png",
+        future=future,
+    )
+
+    backend.store_single_avatar_image(
+        backend.get_avatar_path(file_path, medium=True),
+        user_profile=user_profile,
+        image_data=resize_avatar(image_data, MEDIUM_AVATAR_SIZE),
+        content_type="image/png",
+        future=future,
+    )
+
+
+def write_jdenticon_avatars(
+    file_path: str,
+    user_profile: UserProfile,
+    *,
+    image_data: bytes,
+    image_data_medium: bytes,
+    backend: ZulipUploadBackend | None = None,
+    future: bool = True,
+) -> None:
+    if backend is None:
+        backend = get_upload_backend()
+
+    backend.store_single_avatar_image(
+        backend.get_avatar_path(file_path, medium=False),
+        user_profile=user_profile,
+        image_data=image_data,
+        content_type="image/png",
+        future=future,
+    )
+
+    backend.store_single_avatar_image(
+        backend.get_avatar_path(file_path, medium=True),
+        user_profile=user_profile,
+        image_data=image_data_medium,
+        content_type="image/png",
+        future=future,
+    )
+
+
+def upload_avatar_image(
+    user_file: IO[bytes],
+    user_profile: UserProfile,
+    content_type: str,
+    backend: ZulipUploadBackend | None = None,
+    future: bool = True,
+) -> None:
+    if content_type not in THUMBNAIL_ACCEPT_IMAGE_TYPES:
+        raise BadImageError(_("Invalid image format"))
+    file_path = user_avatar_path(user_profile, future=future)
+
+    image_data = user_file.read()
+    write_avatar_images(
+        file_path,
+        user_profile,
+        image_data,
+        content_type=content_type,
+        backend=backend,
+        future=future,
+    )
+
+
+def copy_avatar(source_profile: UserProfile, target_profile: UserProfile) -> None:
+    source_file_path = user_avatar_path(source_profile, future=False)
+    target_file_path = user_avatar_path(target_profile, future=True)
+
+    image_data, content_type = get_upload_backend().get_avatar_contents(
+        source_file_path, source_profile.avatar_source
+    )
+    write_avatar_images(
+        target_file_path, target_profile, image_data, content_type=content_type, future=True
+    )
+
+
+def ensure_avatar_image(user_profile: UserProfile, medium: bool = False) -> None:
+    file_path = user_avatar_path(user_profile)
+
+    final_file_path = get_upload_backend().get_avatar_path(file_path, medium)
+
+    if settings.LOCAL_AVATARS_DIR is not None:
+        output_path = os.path.join(
+            settings.LOCAL_AVATARS_DIR,
+            final_file_path,
+        )
+
+        if os.path.isfile(output_path):
+            return
+
+    image_data, _ = get_upload_backend().get_avatar_contents(file_path, user_profile.avatar_source)
+
+    if medium:
+        resized_avatar = resize_avatar(image_data, MEDIUM_AVATAR_SIZE)
+    else:
+        resized_avatar = resize_avatar(image_data)
+    get_upload_backend().store_single_avatar_image(
+        final_file_path,
+        user_profile=user_profile,
+        image_data=resized_avatar,
+        content_type="image/png",
+        future=False,
+    )
+
+
+def delete_avatar_image(user_profile: UserProfile, avatar_version: int) -> None:
+    path_id = user_avatar_base_path_from_ids(user_profile.id, avatar_version, user_profile.realm_id)
+    get_upload_backend().delete_avatar_image_from_storage(path_id)
+
+
+# Realm icon and logo uploads
+
+
+def get_uploaded_realm_icon_url(realm_id: int, version: int) -> str:
+    return get_upload_backend().get_realm_icon_url(realm_id, version)
+
+
+def get_uploaded_realm_logo_url(realm_id: int, version: int, night: bool) -> str:
+    return get_upload_backend().get_realm_logo_url(realm_id, version, night)
+
+
+def realm_avatar_and_logo_path(realm: Realm) -> str:
+    return get_upload_backend().realm_avatar_and_logo_path(realm)
+
+
+def upload_icon_image(user_file: IO[bytes], user_profile: UserProfile, content_type: str) -> None:
+    if content_type not in THUMBNAIL_ACCEPT_IMAGE_TYPES:
+        raise BadImageError(_("Invalid image format"))
+    get_upload_backend().store_realm_icon_image(user_file, user_profile, content_type)
+
+
+def upload_logo_image(
+    user_file: IO[bytes], user_profile: UserProfile, night: bool, content_type: str
+) -> None:
+    if content_type not in THUMBNAIL_ACCEPT_IMAGE_TYPES:
+        raise BadImageError(_("Invalid image format"))
+    get_upload_backend().store_realm_logo_image(user_file, user_profile, night, content_type)
+
+
+# Realm emoji uploads
+
+
+def get_emoji_url(emoji_file_name: str, realm_id: int, still: bool = False) -> str:
+    return get_upload_backend().get_emoji_url(emoji_file_name, realm_id, still)
+
+
+def upload_emoji_image(
+    emoji_file: IO[bytes],
+    emoji_file_name: str,
+    user_profile: UserProfile,
+    content_type: str,
+    backend: ZulipUploadBackend | None = None,
+) -> bool:
+    if backend is None:
+        backend = get_upload_backend()
+
+    # Emoji are served in the format that they are uploaded, so must
+    # be _both_ an image format that we're willing to thumbnail, _and_
+    # a format which is widespread enough that we're willing to inline
+    # it.  The latter contains non-image formats, but the former
+    # limits to only images.
+    content_type = bare_content_type(content_type)
+    if content_type not in THUMBNAIL_ACCEPT_IMAGE_TYPES or content_type not in INLINE_MIME_TYPES:
+        raise BadImageError(_("Invalid image format"))
+
+    emoji_path = RealmEmoji.PATH_ID_TEMPLATE.format(
+        realm_id=user_profile.realm_id,
+        emoji_file_name=emoji_file_name,
+    )
+
+    image_data = emoji_file.read()
+    backend.store_single_emoji_image(
+        f"{emoji_path}.original", content_type, user_profile, image_data
+    )
+    resized_image_data, still_image_data = resize_emoji(image_data, emoji_file_name)
+    if still_image_data is not None:
+        if len(still_image_data) > MAX_EMOJI_GIF_FILE_SIZE_BYTES:  # nocoverage
+            raise BadImageError(_("Image size exceeds limit"))
+    elif len(resized_image_data) > MAX_EMOJI_GIF_FILE_SIZE_BYTES:  # nocoverage
+        raise BadImageError(_("Image size exceeds limit"))
+    backend.store_single_emoji_image(emoji_path, content_type, user_profile, resized_image_data)
+    if still_image_data is None:
+        return False
+
+    still_path = RealmEmoji.STILL_PATH_ID_TEMPLATE.format(
+        realm_id=user_profile.realm_id,
+        emoji_filename_without_extension=os.path.splitext(emoji_file_name)[0],
+    )
+    backend.store_single_emoji_image(still_path, "image/png", user_profile, still_image_data)
+    return True
+
+
+def get_emoji_file_content(
+    session: OutgoingSession, emoji_url: str, emoji_id: int, logger: logging.Logger
+) -> tuple[bytes, str]:  # nocoverage
+    original_emoji_url = emoji_url + ".original"
+
+    logger.info("Downloading %s", original_emoji_url)
+    response = session.get(original_emoji_url)
+    if response.status_code == 200:
+        assert isinstance(response.content, bytes)
+        return response.content, response.headers["Content-Type"]
+
+    logger.info("Error fetching emoji from URL %s", original_emoji_url)
+    logger.info("Trying %s instead", emoji_url)
+    response = session.get(emoji_url)
+    if response.status_code == 200:
+        assert isinstance(response.content, bytes)
+        return response.content, response.headers["Content-Type"]
+    logger.info("Error fetching emoji from URL %s", emoji_url)
+    logger.error("Could not fetch emoji %s", emoji_id)
+    raise AssertionError(f"Could not fetch emoji {emoji_id}")
+
+
+def handle_reupload_emojis_event(realm: Realm, logger: logging.Logger) -> None:  # nocoverage
+    session = OutgoingSession(role="reupload_emoji", timeout=3, max_retries=3)
+
+    query = RealmEmoji.objects.filter(realm=realm).order_by("id")
+
+    for realm_emoji in query:
+        logger.info("Processing emoji %s", realm_emoji.id)
+        emoji_filename = realm_emoji.file_name
+        assert emoji_filename is not None
+        emoji_url = get_emoji_url(emoji_filename, realm_emoji.realm_id)
+        if emoji_url.startswith("/"):
+            emoji_url = urljoin(realm_emoji.realm.url, emoji_url)
+
+        emoji_file_content, content_type = get_emoji_file_content(
+            session, emoji_url, realm_emoji.id, logger
+        )
+
+        emoji_bytes_io = io.BytesIO(emoji_file_content)
+
+        user_profile = realm_emoji.author
+        # When this runs, emojis have already been migrated to always have .author set.
+        assert user_profile is not None
+
+        logger.info("Reuploading emoji %s", realm_emoji.id)
+        realm_emoji.is_animated = upload_emoji_image(
+            emoji_bytes_io, emoji_filename, user_profile, content_type
+        )
+        realm_emoji.save(update_fields=["is_animated"])
+
+
+# Export tarballs
+
+
+def get_export_tarball_url(realm: Realm, export_path: str) -> str:
+    return get_upload_backend().get_export_tarball_url(realm, export_path)
+
+
+def upload_export_tarball(
+    realm: Realm, tarball_path: str, percent_callback: Callable[[Any], None] | None = None
+) -> str:
+    return get_upload_backend().store_export_tarball(
+        realm, tarball_path, percent_callback=percent_callback
+    )
+
+
+def delete_export_tarball(export_path: str) -> None:
+    get_upload_backend().delete_export_tarball_from_storage(export_path)

zerver/lib/upload/base.py

@@ -0,0 +1,157 @@
+import os
+from collections.abc import Callable, Iterator
+from contextlib import contextmanager
+from dataclasses import dataclass
+from datetime import datetime
+from typing import IO, Any, Protocol
+
+import pyvips
+
+from zerver.models import Realm, UserProfile
+
+
+class ReadableStream(Protocol):
+    def read(self, size: int = -1) -> bytes: ...
+
+    def close(self) -> None: ...
+
+
+@dataclass
+class StreamingSourceWithSize:
+    size: int
+    vips_source: pyvips.Source
+    reader: Callable[[], ReadableStream]
+
+
+class ZulipUploadBackend:
+    # Message attachment uploads
+    def get_public_upload_root_url(self) -> str:
+        raise NotImplementedError
+
+    def generate_message_upload_path(self, realm_id: str, uploaded_file_name: str) -> str:
+        raise NotImplementedError
+
+    def store_message_attachment(
+        self,
+        path_id: str,
+        filename: str,
+        content_type: str,
+        file_data: bytes,
+        user_profile: UserProfile | None,
+        target_realm: Realm | None,
+    ) -> None:
+        raise NotImplementedError
+
+    def save_attachment_contents(self, path_id: str, filehandle: IO[bytes]) -> None:
+        raise NotImplementedError
+
+    def attachment_source(self, path_id: str) -> StreamingSourceWithSize:
+        raise NotImplementedError
+
+    def delete_message_attachment_from_storage(
+        self, path_id: str, *, raw_path: bool = False
+    ) -> None:
+        """This must delete the attachment, any adjacent .info files, and any thumbnails.
+
+        If raw_path is passed, just delete the one path provided.
+        This is used in rare cases where we're iterating _files_, not
+        path_ids.
+
+        """
+        raise NotImplementedError
+
+    @contextmanager
+    def delete_message_attachments_from_storage(
+        self,
+        *,
+        raw_paths: bool = False,
+        flush: Callable[[list[str]], None] | None = None,
+    ) -> Iterator[Callable[[str], None]]:
+        def delete_one(path_id: str) -> None:
+            self.delete_message_attachment_from_storage(path_id, raw_path=raw_paths)
+            if flush:
+                flush([path_id])
+
+        yield delete_one
+
+    def all_message_attachments(
+        self,
+        include_thumbnails: bool = False,
+        prefix: str = "",
+    ) -> Iterator[tuple[str, datetime]]:
+        raise NotImplementedError
+
+    # Avatar image uploads
+    def get_avatar_url(self, hash_key: str, medium: bool = False) -> str:
+        raise NotImplementedError
+
+    def get_avatar_contents(self, file_path: str, avatar_source: str) -> tuple[bytes, str]:
+        raise NotImplementedError
+
+    def get_avatar_path(self, hash_key: str, medium: bool = False) -> str:
+        if medium:
+            return f"{hash_key}-medium.png"
+        else:
+            return f"{hash_key}.png"
+
+    def store_single_avatar_image(
+        self,
+        file_path: str,
+        *,
+        user_profile: UserProfile,
+        image_data: bytes,
+        content_type: str | None,
+        future: bool = True,
+    ) -> None:
+        raise NotImplementedError
+
+    def delete_avatar_image_from_storage(self, path_id: str) -> None:
+        raise NotImplementedError
+
+    # Realm icon and logo uploads
+    def realm_avatar_and_logo_path(self, realm: Realm) -> str:
+        return os.path.join(str(realm.id), "realm")
+
+    def get_realm_icon_url(self, realm_id: int, version: int) -> str:
+        raise NotImplementedError
+
+    def store_realm_icon_image(
+        self, icon_file: IO[bytes], user_profile: UserProfile, content_type: str
+    ) -> None:
+        raise NotImplementedError
+
+    def get_realm_logo_url(self, realm_id: int, version: int, night: bool) -> str:
+        raise NotImplementedError
+
+    def store_realm_logo_image(
+        self, logo_file: IO[bytes], user_profile: UserProfile, night: bool, content_type: str
+    ) -> None:
+        raise NotImplementedError
+
+    # Realm emoji uploads
+    def get_emoji_url(self, emoji_file_name: str, realm_id: int, still: bool = False) -> str:
+        raise NotImplementedError
+
+    def store_single_emoji_image(
+        self,
+        path: str,
+        content_type: str | None,
+        user_profile: UserProfile,
+        image_data: bytes,
+    ) -> None:
+        raise NotImplementedError
+
+    # Export tarballs
+    def get_export_tarball_url(self, realm: Realm, export_path: str) -> str:
+        raise NotImplementedError
+
+    def store_export_tarball(
+        self,
+        realm: Realm,
+        tarball_path: str,
+        percent_callback: Callable[[Any], None] | None = None,
+    ) -> str:
+        raise NotImplementedError
+
+    def delete_export_tarball_from_storage(self, export_path: str) -> None:
+        raise NotImplementedError

zerver/lib/upload/local.py

@@ -0,0 +1,272 @@
+import os
+import random
+import secrets
+import shutil
+from collections.abc import Callable, Iterator
+from datetime import datetime
+from typing import IO, Any, Literal
+
+import pyvips
+from django.conf import settings
+from typing_extensions import override
+
+from zerver.lib.mime_types import guess_type
+from zerver.lib.thumbnail import resize_logo, resize_realm_icon
+from zerver.lib.timestamp import timestamp_to_datetime
+from zerver.lib.upload.base import StreamingSourceWithSize, ZulipUploadBackend
+from zerver.lib.utils import assert_is_not_none
+from zerver.models import Realm, RealmEmoji, UserProfile
+
+
+def assert_is_local_storage_path(type: Literal["avatars", "files"], full_path: str) -> None:
+    """
+    Verify that we are only reading and writing files under the
+    expected paths.  This is expected to be already enforced at other
+    layers, via cleaning of user input, but we assert it here for
+    defense in depth.
+    """
+    assert settings.LOCAL_UPLOADS_DIR is not None
+    type_path = os.path.normpath(os.path.join(settings.LOCAL_UPLOADS_DIR, type))
+    full_path = os.path.normpath(full_path)
+    assert os.path.commonpath([type_path, full_path]) == type_path
+
+
+def write_local_file(type: Literal["avatars", "files"], path: str, file_data: bytes) -> None:
+    file_path = os.path.join(assert_is_not_none(settings.LOCAL_UPLOADS_DIR), type, path)
+    assert_is_local_storage_path(type, file_path)
+
+    os.makedirs(os.path.dirname(file_path), exist_ok=True)
+    with open(file_path, "wb") as f:
+        f.write(file_data)
+
+
+def read_local_file(type: Literal["avatars", "files"], path: str) -> Iterator[bytes]:
+    file_path = os.path.join(assert_is_not_none(settings.LOCAL_UPLOADS_DIR), type, path)
+    assert_is_local_storage_path(type, file_path)
+
+    with open(file_path, "rb") as f:
+        yield from iter(lambda: f.read(4 * 1024 * 1024), b"")
+
+
+def delete_local_file(
+    type: Literal["avatars", "files"], path: str, *, directory: bool = False
+) -> None:
+    file_path = os.path.join(assert_is_not_none(settings.LOCAL_UPLOADS_DIR), type, path)
+    assert_is_local_storage_path(type, file_path)
+
+    def prune_empty_dirs(file_path: str) -> None:
+        # Remove as many directories up the tree as are now empty
+        directory = os.path.dirname(file_path)
+        while directory != settings.LOCAL_UPLOADS_DIR:
+            try:
+                os.rmdir(directory)
+                directory = os.path.dirname(directory)
+            except OSError:
+                break
+
+    if directory and os.path.isdir(file_path):
+        shutil.rmtree(file_path)
+        prune_empty_dirs(os.path.dirname(file_path))
+    elif os.path.isfile(file_path):
+        os.remove(file_path)
+        prune_empty_dirs(file_path)
+
+
+class LocalUploadBackend(ZulipUploadBackend):
+    @override
+    def get_public_upload_root_url(self) -> str:
+        return "/user_avatars/"
+
+    @override
+    def generate_message_upload_path(self, realm_id: str, sanitized_file_name: str) -> str:
+        # Split into 256 subdirectories to prevent directories from getting too big
+        return "/".join(
+            [
+                realm_id,
+                format(random.randint(0, 255), "x"),
+                secrets.token_urlsafe(18),
+                sanitized_file_name,
+            ]
+        )
+
+    @override
+    def store_message_attachment(
+        self,
+        path_id: str,
+        filename: str,
+        content_type: str,
+        file_data: bytes,
+        user_profile: UserProfile | None,
+        target_realm: Realm | None,
+    ) -> None:
+        write_local_file("files", path_id, file_data)
+
+    @override
+    def save_attachment_contents(self, path_id: str, filehandle: IO[bytes]) -> None:
+        for chunk in read_local_file("files", path_id):
+            filehandle.write(chunk)
+
+    @override
+    def attachment_source(self, path_id: str) -> StreamingSourceWithSize:
+        file_path = os.path.join(assert_is_not_none(settings.LOCAL_UPLOADS_DIR), "files", path_id)
+        assert_is_local_storage_path("files", file_path)
+        vips_source = pyvips.Source.new_from_file(file_path)
+        return StreamingSourceWithSize(
+            size=os.path.getsize(file_path),
+            vips_source=vips_source,
+            reader=lambda: open(file_path, "rb"),
+        )
+
+    @override
+    def delete_message_attachment_from_storage(
+        self, path_id: str, *, raw_path: bool = False
+    ) -> None:
+        delete_local_file("files", path_id)
+        if not raw_path:
+            delete_local_file("files", f"{path_id}.info")
+            delete_local_file("files", f"thumbnail/{path_id}/", directory=True)
+
+    @override
+    def all_message_attachments(
+        self,
+        include_thumbnails: bool = False,
+        prefix: str = "",
+    ) -> Iterator[tuple[str, datetime]]:
+        assert settings.LOCAL_UPLOADS_DIR is not None
+        top = settings.LOCAL_UPLOADS_DIR + "/files"
+        start = top
+        if prefix != "":
+            start += f"/{prefix}"
+        for dirname, subdirnames, files in os.walk(start):
+            if not include_thumbnails and dirname == top and "thumbnail" in subdirnames:
+                subdirnames.remove("thumbnail")
+            for f in files:
+                fullpath = os.path.join(dirname, f)
+                yield (
+                    os.path.relpath(fullpath, top),
+                    timestamp_to_datetime(os.path.getmtime(fullpath)),
+                )
+
+    @override
+    def get_avatar_url(self, hash_key: str, medium: bool = False) -> str:
+        return "/user_avatars/" + self.get_avatar_path(hash_key, medium)
+
+    @override
+    def get_avatar_contents(self, file_path: str, avatar_source: str) -> tuple[bytes, str]:
+        # Currently, only used in codepaths where avatar_source = "U".
+        # We can extend it for avatar_source = "J", if required.
+        assert avatar_source is UserProfile.AVATAR_FROM_USER
+        image_data = b"".join(read_local_file("avatars", file_path + ".original"))
+        content_type = guess_type(file_path)[0]
+        return image_data, content_type or "application/octet-stream"
+
+    @override
+    def store_single_avatar_image(
+        self,
+        file_path: str,
+        *,
+        user_profile: UserProfile,
+        image_data: bytes,
+        content_type: str | None,
+        future: bool = True,
+    ) -> None:
+        write_local_file("avatars", file_path, image_data)
+
+    @override
+    def delete_avatar_image_from_storage(self, path_id: str) -> None:
+        delete_local_file("avatars", path_id + ".original")
+        delete_local_file("avatars", self.get_avatar_path(path_id, True))
+        delete_local_file("avatars", self.get_avatar_path(path_id, False))
+
+    @override
+    def get_realm_icon_url(self, realm_id: int, version: int) -> str:
+        return f"/user_avatars/{realm_id}/realm/icon.png?version={version}"
+
+    @override
+    def store_realm_icon_image(
+        self, icon_file: IO[bytes], user_profile: UserProfile, content_type: str
+    ) -> None:
+        upload_path = self.realm_avatar_and_logo_path(user_profile.realm)
+        image_data = icon_file.read()
+        write_local_file("avatars", os.path.join(upload_path, "icon.original"), image_data)
+
+        resized_data = resize_realm_icon(image_data)
+        write_local_file("avatars", os.path.join(upload_path, "icon.png"), resized_data)
+
+    @override
+    def get_realm_logo_url(self, realm_id: int, version: int, night: bool) -> str:
+        if night:
+            file_name = "night_logo.png"
+        else:
+            file_name = "logo.png"
+        return f"/user_avatars/{realm_id}/realm/{file_name}?version={version}"
+
+    @override
+    def store_realm_logo_image(
+        self, logo_file: IO[bytes], user_profile: UserProfile, night: bool, content_type: str
+    ) -> None:
+        upload_path = self.realm_avatar_and_logo_path(user_profile.realm)
+        if night:
+            original_file = "night_logo.original"
+            resized_file = "night_logo.png"
+        else:
+            original_file = "logo.original"
+            resized_file = "logo.png"
+        image_data = logo_file.read()
+        write_local_file("avatars", os.path.join(upload_path, original_file), image_data)
+
+        resized_data = resize_logo(image_data)
+        write_local_file("avatars", os.path.join(upload_path, resized_file), resized_data)
+
+    @override
+    def get_emoji_url(self, emoji_file_name: str, realm_id: int, still: bool = False) -> str:
+        if still:
+            return os.path.join(
+                "/user_avatars",
+                RealmEmoji.STILL_PATH_ID_TEMPLATE.format(
+                    realm_id=realm_id,
+                    emoji_filename_without_extension=os.path.splitext(emoji_file_name)[0],
+                ),
+            )
+        else:
+            return os.path.join(
+                "/user_avatars",
+                RealmEmoji.PATH_ID_TEMPLATE.format(
+                    realm_id=realm_id, emoji_file_name=emoji_file_name
+                ),
+            )
+
+    @override
+    def store_single_emoji_image(
+        self, path: str, content_type: str | None, user_profile: UserProfile, image_data: bytes
+    ) -> None:
+        write_local_file("avatars", path, image_data)
+
+    @override
+    def get_export_tarball_url(self, realm: Realm, export_path: str) -> str:
+        # export_path has a leading `/`
+        return realm.url + export_path
+
+    @override
+    def store_export_tarball(
+        self,
+        realm: Realm,
+        tarball_path: str,
+        percent_callback: Callable[[Any], None] | None = None,
+    ) -> str:
+        path = os.path.join(
+            "exports",
+            str(realm.id),
+            secrets.token_urlsafe(18),
+            os.path.basename(tarball_path),
+        )
+        abs_path = os.path.join(assert_is_not_none(settings.LOCAL_AVATARS_DIR), path)
+        os.makedirs(os.path.dirname(abs_path), exist_ok=True)
+        shutil.copy(tarball_path, abs_path)
+        return self.get_export_tarball_url(realm, "/user_avatars/" + path)
+
+    @override
+    def delete_export_tarball_from_storage(self, export_path: str) -> None:
+        assert export_path.startswith("/")
+        file_path = export_path.removeprefix("/").split("/", 1)[-1]
+        delete_local_file("avatars", file_path)

zerver/lib/upload/s3.py

@@ -0,0 +1,555 @@
+import os
+import secrets
+from collections.abc import Callable, Iterator
+from contextlib import contextmanager
+from datetime import datetime
+from typing import IO, TYPE_CHECKING, Any, Literal
+from urllib.parse import urljoin, urlsplit, urlunsplit
+
+import botocore
+import pyvips
+from botocore.client import Config
+from botocore.response import StreamingBody
+from django.conf import settings
+from django.utils.http import content_disposition_header
+from typing_extensions import override
+
+from zerver.lib.mime_types import INLINE_MIME_TYPES, bare_content_type
+from zerver.lib.partial import partial
+from zerver.lib.thumbnail import resize_logo, resize_realm_icon
+from zerver.lib.upload.base import StreamingSourceWithSize, ZulipUploadBackend
+from zerver.models import Realm, RealmEmoji, UserProfile
+
+if TYPE_CHECKING:
+    from mypy_boto3_s3.client import S3Client
+    from mypy_boto3_s3.service_resource import Bucket, Object
+
+# Duration that the signed upload URLs that we redirect to when
+# accessing uploaded files are available for clients to fetch before
+# they expire.
+SIGNED_UPLOAD_URL_DURATION = 60
+
+
+DELETE_BATCH_SIZE = 1000
+
+# Performance note:
+#
+# For writing files to S3, the file could either be stored in RAM
+# (if it is less than 2.5MiB or so) or an actual temporary file on disk.
+#
+# Because we set FILE_UPLOAD_MAX_MEMORY_SIZE to 0, only the latter case
+# should occur in practice.
+#
+# This is great, because passing the pseudofile object that Django gives
+# you to boto would be a pain.
+
+# To come up with a s3 key we randomly generate a "directory". The
+# "file name" is the original filename provided by the user run
+# through a sanitization function.
+
+# Note: boto3 is configured to bypass the Smokescreen outgoing proxy in
+# zerver.apps, making sure the setup runs at process startup.
+
+
+def get_bucket(bucket_name: str, authed: bool = True) -> "Bucket":
+    import boto3
+
+    checksum: Literal["when_required", "when_supported"] = (
+        "when_required" if settings.S3_SKIP_CHECKSUM else "when_supported"
+    )
+    return boto3.resource(
+        "s3",
+        aws_access_key_id=settings.S3_KEY if authed else None,
+        aws_secret_access_key=settings.S3_SECRET_KEY if authed else None,
+        region_name=settings.S3_REGION,
+        endpoint_url=settings.S3_ENDPOINT_URL,
+        config=Config(
+            signature_version=None if authed else botocore.UNSIGNED,
+            s3={"addressing_style": settings.S3_ADDRESSING_STYLE},
+            request_checksum_calculation=checksum,
+        ),
+    ).Bucket(bucket_name)
+
+
+def upload_content_to_s3(
+    bucket: "Bucket",
+    path: str,
+    content_type: str | None,
+    user_profile: UserProfile | None,
+    contents: bytes,
+    *,
+    storage_class: Literal[
+        "GLACIER_IR",
+        "INTELLIGENT_TIERING",
+        "ONEZONE_IA",
+        "REDUCED_REDUNDANCY",
+        "STANDARD",
+        "STANDARD_IA",
+    ] = "STANDARD",
+    cache_control: str | None = None,
+    extra_metadata: dict[str, str] | None = None,
+    filename: str | None = None,
+    target_realm: Realm | None = None,
+) -> None:
+    # Note that these steps are also replicated in
+    # handle_upload_pre_finish_hook in zerver.views.tus, to update
+    # properties for files uploaded via TUS.
+
+    key = bucket.Object(path)
+    metadata: dict[str, str] = {}
+    if user_profile:
+        metadata["user_profile_id"] = str(user_profile.id)
+        metadata["realm_id"] = str(user_profile.realm_id)
+    if target_realm:
+        metadata["realm_id"] = str(target_realm.id)
+    if extra_metadata is not None:
+        metadata.update(extra_metadata)
+
+    extras = {}
+    if content_type is None:  # nocoverage
+        content_type = ""
+    is_attachment = bare_content_type(content_type) not in INLINE_MIME_TYPES
+    if filename is not None:
+        extras["ContentDisposition"] = content_disposition_header(is_attachment, filename)
+    elif is_attachment:
+        extras["ContentDisposition"] = "attachment"
+    if cache_control is not None:
+        extras["CacheControl"] = cache_control
+
+    key.put(
+        Body=contents,
+        Metadata=metadata,
+        ContentType=content_type,
+        StorageClass=storage_class,
+        **extras,  # type: ignore[arg-type] # The dynamic kwargs here confuse mypy.
+    )
+
+
+BOTO_CLIENT: "S3Client | None" = None
+
+
+def get_boto_client() -> "S3Client":
+    """
+    Creating the client takes a long time so we need to cache it.
+    """
+    global BOTO_CLIENT
+    if BOTO_CLIENT is None:
+        BOTO_CLIENT = get_bucket(settings.S3_AUTH_UPLOADS_BUCKET).meta.client
+    return BOTO_CLIENT
+
+
+def get_signed_upload_url(path: str, filename: str, force_download: bool = False) -> str:
+    params = {
+        "Bucket": settings.S3_AUTH_UPLOADS_BUCKET,
+        "Key": path,
+    }
+    if force_download:
+        params["ResponseContentDisposition"] = (
+            content_disposition_header(True, filename) or "attachment"
+        )
+
+    return get_boto_client().generate_presigned_url(
+        ClientMethod="get_object",
+        Params=params,
+        ExpiresIn=SIGNED_UPLOAD_URL_DURATION,
+        HttpMethod="GET",
+    )
+
+
+class S3UploadBackend(ZulipUploadBackend):
+    def __init__(self) -> None:
+        from mypy_boto3_s3.service_resource import Bucket
+
+        self.avatar_bucket = get_bucket(settings.S3_AVATAR_BUCKET)
+        self.uploads_bucket = get_bucket(settings.S3_AUTH_UPLOADS_BUCKET)
+        self.export_bucket: Bucket | None = None
+        if settings.S3_EXPORT_BUCKET:
+            self.export_bucket = get_bucket(settings.S3_EXPORT_BUCKET)
+
+        self.public_upload_url_base = self.construct_public_upload_url_base()
+
+    def delete_file_from_s3(self, path_id: str, bucket: "Bucket") -> bool:
+        key = bucket.Object(path_id)
+
+        try:
+            key.load()
+        except botocore.exceptions.ClientError:
+            return False
+
+        key.delete()
+        return True
+
+    def construct_public_upload_url_base(self) -> str:
+        # Return the pattern for public URL for a key in the S3 Avatar bucket.
+        # For Amazon S3 itself, this will return the following:
+        #     f"https://{self.avatar_bucket.name}.{network_location}/{key}"
+        #
+        # However, we need this function to properly handle S3 style
+        # file upload backends that Zulip supports, which can have a
+        # different URL format. Configuring no signature and providing
+        # no access key makes `generate_presigned_url` just return the
+        # normal public URL for a key.
+        #
+        # It unfortunately takes 2ms per query to call
+        # generate_presigned_url. Since we need to potentially compute
+        # hundreds of avatar URLs in single `GET /messages` request,
+        # we instead back-compute the URL pattern here.
+
+        # The S3_AVATAR_PUBLIC_URL_PREFIX setting is used to override
+        # this prefix, for instance if a CloudFront distribution is
+        # used.
+        if settings.S3_AVATAR_PUBLIC_URL_PREFIX is not None:
+            prefix = settings.S3_AVATAR_PUBLIC_URL_PREFIX
+            if not prefix.endswith("/"):
+                prefix += "/"
+            return prefix
+
+        DUMMY_KEY = "dummy_key_ignored"
+
+        # We do not access self.avatar_bucket.meta.client directly,
+        # since that client is auth'd, and we want only the direct
+        # unauthed endpoint here.
+        client = get_bucket(self.avatar_bucket.name, authed=False).meta.client
+        dummy_signed_url = client.generate_presigned_url(
+            ClientMethod="get_object",
+            Params={
+                "Bucket": self.avatar_bucket.name,
+                "Key": DUMMY_KEY,
+            },
+            ExpiresIn=0,
+        )
+        split_url = urlsplit(dummy_signed_url)
+        assert split_url.path.endswith(f"/{DUMMY_KEY}")
+
+        return urlunsplit(
+            (split_url.scheme, split_url.netloc, split_url.path.removesuffix(DUMMY_KEY), "", "")
+        )
+
+    @override
+    def get_public_upload_root_url(self) -> str:
+        return self.public_upload_url_base
+
+    def get_public_upload_url(
+        self,
+        key: str,
+    ) -> str:
+        assert not key.startswith("/")
+        return urljoin(self.public_upload_url_base, key)
+
+    @override
+    def generate_message_upload_path(self, realm_id: str, sanitized_file_name: str) -> str:
+        return "/".join(
+            [
+                realm_id,
+                secrets.token_urlsafe(18),
+                sanitized_file_name,
+            ]
+        )
+
+    @override
+    def store_message_attachment(
+        self,
+        path_id: str,
+        filename: str,
+        content_type: str,
+        file_data: bytes,
+        user_profile: UserProfile | None,
+        target_realm: Realm | None,
+    ) -> None:
+        upload_content_to_s3(
+            self.uploads_bucket,
+            path_id,
+            content_type,
+            user_profile,
+            file_data,
+            storage_class=settings.S3_UPLOADS_STORAGE_CLASS,
+            filename=filename,
+            target_realm=target_realm,
+        )
+
+    @override
+    def save_attachment_contents(self, path_id: str, filehandle: IO[bytes]) -> None:
+        for chunk in self.uploads_bucket.Object(path_id).get()["Body"]:
+            filehandle.write(chunk)
+
+    @override
+    def attachment_source(self, path_id: str) -> StreamingSourceWithSize:
+        metadata = self.uploads_bucket.Object(path_id).get()
+
+        def s3_read(streamingbody: StreamingBody, size: int) -> bytes:
+            return streamingbody.read(amt=size)
+
+        vips_source: pyvips.Source = pyvips.SourceCustom()
+        vips_source.on_read(partial(s3_read, metadata["Body"]))
+        return StreamingSourceWithSize(
+            size=metadata["ContentLength"],
+            vips_source=vips_source,
+            reader=lambda: metadata["Body"],
+        )
+
+    @override
+    def delete_message_attachment_from_storage(
+        self, path_id: str, *, raw_path: bool = False
+    ) -> None:
+        with self.delete_message_attachments_from_storage(raw_paths=raw_path) as delete_one:
+            delete_one(path_id)
+
+    @contextmanager
+    @override
+    def delete_message_attachments_from_storage(
+        self, *, raw_paths: bool = False, flush: Callable[[list[str]], None] | None = None
+    ) -> Iterator[Callable[[str], None]]:
+        paths: list[tuple[str, bool]] = []
+
+        def flush_queue() -> None:
+            nonlocal paths
+            self.uploads_bucket.delete_objects(
+                Delete={
+                    "Objects": [{"Key": path_id} for path_id, _ in paths[:DELETE_BATCH_SIZE]],
+                    "Quiet": True,
+                },
+            )
+            if flush:
+                flush([path for path, is_db_path_id in paths[:DELETE_BATCH_SIZE] if is_db_path_id])
+            paths = paths[DELETE_BATCH_SIZE:]
+
+        def queue_delete(path_id: str) -> None:
+            nonlocal paths
+            paths.append((path_id, True))
+            if not raw_paths:
+                paths.append((f"{path_id}.info", False))
+                paths += [
+                    (thumb_path, False)
+                    for thumb_path, _ in self.all_message_attachments(
+                        include_thumbnails=True, prefix=f"thumbnail/{path_id}/"
+                    )
+                ]
+            if len(paths) > DELETE_BATCH_SIZE:
+                flush_queue()
+
+        yield queue_delete
+        if paths:
+            flush_queue()
+
+    @override
+    def all_message_attachments(
+        self,
+        include_thumbnails: bool = False,
+        prefix: str = "",
+    ) -> Iterator[tuple[str, datetime]]:
+        client = self.uploads_bucket.meta.client
+        paginator = client.get_paginator("list_objects_v2")
+        page_iterator = paginator.paginate(Bucket=self.uploads_bucket.name, Prefix=prefix)
+
+        for page in page_iterator:
+            if page["KeyCount"] > 0:
+                for item in page["Contents"]:
+                    if not include_thumbnails and item["Key"].startswith("thumbnail/"):
+                        continue
+                    yield (
+                        item["Key"],
+                        item["LastModified"],
+                    )
+
+    @override
+    def get_avatar_url(self, hash_key: str, medium: bool = False) -> str:
+        return self.get_public_upload_url(self.get_avatar_path(hash_key, medium))
+
+    @override
+    def get_avatar_contents(self, file_path: str, avatar_source: str) -> tuple[bytes, str]:
+        # Currently, only used in codepaths where avatar_source = "U".
+        # We can extend it for avatar_source = "J", if required.
+        assert avatar_source is UserProfile.AVATAR_FROM_USER
+        key = self.avatar_bucket.Object(file_path + ".original")
+        image_data = key.get()["Body"].read()
+        content_type = key.content_type
+        return image_data, content_type
+
+    @override
+    def store_single_avatar_image(
+        self,
+        file_path: str,
+        *,
+        user_profile: UserProfile,
+        image_data: bytes,
+        content_type: str | None,
+        future: bool = True,
+    ) -> None:
+        extra_metadata = {"avatar_version": str(user_profile.avatar_version + (1 if future else 0))}
+        upload_content_to_s3(
+            self.avatar_bucket,
+            file_path,
+            content_type,
+            user_profile,
+            image_data,
+            extra_metadata=extra_metadata,
+            cache_control="public, max-age=31536000, immutable",
+        )
+
+    @override
+    def delete_avatar_image_from_storage(self, path_id: str) -> None:
+        self.delete_file_from_s3(path_id + ".original", self.avatar_bucket)
+        self.delete_file_from_s3(self.get_avatar_path(path_id, True), self.avatar_bucket)
+        self.delete_file_from_s3(self.get_avatar_path(path_id, False), self.avatar_bucket)
+
+    @override
+    def get_realm_icon_url(self, realm_id: int, version: int) -> str:
+        public_url = self.get_public_upload_url(f"{realm_id}/realm/icon.png")
+        return public_url + f"?version={version}"
+
+    @override
+    def store_realm_icon_image(
+        self, icon_file: IO[bytes], user_profile: UserProfile, content_type: str
+    ) -> None:
+        s3_file_name = os.path.join(self.realm_avatar_and_logo_path(user_profile.realm), "icon")
+
+        image_data = icon_file.read()
+        upload_content_to_s3(
+            self.avatar_bucket,
+            s3_file_name + ".original",
+            content_type,
+            user_profile,
+            image_data,
+        )
+
+        resized_data = resize_realm_icon(image_data)
+        upload_content_to_s3(
+            self.avatar_bucket,
+            s3_file_name + ".png",
+            "image/png",
+            user_profile,
+            resized_data,
+        )
+        # See avatar_url in avatar.py for URL.  (That code also handles the case
+        # that users use gravatar.)
+
+    @override
+    def get_realm_logo_url(self, realm_id: int, version: int, night: bool) -> str:
+        if not night:
+            file_name = "logo.png"
+        else:
+            file_name = "night_logo.png"
+        public_url = self.get_public_upload_url(f"{realm_id}/realm/{file_name}")
+        return public_url + f"?version={version}"
+
+    @override
+    def store_realm_logo_image(
+        self, logo_file: IO[bytes], user_profile: UserProfile, night: bool, content_type: str
+    ) -> None:
+        if night:
+            basename = "night_logo"
+        else:
+            basename = "logo"
+        s3_file_name = os.path.join(self.realm_avatar_and_logo_path(user_profile.realm), basename)
+
+        image_data = logo_file.read()
+        upload_content_to_s3(
+            self.avatar_bucket,
+            s3_file_name + ".original",
+            content_type,
+            user_profile,
+            image_data,
+        )
+
+        resized_data = resize_logo(image_data)
+        upload_content_to_s3(
+            self.avatar_bucket,
+            s3_file_name + ".png",
+            "image/png",
+            user_profile,
+            resized_data,
+        )
+        # See avatar_url in avatar.py for URL.  (That code also handles the case
+        # that users use gravatar.)
+
+    @override
+    def get_emoji_url(self, emoji_file_name: str, realm_id: int, still: bool = False) -> str:
+        if still:
+            emoji_path = RealmEmoji.STILL_PATH_ID_TEMPLATE.format(
+                realm_id=realm_id,
+                emoji_filename_without_extension=os.path.splitext(emoji_file_name)[0],
+            )
+            return self.get_public_upload_url(emoji_path)
+        else:
+            emoji_path = RealmEmoji.PATH_ID_TEMPLATE.format(
+                realm_id=realm_id, emoji_file_name=emoji_file_name
+            )
+            return self.get_public_upload_url(emoji_path)
+
+    @override
+    def store_single_emoji_image(
+        self, path: str, content_type: str | None, user_profile: UserProfile, image_data: bytes
+    ) -> None:
+        upload_content_to_s3(
+            self.avatar_bucket,
+            path,
+            content_type,
+            user_profile,
+            image_data,
+            cache_control="public, max-age=31536000, immutable",
+        )
+
+    @override
+    def get_export_tarball_url(self, realm: Realm, export_path: str) -> str:
+        export_path = export_path.removeprefix("/")
+        if self.export_bucket:
+            # Fix old data if the row was created when an export bucket was not in use.
+            export_path = export_path.removeprefix("exports/")
+            client = self.export_bucket.meta.client
+            return client.generate_presigned_url(
+                ClientMethod="get_object",
+                Params={
+                    "Bucket": self.export_bucket.name,
+                    "Key": export_path,
+                },
+                # Expires in one week, the longest allowed by AWS
+                ExpiresIn=60 * 60 * 24 * 7,
+            )
+        else:
+            if not export_path.startswith("exports/"):
+                export_path = "exports/" + export_path
+            client = self.avatar_bucket.meta.client
+            signed_url = client.generate_presigned_url(
+                ClientMethod="get_object",
+                Params={
+                    "Bucket": self.avatar_bucket.name,
+                    "Key": export_path,
+                },
+                ExpiresIn=0,
+            )
+            # Strip off the signing query parameters, since this URL is public
+            return urlsplit(signed_url)._replace(query="").geturl()
+
+    def export_object(self, tarball_path: str) -> "Object":
+        if self.export_bucket:
+            return self.export_bucket.Object(
+                os.path.join(secrets.token_hex(16), os.path.basename(tarball_path))
+            )
+        else:
+            # We fall back to the avatar bucket, because it's world-readable.
+            return self.avatar_bucket.Object(
+                os.path.join("exports", secrets.token_hex(16), os.path.basename(tarball_path))
+            )
+
+    @override
+    def store_export_tarball(
+        self,
+        realm: Realm,
+        tarball_path: str,
+        percent_callback: Callable[[Any], None] | None = None,
+    ) -> str:
+        key = self.export_object(tarball_path)
+
+        if percent_callback is None:
+            key.upload_file(Filename=tarball_path)
+        else:
+            key.upload_file(Filename=tarball_path, Callback=percent_callback)
+
+        return self.get_export_tarball_url(realm, key.key)
+
+    @override
+    def delete_export_tarball_from_storage(self, export_path: str) -> None:
+        assert export_path.startswith("/")
+        path_id = export_path.removeprefix("/")
+        bucket = self.export_bucket or self.avatar_bucket
+        self.delete_file_from_s3(path_id, bucket)