snapshot of https://github.com/zulip/zulip.git @ 48c69911c7dd4e03b20552eb3bd77ad8151c1357 (2026-09-15)

zerver/tests/test_upload_s3.py

@@ -0,0 +1,861 @@
+import os
+import re
+from io import BytesIO, StringIO
+from unittest.mock import patch
+from urllib.parse import parse_qs, urlsplit
+
+import boto3
+import botocore.exceptions
+import pyvips
+from django.conf import settings
+from django.test import override_settings
+from moto.core.decorator import mock_aws
+from mypy_boto3_s3.type_defs import CopySourceTypeDef
+
+import zerver.lib.upload
+from zerver.actions.create_user import do_create_user
+from zerver.actions.user_settings import do_scrub_avatar_images
+from zerver.lib.avatar_hash import user_avatar_path
+from zerver.lib.create_user import copy_default_settings
+from zerver.lib.test_classes import ZulipTestCase
+from zerver.lib.test_helpers import (
+    create_s3_buckets,
+    get_test_image_file,
+    read_test_image_file,
+    use_s3_backend,
+)
+from zerver.lib.thumbnail import (
+    DEFAULT_AVATAR_SIZE,
+    MEDIUM_AVATAR_SIZE,
+    THUMBNAIL_OUTPUT_FORMATS,
+    BadImageError,
+    ThumbnailFormat,
+    resize_avatar,
+    resize_emoji,
+)
+from zerver.lib.upload import (
+    all_message_attachments,
+    attachment_source,
+    delete_export_tarball,
+    delete_message_attachment,
+    delete_message_attachments,
+    save_attachment_contents,
+    upload_export_tarball,
+    upload_message_attachment,
+)
+from zerver.lib.upload.base import StreamingSourceWithSize
+from zerver.lib.upload.s3 import S3UploadBackend
+from zerver.models import Attachment, OnboardingStep, RealmEmoji, UserProfile
+from zerver.models.realms import get_realm
+from zerver.models.users import get_system_bot
+
+
+class S3Test(ZulipTestCase):
+    @use_s3_backend
+    def test_upload_message_attachment(self) -> None:
+        bucket = create_s3_buckets(settings.S3_AUTH_UPLOADS_BUCKET)[0]
+
+        user_profile = self.example_user("hamlet")
+        url = upload_message_attachment("dummy.txt", "text/plain", b"zulip!", user_profile)[0]
+
+        base = "/user_uploads/"
+        self.assertEqual(base, url[: len(base)])
+        path_id = re.sub(r"/user_uploads/", "", url)
+        content = bucket.Object(path_id).get()["Body"].read()
+        self.assertEqual(b"zulip!", content)
+
+        uploaded_file = Attachment.objects.get(owner=user_profile, path_id=path_id)
+        self.assert_length(b"zulip!", uploaded_file.size)
+
+        self.subscribe(self.example_user("hamlet"), "Denmark")
+        body = f"First message ...[zulip.txt](http://{user_profile.realm.host}{url})"
+        self.send_stream_message(self.example_user("hamlet"), "Denmark", body, "test")
+
+    @use_s3_backend
+    def test_upload_message_attachment_thumbnail(self) -> None:
+        bucket = create_s3_buckets(settings.S3_AUTH_UPLOADS_BUCKET)[0]
+        user_profile = self.example_user("hamlet")
+        with (
+            self.thumbnail_formats(ThumbnailFormat("webp", 100, 75, animated=False)),
+            self.captureOnCommitCallbacks(execute=True),
+        ):
+            url = upload_message_attachment(
+                "img.png", "image/png", read_test_image_file("img.png"), user_profile
+            )[0]
+        self.assertTrue(url.startswith("/user_uploads/"))
+        path_id = url.removeprefix("/user_uploads/")
+        s3_image = bucket.Object(path_id).get()
+        self.assertEqual(s3_image["Body"].read(), read_test_image_file("img.png"))
+        self.assertEqual(
+            s3_image["Metadata"],
+            {"realm_id": str(user_profile.realm_id), "user_profile_id": str(user_profile.id)},
+        )
+
+        s3_thumbnail_image = bucket.Object(f"thumbnail/{path_id}/100x75.webp").get()
+        resized_image = pyvips.Image.new_from_buffer(s3_thumbnail_image["Body"].read(), "")
+        self.assertEqual(75, resized_image.width)
+        self.assertEqual(75, resized_image.height)
+        self.assertEqual(s3_thumbnail_image["Metadata"], {})
+
+    @use_s3_backend
+    def test_save_attachment_contents(self) -> None:
+        create_s3_buckets(settings.S3_AUTH_UPLOADS_BUCKET)
+        user_profile = self.example_user("hamlet")
+        url = upload_message_attachment("dummy.txt", "text/plain", b"zulip!", user_profile)[0]
+
+        path_id = re.sub(r"/user_uploads/", "", url)
+        output = BytesIO()
+        save_attachment_contents(path_id, output)
+        self.assertEqual(output.getvalue(), b"zulip!")
+
+    @use_s3_backend
+    def test_attachment_source(self) -> None:
+        create_s3_buckets(settings.S3_AUTH_UPLOADS_BUCKET)
+        user_profile = self.example_user("hamlet")
+        url = upload_message_attachment(
+            "img.png", "image/png", read_test_image_file("img.png"), user_profile
+        )[0]
+        path_id = re.sub(r"/user_uploads/", "", url)
+
+        source = attachment_source(path_id)
+        self.assertIsInstance(source, StreamingSourceWithSize)
+        self.assertEqual(source.size, len(read_test_image_file("img.png")))
+        image = pyvips.Image.new_from_source(source.vips_source, "", access="sequential")
+        self.assertEqual(128, image.height)
+        self.assertEqual(128, image.width)
+
+    @use_s3_backend
+    def test_upload_message_attachment_s3_cross_realm_path(self) -> None:
+        """
+        Verifies that the path of a file uploaded by a cross-realm bot to another
+        realm is correct.
+        """
+        bucket = create_s3_buckets(settings.S3_AUTH_UPLOADS_BUCKET)[0]
+
+        internal_realm = get_realm(settings.SYSTEM_BOT_REALM)
+        zulip_realm = get_realm("zulip")
+        user_profile = get_system_bot(settings.EMAIL_GATEWAY_BOT, internal_realm.id)
+        self.assertEqual(user_profile.realm, internal_realm)
+
+        url = upload_message_attachment(
+            "dummy.txt", "text/plain", b"zulip!", user_profile, zulip_realm
+        )[0]
+        # Ensure the correct realm id of the target realm is used instead of the bot's realm.
+        self.assertTrue(url.startswith(f"/user_uploads/{zulip_realm.id}/"))
+
+        path_id = re.sub(r"/user_uploads/", "", url)
+        s3_obj = bucket.Object(path_id)
+        s3_obj.load()
+        self.assertEqual(s3_obj.metadata["realm_id"], str(zulip_realm.id))
+
+    @use_s3_backend
+    def test_delete_message_attachment(self) -> None:
+        bucket = create_s3_buckets(settings.S3_AUTH_UPLOADS_BUCKET)[0]
+
+        self.login("hamlet")
+        # Upload an image which will generate multiple thumbnails
+        webp_anim = ThumbnailFormat("webp", 100, 75, animated=True)
+        webp_still = ThumbnailFormat("webp", 100, 75, animated=False)
+        with (
+            self.thumbnail_formats(webp_anim, webp_still),
+            self.captureOnCommitCallbacks(execute=True),
+            get_test_image_file("animated_img.gif") as image_file,
+        ):
+            json_response = self.assert_json_success(
+                self.client_post("/json/user_uploads", {"file": image_file})
+            )
+            path_id = re.sub(r"/user_uploads/", "", json_response["url"])
+            # Exit the block, triggering the thumbnailing worker
+
+        self.assertIsNotNone(bucket.Object(path_id).get())
+        self.assertIsNotNone(bucket.Object(f"thumbnail/{path_id}/{webp_anim}").get())
+        self.assertIsNotNone(bucket.Object(f"thumbnail/{path_id}/{webp_still}").get())
+        delete_message_attachment(path_id)
+        with self.assertRaises(botocore.exceptions.ClientError):
+            bucket.Object(path_id).load()
+        with self.assertRaises(botocore.exceptions.ClientError):
+            bucket.Object(f"thumbnail/{path_id}/{webp_anim}").load()
+        with self.assertRaises(botocore.exceptions.ClientError):
+            bucket.Object(f"thumbnail/{path_id}/{webp_still}").load()
+
+    @use_s3_backend
+    def test_delete_message_attachments(self) -> None:
+        bucket = create_s3_buckets(settings.S3_AUTH_UPLOADS_BUCKET)[0]
+
+        user_profile = self.example_user("hamlet")
+        path_ids = []
+        for n in range(1, 5):
+            url = upload_message_attachment("dummy.txt", "text/plain", b"zulip!", user_profile)[0]
+            path_id = re.sub(r"/user_uploads/", "", url)
+            self.assertIsNotNone(bucket.Object(path_id).get())
+            path_ids.append(path_id)
+
+        with patch.object(
+            S3UploadBackend, "delete_message_attachment_from_storage"
+        ) as single_delete:
+            with delete_message_attachments() as delete_one:
+                for path_id in path_ids:
+                    delete_one(path_id)
+            single_delete.assert_not_called()
+        for path_id in path_ids:
+            with self.assertRaises(botocore.exceptions.ClientError):
+                bucket.Object(path_id).load()
+
+    @use_s3_backend
+    def test_delete_message_attachment_when_file_doesnt_exist(self) -> None:
+        bucket = create_s3_buckets(settings.S3_AUTH_UPLOADS_BUCKET)[0]
+        with self.assertRaises(botocore.exceptions.ClientError):
+            bucket.Object("non-existent-file").load()
+        delete_message_attachment("non-existent-file")
+
+    @use_s3_backend
+    def test_all_message_attachments(self) -> None:
+        create_s3_buckets(settings.S3_AUTH_UPLOADS_BUCKET)
+
+        user_profile = self.example_user("hamlet")
+        path_ids = []
+        for n in range(1, 5):
+            url = upload_message_attachment("dummy.txt", "text/plain", b"zulip!", user_profile)[0]
+            path_ids.append(re.sub(r"/user_uploads/", "", url))
+
+        # Put an image in, which gets thumbnailed
+        with self.captureOnCommitCallbacks(execute=True):
+            url = upload_message_attachment(
+                "img.png", "image/png", read_test_image_file("img.png"), user_profile
+            )[0]
+            image_path_id = re.sub(r"/user_uploads/", "", url)
+            path_ids.append(image_path_id)
+
+        found_paths = [r[0] for r in all_message_attachments()]
+        self.assertEqual(sorted(found_paths), sorted(path_ids))
+
+        found_paths = [r[0] for r in all_message_attachments(prefix=str(user_profile.realm_id))]
+        self.assertEqual(sorted(found_paths), sorted(path_ids))
+
+        found_paths = [r[0] for r in all_message_attachments(prefix=os.path.dirname(path_ids[0]))]
+        self.assertEqual(found_paths, [path_ids[0]])
+
+        found_paths = [r[0] for r in all_message_attachments(prefix="missing")]
+        self.assertEqual(found_paths, [])
+
+        found_paths = [r[0] for r in all_message_attachments(include_thumbnails=True)]
+        for thumbnail_format in THUMBNAIL_OUTPUT_FORMATS:
+            if thumbnail_format.animated:
+                continue
+            path_ids.append(f"thumbnail/{image_path_id}/{thumbnail_format!s}")
+        self.assertEqual(sorted(found_paths), sorted(path_ids))
+
+    @use_s3_backend
+    def test_user_uploads_authed(self) -> None:
+        """
+        A call to /json/user_uploads should return a url and actually create an object.
+        """
+        bucket = create_s3_buckets(settings.S3_AUTH_UPLOADS_BUCKET)[0]
+
+        self.login("hamlet")
+        fp = StringIO("zulip!")
+        fp.name = "zulip with excitement!.txt"
+
+        result = self.client_post("/json/user_uploads", {"file": fp})
+        response_dict = self.assert_json_success(result)
+        self.assertIn("uri", response_dict)
+        self.assertIn("url", response_dict)
+        base = "/user_uploads/"
+        url = response_dict["url"]
+        self.assertEqual(response_dict["uri"], url)
+        self.assertEqual(base, url[: len(base)])
+
+        # In development, this is just a redirect
+        response = self.client_get(url)
+        redirect_url = response["Location"]
+        path = urlsplit(redirect_url).path
+        assert path.startswith("/")
+        key = path.removeprefix("/")
+        self.assertEqual(b"zulip!", bucket.Object(key).get()["Body"].read())
+
+        prefix = f"/internal/s3/{settings.S3_AUTH_UPLOADS_BUCKET}.s3.amazonaws.com/"
+        with self.settings(DEVELOPMENT=False):
+            response = self.client_get(url)
+        redirect_url = response["X-Accel-Redirect"]
+        path = urlsplit(redirect_url).path
+        assert path.startswith(prefix)
+        key = path.removeprefix(prefix)
+        self.assertEqual(b"zulip!", bucket.Object(key).get()["Body"].read())
+
+        # Check the download endpoint
+        download_url = url.replace("/user_uploads/", "/user_uploads/download/")
+        with self.settings(DEVELOPMENT=False):
+            response = self.client_get(download_url)
+        redirect_url = response["X-Accel-Redirect"]
+        path = urlsplit(redirect_url).path
+        content_disposition = parse_qs(urlsplit(redirect_url).query)["response-content-disposition"]
+        self.assertEqual(
+            content_disposition[0], 'attachment; filename="zulip with excitement!.txt"'
+        )
+        assert path.startswith(prefix)
+        key = path.removeprefix(prefix)
+        self.assertEqual(b"zulip!", bucket.Object(key).get()["Body"].read())
+
+        # Now try the endpoint that's supposed to return a temporary URL for access
+        # to the file.
+        result = self.client_get("/json" + url)
+        data = self.assert_json_success(result)
+        url_only_url = data["url"]
+
+        self.assertNotEqual(url_only_url, url)
+        self.assertIn("user_uploads/temporary/", url_only_url)
+        self.assertTrue(url_only_url.endswith("zulip-with-excitement.txt"))
+        # The generated URL has a token authorizing the requester to access the file
+        # without being logged in.
+        self.logout()
+        with self.settings(DEVELOPMENT=False):
+            self.client_get(url_only_url)
+        redirect_url = response["X-Accel-Redirect"]
+        path = urlsplit(redirect_url).path
+        assert path.startswith(prefix)
+        key = path.removeprefix(prefix)
+        self.assertEqual(b"zulip!", bucket.Object(key).get()["Body"].read())
+
+        # The original url shouldn't work when logged out:
+        with self.settings(DEVELOPMENT=False):
+            result = self.client_get(url)
+        self.assertEqual(result.status_code, 302)
+        self.assertTrue(result.headers["Location"].endswith(f"/login/?next={url}"))
+
+        hamlet = self.example_user("hamlet")
+        self.subscribe(hamlet, "Denmark")
+        body = f"First message ...[zulip.txt](http://{hamlet.realm.host}" + url + ")"
+        self.send_stream_message(hamlet, "Denmark", body, "test")
+
+    @use_s3_backend
+    def test_user_uploads_empty_file(self) -> None:
+        bucket = create_s3_buckets(settings.S3_AUTH_UPLOADS_BUCKET)[0]
+
+        self.login("hamlet")
+        fp = StringIO("")
+        fp.name = "empty-file.txt"
+
+        result = self.client_post("/json/user_uploads", {"file": fp})
+        response_dict = self.assert_json_success(result)
+        self.assertIn("url", response_dict)
+        url = response_dict["url"]
+
+        # Despite S3 being configured, we serve the 0-byte file directly
+        response = self.client_get(url)
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.headers["Content-Type"], "text/plain")
+        self.assertEqual(response.headers["Content-Length"], "0")
+        self.assertEqual(
+            response.headers["Content-Disposition"], 'inline; filename="empty-file.txt"'
+        )
+        self.assertEqual(response.headers["Cache-Control"], "private, immutable")
+        key = url.removeprefix("/user_uploads/")
+        self.assertEqual(b"", bucket.Object(key).get()["Body"].read())
+
+    @use_s3_backend
+    def test_user_uploads_text_charset(self) -> None:
+        bucket = create_s3_buckets(settings.S3_AUTH_UPLOADS_BUCKET)[0]
+
+        self.login("hamlet")
+        fp = BytesIO("नाम में क्या रक्खा हे".encode())
+        fp.name = "zulip.txt"
+
+        result = self.client_post("/json/user_uploads", {"file": fp})
+        response_dict = self.assert_json_success(result)
+        self.assertIn("url", response_dict)
+        url = response_dict["url"]
+
+        path_id = re.sub(r"/user_uploads/", "", url)
+        s3_obj = bucket.Object(path_id)
+        s3_obj.load()
+        attachment_obj = Attachment.objects.get(path_id=path_id)
+        self.assertEqual(attachment_obj.content_type, 'text/plain; charset="utf-8"')
+        self.assertEqual(s3_obj.content_type, 'text/plain; charset="utf-8"')
+
+    @use_s3_backend
+    def test_user_avatars_base(self) -> None:
+        backend = zerver.lib.upload.get_upload_backend()
+        assert isinstance(backend, S3UploadBackend)
+        self.assertEqual(
+            backend.construct_public_upload_url_base(),
+            f"https://{settings.S3_AVATAR_BUCKET}.s3.amazonaws.com/",
+        )
+
+        with self.settings(S3_AVATAR_PUBLIC_URL_PREFIX="https://avatars.example.com"):
+            self.assertEqual(
+                backend.construct_public_upload_url_base(), "https://avatars.example.com/"
+            )
+
+        with self.settings(S3_AVATAR_PUBLIC_URL_PREFIX="https://avatars.example.com/"):
+            self.assertEqual(
+                backend.construct_public_upload_url_base(), "https://avatars.example.com/"
+            )
+
+    @use_s3_backend
+    def test_user_avatars_redirect(self) -> None:
+        create_s3_buckets(settings.S3_AVATAR_BUCKET)[0]
+        self.login("hamlet")
+        with get_test_image_file("img.png") as image_file:
+            result = self.client_post("/json/users/me/avatar", {"file": image_file})
+
+        response_dict = self.assert_json_success(result)
+        self.assertIn("avatar_url", response_dict)
+        base = f"https://{settings.S3_AVATAR_BUCKET}.s3.amazonaws.com/"
+        url = self.assert_json_success(result)["avatar_url"]
+        self.assertEqual(base, url[: len(base)])
+
+        # Try hitting the equivalent `/user_avatars` endpoint
+        wrong_url = "/user_avatars/" + url.removeprefix(base)
+        result = self.client_get(wrong_url)
+        self.assertEqual(result.status_code, 301)
+        self.assertEqual(result["Location"], url)
+
+    @use_s3_backend
+    def test_upload_avatar_image(self) -> None:
+        bucket = create_s3_buckets(settings.S3_AVATAR_BUCKET)[0]
+
+        user_profile = self.example_user("hamlet")
+        path_id = user_avatar_path(user_profile)
+        original_image_path_id = path_id + ".original"
+        medium_path_id = path_id + "-medium.png"
+
+        with get_test_image_file("img.png") as image_file:
+            zerver.lib.upload.upload_avatar_image(
+                image_file, user_profile, content_type="image/png", future=False
+            )
+        user_profile.avatar_source = UserProfile.AVATAR_FROM_USER
+        user_profile.save()
+        test_image_data = read_test_image_file("img.png")
+        test_medium_image_data = resize_avatar(test_image_data, MEDIUM_AVATAR_SIZE)
+
+        original_image_key = bucket.Object(original_image_path_id)
+        self.assertEqual(original_image_key.key, original_image_path_id)
+        image_data = original_image_key.get()["Body"].read()
+        self.assertEqual(image_data, test_image_data)
+
+        medium_image_key = bucket.Object(medium_path_id)
+        self.assertEqual(medium_image_key.key, medium_path_id)
+        medium_image_data = medium_image_key.get()["Body"].read()
+        self.assertEqual(medium_image_data, test_medium_image_data)
+
+        bucket.Object(medium_image_key.key).delete()
+        zerver.lib.upload.ensure_avatar_image(user_profile, medium=True)
+        medium_image_key = bucket.Object(medium_path_id)
+        self.assertEqual(medium_image_key.key, medium_path_id)
+
+    @use_s3_backend
+    def test_copy_avatar_image(self) -> None:
+        bucket = create_s3_buckets(settings.S3_AVATAR_BUCKET)[0]
+
+        self.login("hamlet")
+        with get_test_image_file("img.png") as image_file:
+            self.client_post("/json/users/me/avatar", {"file": image_file})
+
+        source_user_profile = self.example_user("hamlet")
+        target_user_profile = do_create_user(
+            "user@zulip.com", "password", get_realm("zulip"), "user", acting_user=None
+        )
+
+        # 'visibility_policy_banner' is already marked as read for new users.
+        # Delete that row to avoid integrity error in copy_default_settings.
+        OnboardingStep.objects.filter(
+            user=target_user_profile, onboarding_step="visibility_policy_banner"
+        ).delete()
+
+        copy_default_settings(source_user_profile, target_user_profile)
+
+        source_path_id = user_avatar_path(source_user_profile)
+        target_path_id = user_avatar_path(target_user_profile)
+        self.assertNotEqual(source_path_id, target_path_id)
+
+        source_image_key = bucket.Object(source_path_id + ".png")
+        target_image_key = bucket.Object(target_path_id + ".png")
+        self.assertEqual(target_image_key.key, target_path_id + ".png")
+        self.assertEqual(source_image_key.content_type, target_image_key.content_type)
+        source_image_data = source_image_key.get()["Body"].read()
+        target_image_data = target_image_key.get()["Body"].read()
+
+        source_original_image_path_id = source_path_id + ".original"
+        target_original_image_path_id = target_path_id + ".original"
+        target_original_image_key = bucket.Object(target_original_image_path_id)
+        self.assertEqual(target_original_image_key.key, target_original_image_path_id)
+        source_original_image_key = bucket.Object(source_original_image_path_id)
+        self.assertEqual(
+            source_original_image_key.content_type, target_original_image_key.content_type
+        )
+        source_image_data = source_original_image_key.get()["Body"].read()
+        target_image_data = target_original_image_key.get()["Body"].read()
+        self.assertEqual(source_image_data, target_image_data)
+
+        target_medium_path_id = target_path_id + "-medium.png"
+        source_medium_path_id = source_path_id + "-medium.png"
+        source_medium_image_key = bucket.Object(source_medium_path_id)
+        target_medium_image_key = bucket.Object(target_medium_path_id)
+        self.assertEqual(target_medium_image_key.key, target_medium_path_id)
+        self.assertEqual(source_medium_image_key.content_type, target_medium_image_key.content_type)
+        source_medium_image_data = source_medium_image_key.get()["Body"].read()
+        target_medium_image_data = target_medium_image_key.get()["Body"].read()
+        self.assertEqual(source_medium_image_data, target_medium_image_data)
+
+    @use_s3_backend
+    def test_ensure_avatar_image(self) -> None:
+        bucket = create_s3_buckets(settings.S3_AVATAR_BUCKET)[0]
+
+        user_profile = self.example_user("hamlet")
+        base_file_path = user_avatar_path(user_profile)
+        file_path = base_file_path + ".png"
+        original_file_path = base_file_path + ".original"
+        medium_file_path = base_file_path + "-medium.png"
+
+        with get_test_image_file("img.png") as image_file:
+            zerver.lib.upload.upload_avatar_image(
+                image_file, user_profile, content_type="image/png", future=False
+            )
+        user_profile.avatar_source = UserProfile.AVATAR_FROM_USER
+        user_profile.save()
+
+        key = bucket.Object(original_file_path)
+        image_data = key.get()["Body"].read()
+
+        zerver.lib.upload.ensure_avatar_image(user_profile)
+        resized_avatar = resize_avatar(image_data)
+        key = bucket.Object(file_path)
+        self.assertEqual(resized_avatar, key.get()["Body"].read())
+
+        zerver.lib.upload.ensure_avatar_image(user_profile, medium=True)
+        resized_avatar = resize_avatar(image_data, MEDIUM_AVATAR_SIZE)
+        key = bucket.Object(medium_file_path)
+        self.assertEqual(resized_avatar, key.get()["Body"].read())
+
+    @use_s3_backend
+    def test_delete_avatar_image(self) -> None:
+        bucket = create_s3_buckets(settings.S3_AVATAR_BUCKET)[0]
+
+        self.login("hamlet")
+        with get_test_image_file("img.png") as image_file:
+            self.client_post("/json/users/me/avatar", {"file": image_file})
+
+        user = self.example_user("hamlet")
+
+        avatar_base_path = user_avatar_path(user)
+        avatar_path_id = avatar_base_path + ".png"
+        avatar_original_image_path_id = avatar_base_path + ".original"
+        avatar_medium_path_id = avatar_base_path + "-medium.png"
+
+        self.assertEqual(user.avatar_source, UserProfile.AVATAR_FROM_USER)
+        self.assertIsNotNone(bucket.Object(avatar_path_id))
+        self.assertIsNotNone(bucket.Object(avatar_original_image_path_id))
+        self.assertIsNotNone(bucket.Object(avatar_medium_path_id))
+
+        do_scrub_avatar_images(user, acting_user=user)
+
+        self.assertEqual(user.avatar_source, UserProfile.AVATAR_FROM_JDENTICON)
+
+        # Confirm that the avatar files no longer exist in S3.
+        with self.assertRaises(botocore.exceptions.ClientError):
+            bucket.Object(avatar_path_id).load()
+        with self.assertRaises(botocore.exceptions.ClientError):
+            bucket.Object(avatar_original_image_path_id).load()
+        with self.assertRaises(botocore.exceptions.ClientError):
+            bucket.Object(avatar_medium_path_id).load()
+
+    @use_s3_backend
+    def test_upload_realm_icon_image(self) -> None:
+        bucket = create_s3_buckets(settings.S3_AVATAR_BUCKET)[0]
+
+        user_profile = self.example_user("hamlet")
+        with get_test_image_file("img.png") as image_file:
+            zerver.lib.upload.get_upload_backend().store_realm_icon_image(
+                image_file, user_profile, content_type="image/png"
+            )
+
+        original_path_id = os.path.join(str(user_profile.realm.id), "realm", "icon.original")
+        original_key = bucket.Object(original_path_id)
+        self.assertEqual(read_test_image_file("img.png"), original_key.get()["Body"].read())
+
+        resized_path_id = os.path.join(str(user_profile.realm.id), "realm", "icon.png")
+        resized_data = bucket.Object(resized_path_id).get()["Body"].read()
+        resized_image = pyvips.Image.new_from_buffer(resized_data, "")
+        self.assertEqual(DEFAULT_AVATAR_SIZE, resized_image.height)
+        self.assertEqual(DEFAULT_AVATAR_SIZE, resized_image.width)
+
+    @use_s3_backend
+    def _test_upload_logo_image(self, night: bool, file_name: str) -> None:
+        bucket = create_s3_buckets(settings.S3_AVATAR_BUCKET)[0]
+
+        user_profile = self.example_user("hamlet")
+        with get_test_image_file("img.png") as image_file:
+            zerver.lib.upload.get_upload_backend().store_realm_logo_image(
+                image_file, user_profile, night, "image/png"
+            )
+
+        original_path_id = os.path.join(
+            str(user_profile.realm.id), "realm", f"{file_name}.original"
+        )
+        original_key = bucket.Object(original_path_id)
+        self.assertEqual(read_test_image_file("img.png"), original_key.get()["Body"].read())
+
+        resized_path_id = os.path.join(str(user_profile.realm.id), "realm", f"{file_name}.png")
+        resized_data = bucket.Object(resized_path_id).get()["Body"].read()
+        resized_image = pyvips.Image.new_from_buffer(resized_data, "")
+        self.assertEqual(DEFAULT_AVATAR_SIZE, resized_image.height)
+        self.assertEqual(DEFAULT_AVATAR_SIZE, resized_image.width)
+
+    def test_store_realm_logo_image(self) -> None:
+        self._test_upload_logo_image(night=False, file_name="logo")
+        self._test_upload_logo_image(night=True, file_name="night_logo")
+
+    @use_s3_backend
+    def test_get_emoji_url(self) -> None:
+        emoji_name = "emoji.png"
+        realm_id = 1
+        bucket = settings.S3_AVATAR_BUCKET
+        path = RealmEmoji.PATH_ID_TEMPLATE.format(realm_id=realm_id, emoji_file_name=emoji_name)
+
+        url = zerver.lib.upload.get_emoji_url("emoji.png", realm_id)
+
+        expected_url = f"https://{bucket}.s3.amazonaws.com/{path}"
+        self.assertEqual(expected_url, url)
+
+        emoji_name = "animated_image.gif"
+
+        path = RealmEmoji.PATH_ID_TEMPLATE.format(realm_id=realm_id, emoji_file_name=emoji_name)
+        still_path = RealmEmoji.STILL_PATH_ID_TEMPLATE.format(
+            realm_id=realm_id, emoji_filename_without_extension=os.path.splitext(emoji_name)[0]
+        )
+
+        url = zerver.lib.upload.get_emoji_url("animated_image.gif", realm_id)
+        still_url = zerver.lib.upload.get_emoji_url("animated_image.gif", realm_id, still=True)
+
+        expected_url = f"https://{bucket}.s3.amazonaws.com/{path}"
+        self.assertEqual(expected_url, url)
+        expected_still_url = f"https://{bucket}.s3.amazonaws.com/{still_path}"
+        self.assertEqual(expected_still_url, still_url)
+
+    @use_s3_backend
+    def test_upload_emoji_image(self) -> None:
+        bucket = create_s3_buckets(settings.S3_AVATAR_BUCKET)[0]
+
+        user_profile = self.example_user("hamlet")
+        emoji_name = "animated_img.gif"
+        with get_test_image_file(emoji_name) as image_file:
+            zerver.lib.upload.upload_emoji_image(image_file, emoji_name, user_profile, "image/gif")
+
+        emoji_path = RealmEmoji.PATH_ID_TEMPLATE.format(
+            realm_id=user_profile.realm_id,
+            emoji_file_name=emoji_name,
+        )
+        original_key = bucket.Object(emoji_path + ".original")
+        self.assertEqual(read_test_image_file(emoji_name), original_key.get()["Body"].read())
+
+        self.assertEqual(os.path.splitext(emoji_path)[1], ".gif")
+        bucket_data = bucket.Object(emoji_path).get()
+        self.assertEqual(bucket_data["ContentType"], "image/gif")
+        resized_image = pyvips.Image.new_from_buffer(bucket_data["Body"].read(), "")
+        self.assertEqual(resized_image.get("vips-loader"), "gifload_buffer")
+
+        still_path = RealmEmoji.STILL_PATH_ID_TEMPLATE.format(
+            realm_id=user_profile.realm_id,
+            emoji_filename_without_extension=os.path.splitext(emoji_name)[0],
+        )
+        self.assertEqual(os.path.splitext(still_path)[1], ".png")
+        bucket_data = bucket.Object(still_path).get()
+        self.assertEqual(bucket_data["ContentType"], "image/png")
+        still_image = pyvips.Image.new_from_buffer(bucket_data["Body"].read(), "")
+        self.assertEqual(still_image.get("vips-loader"), "pngload_buffer")
+
+    @use_s3_backend
+    def test_upload_emoji_non_image(self) -> None:
+        create_s3_buckets(settings.S3_AVATAR_BUCKET)[0]
+
+        user_profile = self.example_user("hamlet")
+        emoji_name = "emoji.png"
+        with get_test_image_file("text.txt") as image_file:
+            with patch("zerver.lib.upload.resize_emoji", side_effect=resize_emoji) as resize_mock:
+                with self.assertRaises(BadImageError):
+                    # We trust the content-type and fail when we try to load the image
+                    zerver.lib.upload.upload_emoji_image(
+                        image_file, emoji_name, user_profile, "image/png"
+                    )
+                resize_mock.assert_called_once()
+
+            with patch("zerver.lib.upload.resize_emoji", side_effect=resize_emoji) as resize_mock:
+                with self.assertRaises(BadImageError):
+                    # We trust the content-type and abort before trying to load
+                    zerver.lib.upload.upload_emoji_image(
+                        image_file, emoji_name, user_profile, "text/plain"
+                    )
+                resize_mock.assert_not_called()
+
+    @use_s3_backend
+    def test_tarball_upload_and_deletion(self) -> None:
+        bucket = create_s3_buckets(settings.S3_EXPORT_BUCKET)[0]
+
+        user_profile = self.example_user("iago")
+        self.assertTrue(user_profile.is_realm_admin)
+
+        tarball_path = os.path.join(settings.TEST_WORKER_DIR, "tarball.tar.gz")
+        with open(tarball_path, "w") as f:
+            f.write("dummy")
+
+        total_bytes_transferred = 0
+
+        def percent_callback(bytes_transferred: int) -> None:
+            nonlocal total_bytes_transferred
+            total_bytes_transferred += bytes_transferred
+
+        url = upload_export_tarball(
+            user_profile.realm, tarball_path, percent_callback=percent_callback
+        )
+        # Verify the percent_callback API works
+        self.assertEqual(total_bytes_transferred, 5)
+
+        parsed_url = urlsplit(url)
+        result = re.search(re.compile(r"/([0-9a-fA-F]{32})/"), url)
+        if result is not None:
+            hex_value = result.group(1)
+        expected_url = (
+            f"https://{bucket.name}.s3.amazonaws.com/{hex_value}/{os.path.basename(tarball_path)}"
+        )
+        self.assertEqual(parsed_url._replace(query="").geturl(), expected_url)
+        params = parse_qs(parsed_url.query)
+        self.assertEqual(params["AWSAccessKeyId"], ["test-key"])
+        self.assertIn("Signature", params)
+        self.assertIn("Expires", params)
+
+        # Delete the tarball.
+        bucket.Object(parsed_url.path.removeprefix("/")).load()
+        delete_export_tarball(parsed_url.path)
+        with self.assertRaises(botocore.exceptions.ClientError):
+            bucket.Object(parsed_url.path.removeprefix("/")).load()
+
+    @override_settings(S3_EXPORT_BUCKET="")
+    @use_s3_backend
+    def test_tarball_upload_and_deletion_no_export_bucket(self) -> None:
+        bucket = create_s3_buckets(settings.S3_AVATAR_BUCKET)[0]
+
+        user_profile = self.example_user("iago")
+        self.assertTrue(user_profile.is_realm_admin)
+
+        tarball_path = os.path.join(settings.TEST_WORKER_DIR, "tarball.tar.gz")
+        with open(tarball_path, "w") as f:
+            f.write("dummy")
+
+        url = upload_export_tarball(user_profile.realm, tarball_path)
+        parsed_url = urlsplit(url)
+        result = re.search(re.compile(r"/([0-9a-fA-F]{32})/"), url)
+        if result is not None:
+            hex_value = result.group(1)
+        expected_url = f"https://{bucket.name}.s3.amazonaws.com/exports/{hex_value}/{os.path.basename(tarball_path)}"
+        self.assertEqual(url, expected_url)
+
+        # Delete the tarball.
+        bucket.Object(parsed_url.path.removeprefix("/")).load()
+        delete_export_tarball(parsed_url.path)
+        with self.assertRaises(botocore.exceptions.ClientError):
+            bucket.Object(parsed_url.path.removeprefix("/")).load()
+
+    @mock_aws
+    def test_tarball_upload_avatar_bucket_download_export_bucket(self) -> None:
+        """Test to verify that tarballs uploaded to avatar bucket can be later
+        accessed via export bucket when server is configured to use export bucket.
+        """
+        user_profile = self.example_user("iago")
+        self.assertTrue(user_profile.is_realm_admin)
+
+        tarball_path = os.path.join(settings.TEST_WORKER_DIR, "tarball.tar.gz")
+        with open(tarball_path, "w") as f:
+            f.write("dummy")
+
+        # Upload export tarball to the avatar bucket.
+        with override_settings(S3_EXPORT_BUCKET=""):
+            backend = S3UploadBackend()
+            avatar_bucket = create_s3_buckets(settings.S3_AVATAR_BUCKET)[0]
+            with patch("zerver.lib.upload._upload_backend", backend):
+                public_url = upload_export_tarball(user_profile.realm, tarball_path)
+                avatar_object_key = urlsplit(public_url).path.removeprefix("/")
+
+        # Verify that old tarballs (uploaded to avatar bucket) can be accessed
+        # from export bucket, once server is configured to use a separate export bucket.
+        with override_settings(S3_EXPORT_BUCKET=settings.S3_EXPORT_BUCKET):
+            backend = S3UploadBackend()
+            export_bucket = create_s3_buckets(settings.S3_EXPORT_BUCKET)[0]
+
+            # Copy existing exports to the new bucket.
+            # This operation is performed as a part of configuring export bucket.
+            session = boto3.session.Session(settings.S3_KEY, settings.S3_SECRET_KEY)
+            s3 = session.resource("s3")
+            copy_source: CopySourceTypeDef = {
+                "Bucket": avatar_bucket.name,
+                "Key": avatar_object_key,
+            }
+            export_object_key = avatar_object_key.removeprefix("exports/")
+            s3.meta.client.copy(copy_source, export_bucket.name, export_object_key)
+
+            # Verify copy operation.
+            object = s3.Object(export_bucket.name, export_object_key)
+            content = object.get()["Body"].read()
+            self.assertEqual(content, b"dummy")
+
+            # Verify that tarball can be accessed using old 'avatar_object_key'.
+            url = backend.get_export_tarball_url(user_profile.realm, avatar_object_key)
+            parsed_url = urlsplit(url)
+            result = re.search(re.compile(r"/([0-9a-fA-F]{32})/"), url)
+            if result is not None:
+                hex_value = result.group(1)
+            expected_url = f"https://{export_bucket.name}.s3.amazonaws.com/{hex_value}/{os.path.basename(tarball_path)}"
+            self.assertEqual(parsed_url._replace(query="").geturl(), expected_url)
+            params = parse_qs(parsed_url.query)
+            self.assertEqual(params["AWSAccessKeyId"], ["test-key"])
+            self.assertIn("Signature", params)
+            self.assertIn("Expires", params)
+
+    @mock_aws
+    def test_tarball_upload_export_bucket_download_avatar_bucket(self) -> None:
+        """Test to verify that tarballs uploaded to export bucket can be later
+        accessed via avatar bucket when server is configured to use ONLY avatar bucket.
+        """
+        user_profile = self.example_user("iago")
+        self.assertTrue(user_profile.is_realm_admin)
+
+        tarball_path = os.path.join(settings.TEST_WORKER_DIR, "tarball.tar.gz")
+        with open(tarball_path, "w") as f:
+            f.write("dummy")
+
+        # Upload export tarball to the export bucket.
+        with override_settings(S3_EXPORT_BUCKET=settings.S3_EXPORT_BUCKET):
+            backend = S3UploadBackend()
+            export_bucket = create_s3_buckets(settings.S3_EXPORT_BUCKET)[0]
+            with patch("zerver.lib.upload._upload_backend", backend):
+                public_url = upload_export_tarball(user_profile.realm, tarball_path)
+                export_object_key = urlsplit(public_url).path.removeprefix("/")
+
+        # Verify that old tarballs (uploaded to export bucket) can be accessed
+        # from avatar bucket, once server is configured to use ONLY avatar bucket.
+        with override_settings(S3_EXPORT_BUCKET=""):
+            backend = S3UploadBackend()
+            avatar_bucket = create_s3_buckets(settings.S3_AVATAR_BUCKET)[0]
+
+            # Copy existing exports to the avatar bucket.
+            # This operation is performed as a part of the changes to use ONLY avatar bucket.
+            session = boto3.session.Session(settings.S3_KEY, settings.S3_SECRET_KEY)
+            s3 = session.resource("s3")
+            copy_source: CopySourceTypeDef = {
+                "Bucket": export_bucket.name,
+                "Key": export_object_key,
+            }
+            avatar_object_key = "exports/" + export_object_key
+            s3.meta.client.copy(copy_source, avatar_bucket.name, avatar_object_key)
+
+            # Verify copy operation.
+            object = s3.Object(avatar_bucket.name, avatar_object_key)
+            content = object.get()["Body"].read()
+            self.assertEqual(content, b"dummy")
+
+            # Verify that tarball can still be accessed using old 'export_object_key'.
+            url = backend.get_export_tarball_url(user_profile.realm, export_object_key)
+            result = re.search(re.compile(r"/([0-9a-fA-F]{32})/"), url)
+            if result is not None:
+                hex_value = result.group(1)
+            expected_url = f"https://{avatar_bucket.name}.s3.amazonaws.com/exports/{hex_value}/{os.path.basename(tarball_path)}"
+            self.assertEqual(url, expected_url)