snapshot of https://github.com/zulip/zulip.git @ 48c69911c7dd4e03b20552eb3bd77ad8151c1357 (2026-09-15)

zerver/tests/test_realm_export.py

@@ -0,0 +1,534 @@
+import os
+from datetime import datetime, timezone
+from unittest.mock import patch
+from urllib.parse import urlsplit
+
+import botocore.exceptions
+import orjson
+import time_machine
+from django.conf import settings
+from django.utils.timezone import now as timezone_now
+
+from analytics.models import RealmCount
+from zerver.actions.user_settings import do_change_user_setting
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.export import export_tarball_prefix
+from zerver.lib.queue import queue_json_publish_rollback_unsafe
+from zerver.lib.test_classes import ZulipTestCase
+from zerver.lib.test_helpers import (
+    HostRequestMock,
+    create_dummy_file,
+    create_s3_buckets,
+    stdout_suppressed,
+    use_s3_backend,
+)
+from zerver.models import Realm, RealmExport, UserProfile
+from zerver.models.realms import RealmExportSlug
+from zerver.views.realm_export import export_realm
+
+
+class RealmExportTest(ZulipTestCase):
+    """
+    API endpoint testing covers the full end-to-end flow
+    from both the S3 and local uploads perspective.
+
+    `test_endpoint_s3` and `test_endpoint_local_uploads` follow
+    an identical pattern, which is documented in both test
+    functions.
+    """
+
+    def test_export_as_not_admin(self) -> None:
+        user = self.example_user("hamlet")
+        self.login_user(user)
+        with self.assertRaises(JsonableError):
+            export_realm(HostRequestMock(), user)
+
+    @use_s3_backend
+    def test_endpoint_s3(self) -> None:
+        admin = self.example_user("iago")
+        self.login_user(admin)
+        bucket = create_s3_buckets(settings.S3_EXPORT_BUCKET)[0]
+        tarball_path = create_dummy_file("test-export.tar.gz")
+
+        # Test the export logic.
+        export_time = datetime(2026, 5, 25, 10, 11, 16, tzinfo=timezone.utc)
+        with patch(
+            "zerver.lib.export.do_export_realm", return_value=(tarball_path, dict())
+        ) as mock_export:
+            with (
+                self.settings(LOCAL_UPLOADS_DIR=None),
+                stdout_suppressed(),
+                self.assertLogs(level="INFO") as info_logs,
+                time_machine.travel(export_time, tick=False),
+                self.captureOnCommitCallbacks(execute=True),
+            ):
+                result = self.client_post("/json/export/realm")
+            self.assertTrue("INFO:root:Completed data export for zulip in " in info_logs.output[0])
+        self.assert_json_success(result)
+        self.assertFalse(os.path.exists(tarball_path))
+        args = mock_export.call_args_list[0][1]
+        self.assertEqual(args["realm"], admin.realm)
+        self.assertEqual(args["export_type"], RealmExport.EXPORT_PUBLIC)
+        self.assertTrue(
+            os.path.basename(args["output_dir"]).startswith(
+                "zulip-export-zulip-2026-05-25-10-11-16-"
+            )
+        )
+        self.assertEqual(args["processes"], 6)
+
+        # Get the entry and test that iago initiated it.
+        export_row = RealmExport.objects.first()
+        assert export_row is not None
+        self.assertEqual(export_row.acting_user_id, admin.id)
+        self.assertEqual(export_row.status, RealmExport.SUCCEEDED)
+
+        # Test that the file is hosted, and the contents are as expected.
+        export_path = export_row.export_path
+        assert export_path is not None
+        assert export_path.startswith("/")
+        path_id = export_path.removeprefix("/")
+        self.assertEqual(bucket.Object(path_id).get()["Body"].read(), b"zulip!")
+
+        result = self.client_get("/json/export/realm")
+        response_dict = self.assert_json_success(result)
+
+        # Test that the export we have is the export we created.
+        export_dict = response_dict["exports"]
+        self.assertEqual(export_dict[0]["id"], export_row.id)
+        parsed_url = urlsplit(export_dict[0]["export_url"])
+        self.assertEqual(
+            parsed_url._replace(query="").geturl(),
+            "https://test-export-bucket.s3.amazonaws.com" + export_path,
+        )
+        self.assertEqual(export_dict[0]["acting_user_id"], admin.id)
+        self.assert_length(
+            export_dict,
+            RealmExport.objects.filter(realm=admin.realm).count(),
+        )
+
+        # Finally, delete the file.
+        result = self.client_delete(f"/json/export/realm/{export_row.id}")
+        self.assert_json_success(result)
+        with self.assertRaises(botocore.exceptions.ClientError):
+            bucket.Object(path_id).load()
+
+        # Try to delete an export with a `DELETED` status.
+        export_row.refresh_from_db()
+        self.assertEqual(export_row.status, RealmExport.DELETED)
+        self.assertIsNotNone(export_row.date_deleted)
+        result = self.client_delete(f"/json/export/realm/{export_row.id}")
+        self.assert_json_error(result, "Export already deleted")
+
+        # Now try to delete a non-existent export.
+        result = self.client_delete("/json/export/realm/0")
+        self.assert_json_error(result, "Invalid data export ID")
+
+    def test_endpoint_local_uploads(self) -> None:
+        admin = self.example_user("iago")
+        self.login_user(admin)
+        tarball_path = create_dummy_file("test-export.tar.gz")
+
+        # Test the export logic.
+        def fake_export_realm(
+            realm: Realm,
+            output_dir: str,
+            processes: int,
+            export_type: int,
+            exportable_user_ids: set[int] | None = None,
+            export_as_active: bool | None = None,
+        ) -> tuple[str, dict[str, int | dict[str, int]]]:
+            self.assertEqual(realm, admin.realm)
+            self.assertEqual(export_type, RealmExport.EXPORT_PUBLIC)
+            self.assertTrue(
+                os.path.basename(output_dir).startswith("zulip-export-zulip-2026-05-25-10-11-16-")
+            )
+            self.assertEqual(processes, 6)
+
+            # Check that the export shows up as in progress
+            result = self.client_get("/json/export/realm")
+            response_dict = self.assert_json_success(result)
+            export_dict = response_dict["exports"]
+            self.assert_length(export_dict, 1)
+            id = export_dict[0]["id"]
+            self.assertEqual(export_dict[0]["pending"], True)
+            self.assertIsNone(export_dict[0]["export_url"])
+            self.assertIsNone(export_dict[0]["deleted_timestamp"])
+            self.assertIsNone(export_dict[0]["failed_timestamp"])
+            self.assertEqual(export_dict[0]["acting_user_id"], admin.id)
+
+            # While the export is in progress, we can't delete it
+            result = self.client_delete(f"/json/export/realm/{id}")
+            self.assert_json_error(result, "Export still in progress")
+
+            return tarball_path, dict()
+
+        export_time = datetime(2026, 5, 25, 10, 11, 16, tzinfo=timezone.utc)
+        with patch(
+            "zerver.lib.export.do_export_realm", side_effect=fake_export_realm
+        ) as mock_export:
+            with (
+                stdout_suppressed(),
+                self.assertLogs(level="INFO") as info_logs,
+                time_machine.travel(export_time, tick=False),
+                self.captureOnCommitCallbacks(execute=True),
+            ):
+                result = self.client_post("/json/export/realm")
+            self.assertTrue("INFO:root:Completed data export for zulip in " in info_logs.output[0])
+        mock_export.assert_called_once()
+        data = self.assert_json_success(result)
+        self.assertFalse(os.path.exists(tarball_path))
+
+        # Get the entry and test that iago initiated it.
+        export_row = RealmExport.objects.first()
+        assert export_row is not None
+        self.assertEqual(export_row.id, data["id"])
+        self.assertEqual(export_row.acting_user_id, admin.id)
+        self.assertEqual(export_row.status, RealmExport.SUCCEEDED)
+
+        # Test that the file is hosted, and the contents are as expected.
+        export_path = export_row.export_path
+        assert export_path is not None
+        response = self.client_get(export_path)
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.getvalue(), b"zulip!")
+
+        result = self.client_get("/json/export/realm")
+        response_dict = self.assert_json_success(result)
+
+        # Test that the export we have is the export we created.
+        export_dict = response_dict["exports"]
+        self.assertEqual(export_dict[0]["id"], export_row.id)
+        self.assertEqual(export_dict[0]["export_url"], admin.realm.url + export_path)
+        self.assertEqual(export_dict[0]["acting_user_id"], admin.id)
+        self.assert_length(export_dict, RealmExport.objects.filter(realm=admin.realm).count())
+
+        # Finally, delete the file.
+        result = self.client_delete(f"/json/export/realm/{export_row.id}")
+        self.assert_json_success(result)
+        response = self.client_get(export_path)
+        self.assertEqual(response.status_code, 404)
+
+        # Try to delete an export with a `DELETED` status.
+        export_row.refresh_from_db()
+        self.assertEqual(export_row.status, RealmExport.DELETED)
+        self.assertIsNotNone(export_row.date_deleted)
+        result = self.client_delete(f"/json/export/realm/{export_row.id}")
+        self.assert_json_error(result, "Export already deleted")
+
+        # Now try to delete a non-existent export.
+        result = self.client_delete("/json/export/realm/0")
+        self.assert_json_error(result, "Invalid data export ID")
+
+    def test_export_tarball_prefix(self) -> None:
+        realm = self.example_user("iago").realm
+        with time_machine.travel(
+            datetime(2026, 5, 25, 10, 11, 16, tzinfo=timezone.utc), tick=False
+        ):
+            self.assertEqual(
+                export_tarball_prefix(realm), "zulip-export-zulip-2026-05-25-10-11-16-"
+            )
+            realm.string_id = ""
+            self.assertEqual(export_tarball_prefix(realm), "zulip-export-2026-05-25-10-11-16-")
+
+    def test_export_failure(self) -> None:
+        admin = self.example_user("iago")
+        self.login_user(admin)
+
+        with (
+            patch(
+                "zerver.lib.export.do_export_realm", side_effect=Exception("failure")
+            ) as mock_export,
+            stdout_suppressed(),
+            self.assertLogs(level="INFO") as info_logs,
+            self.captureOnCommitCallbacks(execute=True),
+        ):
+            result = self.client_post("/json/export/realm")
+        self.assertTrue(
+            info_logs.output[0].startswith("ERROR:root:Data export for zulip failed after ")
+        )
+        mock_export.assert_called_once()
+        # This is a success because the failure is swallowed in the queue worker
+        data = self.assert_json_success(result)
+        export_id = data["id"]
+
+        # Check that the export shows up as failed
+        result = self.client_get("/json/export/realm")
+        response_dict = self.assert_json_success(result)
+        export_dict = response_dict["exports"]
+        self.assert_length(export_dict, 1)
+        self.assertEqual(export_dict[0]["id"], export_id)
+        self.assertEqual(export_dict[0]["pending"], False)
+        self.assertIsNone(export_dict[0]["export_url"])
+        self.assertIsNone(export_dict[0]["deleted_timestamp"])
+        self.assertIsNotNone(export_dict[0]["failed_timestamp"])
+        self.assertEqual(export_dict[0]["acting_user_id"], admin.id)
+
+        export_row = RealmExport.objects.get(id=export_id)
+        self.assertEqual(export_row.status, RealmExport.FAILED)
+
+        # Check that we can't delete it
+        result = self.client_delete(f"/json/export/realm/{export_id}")
+        self.assert_json_error(result, "Export failed, nothing to delete")
+
+        # If the queue worker sees the same export-id again, it aborts
+        # instead of retrying
+        with (
+            patch("zerver.lib.export.do_export_realm") as mock_export,
+            self.assertLogs(level="INFO") as info_logs,
+        ):
+            queue_json_publish_rollback_unsafe(
+                "deferred_work",
+                {
+                    "type": "realm_export",
+                    "user_profile_id": admin.id,
+                    "realm_export_id": export_id,
+                },
+            )
+        mock_export.assert_not_called()
+        self.assertEqual(
+            info_logs.output,
+            [
+                (
+                    "ERROR:zerver.worker.deferred_work:Marking export for realm zulip "
+                    "as failed due to retry -- possible OOM during export?"
+                )
+            ],
+        )
+
+    def test_export_from_prior_server_status_in_api(self) -> None:
+        admin = self.example_user("iago")
+        self.login_user(admin)
+
+        # A RealmExport row whose tarball is no longer stored on this
+        # server - the result of the record being carried
+        # across a realm export->import.
+        prior_server_export = RealmExport.objects.create(
+            realm=admin.realm,
+            type=RealmExport.EXPORT_PUBLIC,
+            status=RealmExport.EXPORT_FROM_PRIOR_SERVER,
+            date_requested=timezone_now(),
+            date_succeeded=timezone_now(),
+            acting_user=admin,
+            export_path=None,
+        )
+
+        result = self.client_get("/json/export/realm")
+        response_dict = self.assert_json_success(result)
+        export_dict = response_dict["exports"]
+        self.assert_length(export_dict, 1)
+        self.assertEqual(export_dict[0]["id"], prior_server_export.id)
+        self.assertIsNone(export_dict[0]["export_url"])
+        self.assertEqual(export_dict[0]["pending"], False)
+        self.assertIsNone(export_dict[0]["deleted_timestamp"])
+        self.assertIsNone(export_dict[0]["failed_timestamp"])
+        self.assertEqual(export_dict[0]["export_from_prior_server"], True)
+        self.assertEqual(export_dict[0]["acting_user_id"], admin.id)
+
+        # Trying to delete such a row reports it as already deleted.
+        result = self.client_delete(f"/json/export/realm/{prior_server_export.id}")
+        self.assert_json_error(result, "Export already deleted")
+
+    def test_realm_export_rate_limited(self) -> None:
+        admin = self.example_user("iago")
+        self.login_user(admin)
+
+        export_rows = RealmExport.objects.all()
+        self.assert_length(export_rows, 0)
+
+        exports = [
+            RealmExport(
+                realm=admin.realm,
+                type=RealmExport.EXPORT_PUBLIC,
+                date_requested=timezone_now(),
+                acting_user=admin,
+            )
+            for i in range(5)
+        ]
+        RealmExport.objects.bulk_create(exports)
+
+        with self.assertRaises(JsonableError) as error:
+            export_realm(HostRequestMock(), admin)
+        self.assertEqual(str(error.exception), "Exceeded rate limit.")
+
+    def test_upload_and_message_limit(self) -> None:
+        admin = self.example_user("iago")
+        self.login_user(admin)
+        public_realm_count = RealmCount.objects.create(
+            realm_id=admin.realm.id,
+            end_time=timezone_now(),
+            value=0,
+            property="messages_sent:message_type:day",
+            subgroup="public_stream",
+        )
+        private_realm_count = RealmCount.objects.create(
+            realm_id=admin.realm.id,
+            end_time=timezone_now(),
+            value=0,
+            property="messages_sent:message_type:day",
+            subgroup="private_stream",
+        )
+
+        # Space limit is set as 20 GiB
+        with patch(
+            "zerver.models.Realm.currently_used_upload_space_bytes",
+            return_value=21 * 1024 * 1024 * 1024,
+        ):
+            result = self.client_post("/json/export/realm")
+        self.assert_json_error(
+            result,
+            f"The export you requested is too large for automatic processing. Please request a manual export by contacting {settings.ZULIP_ADMINISTRATOR}.",
+        )
+
+        # Message limit is set as 250000
+        public_realm_count.value = 250001
+        public_realm_count.save(update_fields=["value"])
+        result = self.client_post("/json/export/realm")
+        self.assert_json_error(
+            result,
+            f"The export you requested is too large for automatic processing. Please request a manual export by contacting {settings.ZULIP_ADMINISTRATOR}.",
+        )
+
+        # Test when public message count is within the limit but total
+        # count for public and private messages exceed the limit.
+        public_realm_count.value = 150000
+        public_realm_count.save(update_fields=["value"])
+        private_realm_count.value = 100001
+        private_realm_count.save(update_fields=["value"])
+        result = self.client_post(
+            "/json/export/realm",
+            {
+                "export_type": self.FULL_WITH_CONSENT_EXPORT_TYPE,
+            },
+        )
+        self.assert_json_error(
+            result,
+            f"The export you requested is too large for automatic processing. Please request a manual export by contacting {settings.ZULIP_ADMINISTRATOR}.",
+        )
+
+        self.check_success_realm_export(admin, self.PUBLIC_EXPORT_TYPE)
+
+    def test_get_users_export_consents(self) -> None:
+        admin = self.example_user("iago")
+        self.login_user(admin)
+
+        # By default, export consent is set to False.
+        self.assertFalse(
+            UserProfile.objects.filter(
+                realm=admin.realm, is_active=True, is_bot=False, allow_private_data_export=True
+            ).exists()
+        )
+
+        # Hamlet and Aaron consented to export their private data.
+        hamlet = self.example_user("hamlet")
+        aaron = self.example_user("aaron")
+        for user in [hamlet, aaron]:
+            do_change_user_setting(user, "allow_private_data_export", True, acting_user=None)
+
+        # Verify export consents of users.
+        aaron.role = UserProfile.ROLE_REALM_ADMINISTRATOR
+        aaron.save()
+        do_change_user_setting(
+            aaron,
+            "email_address_visibility",
+            UserProfile.EMAIL_ADDRESS_VISIBILITY_NOBODY,
+            acting_user=aaron,
+        )
+        result = self.client_get("/json/export/realm/consents")
+        response_dict = self.assert_json_success(result)
+        export_consents = response_dict["export_consents"]
+        for export_consent in export_consents:
+            if export_consent["user_id"] == aaron.id:
+                self.assertEqual(
+                    export_consent["email_address_visibility"],
+                    UserProfile.EMAIL_ADDRESS_VISIBILITY_NOBODY,
+                )
+            if export_consent["user_id"] in [hamlet.id, aaron.id]:
+                self.assertTrue(export_consent["consented"])
+                continue
+            self.assertFalse(export_consent["consented"])
+
+    PUBLIC_EXPORT_TYPE: RealmExportSlug = "public"
+    FULL_WITH_CONSENT_EXPORT_TYPE: RealmExportSlug = "full_with_consent"
+    FULL_WITHOUT_CONSENT_EXPORT_TYPE: RealmExportSlug = "full_without_consent"
+
+    def check_success_realm_export(
+        self, acting_user: UserProfile, export_type: RealmExportSlug
+    ) -> None:
+        expected_realm_export_type = RealmExport.EXPORT_TYPES[export_type]
+        with patch("zerver.views.realm_export.queue_event_on_commit") as mock_event_on_commit:
+            result = self.client_post(
+                "/json/export/realm",
+                {
+                    "export_type": export_type,
+                },
+            )
+        self.assert_json_success(result)
+        response = orjson.loads(result.content)
+        realm_export_id = response["id"]
+        expected_event = {
+            "type": "realm_export",
+            "user_profile_id": acting_user.id,
+            "realm_export_id": realm_export_id,
+        }
+        mock_event_on_commit.assert_called_once_with("deferred_work", expected_event)
+        realm_export = RealmExport.objects.get(id=realm_export_id)
+        self.assertEqual(realm_export.type, expected_realm_export_type)
+
+    def test_allow_export_with_no_usable_user_accounts(self) -> None:
+        """
+        Generating export with no usable accounts should be allowed.
+        """
+        admin = self.example_user("iago")
+        self.login_user(admin)
+
+        # For standard export, this means no one consented to their
+        # private data being shared.
+        UserProfile.objects.filter(
+            role=UserProfile.ROLE_REALM_OWNER,
+            realm=admin.realm,
+        ).update(
+            allow_private_data_export=False,
+        )
+        self.check_success_realm_export(admin, self.FULL_WITH_CONSENT_EXPORT_TYPE)
+
+        # For public export, this means everyone has set their email
+        # address visibility policy to nobody.
+        UserProfile.objects.filter(
+            role=UserProfile.ROLE_REALM_OWNER,
+            realm=admin.realm,
+        ).update(
+            email_address_visibility=UserProfile.EMAIL_ADDRESS_VISIBILITY_NOBODY,
+        )
+
+        self.check_success_realm_export(admin, self.PUBLIC_EXPORT_TYPE)
+
+    def test_full_without_consent_export_requires_org_permission(self) -> None:
+        admin = self.example_user("iago")
+        owner = self.example_user("desdemona")
+        self.login_user(admin)
+
+        admin.realm.owner_full_content_access = False
+        admin.realm.save(update_fields=["owner_full_content_access"])
+        result = self.client_post(
+            "/json/export/realm",
+            {
+                "export_type": self.FULL_WITHOUT_CONSENT_EXPORT_TYPE,
+            },
+        )
+        self.assert_json_error(
+            result,
+            "Exports of all public and private data are not enabled for this organization.",
+        )
+
+        admin.realm.owner_full_content_access = True
+        admin.realm.save(update_fields=["owner_full_content_access"])
+        result = self.client_post(
+            "/json/export/realm",
+            {
+                "export_type": self.FULL_WITHOUT_CONSENT_EXPORT_TYPE,
+            },
+        )
+        self.assert_json_error(result, "Must be an organization owner")
+
+        self.login_user(owner)
+        self.check_success_realm_export(owner, self.FULL_WITHOUT_CONSENT_EXPORT_TYPE)