snapshot of https://github.com/zulip/zulip.git @ 48c69911c7dd4e03b20552eb3bd77ad8151c1357 (2026-09-15)

zilencer/README.md

@@ -0,0 +1,4 @@
+# ZILENCER -- The Zulip license manager
+
+This app is the place for storing state about various deployments of
+Zulip that exist in the world.

zilencer/auth.py

@@ -0,0 +1,178 @@
+import base64
+import binascii
+import logging
+from collections.abc import Callable
+from functools import wraps
+from typing import Any, Concatenate
+
+import sentry_sdk
+from django.conf import settings
+from django.core.signing import BadSignature, SignatureExpired, TimestampSigner
+from django.http import HttpRequest, HttpResponse
+from django.urls import path
+from django.urls.resolvers import URLPattern
+from django.utils.crypto import constant_time_compare
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+from django.views.decorators.csrf import csrf_exempt
+from typing_extensions import ParamSpec, override
+
+from zerver.decorator import get_basic_credentials, process_client
+from zerver.lib.exceptions import (
+    ErrorCode,
+    JsonableError,
+    RateLimitedError,
+    RemoteServerDeactivatedError,
+    UnauthorizedError,
+)
+from zerver.lib.rate_limiter import should_rate_limit
+from zerver.lib.request import RequestNotes
+from zerver.lib.rest import default_never_cache_responses, get_target_view_function_or_response
+from zerver.lib.subdomains import get_subdomain
+from zerver.models import Realm
+from zilencer.models import (
+    RateLimitedRemoteZulipServer,
+    RemoteZulipServer,
+    get_remote_server_by_uuid,
+)
+
+logger = logging.getLogger(__name__)
+
+ParamT = ParamSpec("ParamT")
+
+REMOTE_SERVER_TAKEOVER_TOKEN_SALT = "remote_server_transfer"
+REMOTE_SERVER_TAKEOVER_TOKEN_VALIDITY_SECONDS = 10
+
+
+def generate_registration_transfer_verification_secret(hostname: str) -> str:
+    signer = TimestampSigner(salt=REMOTE_SERVER_TAKEOVER_TOKEN_SALT)
+    secret = base64.b16encode(signer.sign(hostname).encode()).decode()
+    return secret
+
+
+def validate_registration_transfer_verification_secret(secret: str, hostname: str) -> None:
+    signer = TimestampSigner(salt=REMOTE_SERVER_TAKEOVER_TOKEN_SALT)
+    try:
+        signed_data = base64.b16decode(secret).decode()
+        hostname_from_secret = signer.unsign(
+            signed_data, max_age=REMOTE_SERVER_TAKEOVER_TOKEN_VALIDITY_SECONDS
+        )
+    except SignatureExpired:
+        raise JsonableError(_("The verification secret has expired"))
+    except BadSignature:
+        raise JsonableError(_("The verification secret is invalid"))
+    except binascii.Error:
+        raise JsonableError(_("The verification secret is malformed"))
+    if hostname_from_secret != hostname:
+        raise JsonableError(_("The verification secret is for a different hostname"))
+
+
+class InvalidZulipServerError(JsonableError):
+    code = ErrorCode.INVALID_ZULIP_SERVER
+    data_fields = ["role"]
+
+    def __init__(self, role: str) -> None:
+        self.role: str = role
+
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return "Zulip server auth failure: {role} is not registered -- did you run `manage.py register_server`?"
+
+
+class InvalidZulipServerKeyError(InvalidZulipServerError):
+    @staticmethod
+    @override
+    def msg_format() -> str:
+        return "Zulip server auth failure: key does not match role {role}"
+
+
+def rate_limit_remote_server(
+    request: HttpRequest, remote_server: RemoteZulipServer, domain: str
+) -> None:
+    if not should_rate_limit(request):
+        return
+
+    try:
+        RateLimitedRemoteZulipServer(remote_server, domain=domain).rate_limit_request(request)
+    except RateLimitedError:
+        logger.warning("Remote server %s exceeded rate limits on domain %s", remote_server, domain)
+        raise
+
+
+def validate_remote_server(
+    request: HttpRequest,
+    role: str,
+    api_key: str,
+) -> RemoteZulipServer:
+    log_data = RequestNotes.get_notes(request).log_data
+    assert log_data is not None
+    try:
+        remote_server = get_remote_server_by_uuid(role)
+    except RemoteZulipServer.DoesNotExist:
+        log_data["extra"] = "[invalid-server]"
+        raise InvalidZulipServerError(role)
+    if not constant_time_compare(api_key, remote_server.api_key):
+        log_data["extra"] = "[invalid-server-key]"
+        raise InvalidZulipServerKeyError(role)
+
+    if remote_server.deactivated:
+        log_data["extra"] = "[deactivated-server]"
+        raise RemoteServerDeactivatedError
+    if (
+        get_subdomain(request) != Realm.SUBDOMAIN_FOR_ROOT_DOMAIN
+        and not settings.DEVELOPMENT_DISABLE_PUSH_BOUNCER_DOMAIN_CHECK
+    ):
+        # Sometimes we may want to test push bouncer logic in development.
+        log_data["extra"] = "[invalid-domain]"
+        raise JsonableError(_("Invalid subdomain for push notifications bouncer"))
+    RequestNotes.get_notes(request).remote_server = remote_server
+    process_client(request)
+    sentry_sdk.set_user({"server": remote_server.uuid})
+    return remote_server
+
+
+def authenticated_remote_server_view(
+    view_func: Callable[Concatenate[HttpRequest, RemoteZulipServer, ParamT], HttpResponse],
+) -> Callable[Concatenate[HttpRequest, ParamT], HttpResponse]:
+    @wraps(view_func)
+    def _wrapped_view_func(
+        request: HttpRequest, /, *args: ParamT.args, **kwargs: ParamT.kwargs
+    ) -> HttpResponse:
+        role, api_key = get_basic_credentials(request)
+        if "@" in role:
+            log_data = RequestNotes.get_notes(request).log_data
+            assert log_data is not None
+            log_data["extra"] = "[non-server-key]"
+            raise JsonableError(_("Must validate with valid Zulip server API key"))
+        try:
+            remote_server = validate_remote_server(request, role, api_key)
+        except JsonableError as e:
+            raise UnauthorizedError(e.msg)
+
+        rate_limit_remote_server(request, remote_server, domain="api_by_remote_server")
+
+        remote_server.last_request_datetime = timezone_now()
+        remote_server.save(update_fields=["last_request_datetime"])
+
+        return view_func(request, remote_server, *args, **kwargs)
+
+    return _wrapped_view_func
+
+
+@default_never_cache_responses
+@csrf_exempt
+def remote_server_dispatch(request: HttpRequest, /, **kwargs: Any) -> HttpResponse:
+    result = get_target_view_function_or_response(request, kwargs)
+    if isinstance(result, HttpResponse):
+        return result
+    target_function, _view_flags = result
+    return authenticated_remote_server_view(target_function)(request, **kwargs)
+
+
+def remote_server_path(
+    route: str,
+    **handlers: Callable[Concatenate[HttpRequest, RemoteZulipServer, ParamT], HttpResponse]
+    | tuple[Callable[Concatenate[HttpRequest, RemoteZulipServer, ParamT], HttpResponse], set[str]],
+) -> URLPattern:
+    return path(route, remote_server_dispatch, handlers)

zilencer/lib/push_notifications.py

@@ -0,0 +1,252 @@
+import asyncio
+import logging
+from collections.abc import Iterable
+from dataclasses import asdict, dataclass
+
+from aioapns import NotificationRequest
+from django.utils.timezone import now as timezone_now
+from firebase_admin import exceptions as firebase_exceptions
+from firebase_admin import messaging as firebase_messaging
+from firebase_admin.messaging import UnregisteredError as FCMUnregisteredError
+
+from zerver.lib.devices import b64decode_token_id_base64, b64encode_token_id_int
+from zerver.lib.push_notifications import (
+    APNsPushRequest,
+    FCMPushRequest,
+    SendNotificationResponseData,
+    fcm_app,
+    get_apns_context,
+    get_info_from_apns_result,
+)
+from zerver.models.realms import Realm
+from zilencer.models import RemotePushDevice, RemoteRealm
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class SentPushNotificationResult:
+    successfully_sent_count: int
+    delete_token_ids_base64: list[str]
+
+
+def send_e2ee_push_notification_apple(
+    apns_requests: list[NotificationRequest],
+    apns_remote_push_devices: list[RemotePushDevice],
+    log_context: str,
+) -> SentPushNotificationResult:
+    import aioapns
+
+    successfully_sent_count = 0
+    delete_token_ids_base64: list[str] = []
+    apns_context = get_apns_context()
+
+    if apns_context is None:
+        logger.error(
+            "APNs: Dropping push notifications since "
+            "neither APNS_TOKEN_KEY_FILE nor APNS_CERT_FILE is set."
+        )
+        return SentPushNotificationResult(
+            successfully_sent_count=successfully_sent_count,
+            delete_token_ids_base64=delete_token_ids_base64,
+        )
+
+    async def send_all_notifications() -> Iterable[
+        tuple[RemotePushDevice, aioapns.common.NotificationResult | BaseException]
+    ]:
+        results = await asyncio.gather(
+            *(apns_context.apns.send_notification(request) for request in apns_requests),
+            return_exceptions=True,
+        )
+        return zip(apns_remote_push_devices, results, strict=False)
+
+    results = apns_context.loop.run_until_complete(send_all_notifications())
+
+    for remote_push_device, result in results:
+        result_info = get_info_from_apns_result(
+            result,
+            remote_push_device,
+            log_context % remote_push_device.token,
+        )
+
+        if result_info.successfully_sent:
+            successfully_sent_count += 1
+        elif result_info.delete_token_id_base64 is not None:
+            remote_push_device.expired_time = timezone_now()
+            remote_push_device.save(update_fields=["expired_time"])
+            delete_token_ids_base64.append(result_info.delete_token_id_base64)
+
+    return SentPushNotificationResult(
+        successfully_sent_count=successfully_sent_count,
+        delete_token_ids_base64=delete_token_ids_base64,
+    )
+
+
+def send_e2ee_push_notification_android(
+    fcm_requests: list[firebase_messaging.Message],
+    fcm_remote_push_devices: list[RemotePushDevice],
+    log_context: str,
+) -> SentPushNotificationResult:
+    successfully_sent_count = 0
+    delete_token_ids_base64: list[str] = []
+
+    if fcm_app is None:
+        logger.error("FCM: Dropping push notifications since ANDROID_FCM_CREDENTIALS_PATH is unset")
+        return SentPushNotificationResult(
+            successfully_sent_count=successfully_sent_count,
+            delete_token_ids_base64=delete_token_ids_base64,
+        )
+
+    try:
+        batch_response = firebase_messaging.send_each(fcm_requests, app=fcm_app)
+    except firebase_exceptions.FirebaseError:
+        logger.warning("Error while pushing to FCM", exc_info=True)
+        return SentPushNotificationResult(
+            successfully_sent_count=successfully_sent_count,
+            delete_token_ids_base64=delete_token_ids_base64,
+        )
+
+    for idx, response in enumerate(batch_response.responses):
+        # We enumerate to have idx to track which token the response
+        # corresponds to. send_each() preserves the order of the messages,
+        # so this works.
+
+        remote_push_device = fcm_remote_push_devices[idx]
+        token = remote_push_device.token
+
+        if response.success:
+            successfully_sent_count += 1
+            logger.info(
+                "FCM: Sent message with ID: %s %s", response.message_id, log_context % token
+            )
+        else:
+            error = response.exception
+            if isinstance(error, FCMUnregisteredError):
+                remote_push_device.expired_time = timezone_now()
+                remote_push_device.save(update_fields=["expired_time"])
+                delete_token_ids_base64.append(b64encode_token_id_int(remote_push_device.token_id))
+
+                logger.info("FCM: Removing %s due to %s", token, error.code)
+            else:
+                logger.warning(
+                    "FCM: Delivery failed %s: %s:%s",
+                    log_context % token,
+                    error.__class__,
+                    error,
+                )
+
+    return SentPushNotificationResult(
+        successfully_sent_count=successfully_sent_count,
+        delete_token_ids_base64=delete_token_ids_base64,
+    )
+
+
+def send_e2ee_push_notifications(
+    push_requests: list[APNsPushRequest | FCMPushRequest],
+    *,
+    realm: Realm | None = None,
+    remote_realm: RemoteRealm | None = None,
+) -> SendNotificationResponseData:
+    assert (realm is None) ^ (remote_realm is None)
+
+    import aioapns
+
+    token_ids_base64 = set()
+    token_ids_int = set()
+    for push_request in push_requests:
+        token_ids_base64.add(push_request.token_id)
+        token_ids_int.add(b64decode_token_id_base64(push_request.token_id))
+
+    remote_push_devices = RemotePushDevice.objects.filter(
+        token_id__in=token_ids_int,
+        realm=realm,
+        remote_realm=remote_realm,
+        expired_time__isnull=True,
+    )
+    token_id_base64_to_remote_push_device = {
+        b64encode_token_id_int(remote_push_device.token_id): remote_push_device
+        for remote_push_device in remote_push_devices
+    }
+    unexpired_token_ids_base64 = set(token_id_base64_to_remote_push_device.keys())
+
+    # Token IDs which should be deleted on server.
+    # Either the token ID is invalid or the token
+    # associated has been marked invalid/expired by APNs/FCM.
+    delete_token_ids_base64 = list(
+        filter(
+            lambda token_id_base64: token_id_base64 not in unexpired_token_ids_base64,
+            token_ids_base64,
+        )
+    )
+
+    apns_requests = []
+    apns_remote_push_devices: list[RemotePushDevice] = []
+
+    fcm_requests = []
+    fcm_remote_push_devices: list[RemotePushDevice] = []
+
+    for push_request in push_requests:
+        token_id_base64 = push_request.token_id
+        if token_id_base64 not in unexpired_token_ids_base64:
+            continue
+
+        remote_push_device = token_id_base64_to_remote_push_device[token_id_base64]
+        if remote_push_device.token_kind == RemotePushDevice.TokenKind.APNS:
+            assert isinstance(push_request, APNsPushRequest)
+            apns_requests.append(
+                aioapns.NotificationRequest(
+                    apns_topic=remote_push_device.ios_app_id,
+                    device_token=remote_push_device.token,
+                    message=asdict(push_request.payload),
+                    priority=push_request.http_headers.apns_priority,
+                    push_type=push_request.http_headers.apns_push_type,
+                )
+            )
+            apns_remote_push_devices.append(remote_push_device)
+        else:
+            assert isinstance(push_request, FCMPushRequest)
+            fcm_payload = dict(
+                # FCM only allows string values, so we stringify push_key_id.
+                push_key_id=str(push_request.payload.push_key_id),
+                encrypted_data=push_request.payload.encrypted_data,
+            )
+            fcm_requests.append(
+                firebase_messaging.Message(
+                    data=fcm_payload,
+                    token=remote_push_device.token,
+                    android=firebase_messaging.AndroidConfig(priority=push_request.fcm_priority),
+                )
+            )
+            fcm_remote_push_devices.append(remote_push_device)
+
+    if realm is not None:
+        log_context = f"to (realm={realm.uuid}, device=%s)"
+    else:
+        assert remote_realm is not None
+        log_context = f"to (remote_realm={remote_realm.uuid}, device=%s)"
+
+    apple_successfully_sent_count = 0
+    if len(apns_requests) > 0:
+        sent_push_notification_result = send_e2ee_push_notification_apple(
+            apns_requests,
+            apns_remote_push_devices,
+            log_context,
+        )
+        apple_successfully_sent_count = sent_push_notification_result.successfully_sent_count
+        delete_token_ids_base64.extend(sent_push_notification_result.delete_token_ids_base64)
+
+    android_successfully_sent_count = 0
+    if len(fcm_requests) > 0:
+        sent_push_notification_result = send_e2ee_push_notification_android(
+            fcm_requests,
+            fcm_remote_push_devices,
+            log_context,
+        )
+        android_successfully_sent_count = sent_push_notification_result.successfully_sent_count
+        delete_token_ids_base64.extend(sent_push_notification_result.delete_token_ids_base64)
+
+    return {
+        "apple_successfully_sent_count": apple_successfully_sent_count,
+        "android_successfully_sent_count": android_successfully_sent_count,
+        "delete_token_ids": delete_token_ids_base64,
+    }

zilencer/lib/remote_counts.py

@@ -0,0 +1,77 @@
+from datetime import timedelta
+
+from django.db import connection
+from django.utils.timezone import now as timezone_now
+from psycopg2.sql import SQL, Literal
+
+from zilencer.models import RemoteInstallationCount, RemoteZulipServer
+
+
+class MissingDataError(Exception):
+    pass
+
+
+def compute_max_monthly_messages(remote_server: RemoteZulipServer) -> int:
+    # Calculate the maximum amount of messages that the server had within a month.
+    # out of the last 3 months.
+
+    # We would like to just check whether we have current data for the
+    # actual property we care about
+    # ('messages_sent:message_type:day'). But because our analytics
+    # tables have implicit zeros, that can't distinguish missing data
+    # from days with no messages. So we filter on `active_users_audit`
+    # instead, which will never be zero for an initialized server.
+    if not RemoteInstallationCount.objects.filter(
+        server=remote_server,
+        property="active_users_audit:is_bot:day",
+        end_time__lte=timezone_now() - timedelta(days=3),
+    ).exists():
+        raise MissingDataError
+
+    query = SQL(
+        """
+    WITH server_message_stats_daily AS -- Up to 4 rows per day for different subgroups
+    (
+        SELECT
+            r.end_time,
+            r.value AS message_count
+        FROM
+            zilencer_remoteinstallationcount r
+        WHERE
+            r.property = 'messages_sent:message_type:day'
+            AND end_time >= CURRENT_TIMESTAMP(0) - INTERVAL '90 days'
+            AND r.server_id = {server_id}
+    ),
+    server_message_stats_monthly AS (
+        SELECT
+            CASE
+                WHEN current_timestamp(0) - end_time <= INTERVAL '30 days' THEN 0
+                WHEN current_timestamp(0) - end_time <= INTERVAL '60 days' THEN 1
+                WHEN current_timestamp(0) - end_time <= INTERVAL '90 days' THEN 2
+            END AS billing_month,
+            SUM(message_count) AS message_count
+        FROM
+            server_message_stats_daily
+        GROUP BY
+            1
+    ),
+    server_max_monthly_messages AS (
+        SELECT
+            MAX(message_count) AS message_count
+        FROM
+            server_message_stats_monthly
+        WHERE
+            billing_month IS NOT NULL
+    )
+    SELECT
+        -- Return zeros, rather than nulls,
+        -- for reporting servers with zero messages.
+        COALESCE(server_max_monthly_messages.message_count, 0) AS message_count
+    FROM
+        server_max_monthly_messages;
+        """
+    ).format(server_id=Literal(remote_server.id))
+    with connection.cursor() as cursor:
+        cursor.execute(query)
+        result = cursor.fetchone()[0]
+    return int(result)

zilencer/management/commands/add_mock_conversation.py

@@ -0,0 +1,156 @@
+from typing import Any
+
+from typing_extensions import override
+
+from zerver.actions.create_user import do_create_user
+from zerver.actions.message_send import do_send_messages, internal_prep_stream_message
+from zerver.actions.reactions import do_add_reaction
+from zerver.actions.streams import bulk_add_subscriptions
+from zerver.actions.user_settings import do_change_avatar_fields
+from zerver.lib.emoji import get_emoji_data
+from zerver.lib.management import ZulipBaseCommand
+from zerver.lib.streams import ensure_stream
+from zerver.lib.upload import upload_avatar_image
+from zerver.models import Message, UserProfile
+from zerver.models.realms import get_realm
+
+
+class Command(ZulipBaseCommand):
+    help = """Add a mock conversation to the development environment.
+
+Usage: ./manage.py add_mock_conversation
+
+After running the script:
+
+From browser (ideally on high resolution screen):
+* Refresh to get the rendered tweet
+* Check that the whale emoji reaction comes before the thumbs_up emoji reaction
+* Remove the blue box (it's a box shadow on .selected_message .messagebox-content;
+  inspecting the selected element will find it fairly quickly)
+* Change the color of the stream to #a6c7e5
+* Shrink screen till the mypy link only just fits
+* Take screenshot that does not include the timestamps or bottom edge
+
+From image editing program:
+* Remove mute (and edit) icons from recipient bar
+"""
+
+    def set_avatar(self, user: UserProfile, filename: str) -> None:
+        with open(filename, "rb") as f:
+            upload_avatar_image(f, user, content_type="image/png")
+        do_change_avatar_fields(user, UserProfile.AVATAR_FROM_USER, acting_user=None)
+
+    def add_message_formatting_conversation(self) -> None:
+        realm = get_realm("zulip")
+        stream = ensure_stream(realm, "zulip features", acting_user=None)
+
+        UserProfile.objects.filter(email__contains="stage").delete()
+        starr = do_create_user(
+            "1@stage.example.com", "password", realm, "Ada Starr", acting_user=None
+        )
+        self.set_avatar(starr, "static/images/characters/starr.png")
+        fisher = do_create_user(
+            "2@stage.example.com", "password", realm, "Bel Fisher", acting_user=None
+        )
+        self.set_avatar(fisher, "static/images/characters/fisher.png")
+        twitter_bot = do_create_user(
+            "3@stage.example.com",
+            "password",
+            realm,
+            "Twitter Bot",
+            bot_type=UserProfile.DEFAULT_BOT,
+            acting_user=None,
+        )
+        self.set_avatar(twitter_bot, "static/images/characters/coral.png")
+
+        bulk_add_subscriptions(
+            realm, [stream], list(UserProfile.objects.filter(realm=realm)), acting_user=None
+        )
+
+        staged_messages: list[dict[str, Any]] = [
+            {
+                "sender": starr,
+                "content": "Hey @**Bel Fisher**, check out Zulip's Markdown formatting! "
+                "You can have:\n* bulleted lists\n  * with sub-bullets too\n"
+                "* **bold**, *italic*, and ~~strikethrough~~ text\n"
+                "* LaTeX for mathematical formulas, both inline -- $$O(n^2)$$ -- and displayed:\n"
+                "```math\n\\int_a^b f(t)\\, dt=F(b)-F(a)\n```",
+            },
+            {
+                "sender": fisher,
+                "content": "My favorite is the syntax highlighting for code blocks\n"
+                "```python\ndef fib(n: int) -> int:\n    # returns the n-th Fibonacci number\n"
+                "    return fib(n-1) + fib(n-2)\n```",
+            },
+            {
+                "sender": starr,
+                "content": "I think you forgot your base case there, Bel :laughing:\n"
+                "```quote\n```python\ndef fib(n: int) -> int:\n    # returns the n-th Fibonacci number\n"
+                "    return fib(n-1) + fib(n-2)\n```\n```",
+            },
+            {
+                "sender": fisher,
+                "content": "I'm also a big fan of inline link, tweet, video, and image previews. "
+                "Check out this picture of seahorse[](/static/images/characters/seahorse.png)!",
+            },
+            {
+                "sender": starr,
+                "content": "I just set up a custom linkifier, "
+                "so `#1234` becomes [#1234](github.com/zulip/zulip/1234), "
+                "a link to the corresponding GitHub issue.",
+            },
+            {
+                "sender": twitter_bot,
+                "content": "https://twitter.com/gvanrossum/status/786661035637772288",
+            },
+            {
+                "sender": fisher,
+                "content": "Oops, the Twitter bot I set up shouldn't be posting here. Let me go fix that.",
+            },
+        ]
+
+        messages = [
+            internal_prep_stream_message(
+                message["sender"],
+                stream,
+                "message formatting",
+                message["content"],
+            )
+            for message in staged_messages
+        ]
+
+        message_ids = [
+            sent_message_result.message_id for sent_message_result in do_send_messages(messages)
+        ]
+
+        preview_message = Message.objects.get(
+            id__in=message_ids, content__icontains="image previews"
+        )
+        whale = get_emoji_data(realm.id, "whale")
+        do_add_reaction(starr, preview_message, "whale", whale.emoji_code, whale.reaction_type)
+
+        twitter_message = Message.objects.get(id__in=message_ids, content__icontains="gvanrossum")
+        # Setting up a twitter integration in dev is a decent amount of work. If you need
+        # to update this tweet, either copy the format below, or send the link to the tweet
+        # to chat.zulip.org and ask an admin of that server to get you the rendered_content.
+        twitter_message.rendered_content = (
+            "<p><a>https://twitter.com/gvanrossum/status/786661035637772288</a></p>\n"
+            '<div class="inline-preview-twitter"><div class="twitter-tweet">'
+            '<a><img class="twitter-avatar" '
+            'src="https://pbs.twimg.com/profile_images/424495004/GuidoAvatar_bigger.jpg"></a>'
+            "<p>Great blog post about Zulip's use of mypy: "
+            "<a>http://blog.zulip.org/2016/10/13/static-types-in-python-oh-mypy/</a></p>"
+            "<span>- Guido van Rossum (@gvanrossum)</span></div></div>"
+        )
+        twitter_message.save(update_fields=["rendered_content"])
+
+        # Put a short pause between the whale reaction and this, so that the
+        # thumbs_up shows up second
+        thumbs_up = get_emoji_data(realm.id, "thumbs_up")
+        do_add_reaction(
+            starr, preview_message, "thumbs_up", thumbs_up.emoji_code, thumbs_up.reaction_type
+        )
+
+    @override
+    def handle(self, *args: Any, **options: str) -> None:
+        self.add_message_formatting_conversation()

zilencer/management/commands/calculate_first_visible_message_id.py

@@ -0,0 +1,37 @@
+from collections.abc import Iterable
+from typing import Any
+
+from django.core.management.base import CommandParser
+from typing_extensions import override
+
+from zerver.lib.management import ZulipBaseCommand, abort_unless_locked
+from zerver.lib.message import maybe_update_first_visible_message_id
+from zerver.models import Realm
+
+
+class Command(ZulipBaseCommand):
+    help = """Calculate the value of first visible message ID and store it in cache"""
+
+    @override
+    def add_arguments(self, parser: CommandParser) -> None:
+        self.add_realm_args(parser)
+        parser.add_argument(
+            "--lookback-hours",
+            type=int,
+            help="Period a bit larger than that of the cron job that runs "
+            "this command so that the lookback periods are sure to overlap.",
+            required=True,
+        )
+
+    @override
+    @abort_unless_locked
+    def handle(self, *args: Any, **options: Any) -> None:
+        target_realm = self.get_realm(options)
+
+        if target_realm is None:
+            realms: Iterable[Realm] = Realm.objects.all().iterator()
+        else:
+            realms = [target_realm]
+
+        for realm in realms:
+            maybe_update_first_visible_message_id(realm, options["lookback_hours"])

zilencer/management/commands/compare_messages.py

@@ -0,0 +1,37 @@
+from typing import Any
+
+import orjson
+from django.core.management.base import CommandParser
+from typing_extensions import override
+
+from zerver.lib.management import ZulipBaseCommand
+
+
+class Command(ZulipBaseCommand):
+    help = """
+    Compare rendered messages from files.
+    Usage: ./manage.py compare_messages <dump1> <dump2>
+    """
+
+    @override
+    def add_arguments(self, parser: CommandParser) -> None:
+        parser.add_argument("dump1", help="First file to compare")
+        parser.add_argument("dump2", help="Second file to compare")
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        total_count = 0
+        changed_count = 0
+        with open(options["dump1"]) as dump1, open(options["dump2"]) as dump2:
+            for line1, line2 in zip(dump1, dump2, strict=False):
+                m1 = orjson.loads(line1)
+                m2 = orjson.loads(line2)
+                total_count += 1
+                if m1["id"] != m2["id"]:
+                    self.stderr.write("Inconsistent messages dump")
+                    break
+                if m1["content"] != m2["content"]:
+                    changed_count += 1
+                    self.stdout.write("Changed message id: {id}".format(id=m1["id"]))
+        self.stdout.write(f"Total messages: {total_count}")
+        self.stdout.write(f"Changed messages: {changed_count}")

zilencer/management/commands/downgrade_small_realms_behind_on_payments.py

@@ -0,0 +1,15 @@
+from typing import Any
+
+from typing_extensions import override
+
+from corporate.lib.stripe import downgrade_small_realms_behind_on_payments_as_needed
+from zerver.lib.management import ZulipBaseCommand, abort_unless_locked
+
+
+class Command(ZulipBaseCommand):
+    help = "Downgrade small realms that are running behind on payments"
+
+    @override
+    @abort_unless_locked
+    def handle(self, *args: Any, **options: Any) -> None:
+        downgrade_small_realms_behind_on_payments_as_needed()

zilencer/management/commands/invoice_plans.py

@@ -0,0 +1,32 @@
+import datetime
+from typing import Any
+
+from django.conf import settings
+from django.core.management.base import CommandError, CommandParser
+from typing_extensions import override
+
+from zerver.lib.management import ZulipBaseCommand, abort_unless_locked
+
+if settings.BILLING_ENABLED:
+    from corporate.lib.stripe import invoice_plans_as_needed
+
+
+class Command(ZulipBaseCommand):
+    help = """Generates invoices for customers if needed."""
+
+    @override
+    def add_arguments(self, parser: CommandParser) -> None:
+        if settings.DEVELOPMENT:
+            parser.add_argument("--date", type=datetime.datetime.fromisoformat)
+
+    @override
+    @abort_unless_locked
+    def handle(self, *args: Any, **options: Any) -> None:
+        if not settings.BILLING_ENABLED:
+            raise CommandError("Billing is not enabled!")
+
+        for_date = None
+        if settings.DEVELOPMENT and options["date"]:
+            for_date = options["date"].replace(tzinfo=datetime.timezone.utc)
+
+        invoice_plans_as_needed(for_date)

zilencer/management/commands/manage_push_registration_encryption_keys.py

@@ -0,0 +1,86 @@
+import configparser
+import json
+from argparse import ArgumentParser
+from typing import Any
+
+from django.conf import settings
+from nacl.encoding import Base64Encoder
+from nacl.public import PrivateKey
+from typing_extensions import override
+
+from zerver.lib.management import ZulipBaseCommand
+
+
+class Command(ZulipBaseCommand):
+    help = """
+Add or remove a key pair from the `push_registration_encryption_keys` map.
+
+Usage:
+./manage.py manage_push_registration_encryption_keys --add
+./manage.py manage_push_registration_encryption_keys --remove-key <public-key>
+"""
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        parser.add_argument(
+            "--add",
+            action="store_true",
+            help="Add a new key pair to the `push_registration_encryption_keys` map.",
+        )
+        parser.add_argument(
+            "--remove-key",
+            metavar="PUBLIC_KEY",
+            help="Remove the key pair associated with the given public key from the `push_registration_encryption_keys` map.",
+        )
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        if not options["add"] and options["remove_key"] is None:
+            print("Error: Please provide either --add or --remove-key <public-key>.")
+            return
+
+        if settings.DEVELOPMENT:
+            SECRETS_FILENAME = "zproject/dev-secrets.conf"
+        else:
+            SECRETS_FILENAME = "/etc/zulip/zulip-secrets.conf"
+
+        config = configparser.ConfigParser()
+        config.read(SECRETS_FILENAME)
+        push_registration_encryption_keys: dict[str, str] = json.loads(
+            config.get("secrets", "push_registration_encryption_keys", fallback="{}")
+        )
+
+        added_key_pair: tuple[str, str] | None = None
+        if options["add"]:
+            # Generate a new key-pair and store.
+            private_key = PrivateKey.generate()
+            private_key_str = Base64Encoder.encode(bytes(private_key)).decode("utf-8")
+            public_key_str = Base64Encoder.encode(bytes(private_key.public_key)).decode("utf-8")
+            push_registration_encryption_keys[public_key_str] = private_key_str
+            added_key_pair = (public_key_str, private_key_str)
+
+        if options["remove_key"] is not None:
+            # Remove the key-pair for the given public key.
+            remove_key = options["remove_key"]
+            if remove_key not in push_registration_encryption_keys:
+                print("Error: No key pair found for the given public key.")
+                return
+
+            del push_registration_encryption_keys[remove_key]
+
+        config.set(
+            "secrets",
+            "push_registration_encryption_keys",
+            json.dumps(push_registration_encryption_keys),
+        )
+        with open(SECRETS_FILENAME, "w") as secrets_file:
+            config.write(secrets_file)
+
+        if added_key_pair is not None:
+            public_key_str, private_key_str = added_key_pair
+            print("Added a new key pair:")
+            print(f"- Public key:  {public_key_str}")
+            print(f"- Private key: {private_key_str}")
+
+        if options["remove_key"] is not None:
+            print(f"Removed the key pair for public key: {options['remove_key']}")

zilencer/management/commands/mark_all_messages_unread.py

@@ -0,0 +1,22 @@
+from typing import Any
+
+import bmemcached
+from django.conf import settings
+from django.core.cache import cache
+from django.db.models import F
+from typing_extensions import override
+
+from zerver.lib.management import ZulipBaseCommand
+from zerver.models import UserMessage
+
+
+class Command(ZulipBaseCommand):
+    help = """Script to mark all messages as unread."""
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        assert settings.DEVELOPMENT
+        UserMessage.objects.all().update(flags=F("flags").bitand(~UserMessage.flags.read))
+        _cache = cache._cache  # type: ignore[attr-defined] # not in stubs
+        assert isinstance(_cache, bmemcached.Client)
+        _cache.flush_all()

zilencer/management/commands/populate_billing_realms.py

@@ -0,0 +1,586 @@
+import contextlib
+import uuid
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import Any
+
+import stripe
+from django.conf import settings
+from django.core.management.base import CommandParser
+from django.utils.timezone import now as timezone_now
+from typing_extensions import override
+
+from corporate.lib.stripe import (
+    RealmBillingSession,
+    RemoteRealmBillingSession,
+    RemoteServerBillingSession,
+    UpgradeRequest,
+    add_months,
+    sign_string,
+)
+from corporate.models.customers import Customer
+from corporate.models.licenses import LicenseLedger
+from corporate.models.plans import CustomerPlan
+from scripts.lib.zulip_tools import TIMESTAMP_FORMAT
+from zerver.actions.create_realm import do_create_realm
+from zerver.actions.create_user import do_create_user
+from zerver.actions.streams import bulk_add_subscriptions
+from zerver.apps import flush_cache
+from zerver.lib.management import ZulipBaseCommand
+from zerver.lib.remote_server import get_realms_info_for_push_bouncer
+from zerver.lib.streams import create_stream_if_needed
+from zerver.models import Realm, UserProfile
+from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.models.realms import get_realm
+from zilencer.models import (
+    RemoteRealm,
+    RemoteRealmBillingUser,
+    RemoteServerBillingUser,
+    RemoteZulipServer,
+    RemoteZulipServerAuditLog,
+)
+from zilencer.views import update_remote_realm_data_for_server
+from zproject.config import get_secret
+
+current_time = timezone_now().strftime(TIMESTAMP_FORMAT)
+communicate_with_stripe = get_secret("stripe_secret_key") is not None
+
+
+@dataclass
+class CustomerProfile:
+    unique_id: str
+    billing_schedule: int = CustomerPlan.BILLING_SCHEDULE_ANNUAL
+    tier: int | None = None
+    new_plan_tier: int | None = None
+    automanage_licenses: bool = False
+    status: int = CustomerPlan.ACTIVE
+    sponsorship_pending: bool = False
+    is_sponsored: bool = False
+    card: str = ""
+    charge_automatically: bool = True
+    is_remote_realm: bool = False
+    is_remote_server: bool = False
+    renewal_date: str = current_time
+    # Use (timezone_now() + timedelta(minutes=1)).strftime(TIMESTAMP_FORMAT) as `end_date` for testing.
+    # `invoice_plan` is not implemented yet for remote servers and realms so no payment is generated in stripe.
+    end_date: str = "2030-10-10-01-10-10"
+    remote_server_plan_start_date: str = "billing_cycle_end_date"
+
+
+class Command(ZulipBaseCommand):
+    help = "Populate database with different types of realms that can exist."
+
+    @override
+    def add_arguments(self, parser: CommandParser) -> None:
+        parser.add_argument(
+            "--only-remote-server",
+            action="store_true",
+            help="Whether to only run for remote servers",
+        )
+
+        parser.add_argument(
+            "--only-remote-realm",
+            action="store_true",
+            help="Whether to only run for remote realms",
+        )
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        # Create a realm for each plan type
+
+        customer_profiles = [
+            # NOTE: The unique_id has to be less than 40 characters.
+            CustomerProfile(unique_id="sponsorship-pending", sponsorship_pending=True),
+            CustomerProfile(
+                unique_id="annual-free",
+                billing_schedule=CustomerPlan.BILLING_SCHEDULE_ANNUAL,
+            ),
+            CustomerProfile(
+                unique_id="annual-standard",
+                billing_schedule=CustomerPlan.BILLING_SCHEDULE_ANNUAL,
+                tier=CustomerPlan.TIER_CLOUD_STANDARD,
+            ),
+            CustomerProfile(
+                unique_id="annual-plus",
+                billing_schedule=CustomerPlan.BILLING_SCHEDULE_ANNUAL,
+                tier=CustomerPlan.TIER_CLOUD_PLUS,
+            ),
+            CustomerProfile(
+                unique_id="monthly-free",
+                billing_schedule=CustomerPlan.BILLING_SCHEDULE_MONTHLY,
+            ),
+            CustomerProfile(
+                unique_id="monthly-standard",
+                billing_schedule=CustomerPlan.BILLING_SCHEDULE_MONTHLY,
+                tier=CustomerPlan.TIER_CLOUD_STANDARD,
+            ),
+            CustomerProfile(
+                unique_id="monthly-plus",
+                billing_schedule=CustomerPlan.BILLING_SCHEDULE_MONTHLY,
+                tier=CustomerPlan.TIER_CLOUD_PLUS,
+            ),
+            CustomerProfile(
+                unique_id="downgrade-end-of-cycle",
+                billing_schedule=CustomerPlan.BILLING_SCHEDULE_MONTHLY,
+                tier=CustomerPlan.TIER_CLOUD_STANDARD,
+                status=CustomerPlan.DOWNGRADE_AT_END_OF_CYCLE,
+            ),
+            CustomerProfile(
+                unique_id="standard-automanage-licenses",
+                billing_schedule=CustomerPlan.BILLING_SCHEDULE_MONTHLY,
+                tier=CustomerPlan.TIER_CLOUD_STANDARD,
+                automanage_licenses=True,
+            ),
+            CustomerProfile(
+                unique_id="standard-automatic-card",
+                billing_schedule=CustomerPlan.BILLING_SCHEDULE_MONTHLY,
+                tier=CustomerPlan.TIER_CLOUD_STANDARD,
+                card="pm_card_visa",
+            ),
+            CustomerProfile(
+                unique_id="standard-invoice-payment",
+                billing_schedule=CustomerPlan.BILLING_SCHEDULE_MONTHLY,
+                tier=CustomerPlan.TIER_CLOUD_STANDARD,
+                charge_automatically=False,
+            ),
+            CustomerProfile(
+                unique_id="standard-switch-to-annual-eoc",
+                billing_schedule=CustomerPlan.BILLING_SCHEDULE_MONTHLY,
+                tier=CustomerPlan.TIER_CLOUD_STANDARD,
+                status=CustomerPlan.SWITCH_TO_ANNUAL_AT_END_OF_CYCLE,
+            ),
+            CustomerProfile(
+                unique_id="sponsored",
+                is_sponsored=True,
+                billing_schedule=CustomerPlan.BILLING_SCHEDULE_MONTHLY,
+                # Customer plan might not exist for sponsored realms.
+                tier=CustomerPlan.TIER_CLOUD_STANDARD,
+            ),
+            CustomerProfile(
+                unique_id="free-trial",
+                tier=CustomerPlan.TIER_CLOUD_STANDARD,
+                status=CustomerPlan.FREE_TRIAL,
+            ),
+            CustomerProfile(
+                unique_id="legacy-server",
+                tier=CustomerPlan.TIER_SELF_HOSTED_LEGACY,
+                is_remote_server=True,
+            ),
+            CustomerProfile(
+                unique_id="legacy-server-upgrade-scheduled",
+                tier=CustomerPlan.TIER_SELF_HOSTED_LEGACY,
+                status=CustomerPlan.SWITCH_PLAN_TIER_AT_PLAN_END,
+                new_plan_tier=CustomerPlan.TIER_SELF_HOSTED_BASIC,
+                is_remote_server=True,
+            ),
+            CustomerProfile(
+                unique_id="business-server",
+                tier=CustomerPlan.TIER_SELF_HOSTED_BUSINESS,
+                is_remote_server=True,
+            ),
+            CustomerProfile(
+                unique_id="business-server-free-trial",
+                tier=CustomerPlan.TIER_SELF_HOSTED_BUSINESS,
+                status=CustomerPlan.FREE_TRIAL,
+                is_remote_server=True,
+            ),
+            CustomerProfile(
+                unique_id="business-server-sponsorship-pending",
+                tier=CustomerPlan.TIER_SELF_HOSTED_BUSINESS,
+                sponsorship_pending=True,
+                is_remote_server=True,
+            ),
+            CustomerProfile(
+                unique_id="self-hosted-server-sponsorship-pending",
+                tier=CustomerPlan.TIER_SELF_HOSTED_BASE,
+                sponsorship_pending=True,
+                is_remote_server=True,
+            ),
+            CustomerProfile(
+                unique_id="server-sponsored",
+                tier=CustomerPlan.TIER_SELF_HOSTED_COMMUNITY,
+                is_sponsored=True,
+                is_remote_server=True,
+            ),
+            CustomerProfile(
+                unique_id="legacy-remote-realm",
+                tier=CustomerPlan.TIER_SELF_HOSTED_LEGACY,
+                is_remote_realm=True,
+            ),
+            CustomerProfile(
+                unique_id="free-tier-remote-realm",
+                is_remote_realm=True,
+            ),
+            CustomerProfile(
+                unique_id="business-remote-realm",
+                tier=CustomerPlan.TIER_SELF_HOSTED_BUSINESS,
+                is_remote_realm=True,
+            ),
+            CustomerProfile(
+                unique_id="business-remote-free-trial",
+                tier=CustomerPlan.TIER_SELF_HOSTED_BUSINESS,
+                status=CustomerPlan.FREE_TRIAL,
+                is_remote_realm=True,
+            ),
+            CustomerProfile(
+                unique_id="business-remote-sponsorship-pending",
+                tier=CustomerPlan.TIER_SELF_HOSTED_BUSINESS,
+                sponsorship_pending=True,
+                is_remote_realm=True,
+            ),
+            CustomerProfile(
+                unique_id="remote-sponsorship-pending",
+                sponsorship_pending=True,
+                is_remote_realm=True,
+            ),
+            CustomerProfile(
+                unique_id="remote-realm-sponsored",
+                tier=CustomerPlan.TIER_SELF_HOSTED_COMMUNITY,
+                is_sponsored=True,
+                is_remote_realm=True,
+            ),
+            CustomerProfile(
+                unique_id="basic-remote-realm",
+                tier=CustomerPlan.TIER_SELF_HOSTED_BASIC,
+                is_remote_realm=True,
+            ),
+            CustomerProfile(
+                unique_id="basic-remote-free-trial",
+                tier=CustomerPlan.TIER_SELF_HOSTED_BASIC,
+                status=CustomerPlan.FREE_TRIAL,
+                is_remote_realm=True,
+            ),
+        ]
+
+        servers = []
+        remote_realms = []
+        for customer_profile in customer_profiles:
+            if customer_profile.is_remote_server and not options.get("only_remote_realm"):
+                server_conf = populate_remote_server(customer_profile)
+                servers.append(server_conf)
+            elif customer_profile.is_remote_realm and not options.get("only_remote_server"):
+                remote_realm_conf = populate_remote_realms(customer_profile)
+                remote_realms.append(remote_realm_conf)
+            elif not options.get("only_remote_server") and not options.get("only_remote_realm"):
+                populate_realm(customer_profile)
+
+        print("-" * 40)
+        for server in servers:
+            for key, value in server.items():
+                print(f"{key}: {value}")
+            print("-" * 40)
+
+        for remote_realm_conf in remote_realms:
+            for key, value in remote_realm_conf.items():
+                print(f"{key}: {value}")
+            print("-" * 40)
+
+
+def add_card_to_customer(customer: Customer) -> None:
+    if not communicate_with_stripe:
+        return
+
+    assert customer.stripe_customer_id is not None
+    # Set the Stripe API key
+    stripe.api_key = get_secret("stripe_secret_key")
+
+    # Create a card payment method and attach it to the customer
+    payment_method = stripe.PaymentMethod.create(
+        type="card",
+        card={"token": "tok_visa"},
+    )
+
+    # Attach the payment method to the customer
+    stripe.PaymentMethod.attach(payment_method.id, customer=customer.stripe_customer_id)
+
+    # Set the default payment method for the customer
+    stripe.Customer.modify(
+        customer.stripe_customer_id,
+        invoice_settings={"default_payment_method": payment_method.id},
+    )
+
+
+def create_plan_for_customer(customer: Customer, customer_profile: CustomerProfile) -> None:
+    assert customer_profile.tier is not None
+    if customer_profile.status == CustomerPlan.FREE_TRIAL:
+        # 2 months free trial.
+        next_invoice_date = add_months(timezone_now(), 2)
+    else:
+        months = 12
+        if customer_profile.billing_schedule == CustomerPlan.BILLING_SCHEDULE_MONTHLY:
+            months = 1
+        next_invoice_date = add_months(timezone_now(), months)
+
+    customer_plan = CustomerPlan.objects.create(
+        customer=customer,
+        billing_cycle_anchor=timezone_now(),
+        billing_schedule=customer_profile.billing_schedule,
+        tier=customer_profile.tier,
+        price_per_license=1200,
+        automanage_licenses=customer_profile.automanage_licenses,
+        status=customer_profile.status,
+        charge_automatically=customer_profile.charge_automatically,
+        next_invoice_date=next_invoice_date,
+    )
+
+    LicenseLedger.objects.create(
+        licenses=25,
+        licenses_at_next_renewal=25,
+        event_time=timezone_now(),
+        is_renewal=True,
+        plan=customer_plan,
+    )
+
+
+def populate_realm(customer_profile: CustomerProfile) -> Realm | None:
+    unique_id = customer_profile.unique_id
+    if customer_profile.is_remote_realm:
+        plan_type = Realm.PLAN_TYPE_SELF_HOSTED
+    elif customer_profile.is_sponsored:
+        plan_type = Realm.PLAN_TYPE_STANDARD_FREE
+    elif customer_profile.tier is None:
+        plan_type = Realm.PLAN_TYPE_LIMITED
+    elif customer_profile.tier == CustomerPlan.TIER_CLOUD_STANDARD:
+        plan_type = Realm.PLAN_TYPE_STANDARD
+    elif customer_profile.tier == CustomerPlan.TIER_CLOUD_PLUS:
+        plan_type = Realm.PLAN_TYPE_PLUS
+    else:
+        raise AssertionError("Unexpected tier!")
+    plan_name = Realm.ALL_PLAN_TYPES[plan_type]
+
+    # Delete existing realm with this name
+    with contextlib.suppress(Realm.DoesNotExist):
+        get_realm(unique_id).delete()
+        # Because we just deleted a bunch of objects in the database
+        # directly (rather than deleting individual objects in Django,
+        # in which case our post_save hooks would have flushed the
+        # individual objects from memcached for us), we need to flush
+        # memcached in order to ensure deleted objects aren't still
+        # present in the memcached cache.
+        flush_cache(None)
+
+    realm = do_create_realm(
+        string_id=unique_id,
+        name=unique_id,
+        description=unique_id,
+        plan_type=plan_type,
+    )
+
+    # Create a user with billing access
+    full_name = f"{plan_name}-admin"
+    email = f"{full_name}@zulip.com"
+    user = do_create_user(
+        email,
+        full_name,
+        realm,
+        full_name,
+        role=UserProfile.ROLE_REALM_OWNER,
+        acting_user=None,
+        tos_version=settings.TERMS_OF_SERVICE_VERSION,
+    )
+
+    stream, _ = create_stream_if_needed(
+        realm,
+        "all",
+    )
+
+    bulk_add_subscriptions(realm, [stream], [user], acting_user=None)
+
+    if customer_profile.is_remote_realm:
+        # Remote realm billing data on their local server is irrelevant.
+        return realm
+
+    if customer_profile.sponsorship_pending or customer_profile.is_sponsored:
+        # plan_type is already set correctly above for sponsored realms.
+        customer = Customer.objects.create(
+            realm=realm,
+            sponsorship_pending=customer_profile.sponsorship_pending,
+        )
+        return realm
+
+    if customer_profile.tier is None:
+        return realm
+
+    billing_session = RealmBillingSession(user)
+    if communicate_with_stripe:
+        # This attaches stripe_customer_id to customer.
+        customer = billing_session.update_or_create_stripe_customer()
+        assert customer.stripe_customer_id is not None
+    else:
+        customer = billing_session.update_or_create_customer()
+
+    if customer_profile.card:
+        add_card_to_customer(customer)
+
+    create_plan_for_customer(customer, customer_profile)
+    return realm
+
+
+def populate_remote_server(customer_profile: CustomerProfile) -> dict[str, str]:
+    unique_id = customer_profile.unique_id
+
+    if (
+        customer_profile.is_sponsored
+        and customer_profile.tier == CustomerPlan.TIER_SELF_HOSTED_COMMUNITY
+    ):
+        plan_type = RemoteZulipServer.PLAN_TYPE_COMMUNITY
+    elif customer_profile.tier == CustomerPlan.TIER_SELF_HOSTED_LEGACY:
+        plan_type = RemoteZulipServer.PLAN_TYPE_SELF_MANAGED_LEGACY
+    elif customer_profile.tier == CustomerPlan.TIER_SELF_HOSTED_BUSINESS:
+        plan_type = RemoteZulipServer.PLAN_TYPE_BUSINESS
+    elif customer_profile.tier is CustomerPlan.TIER_SELF_HOSTED_BASE:
+        plan_type = RemoteZulipServer.PLAN_TYPE_SELF_MANAGED
+    else:
+        raise AssertionError("Unexpected tier!")
+
+    server_uuid = str(uuid.uuid4())
+    api_key = server_uuid
+    hostname = f"{unique_id}.example.com"
+
+    # Delete existing remote server.
+    RemoteZulipServer.objects.filter(hostname=hostname).delete()
+    flush_cache(None)
+
+    remote_server = RemoteZulipServer.objects.create(
+        uuid=server_uuid,
+        api_key=api_key,
+        hostname=f"{unique_id}.example.com",
+        contact_email=f"{unique_id}@example.com",
+        plan_type=plan_type,
+        # TODO: Save property audit log data for server.
+        last_audit_log_update=timezone_now(),
+    )
+
+    RemoteZulipServerAuditLog.objects.create(
+        event_type=AuditLogEventType.REMOTE_SERVER_CREATED,
+        server=remote_server,
+        event_time=remote_server.last_updated,
+    )
+
+    billing_user = RemoteServerBillingUser.objects.create(
+        full_name="Server user",
+        remote_server=remote_server,
+        email=f"{unique_id}@example.com",
+    )
+    billing_session = RemoteServerBillingSession(remote_server, billing_user)
+    if customer_profile.tier == CustomerPlan.TIER_SELF_HOSTED_LEGACY:
+        # Create customer plan for these servers for temporary period.
+        renewal_date = datetime.strptime(customer_profile.renewal_date, TIMESTAMP_FORMAT).replace(
+            tzinfo=timezone.utc
+        )
+        end_date = datetime.strptime(customer_profile.end_date, TIMESTAMP_FORMAT).replace(
+            tzinfo=timezone.utc
+        )
+        billing_session.create_complimentary_access_plan(renewal_date, end_date)
+
+        if not communicate_with_stripe:
+            # We need to communicate with stripe to upgrade here.
+            return {
+                "unique_id": unique_id,
+                "server_uuid": server_uuid,
+                "api_key": api_key,
+                "ERROR": "Need to communicate with stripe to populate this profile.",
+            }
+
+        # Scheduled server to upgrade to business plan.
+        if customer_profile.status == CustomerPlan.SWITCH_PLAN_TIER_AT_PLAN_END:
+            # This attaches stripe_customer_id to customer.
+            customer = billing_session.update_or_create_stripe_customer()
+            add_card_to_customer(customer)
+            seat_count = 30
+            signed_seat_count, salt = sign_string(str(seat_count))
+            upgrade_request = UpgradeRequest(
+                billing_modality="charge_automatically",
+                schedule="annual",
+                signed_seat_count=signed_seat_count,
+                salt=salt,
+                license_management="automatic",
+                licenses=seat_count,
+                tier=CustomerPlan.TIER_SELF_HOSTED_BUSINESS,
+                remote_server_plan_start_date=customer_profile.remote_server_plan_start_date,
+            )
+            billing_session.do_upgrade(upgrade_request)
+
+    elif customer_profile.tier == CustomerPlan.TIER_SELF_HOSTED_BUSINESS:
+        if communicate_with_stripe:
+            # This attaches stripe_customer_id to customer.
+            customer = billing_session.update_or_create_stripe_customer()
+            assert customer.stripe_customer_id is not None
+        else:
+            customer = billing_session.update_or_create_customer()
+        add_card_to_customer(customer)
+        create_plan_for_customer(customer, customer_profile)
+
+    if customer_profile.sponsorship_pending:
+        billing_session.update_customer_sponsorship_status(True)
+    elif customer_profile.is_sponsored:
+        billing_session.do_change_plan_type(tier=None, is_sponsored=True)
+
+    return {
+        "unique_id": unique_id,
+        "server_uuid": server_uuid,
+        "api_key": api_key,
+    }
+
+
+def populate_remote_realms(customer_profile: CustomerProfile) -> dict[str, str]:
+    # Delete existing remote realm.
+    RemoteRealm.objects.filter(name=customer_profile.unique_id).delete()
+    flush_cache(None)
+
+    local_realm = populate_realm(customer_profile)
+    assert local_realm is not None
+
+    remote_server_uuid = settings.ZULIP_ORG_ID
+    assert remote_server_uuid is not None
+    remote_server = RemoteZulipServer.objects.filter(
+        uuid=remote_server_uuid,
+    ).first()
+
+    if remote_server is None:
+        raise AssertionError("Remote server not found! Please run manage.py register_server")
+
+    update_remote_realm_data_for_server(remote_server, get_realms_info_for_push_bouncer())
+
+    remote_realm = RemoteRealm.objects.get(uuid=local_realm.uuid)
+    user = UserProfile.objects.filter(realm=local_realm).first()
+    assert user is not None
+    billing_user = RemoteRealmBillingUser.objects.create(
+        full_name=user.full_name,
+        remote_realm=remote_realm,
+        user_uuid=user.uuid,
+        email=user.email,
+    )
+    billing_session = RemoteRealmBillingSession(remote_realm, billing_user)
+    # TODO: Save property audit log  data for server.
+    remote_realm.server.last_audit_log_update = timezone_now()
+    remote_realm.server.save(update_fields=["last_audit_log_update"])
+    if communicate_with_stripe:
+        # This attaches stripe_customer_id to customer.
+        customer = billing_session.update_or_create_stripe_customer()
+        assert customer.stripe_customer_id is not None
+    else:
+        customer = billing_session.update_or_create_customer()
+    add_card_to_customer(customer)
+    if customer_profile.tier == CustomerPlan.TIER_SELF_HOSTED_LEGACY:
+        renewal_date = datetime.strptime(customer_profile.renewal_date, TIMESTAMP_FORMAT).replace(
+            tzinfo=timezone.utc
+        )
+        end_date = datetime.strptime(customer_profile.end_date, TIMESTAMP_FORMAT).replace(
+            tzinfo=timezone.utc
+        )
+        billing_session.create_complimentary_access_plan(renewal_date, end_date)
+    elif customer_profile.tier is not None:
+        billing_session.do_change_plan_type(
+            tier=customer_profile.tier, is_sponsored=customer_profile.is_sponsored
+        )
+        if not customer_profile.is_sponsored:
+            create_plan_for_customer(customer, customer_profile)
+
+    if customer_profile.sponsorship_pending:
+        billing_session.update_customer_sponsorship_status(True)
+
+    return {
+        "unique_id": customer_profile.unique_id,
+        "login_url": local_realm.url + "/self-hosted-billing/",
+    }

zilencer/management/commands/print_initial_password.py

@@ -0,0 +1,34 @@
+from argparse import ArgumentParser
+from typing import Any
+
+from typing_extensions import override
+
+from zerver.lib.initial_password import initial_password
+from zerver.lib.management import ZulipBaseCommand
+
+
+class Command(ZulipBaseCommand):
+    help = "Print the initial password and API key for accounts as created by populate_db"
+
+    fmt = "%-30s %-16s  %-32s"
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        parser.add_argument(
+            "emails",
+            metavar="<email>",
+            nargs="*",
+            help="email of user to show password and API key for",
+        )
+        self.add_realm_args(parser)
+
+    @override
+    def handle(self, *args: Any, **options: str) -> None:
+        realm = self.get_realm(options)
+        print(self.fmt % ("email", "password", "API key"))
+        for email in options["emails"]:
+            if "@" not in email:
+                print(f"ERROR: {email} does not look like an email address")
+                continue
+            user = self.get_user(email, realm)
+            print(self.fmt % (email, initial_password(email), user.api_key))

zilencer/management/commands/profile_request.py

@@ -0,0 +1,68 @@
+import cProfile
+import logging
+import tempfile
+from typing import Any
+
+from django.contrib.sessions.backends.base import SessionBase
+from django.core.management.base import CommandParser
+from django.http import HttpRequest, HttpResponseBase
+from typing_extensions import override
+
+from zerver.lib.management import ZulipBaseCommand
+from zerver.lib.request import RequestNotes
+from zerver.lib.test_helpers import HostRequestMock
+from zerver.middleware import LogRequests
+from zerver.models import UserMessage
+from zerver.views.message_fetch import get_messages_backend
+
+
+class MockSession(SessionBase):
+    def __init__(self) -> None:
+        self.modified = False
+
+
+def profile_request(request: HttpRequest, num_before: int, num_after: int) -> HttpResponseBase:
+    def get_response(request: HttpRequest) -> HttpResponseBase:
+        return prof.runcall(
+            get_messages_backend,
+            request,
+            request.user,
+            num_before=num_before,
+            num_after=num_after,
+            apply_markdown=True,
+        )
+
+    prof = cProfile.Profile()
+    with tempfile.NamedTemporaryFile(prefix="profile.data.", delete=False) as stats_file:
+        response = LogRequests(get_response)(request)
+        assert isinstance(response, HttpResponseBase)  # async responses not supported here for now
+        prof.dump_stats(stats_file.name)
+        logging.info("Profiling data written to %s", stats_file.name)
+    return response
+
+
+class Command(ZulipBaseCommand):
+    @override
+    def add_arguments(self, parser: CommandParser) -> None:
+        parser.add_argument("email", metavar="<email>", help="Email address of the user")
+        self.add_realm_args(parser)
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        realm = self.get_realm(options)
+        user = self.get_user(options["email"], realm)
+        anchor = UserMessage.objects.filter(user_profile=user).order_by("-message")[200].message_id
+        mock_request = HostRequestMock(
+            post_data={
+                "anchor": anchor,
+                "num_before": 1200,
+                "num_after": 200,
+            },
+            user_profile=user,
+            meta_data={"REMOTE_ADDR": "127.0.0.1"},
+            path="/",
+        )
+        mock_request.session = MockSession()
+        RequestNotes.get_notes(mock_request).log_data = None
+
+        profile_request(mock_request, num_before=1200, num_after=200)

zilencer/management/commands/queue_rate.py

@@ -0,0 +1,89 @@
+import csv
+from timeit import timeit
+from typing import Any
+
+from django.core.management.base import CommandParser
+from typing_extensions import override
+
+from zerver.lib.management import ZulipBaseCommand
+from zerver.lib.queue import SimpleQueueClient, queue_json_publish_rollback_unsafe
+from zerver.worker.test import BatchNoopWorker, NoopWorker
+
+
+class Command(ZulipBaseCommand):
+    help = """Times the overhead of enqueuing and dequeuing messages from RabbitMQ."""
+
+    @override
+    def add_arguments(self, parser: CommandParser) -> None:
+        parser.add_argument(
+            "--count", help="Number of messages to enqueue", default=10000, type=int
+        )
+        parser.add_argument("--reps", help="Iterations of enqueue/dequeue", default=1, type=int)
+        parser.add_argument("--batch", help="Enables batch dequeuing", action="store_true")
+        parser.add_argument("--csv", help="Path to CSV output", default="rabbitmq-timings.csv")
+        parser.add_argument(
+            "--prefetches",
+            help="Limits the prefetch size; RabbitMQ defaults to unbounded (0)",
+            default=[0],
+            nargs="+",
+            type=int,
+        )
+        parser.add_argument(
+            "--slow",
+            help="Which request numbers should take 60s (1-based)",
+            action="append",
+            type=int,
+            default=[],
+        )
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        print("Purging queue...")
+        queue = SimpleQueueClient()
+        queue_name = "noop_batch" if options["batch"] else "noop"
+        queue.ensure_queue(queue_name, lambda channel: channel.queue_purge("noop"))
+        count = options["count"]
+        reps = options["reps"]
+
+        with open(options["csv"], "w", newline="") as csvfile:
+            writer = csv.DictWriter(
+                csvfile, fieldnames=["Queue size", "Queue type", "Prefetch", "Rate"]
+            )
+            writer.writeheader()
+
+            for prefetch in options["prefetches"]:
+                print(f"Queue size {count}, prefetch {prefetch}...")
+                worker: NoopWorker | BatchNoopWorker = NoopWorker(count, options["slow"])
+                if options["batch"]:
+                    worker = BatchNoopWorker(count, options["slow"])
+                    if prefetch > 0 and prefetch < worker.batch_size:
+                        print(
+                            f"    Skipping, as prefetch {prefetch} is less than batch size {worker.batch_size}"
+                        )
+                        continue
+                worker.setup()
+
+                assert worker.q is not None
+                assert worker.q.channel is not None
+                worker.q.channel.basic_qos(prefetch_count=prefetch)
+
+                total_time = 0.0
+                for i in range(1, reps + 1):
+                    worker.consumed = 0
+                    timeit(
+                        lambda: queue_json_publish_rollback_unsafe(queue_name, {}),
+                        number=count,
+                    )
+                    duration = timeit(worker.start, number=1)
+                    print(f"    {i}/{reps}: {count}/{duration}s = {count / duration}/s")
+                    total_time += duration
+                    writer.writerow(
+                        {
+                            "Queue size": count,
+                            "Queue type": queue_name,
+                            "Prefetch": prefetch,
+                            "Rate": count / duration,
+                        }
+                    )
+                    csvfile.flush()
+                print(f"  Overall: {reps * count}/{total_time}s = {(reps * count) / total_time}/s")

zilencer/management/commands/render_messages.py

@@ -0,0 +1,69 @@
+import os
+from collections.abc import Iterator
+from typing import Any
+
+import orjson
+from django.core.management.base import CommandParser
+from django.db.models import QuerySet
+from typing_extensions import override
+
+from zerver.lib.management import ZulipBaseCommand
+from zerver.lib.markdown import render_message_markdown
+from zerver.models import Message
+
+
+def queryset_iterator(queryset: QuerySet[Message], chunksize: int = 5000) -> Iterator[Message]:
+    queryset = queryset.order_by("id")
+    while queryset.exists():
+        for row in queryset[:chunksize]:
+            msg_id = row.id
+            yield row
+        queryset = queryset.filter(id__gt=msg_id)
+
+
+class Command(ZulipBaseCommand):
+    help = """
+    Render messages to a file.
+    Usage: ./manage.py render_messages <destination> [--amount=10000]
+    """
+
+    @override
+    def add_arguments(self, parser: CommandParser) -> None:
+        parser.add_argument("destination", help="Destination file path")
+        parser.add_argument("--amount", default=100000, help="Number of messages to render")
+        parser.add_argument("--latest_id", default=0, help="Last message id to render")
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        dest_dir = os.path.realpath(os.path.dirname(options["destination"]))
+        amount = int(options["amount"])
+        latest = int(options["latest_id"]) or Message.objects.latest("id").id
+        self.stdout.write(f"Latest message id: {latest}")
+        if not os.path.exists(dest_dir):
+            os.makedirs(dest_dir)
+
+        with open(options["destination"], "wb") as result:
+            messages = Message.objects.filter(id__gt=latest - amount, id__lte=latest).order_by("id")
+            for message in queryset_iterator(messages):
+                content = message.content
+                # In order to ensure that the output of this tool is
+                # consistent across the time, even if messages are
+                # edited, we always render the original content
+                # version, extracting it from the edit history if
+                # necessary.
+                if message.edit_history:
+                    history = orjson.loads(message.edit_history)
+                    history = sorted(history, key=lambda i: i["timestamp"])
+                    for entry in history:
+                        if "prev_content" in entry:
+                            content = entry["prev_content"]
+                            break
+                result.write(
+                    orjson.dumps(
+                        {
+                            "id": message.id,
+                            "content": render_message_markdown(message, content),
+                        },
+                        option=orjson.OPT_APPEND_NEWLINE,
+                    )
+                )

zilencer/management/commands/rundjangoserver.py

@@ -0,0 +1,32 @@
+from collections.abc import Callable
+from datetime import datetime
+from functools import wraps
+
+from django.core.management.commands.runserver import Command as DjangoCommand
+from typing_extensions import override
+
+
+def output_styler(style_func: Callable[[str], str]) -> Callable[[str], str]:
+    # Might fail to suppress the date line around midnight, but, whatever.
+    date_prefix = datetime.now().astimezone().strftime("%B %d, %Y - ")
+
+    @wraps(style_func)
+    def _wrapped_style_func(message: str) -> str:
+        if message == "Performing system checks...\n\n" or message.startswith(
+            ("System check identified no issues", date_prefix)
+        ):
+            message = ""
+        elif "Quit the server with " in message:
+            message = (
+                "Django process (re)started. " + message[message.index("Quit the server with ") :]
+            )
+        return style_func(message)
+
+    return _wrapped_style_func
+
+
+class Command(DjangoCommand):
+    @override
+    def inner_run(self, *args: object, **options: object) -> None:
+        self.stdout.style_func = output_styler(self.stdout.style_func)
+        super().inner_run(*args, **options)

zilencer/management/commands/switch_realm_from_standard_to_plus_plan.py

@@ -0,0 +1,28 @@
+from typing import Any
+
+from django.conf import settings
+from django.core.management.base import CommandError, CommandParser
+from typing_extensions import override
+
+from zerver.lib.management import ZulipBaseCommand
+
+if settings.BILLING_ENABLED:
+    from corporate.lib.stripe import RealmBillingSession
+    from corporate.models.plans import CustomerPlan
+
+
+class Command(ZulipBaseCommand):
+    @override
+    def add_arguments(self, parser: CommandParser) -> None:
+        self.add_realm_args(parser)
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        realm = self.get_realm(options)
+
+        if not realm:
+            raise CommandError("No realm found.")
+
+        if settings.BILLING_ENABLED:
+            billing_session = RealmBillingSession(realm=realm)
+            billing_session.do_change_plan_to_new_tier(new_plan_tier=CustomerPlan.TIER_CLOUD_PLUS)

zilencer/management/commands/sync_api_key.py

@@ -0,0 +1,33 @@
+import os
+from configparser import ConfigParser
+from typing import Any
+
+from typing_extensions import override
+
+from zerver.lib.management import ZulipBaseCommand
+from zerver.models import UserProfile
+from zerver.models.realms import get_realm
+from zerver.models.users import get_user_by_delivery_email
+
+
+class Command(ZulipBaseCommand):
+    help = """Sync your API key from ~/.zuliprc into your development instance"""
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        config_file = os.path.join(os.environ["HOME"], ".zuliprc")
+        if not os.path.exists(config_file):
+            raise RuntimeError("No ~/.zuliprc found")
+        config = ConfigParser()
+        with open(config_file) as f:
+            config.read_file(f, config_file)
+        api_key = config.get("api", "key")
+        email = config.get("api", "email")
+
+        try:
+            realm = get_realm("zulip")
+            user_profile = get_user_by_delivery_email(email, realm)
+            user_profile.api_key = api_key
+            user_profile.save(update_fields=["api_key"])
+        except UserProfile.DoesNotExist:
+            print(f"User {email} does not exist; not syncing API key")

zilencer/migrations/0001_initial.py

@@ -0,0 +1,28 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zerver", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Deployment",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        verbose_name="ID", serialize=False, auto_created=True, primary_key=True
+                    ),
+                ),
+                ("is_active", models.BooleanField(default=True)),
+                ("api_key", models.CharField(max_length=32, null=True)),
+                ("base_api_url", models.CharField(max_length=128)),
+                ("base_site_url", models.CharField(max_length=128)),
+                ("realms", models.ManyToManyField(related_name="_deployments", to="zerver.Realm")),
+            ],
+            options={},
+            bases=(models.Model,),
+        ),
+    ]

zilencer/migrations/0001_squashed_0064_remotezulipserver_last_merge_base.py

@@ -0,0 +1,595 @@
+# Generated by Django 5.0.7 on 2024-08-13 20:11
+
+import django.core.serializers.json
+import django.db.models.deletion
+import django.utils.timezone
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    replaces = [
+        ("zilencer", "0001_initial"),
+        ("zilencer", "0002_remote_zulip_server"),
+        ("zilencer", "0003_add_default_for_remotezulipserver_last_updated_field"),
+        ("zilencer", "0004_remove_deployment_model"),
+        ("zilencer", "0005_remotepushdevicetoken_fix_uniqueness"),
+        ("zilencer", "0006_customer"),
+        ("zilencer", "0007_remotezulipserver_fix_uniqueness"),
+        ("zilencer", "0008_customer_billing_user"),
+        ("zilencer", "0009_plan"),
+        ("zilencer", "0010_billingprocessor"),
+        ("zilencer", "0011_customer_has_billing_relationship"),
+        ("zilencer", "0012_coupon"),
+        ("zilencer", "0013_remove_customer_billing_user"),
+        ("zilencer", "0014_cleanup_pushdevicetoken"),
+        ("zilencer", "0015_delete_billing"),
+        ("zilencer", "0016_remote_counts"),
+        ("zilencer", "0017_installationcount_indexes"),
+        ("zilencer", "0018_remoterealmauditlog"),
+        ("zilencer", "0019_remotezulipserver_plan_type"),
+        ("zilencer", "0020_remotezulipserverauditlog"),
+        ("zilencer", "0021_alter_remotezulipserver_uuid"),
+        ("zilencer", "0022_remotezulipserver_create_audit_log_backfill"),
+        ("zilencer", "0023_remotezulipserver_deactivated"),
+        ("zilencer", "0024_remotepushdevicetoken_user_uuid"),
+        ("zilencer", "0025_alter_remotepushdevicetoken_user_id_drop_index"),
+        ("zilencer", "0026_auditlog_models_extra_data_json"),
+        ("zilencer", "0027_backfill_remote_realmauditlog_extradata_to_json_field"),
+        ("zilencer", "0028_rename_extradatajson_remoteauditlog_extra_data"),
+        ("zilencer", "0029_update_remoterealm_indexes"),
+        ("zilencer", "0030_alter_remoteinstallationcount_remote_id"),
+        ("zilencer", "0031_alter_remoteinstallationcount_remote_id_and_more"),
+        ("zilencer", "0032_remotepushdevicetoken_backfill_ios_app_id"),
+        ("zilencer", "0033_remoterealm"),
+        ("zilencer", "0034_remoterealmauditlog_remote_realm_and_more"),
+        ("zilencer", "0035_remoterealmcount_remote_realm_and_more"),
+        ("zilencer", "0036_remotezulipserver_last_version"),
+        ("zilencer", "0037_alter_remoteinstallationcount_unique_together_and_more"),
+        ("zilencer", "0038_unique_server_remote_id"),
+        ("zilencer", "0039_remoterealm_org_type"),
+        ("zilencer", "0040_remoterealm_authentication_methods_remoterealm_name"),
+        ("zilencer", "0041_remotezulipserver_org_type"),
+        ("zilencer", "0042_alter_remoterealmauditlog_realm_id"),
+        ("zilencer", "0043_remotepushdevicetoken_remote_realm"),
+        ("zilencer", "0044_remoterealmbillinguser"),
+        ("zilencer", "0045_remoterealmauditlog_zilencer_remoterealmauditlog_server_realm_and_more"),
+        ("zilencer", "0046_remotezulipserver_last_audit_log_update"),
+        ("zilencer", "0047_preregistrationremoteserverbillinguser_and_more"),
+        ("zilencer", "0048_remotezulipserver_last_api_feature_level"),
+        ("zilencer", "0049_alter_remoterealmbillinguser_unique_together_and_more"),
+        ("zilencer", "0050_preregistrationremoterealmbillinguser_created_user_and_more"),
+        ("zilencer", "0051_remoterealm_is_system_bot_realm"),
+        ("zilencer", "0052_alter_remoterealm_plan_type_and_more"),
+        ("zilencer", "0053_remoterealmauditlog_acting_remote_user_and_more"),
+        ("zilencer", "0054_remoterealmbillinguser_enable_maintenance_release_emails_and_more"),
+        ("zilencer", "0055_remoteserverbillinguser_tos_version"),
+        ("zilencer", "0056_remoterealm_realm_locally_deleted"),
+        ("zilencer", "0057_remoterealm_last_request_timestamp_and_more"),
+        ("zilencer", "0058_remoteinstallationcount_add_mobile_pushes_forwarded_index"),
+        ("zilencer", "0059_remoterealmauditlog_add_synced_billing_event_type_index"),
+        (
+            "zilencer",
+            "0060_remove_remoterealmcount_unique_remote_realm_installation_count_and_more",
+        ),
+        ("zilencer", "0061_clean_count_tables"),
+        ("zilencer", "0062_alter_remoteinstallationcount_id_and_more"),
+        ("zilencer", "0063_convert_ids_to_bigints"),
+        ("zilencer", "0064_remotezulipserver_last_merge_base"),
+    ]
+
+    initial = True
+
+    dependencies = []
+
+    operations = [
+        migrations.CreateModel(
+            name="RemoteZulipServer",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                ("uuid", models.UUIDField(unique=True)),
+                ("api_key", models.CharField(max_length=64)),
+                ("hostname", models.CharField(max_length=128)),
+                ("contact_email", models.EmailField(blank=True, max_length=254)),
+                ("last_updated", models.DateTimeField(auto_now=True, verbose_name="last updated")),
+                ("plan_type", models.PositiveSmallIntegerField(default=100)),
+                ("deactivated", models.BooleanField(default=False)),
+                ("last_version", models.CharField(max_length=128, null=True)),
+                (
+                    "org_type",
+                    models.PositiveSmallIntegerField(
+                        choices=[
+                            (0, "Unspecified"),
+                            (10, "Business"),
+                            (20, "Open-source project"),
+                            (30, "Education (non-profit)"),
+                            (35, "Education (for-profit)"),
+                            (40, "Research"),
+                            (50, "Event or conference"),
+                            (60, "Non-profit (registered)"),
+                            (70, "Government"),
+                            (80, "Political group"),
+                            (90, "Community"),
+                            (100, "Personal"),
+                            (1000, "Other"),
+                        ],
+                        default=0,
+                    ),
+                ),
+                ("last_audit_log_update", models.DateTimeField(null=True)),
+                ("last_api_feature_level", models.PositiveIntegerField(null=True)),
+                ("last_request_datetime", models.DateTimeField(null=True)),
+                ("last_merge_base", models.CharField(max_length=128, null=True)),
+            ],
+        ),
+        migrations.CreateModel(
+            name="RemoteRealm",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                ("uuid", models.UUIDField(unique=True)),
+                ("uuid_owner_secret", models.TextField()),
+                ("host", models.TextField()),
+                ("last_updated", models.DateTimeField(auto_now=True, verbose_name="last updated")),
+                ("registration_deactivated", models.BooleanField(default=False)),
+                ("realm_deactivated", models.BooleanField(default=False)),
+                ("realm_date_created", models.DateTimeField()),
+                ("plan_type", models.PositiveSmallIntegerField(db_index=True, default=100)),
+                (
+                    "server",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="zilencer.remotezulipserver"
+                    ),
+                ),
+                (
+                    "org_type",
+                    models.PositiveSmallIntegerField(
+                        choices=[
+                            (0, "Unspecified"),
+                            (10, "Business"),
+                            (20, "Open-source project"),
+                            (30, "Education (non-profit)"),
+                            (35, "Education (for-profit)"),
+                            (40, "Research"),
+                            (50, "Event or conference"),
+                            (60, "Non-profit (registered)"),
+                            (70, "Government"),
+                            (80, "Political group"),
+                            (90, "Community"),
+                            (100, "Personal"),
+                            (1000, "Other"),
+                        ],
+                        default=0,
+                    ),
+                ),
+                ("authentication_methods", models.JSONField(default=dict)),
+                ("name", models.TextField(default="")),
+                ("is_system_bot_realm", models.BooleanField(default=False)),
+                ("realm_locally_deleted", models.BooleanField(default=False)),
+                ("last_request_datetime", models.DateTimeField(null=True)),
+            ],
+        ),
+        migrations.CreateModel(
+            name="RemoteServerBillingUser",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                ("email", models.EmailField(max_length=254)),
+                ("full_name", models.TextField(default="")),
+                (
+                    "remote_server",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="zilencer.remotezulipserver"
+                    ),
+                ),
+                ("date_joined", models.DateTimeField(default=django.utils.timezone.now)),
+                ("is_active", models.BooleanField(default=True)),
+                ("last_login", models.DateTimeField(null=True)),
+                ("enable_maintenance_release_emails", models.BooleanField(default=True)),
+                ("enable_major_release_emails", models.BooleanField(default=True)),
+                ("tos_version", models.TextField(default="-1")),
+            ],
+            options={
+                "unique_together": {("remote_server", "email")},
+            },
+        ),
+        migrations.CreateModel(
+            name="RemoteRealmBillingUser",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                ("user_uuid", models.UUIDField()),
+                ("full_name", models.TextField(default="")),
+                ("email", models.EmailField(max_length=254)),
+                ("tos_version", models.TextField(default="-1")),
+                (
+                    "remote_realm",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="zilencer.remoterealm"
+                    ),
+                ),
+                ("date_joined", models.DateTimeField(default=django.utils.timezone.now)),
+                ("is_active", models.BooleanField(default=True)),
+                ("last_login", models.DateTimeField(null=True)),
+                ("enable_maintenance_release_emails", models.BooleanField(default=True)),
+                ("enable_major_release_emails", models.BooleanField(default=True)),
+            ],
+            options={
+                "unique_together": {("remote_realm", "user_uuid")},
+            },
+        ),
+        migrations.CreateModel(
+            name="RemoteInstallationCount",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                ("property", models.CharField(max_length=32)),
+                ("subgroup", models.CharField(max_length=16, null=True)),
+                ("end_time", models.DateTimeField()),
+                ("value", models.BigIntegerField()),
+                ("remote_id", models.IntegerField(null=True)),
+                (
+                    "server",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="zilencer.remotezulipserver"
+                    ),
+                ),
+            ],
+            options={
+                "unique_together": set(),
+                "indexes": [
+                    models.Index(
+                        condition=models.Q(("property", "mobile_pushes_forwarded::day")),
+                        fields=["server_id", "end_time"],
+                        name="zilencer_remoteinstallationcount_server_end_time_mobile_pushes_forwarded",
+                    )
+                ],
+                "constraints": [
+                    models.UniqueConstraint(
+                        condition=models.Q(("subgroup__isnull", False)),
+                        fields=("server", "property", "subgroup", "end_time"),
+                        name="unique_remote_installation_count",
+                    ),
+                    models.UniqueConstraint(
+                        condition=models.Q(("subgroup__isnull", True)),
+                        fields=("server", "property", "end_time"),
+                        name="unique_remote_installation_count_null_subgroup",
+                    ),
+                    models.UniqueConstraint(
+                        condition=models.Q(("remote_id__isnull", False)),
+                        fields=("server", "remote_id"),
+                        name="unique_remote_installation_count_server_id_remote_id",
+                    ),
+                ],
+            },
+        ),
+        migrations.CreateModel(
+            name="RemoteRealmCount",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                ("property", models.CharField(max_length=32)),
+                ("subgroup", models.CharField(max_length=16, null=True)),
+                ("end_time", models.DateTimeField()),
+                ("value", models.BigIntegerField()),
+                ("realm_id", models.IntegerField(null=True)),
+                ("remote_id", models.IntegerField(null=True)),
+                (
+                    "server",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="zilencer.remotezulipserver"
+                    ),
+                ),
+                (
+                    "remote_realm",
+                    models.ForeignKey(
+                        null=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="zilencer.remoterealm",
+                    ),
+                ),
+            ],
+            options={
+                "unique_together": set(),
+                "indexes": [
+                    models.Index(
+                        fields=["property", "end_time"],
+                        name="zilencer_remoterealmcount_property_end_time_506a0b38_idx",
+                    ),
+                    models.Index(
+                        condition=models.Q(("remote_realm__isnull", True)),
+                        fields=["server", "realm_id"],
+                        name="zilencer_remoterealmcount_server_realm",
+                    ),
+                    models.Index(
+                        condition=models.Q(("remote_realm__isnull", True)),
+                        fields=["server"],
+                        name="zilencer_remoterealmcount_server",
+                    ),
+                ],
+                "constraints": [
+                    models.UniqueConstraint(
+                        condition=models.Q(("remote_id__isnull", False)),
+                        fields=("server", "remote_id"),
+                        name="unique_remote_realm_installation_count_server_id_remote_id",
+                    ),
+                    models.UniqueConstraint(
+                        condition=models.Q(("subgroup__isnull", False)),
+                        fields=("server", "realm_id", "property", "subgroup", "end_time"),
+                        name="unique_server_realm_installation_count",
+                    ),
+                    models.UniqueConstraint(
+                        condition=models.Q(("subgroup__isnull", True)),
+                        fields=("server", "realm_id", "property", "end_time"),
+                        name="unique_server_realm_installation_count_null_subgroup",
+                    ),
+                    models.UniqueConstraint(
+                        condition=models.Q(("subgroup__isnull", False)),
+                        fields=("remote_realm_id", "property", "subgroup", "end_time"),
+                        name="unique_remote_realm_installation_count",
+                    ),
+                    models.UniqueConstraint(
+                        condition=models.Q(("subgroup__isnull", True)),
+                        fields=("remote_realm_id", "property", "end_time"),
+                        name="unique_remote_realm_installation_count_null_subgroup",
+                    ),
+                ],
+            },
+        ),
+        migrations.CreateModel(
+            name="PreregistrationRemoteRealmBillingUser",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                ("user_uuid", models.UUIDField()),
+                ("email", models.EmailField(max_length=254)),
+                ("status", models.IntegerField(default=0)),
+                ("next_page", models.TextField(null=True)),
+                ("uri_scheme", models.TextField()),
+                (
+                    "remote_realm",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="zilencer.remoterealm"
+                    ),
+                ),
+                (
+                    "created_user",
+                    models.ForeignKey(
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        to="zilencer.remoterealmbillinguser",
+                    ),
+                ),
+                ("date_joined", models.DateTimeField(default=django.utils.timezone.now)),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+        migrations.CreateModel(
+            name="PreregistrationRemoteServerBillingUser",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                ("email", models.EmailField(max_length=254)),
+                ("status", models.IntegerField(default=0)),
+                ("next_page", models.TextField(null=True)),
+                (
+                    "remote_server",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="zilencer.remotezulipserver"
+                    ),
+                ),
+                (
+                    "created_user",
+                    models.ForeignKey(
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        to="zilencer.remoteserverbillinguser",
+                    ),
+                ),
+                ("date_joined", models.DateTimeField(default=django.utils.timezone.now)),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+        migrations.CreateModel(
+            name="RemotePushDeviceToken",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                ("user_id", models.BigIntegerField(null=True)),
+                ("kind", models.PositiveSmallIntegerField(choices=[(1, "apns"), (2, "gcm")])),
+                ("token", models.CharField(db_index=True, max_length=4096)),
+                ("last_updated", models.DateTimeField(auto_now=True)),
+                ("ios_app_id", models.TextField(null=True)),
+                (
+                    "server",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="zilencer.remotezulipserver"
+                    ),
+                ),
+                ("user_uuid", models.UUIDField(null=True)),
+                (
+                    "remote_realm",
+                    models.ForeignKey(
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        to="zilencer.remoterealm",
+                    ),
+                ),
+            ],
+            options={
+                "unique_together": {
+                    ("server", "user_id", "kind", "token"),
+                    ("server", "user_uuid", "kind", "token"),
+                },
+            },
+        ),
+        migrations.CreateModel(
+            name="RemoteRealmAuditLog",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                ("realm_id", models.IntegerField(blank=True, null=True)),
+                ("remote_id", models.IntegerField(null=True)),
+                ("event_time", models.DateTimeField(db_index=True)),
+                ("backfilled", models.BooleanField(default=False)),
+                ("event_type", models.PositiveSmallIntegerField()),
+                (
+                    "server",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="zilencer.remotezulipserver"
+                    ),
+                ),
+                (
+                    "extra_data",
+                    models.JSONField(
+                        default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder
+                    ),
+                ),
+                (
+                    "remote_realm",
+                    models.ForeignKey(
+                        null=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="zilencer.remoterealm",
+                    ),
+                ),
+                (
+                    "acting_remote_user",
+                    models.ForeignKey(
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        to="zilencer.remoterealmbillinguser",
+                    ),
+                ),
+                (
+                    "acting_support_user",
+                    models.ForeignKey(
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+            options={
+                "constraints": [
+                    models.UniqueConstraint(
+                        fields=("server", "remote_id"),
+                        name="zilencer_remoterealmauditlog_server_remote",
+                    )
+                ],
+                "indexes": [
+                    models.Index(
+                        fields=["server", "realm_id", "remote_id"],
+                        name="zilencer_remoterealmauditlog_server_realm_remote",
+                    ),
+                    models.Index(
+                        condition=models.Q(("remote_realm__isnull", True)),
+                        fields=["server", "realm_id"],
+                        name="zilencer_remoterealmauditlog_server_realm",
+                    ),
+                    models.Index(
+                        condition=models.Q(("remote_realm__isnull", True)),
+                        fields=["server"],
+                        name="zilencer_remoterealmauditlog_server",
+                    ),
+                    models.Index(
+                        condition=models.Q(
+                            ("event_type__in", [101, 102, 103, 104, 105, 201, 202, 229])
+                        ),
+                        fields=["remote_realm_id", "id"],
+                        name="zilencer_remoterealmauditlog_synced_billing_events",
+                    ),
+                ],
+            },
+        ),
+        migrations.CreateModel(
+            name="RemoteZulipServerAuditLog",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                ("event_time", models.DateTimeField(db_index=True)),
+                ("backfilled", models.BooleanField(default=False)),
+                ("event_type", models.PositiveSmallIntegerField()),
+                (
+                    "server",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="zilencer.remotezulipserver"
+                    ),
+                ),
+                (
+                    "extra_data",
+                    models.JSONField(
+                        default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder
+                    ),
+                ),
+                (
+                    "acting_remote_user",
+                    models.ForeignKey(
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        to="zilencer.remoteserverbillinguser",
+                    ),
+                ),
+                (
+                    "acting_support_user",
+                    models.ForeignKey(
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+    ]

zilencer/migrations/0002_remote_zulip_server.py

@@ -0,0 +1,50 @@
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="RemotePushDeviceToken",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        serialize=False, auto_created=True, verbose_name="ID", primary_key=True
+                    ),
+                ),
+                ("user_id", models.BigIntegerField()),
+                ("kind", models.PositiveSmallIntegerField(choices=[(1, "apns"), (2, "gcm")])),
+                ("token", models.CharField(unique=True, max_length=4096)),
+                ("last_updated", models.DateTimeField(auto_now=True)),
+                ("ios_app_id", models.TextField(null=True)),
+            ],
+        ),
+        migrations.CreateModel(
+            name="RemoteZulipServer",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        serialize=False, auto_created=True, verbose_name="ID", primary_key=True
+                    ),
+                ),
+                ("uuid", models.CharField(unique=True, max_length=36)),
+                ("api_key", models.CharField(max_length=64)),
+                ("hostname", models.CharField(unique=True, max_length=128)),
+                ("contact_email", models.EmailField(max_length=254, blank=True)),
+                ("last_updated", models.DateTimeField(verbose_name="last updated")),
+            ],
+        ),
+        migrations.AddField(
+            model_name="remotepushdevicetoken",
+            name="server",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.CASCADE, to="zilencer.RemoteZulipServer"
+            ),
+        ),
+    ]

zilencer/migrations/0003_add_default_for_remotezulipserver_last_updated_field.py

@@ -0,0 +1,16 @@
+# Generated by Django 1.10.5 on 2017-05-16 00:03
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0002_remote_zulip_server"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="remotezulipserver",
+            name="last_updated",
+            field=models.DateTimeField(auto_now=True, verbose_name="last updated"),
+        ),
+    ]

zilencer/migrations/0004_remove_deployment_model.py

@@ -0,0 +1,18 @@
+# Generated by Django 1.11.5 on 2017-10-12 06:27
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0003_add_default_for_remotezulipserver_last_updated_field"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="deployment",
+            name="realms",
+        ),
+        migrations.DeleteModel(
+            name="Deployment",
+        ),
+    ]

zilencer/migrations/0005_remotepushdevicetoken_fix_uniqueness.py

@@ -0,0 +1,26 @@
+# Generated by Django 1.11.5 on 2017-10-19 04:23
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0004_remove_deployment_model"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="remotepushdevicetoken",
+            name="token",
+            field=models.CharField(db_index=True, max_length=4096),
+        ),
+        migrations.AlterField(
+            model_name="remotepushdevicetoken",
+            name="user_id",
+            field=models.BigIntegerField(db_index=True),
+        ),
+        migrations.AlterUniqueTogether(
+            name="remotepushdevicetoken",
+            unique_together={("server", "token")},
+        ),
+    ]

zilencer/migrations/0006_customer.py

@@ -0,0 +1,32 @@
+# Generated by Django 1.11.6 on 2018-01-13 11:54
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0005_remotepushdevicetoken_fix_uniqueness"),
+        ("zerver", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Customer",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                ("stripe_customer_id", models.CharField(max_length=255, unique=True)),
+                (
+                    "realm",
+                    models.OneToOneField(
+                        on_delete=django.db.models.deletion.CASCADE, to="zerver.Realm"
+                    ),
+                ),
+            ],
+        ),
+    ]

zilencer/migrations/0007_remotezulipserver_fix_uniqueness.py

@@ -0,0 +1,17 @@
+# Generated by Django 1.11.11 on 2018-04-30 06:55
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0006_customer"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="remotezulipserver",
+            name="hostname",
+            field=models.CharField(max_length=128),
+        ),
+    ]

zilencer/migrations/0008_customer_billing_user.py

@@ -0,0 +1,22 @@
+# Generated by Django 1.11.11 on 2018-04-12 01:14
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ("zilencer", "0007_remotezulipserver_fix_uniqueness"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="customer",
+            name="billing_user",
+            field=models.ForeignKey(
+                null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL
+            ),
+        ),
+    ]

zilencer/migrations/0009_plan.py

@@ -0,0 +1,25 @@
+# Generated by Django 1.11.11 on 2018-04-12 01:19
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0008_customer_billing_user"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Plan",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                ("nickname", models.CharField(max_length=40, unique=True)),
+                ("stripe_plan_id", models.CharField(max_length=255, unique=True)),
+            ],
+        ),
+    ]

zilencer/migrations/0010_billingprocessor.py

@@ -0,0 +1,39 @@
+# Generated by Django 1.11.14 on 2018-08-13 23:15
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zerver", "0182_set_initial_value_is_private_flag"),
+        ("zilencer", "0009_plan"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="BillingProcessor",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                ("state", models.CharField(max_length=20)),
+                ("last_modified", models.DateTimeField(auto_now=True)),
+                (
+                    "log_row",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="zerver.RealmAuditLog"
+                    ),
+                ),
+                (
+                    "realm",
+                    models.OneToOneField(
+                        null=True, on_delete=django.db.models.deletion.CASCADE, to="zerver.Realm"
+                    ),
+                ),
+            ],
+        ),
+    ]

zilencer/migrations/0011_customer_has_billing_relationship.py

@@ -0,0 +1,17 @@
+# Generated by Django 1.11.14 on 2018-08-14 01:32
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0010_billingprocessor"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="customer",
+            name="has_billing_relationship",
+            field=models.BooleanField(default=False),
+        ),
+    ]

zilencer/migrations/0012_coupon.py

@@ -0,0 +1,25 @@
+# Generated by Django 1.11.14 on 2018-08-23 05:40
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0011_customer_has_billing_relationship"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Coupon",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                ("percent_off", models.SmallIntegerField(unique=True)),
+                ("stripe_coupon_id", models.CharField(max_length=255, unique=True)),
+            ],
+        ),
+    ]

zilencer/migrations/0013_remove_customer_billing_user.py

@@ -0,0 +1,16 @@
+# Generated by Django 1.11.14 on 2018-08-22 06:31
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0012_coupon"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="customer",
+            name="billing_user",
+        ),
+    ]

zilencer/migrations/0014_cleanup_pushdevicetoken.py

@@ -0,0 +1,16 @@
+# Generated by Django 1.11.14 on 2018-10-10 22:52
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0013_remove_customer_billing_user"),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name="remotepushdevicetoken",
+            unique_together={("server", "user_id", "kind", "token")},
+        ),
+    ]

zilencer/migrations/0015_delete_billing.py

@@ -0,0 +1,36 @@
+# Generated by Django 1.11.14 on 2018-09-25 12:01
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0014_cleanup_pushdevicetoken"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="billingprocessor",
+            name="log_row",
+        ),
+        migrations.RemoveField(
+            model_name="billingprocessor",
+            name="realm",
+        ),
+        migrations.DeleteModel(
+            name="Coupon",
+        ),
+        migrations.RemoveField(
+            model_name="customer",
+            name="realm",
+        ),
+        migrations.DeleteModel(
+            name="Plan",
+        ),
+        migrations.DeleteModel(
+            name="BillingProcessor",
+        ),
+        migrations.DeleteModel(
+            name="Customer",
+        ),
+    ]

zilencer/migrations/0016_remote_counts.py

@@ -0,0 +1,73 @@
+# Generated by Django 1.11.18 on 2019-02-02 06:02
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0015_delete_billing"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="RemoteInstallationCount",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                ("property", models.CharField(max_length=32)),
+                ("subgroup", models.CharField(max_length=16, null=True)),
+                ("end_time", models.DateTimeField()),
+                ("value", models.BigIntegerField()),
+                ("remote_id", models.IntegerField(db_index=True)),
+                (
+                    "server",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="zilencer.RemoteZulipServer"
+                    ),
+                ),
+            ],
+        ),
+        migrations.CreateModel(
+            name="RemoteRealmCount",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                ("property", models.CharField(max_length=32)),
+                ("subgroup", models.CharField(max_length=16, null=True)),
+                ("end_time", models.DateTimeField()),
+                ("value", models.BigIntegerField()),
+                ("realm_id", models.IntegerField(db_index=True)),
+                ("remote_id", models.IntegerField(db_index=True)),
+                (
+                    "server",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="zilencer.RemoteZulipServer"
+                    ),
+                ),
+            ],
+        ),
+        migrations.AlterUniqueTogether(
+            name="remoterealmcount",
+            unique_together={("server", "realm_id", "property", "subgroup", "end_time")},
+        ),
+        migrations.AddIndex(
+            model_name="remoterealmcount",
+            index=models.Index(
+                fields=["property", "end_time"],
+                name="zilencer_remoterealmcount_property_end_time_506a0b38_idx",
+            ),
+        ),
+        migrations.AlterUniqueTogether(
+            name="remoteinstallationcount",
+            unique_together={("server", "property", "subgroup", "end_time")},
+        ),
+    ]

zilencer/migrations/0017_installationcount_indexes.py

@@ -0,0 +1,26 @@
+# Generated by Django 1.11.20 on 2019-04-23 20:17
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0016_remote_counts"),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="remoteinstallationcount",
+            index=models.Index(
+                fields=["server", "remote_id"],
+                name="zilencer_remoteinstallat_server_id_remote_id_f72e4c30_idx",
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="remoterealmcount",
+            index=models.Index(
+                fields=["server", "remote_id"],
+                name="zilencer_remoterealmcount_server_id_remote_id_de1573d8_idx",
+            ),
+        ),
+    ]

zilencer/migrations/0018_remoterealmauditlog.py

@@ -0,0 +1,36 @@
+# Generated by Django 1.11.24 on 2019-10-03 00:10
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0017_installationcount_indexes"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="RemoteRealmAuditLog",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                ("realm_id", models.IntegerField(db_index=True)),
+                ("remote_id", models.IntegerField(db_index=True)),
+                ("event_time", models.DateTimeField(db_index=True)),
+                ("backfilled", models.BooleanField(default=False)),
+                ("extra_data", models.TextField(null=True)),
+                ("event_type", models.PositiveSmallIntegerField()),
+                (
+                    "server",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="zilencer.RemoteZulipServer"
+                    ),
+                ),
+            ],
+        ),
+    ]

zilencer/migrations/0019_remotezulipserver_plan_type.py

@@ -0,0 +1,17 @@
+# Generated by Django 3.2.9 on 2021-12-01 16:24
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0018_remoterealmauditlog"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="remotezulipserver",
+            name="plan_type",
+            field=models.PositiveSmallIntegerField(default=1),
+        ),
+    ]

zilencer/migrations/0020_remotezulipserverauditlog.py

@@ -0,0 +1,37 @@
+# Generated by Django 3.2.9 on 2021-12-06 18:35
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0019_remotezulipserver_plan_type"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="RemoteZulipServerAuditLog",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                ("event_time", models.DateTimeField(db_index=True)),
+                ("backfilled", models.BooleanField(default=False)),
+                ("extra_data", models.TextField(null=True)),
+                ("event_type", models.PositiveSmallIntegerField()),
+                (
+                    "server",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="zilencer.remotezulipserver"
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+    ]

zilencer/migrations/0021_alter_remotezulipserver_uuid.py

@@ -0,0 +1,17 @@
+# Generated by Django 3.2.9 on 2021-12-22 10:06
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0020_remotezulipserverauditlog"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="remotezulipserver",
+            name="uuid",
+            field=models.UUIDField(unique=True),
+        ),
+    ]

zilencer/migrations/0022_remotezulipserver_create_audit_log_backfill.py

@@ -0,0 +1,44 @@
+from django.db import migrations
+from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+from django.db.migrations.state import StateApps
+
+
+def backfill_remote_zulip_server_creation_log_events(
+    apps: StateApps, schema_editor: BaseDatabaseSchemaEditor
+) -> None:
+    RemoteZulipServer = apps.get_model("zilencer", "RemoteZulipServer")
+    RemoteZulipServerAuditLog = apps.get_model("zilencer", "RemoteZulipServerAuditLog")
+    RemoteZulipServerAuditLog.REMOTE_SERVER_CREATED = 10215
+
+    objects_to_create = []
+    for remote_server in RemoteZulipServer.objects.all().iterator():
+        entry = RemoteZulipServerAuditLog(
+            server=remote_server,
+            event_type=RemoteZulipServerAuditLog.REMOTE_SERVER_CREATED,
+            event_time=remote_server.last_updated,
+            backfilled=True,
+        )
+        objects_to_create.append(entry)
+    RemoteZulipServerAuditLog.objects.bulk_create(objects_to_create)
+
+
+def reverse_code(apps: StateApps, schema_editor: BaseDatabaseSchemaEditor) -> None:
+    RemoteZulipServerAuditLog = apps.get_model("zilencer", "RemoteZulipServerAuditLog")
+    RemoteZulipServerAuditLog.REMOTE_SERVER_CREATED = 10215
+    RemoteZulipServerAuditLog.objects.filter(
+        event_type=RemoteZulipServerAuditLog.REMOTE_SERVER_CREATED, backfilled=True
+    ).delete()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0021_alter_remotezulipserver_uuid"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            backfill_remote_zulip_server_creation_log_events,
+            reverse_code=reverse_code,
+            elidable=True,
+        )
+    ]

zilencer/migrations/0023_remotezulipserver_deactivated.py

@@ -0,0 +1,17 @@
+# Generated by Django 3.2.9 on 2021-12-15 17:26
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0022_remotezulipserver_create_audit_log_backfill"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="remotezulipserver",
+            name="deactivated",
+            field=models.BooleanField(default=False),
+        ),
+    ]

zilencer/migrations/0024_remotepushdevicetoken_user_uuid.py

@@ -0,0 +1,29 @@
+# Generated by Django 3.2.9 on 2021-12-27 21:10
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0023_remotezulipserver_deactivated"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="remotepushdevicetoken",
+            name="user_id",
+            field=models.BigIntegerField(db_index=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="remotepushdevicetoken",
+            name="user_uuid",
+            field=models.UUIDField(null=True),
+        ),
+        migrations.AlterUniqueTogether(
+            name="remotepushdevicetoken",
+            unique_together={
+                ("server", "user_uuid", "kind", "token"),
+                ("server", "user_id", "kind", "token"),
+            },
+        ),
+    ]

zilencer/migrations/0025_alter_remotepushdevicetoken_user_id_drop_index.py

@@ -0,0 +1,17 @@
+# Generated by Django 3.2.12 on 2022-03-10 12:34
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0024_remotepushdevicetoken_user_uuid"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="remotepushdevicetoken",
+            name="user_id",
+            field=models.BigIntegerField(null=True),
+        ),
+    ]

zilencer/migrations/0026_auditlog_models_extra_data_json.py

@@ -0,0 +1,27 @@
+# Generated by Django 4.0.7 on 2022-09-30 20:25
+
+import django
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0025_alter_remotepushdevicetoken_user_id_drop_index"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="remoterealmauditlog",
+            name="extra_data_json",
+            field=models.JSONField(
+                default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder
+            ),
+        ),
+        migrations.AddField(
+            model_name="remotezulipserverauditlog",
+            name="extra_data_json",
+            field=models.JSONField(
+                default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder
+            ),
+        ),
+    ]

zilencer/migrations/0027_backfill_remote_realmauditlog_extradata_to_json_field.py

@@ -0,0 +1,152 @@
+# Generated by Django 4.0.7 on 2022-09-30 20:30
+
+import ast
+from collections.abc import Callable
+
+import orjson
+from django.db import migrations, transaction
+from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+from django.db.migrations.state import StateApps
+from django.db.models import F, JSONField, Model
+from django.db.models.functions import Cast
+
+# This migration is mostly the same as
+# backfill_realmauditlog_extradata_to_json_field in zerver.
+
+OLD_VALUE = "1"
+NEW_VALUE = "2"
+USER_FULL_NAME_CHANGED = 124
+REALM_DISCOUNT_CHANGED = 209
+BATCH_SIZE = 5000
+
+OVERWRITE_TEMPLATE = """Audit log entry with id {id} has extra_data_json been inconsistently overwritten.
+  The old value is:
+{old_value}
+  The new value is:
+{new_value}
+"""
+
+
+@transaction.atomic
+def do_bulk_backfill_extra_data(
+    audit_log_model: type[Model], id_lower_bound: int, id_upper_bound: int
+) -> None:
+    inconsistent_extra_data_json: list[tuple[int, str, object, object]] = []
+    # A dict converted with str() will start with a open bracket followed by a
+    # single quote, as opposed to a JSON-encoded value, which will use a
+    # _double_ quote. We use this to filter out those entries with malformed
+    # extra_data to be handled later. This should only update rows with
+    # extra_data populated with orjson.dumps.
+
+    # The first query below checks for entries that would have extra_data_json
+    # being overwritten by the migration with a value inconsistent with its
+    # previous value.
+    inconsistent_extra_data_json.extend(
+        audit_log_model._default_manager.filter(
+            extra_data__isnull=False, id__range=(id_lower_bound, id_upper_bound)
+        )
+        .annotate(new_extra_data_json=Cast("extra_data", output_field=JSONField()))
+        .exclude(extra_data__startswith="{'")
+        .exclude(extra_data_json={})
+        .exclude(extra_data_json=F("new_extra_data_json"))
+        .values_list("id", "extra_data", "extra_data_json", "new_extra_data_json")
+    )
+    (
+        audit_log_model._default_manager.filter(
+            extra_data__isnull=False,
+            id__range=(id_lower_bound, id_upper_bound),
+            extra_data_json__inconsistent_old_extra_data__isnull=True,
+        )
+        .exclude(extra_data__startswith="{'")
+        .update(extra_data_json=Cast("extra_data", output_field=JSONField()))
+    )
+
+    python_valued_audit_log_entries = audit_log_model._default_manager.filter(
+        extra_data__startswith="{'",
+        id__range=(id_lower_bound, id_upper_bound),
+        extra_data_json__inconsistent_old_extra_data__isnull=True,
+    )
+    for audit_log_entry in python_valued_audit_log_entries:
+        # extra_data for entries that store dict stringified with builtins.str()
+        # are converted back with ast.literal_eval for safety and efficiency.
+        old_value = audit_log_entry.extra_data_json  # type: ignore[attr-defined] # The migration cannot depend on zerver.models, which contains the real type of the RealmAuditLog model, so it cannot be properly typed.
+        new_value = ast.literal_eval(audit_log_entry.extra_data)  # type: ignore[attr-defined] # Explained above.
+        if old_value not in ({}, new_value):
+            inconsistent_extra_data_json.append(
+                (audit_log_entry.id, audit_log_entry.extra_data, old_value, new_value)  # type: ignore[attr-defined] # Explained above.
+            )
+        audit_log_entry.extra_data_json = new_value  # type: ignore[attr-defined] # Explained above.
+    audit_log_model._default_manager.bulk_update(
+        python_valued_audit_log_entries, fields=["extra_data_json"]
+    )
+
+    if inconsistent_extra_data_json:
+        audit_log_entries = []
+        for (
+            audit_log_entry_id,
+            old_extra_data,
+            old_extra_data_json,
+            new_extra_data_json,
+        ) in inconsistent_extra_data_json:
+            audit_log_entry = audit_log_model._default_manager.get(id=audit_log_entry_id)
+            assert isinstance(old_extra_data_json, dict)
+            if "inconsistent_old_extra_data" in old_extra_data_json:
+                # Skip entries that have been backfilled and detected as
+                # anomalies before.
+                continue
+            assert isinstance(new_extra_data_json, dict)
+            audit_log_entry.extra_data_json = {  # type: ignore[attr-defined] # Explained above.
+                **new_extra_data_json,
+                "inconsistent_old_extra_data": old_extra_data,
+                "inconsistent_old_extra_data_json": old_extra_data_json,
+            }
+            audit_log_entries.append(audit_log_entry)
+            print(
+                OVERWRITE_TEMPLATE.format(
+                    id=audit_log_entry_id,
+                    old_value=orjson.dumps(old_extra_data_json).decode(),
+                    new_value=orjson.dumps(new_extra_data_json).decode(),
+                )
+            )
+        audit_log_model._default_manager.bulk_update(audit_log_entries, fields=["extra_data_json"])
+
+
+def backfill_extra_data(model_name: str) -> Callable[[StateApps, BaseDatabaseSchemaEditor], None]:
+    def inner(apps: StateApps, schema_editor: BaseDatabaseSchemaEditor) -> None:
+        audit_log_model = apps.get_model("zilencer", model_name)
+        if not audit_log_model.objects.filter(extra_data__isnull=False).exists():
+            return
+
+        audit_log_entries = audit_log_model.objects.filter(extra_data__isnull=False)
+        id_lower_bound = audit_log_entries.earliest("id").id
+        id_upper_bound = audit_log_entries.latest("id").id
+        while id_lower_bound <= id_upper_bound:
+            do_bulk_backfill_extra_data(
+                audit_log_model, id_lower_bound, min(id_lower_bound + BATCH_SIZE, id_upper_bound)
+            )
+            id_lower_bound += BATCH_SIZE + 1
+
+        do_bulk_backfill_extra_data(audit_log_model, id_lower_bound, id_upper_bound)
+
+    return inner
+
+
+class Migration(migrations.Migration):
+    atomic = False
+
+    dependencies = [
+        ("zilencer", "0026_auditlog_models_extra_data_json"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            backfill_extra_data("RemoteRealmAuditLog"),
+            reverse_code=migrations.RunPython.noop,
+            elidable=True,
+        ),
+        migrations.RunPython(
+            backfill_extra_data("RemoteZulipServerAuditLog"),
+            reverse_code=migrations.RunPython.noop,
+            elidable=True,
+        ),
+    ]

zilencer/migrations/0028_rename_extradatajson_remoteauditlog_extra_data.py

@@ -0,0 +1,30 @@
+# Generated by Django 4.0.7 on 2022-10-02 17:02
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0027_backfill_remote_realmauditlog_extradata_to_json_field"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="remoterealmauditlog",
+            name="extra_data",
+        ),
+        migrations.RemoveField(
+            model_name="remotezulipserverauditlog",
+            name="extra_data",
+        ),
+        migrations.RenameField(
+            model_name="remoterealmauditlog",
+            old_name="extra_data_json",
+            new_name="extra_data",
+        ),
+        migrations.RenameField(
+            model_name="remotezulipserverauditlog",
+            old_name="extra_data_json",
+            new_name="extra_data",
+        ),
+    ]

zilencer/migrations/0029_update_remoterealm_indexes.py

@@ -0,0 +1,45 @@
+# Generated by Django 4.2.5 on 2023-09-13 05:02
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0028_rename_extradatajson_remoteauditlog_extra_data"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="remoterealmauditlog",
+            name="realm_id",
+            field=models.IntegerField(),
+        ),
+        migrations.AlterField(
+            model_name="remoterealmauditlog",
+            name="remote_id",
+            field=models.IntegerField(),
+        ),
+        migrations.AlterField(
+            model_name="remoterealmcount",
+            name="realm_id",
+            field=models.IntegerField(),
+        ),
+        migrations.AlterField(
+            model_name="remoterealmcount",
+            name="remote_id",
+            field=models.IntegerField(),
+        ),
+        migrations.AddConstraint(
+            model_name="remoterealmauditlog",
+            constraint=models.UniqueConstraint(
+                fields=("server", "remote_id"), name="zilencer_remoterealmauditlog_server_remote"
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="remoterealmauditlog",
+            index=models.Index(
+                fields=["server", "realm_id", "remote_id"],
+                name="zilencer_remoterealmauditlog_server_realm_remote",
+            ),
+        ),
+    ]

zilencer/migrations/0030_alter_remoteinstallationcount_remote_id.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.6 on 2023-10-20 00:10
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0029_update_remoterealm_indexes"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="remoteinstallationcount",
+            name="remote_id",
+            field=models.IntegerField(),
+        ),
+    ]

zilencer/migrations/0031_alter_remoteinstallationcount_remote_id_and_more.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.2.6 on 2023-10-23 20:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0030_alter_remoteinstallationcount_remote_id"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="remoteinstallationcount",
+            name="remote_id",
+            field=models.IntegerField(null=True),
+        ),
+        migrations.AlterField(
+            model_name="remoterealmcount",
+            name="remote_id",
+            field=models.IntegerField(null=True),
+        ),
+    ]

zilencer/migrations/0032_remotepushdevicetoken_backfill_ios_app_id.py

@@ -0,0 +1,37 @@
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    """
+    Previous versions of zilencer dropped the ios_app_id on the floor
+    when registering a push token, and then effectively assumed
+    the value was "org.zulip.Zulip".
+
+    We're going to start actually using that parameter.  To preserve
+    the existing behavior for existing tokens, backfill the value.
+    """
+
+    dependencies = [
+        ("zilencer", "0031_alter_remoteinstallationcount_remote_id_and_more"),
+    ]
+
+    FORMERLY_IMPLICIT_IOS_APP_ID = "org.zulip.Zulip"
+
+    operations = [
+        migrations.RunSQL(
+            sql=[
+                (
+                    """
+            UPDATE zilencer_remotepushdevicetoken
+            SET ios_app_id = %s
+            WHERE kind = 1 AND ios_app_id IS NULL
+            """,
+                    [FORMERLY_IMPLICIT_IOS_APP_ID],
+                )
+            ],
+            # The updated table is still valid with the old schema;
+            # so to reverse, a no-op suffices.
+            reverse_sql=[],
+            elidable=True,
+        ),
+    ]

zilencer/migrations/0033_remoterealm.py

@@ -0,0 +1,38 @@
+# Generated by Django 4.2.6 on 2023-11-02 23:22
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0032_remotepushdevicetoken_backfill_ios_app_id"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="RemoteRealm",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                ("uuid", models.UUIDField(unique=True)),
+                ("uuid_owner_secret", models.TextField()),
+                ("host", models.TextField()),
+                ("last_updated", models.DateTimeField(auto_now=True, verbose_name="last updated")),
+                ("registration_deactivated", models.BooleanField(default=False)),
+                ("realm_deactivated", models.BooleanField(default=False)),
+                ("realm_date_created", models.DateTimeField()),
+                ("plan_type", models.PositiveSmallIntegerField(db_index=True, default=1)),
+                (
+                    "server",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="zilencer.remotezulipserver"
+                    ),
+                ),
+            ],
+        ),
+    ]

zilencer/migrations/0034_remoterealmauditlog_remote_realm_and_more.py

@@ -0,0 +1,25 @@
+# Generated by Django 4.2.6 on 2023-11-08 23:17
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0033_remoterealm"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="remoterealmauditlog",
+            name="remote_realm",
+            field=models.ForeignKey(
+                null=True, on_delete=django.db.models.deletion.CASCADE, to="zilencer.remoterealm"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="remoterealmauditlog",
+            name="remote_id",
+            field=models.IntegerField(null=True),
+        ),
+    ]

zilencer/migrations/0035_remoterealmcount_remote_realm_and_more.py

@@ -0,0 +1,25 @@
+# Generated by Django 4.2.6 on 2023-11-09 17:23
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0034_remoterealmauditlog_remote_realm_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="remoterealmcount",
+            name="remote_realm",
+            field=models.ForeignKey(
+                null=True, on_delete=django.db.models.deletion.CASCADE, to="zilencer.remoterealm"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="remoterealmcount",
+            name="realm_id",
+            field=models.IntegerField(null=True),
+        ),
+    ]

zilencer/migrations/0036_remotezulipserver_last_version.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.7 on 2023-11-17 20:59
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0035_remoterealmcount_remote_realm_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="remotezulipserver",
+            name="last_version",
+            field=models.CharField(max_length=128, null=True),
+        ),
+    ]

zilencer/migrations/0037_alter_remoteinstallationcount_unique_together_and_more.py

@@ -0,0 +1,95 @@
+from django.db import migrations, models
+from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+from django.db.migrations.state import StateApps
+from django.db.models import Count, Min
+
+
+def clear_duplicate_counts(apps: StateApps, schema_editor: BaseDatabaseSchemaEditor) -> None:
+    """Clean up duplicated RemoteRealmCount and RemoteInstallationCount rows.
+
+    This is the equivalent of analytics' 0015_clear_duplicate_counts
+    migration -- but it also has additional duplicates if there are
+    multiple servers submitting information with the same UUID.
+
+    We drop the behaviour of rolling up and updating the value to the
+    sum, since the active_users_log:is_bot:day field has a subgroup
+    (and is thus not affected by the bug), and the few cases for
+    `invites_sent::day` seem more likely to be re-submissions of the
+    same data, not duplicates to roll up.
+
+    We must do this step before switching the non-unique indexes to be
+    unique, as there are currently violations.
+
+    """
+    count_tables = dict(
+        realm=apps.get_model("zilencer", "RemoteRealmCount"),
+        installation=apps.get_model("zilencer", "RemoteInstallationCount"),
+    )
+
+    for name, count_table in count_tables.items():
+        value = ["realm_id", "server_id", "property", "end_time"]
+        if name == "installation":
+            value = ["server_id", "property", "end_time"]
+        duplicated_rows = (
+            count_table.objects.filter(subgroup=None)
+            .values(*value)
+            .annotate(Count("id"), Min("id"))
+            .filter(id__count__gt=1)
+        )
+
+        for duplicated_row in duplicated_rows:
+            duplicated_row.pop("id__count")
+            first_id = duplicated_row.pop("id__min")
+            count_table.objects.filter(**duplicated_row, id__gt=first_id).delete()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0036_remotezulipserver_last_version"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            clear_duplicate_counts, reverse_code=migrations.RunPython.noop, elidable=True
+        ),
+        migrations.AddConstraint(
+            model_name="remoteinstallationcount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(("subgroup__isnull", False)),
+                fields=("server", "property", "subgroup", "end_time"),
+                name="unique_remote_installation_count",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="remoteinstallationcount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(("subgroup__isnull", True)),
+                fields=("server", "property", "end_time"),
+                name="unique_remote_installation_count_null_subgroup",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="remoterealmcount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(("subgroup__isnull", False)),
+                fields=("server", "realm_id", "property", "subgroup", "end_time"),
+                name="unique_remote_realm_installation_count",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="remoterealmcount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(("subgroup__isnull", True)),
+                fields=("server", "realm_id", "property", "end_time"),
+                name="unique_remote_realm_installation_count_null_subgroup",
+            ),
+        ),
+        migrations.AlterUniqueTogether(
+            name="remoteinstallationcount",
+            unique_together=set(),
+        ),
+        migrations.AlterUniqueTogether(
+            name="remoterealmcount",
+            unique_together=set(),
+        ),
+    ]

zilencer/migrations/0038_unique_server_remote_id.py

@@ -0,0 +1,34 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0037_alter_remoteinstallationcount_unique_together_and_more"),
+    ]
+
+    operations = [
+        migrations.AddConstraint(
+            model_name="remoteinstallationcount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(("remote_id__isnull", False)),
+                fields=("server", "remote_id"),
+                name="unique_remote_installation_count_server_id_remote_id",
+            ),
+        ),
+        migrations.RemoveIndex(
+            model_name="remoteinstallationcount",
+            name="zilencer_remoteinstallat_server_id_remote_id_f72e4c30_idx",
+        ),
+        migrations.AddConstraint(
+            model_name="remoterealmcount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(("remote_id__isnull", False)),
+                fields=("server", "remote_id"),
+                name="unique_remote_realm_installation_count_server_id_remote_id",
+            ),
+        ),
+        migrations.RemoveIndex(
+            model_name="remoterealmcount",
+            name="zilencer_remoterealmcount_server_id_remote_id_de1573d8_idx",
+        ),
+    ]

zilencer/migrations/0039_remoterealm_org_type.py

@@ -0,0 +1,34 @@
+# Generated by Django 4.2.7 on 2023-11-27 00:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0038_unique_server_remote_id"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="remoterealm",
+            name="org_type",
+            field=models.PositiveSmallIntegerField(
+                choices=[
+                    (0, "Unspecified"),
+                    (10, "Business"),
+                    (20, "Open-source project"),
+                    (30, "Education (non-profit)"),
+                    (35, "Education (for-profit)"),
+                    (40, "Research"),
+                    (50, "Event or conference"),
+                    (60, "Non-profit (registered)"),
+                    (70, "Government"),
+                    (80, "Political group"),
+                    (90, "Community"),
+                    (100, "Personal"),
+                    (1000, "Other"),
+                ],
+                default=0,
+            ),
+        ),
+    ]

zilencer/migrations/0040_remoterealm_authentication_methods_remoterealm_name.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.2.7 on 2023-11-29 22:43
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0039_remoterealm_org_type"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="remoterealm",
+            name="authentication_methods",
+            field=models.JSONField(default=dict),
+        ),
+        migrations.AddField(
+            model_name="remoterealm",
+            name="name",
+            field=models.TextField(default=""),
+        ),
+    ]

zilencer/migrations/0041_remotezulipserver_org_type.py

@@ -0,0 +1,34 @@
+# Generated by Django 4.2.7 on 2023-11-29 05:16
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0040_remoterealm_authentication_methods_remoterealm_name"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="remotezulipserver",
+            name="org_type",
+            field=models.PositiveSmallIntegerField(
+                choices=[
+                    (0, "Unspecified"),
+                    (10, "Business"),
+                    (20, "Open-source project"),
+                    (30, "Education (non-profit)"),
+                    (35, "Education (for-profit)"),
+                    (40, "Research"),
+                    (50, "Event or conference"),
+                    (60, "Non-profit (registered)"),
+                    (70, "Government"),
+                    (80, "Political group"),
+                    (90, "Community"),
+                    (100, "Personal"),
+                    (1000, "Other"),
+                ],
+                default=0,
+            ),
+        ),
+    ]

zilencer/migrations/0042_alter_remoterealmauditlog_realm_id.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.7 on 2023-11-30 11:24
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0041_remotezulipserver_org_type"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="remoterealmauditlog",
+            name="realm_id",
+            field=models.IntegerField(blank=True, null=True),
+        ),
+    ]

zilencer/migrations/0043_remotepushdevicetoken_remote_realm.py

@@ -0,0 +1,20 @@
+# Generated by Django 4.2.7 on 2023-12-01 15:52
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0042_alter_remoterealmauditlog_realm_id"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="remotepushdevicetoken",
+            name="remote_realm",
+            field=models.ForeignKey(
+                null=True, on_delete=django.db.models.deletion.SET_NULL, to="zilencer.remoterealm"
+            ),
+        ),
+    ]

zilencer/migrations/0044_remoterealmbillinguser.py

@@ -0,0 +1,34 @@
+# Generated by Django 4.2.7 on 2023-12-05 01:15
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0043_remotepushdevicetoken_remote_realm"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="RemoteRealmBillingUser",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                ("user_uuid", models.UUIDField()),
+                ("full_name", models.TextField(default="")),
+                ("email", models.EmailField(max_length=254)),
+                ("tos_version", models.TextField(default="-1")),
+                (
+                    "remote_realm",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="zilencer.remoterealm"
+                    ),
+                ),
+            ],
+        ),
+    ]

zilencer/migrations/0045_remoterealmauditlog_zilencer_remoterealmauditlog_server_realm_and_more.py

@@ -0,0 +1,47 @@
+# Generated by Django 4.2.7 on 2023-12-05 19:33
+
+from django.contrib.postgres.operations import AddIndexConcurrently
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    atomic = False
+
+    dependencies = [
+        ("zilencer", "0044_remoterealmbillinguser"),
+    ]
+
+    operations = [
+        AddIndexConcurrently(
+            model_name="remoterealmauditlog",
+            index=models.Index(
+                condition=models.Q(("remote_realm__isnull", True)),
+                fields=["server", "realm_id"],
+                name="zilencer_remoterealmauditlog_server_realm",
+            ),
+        ),
+        AddIndexConcurrently(
+            model_name="remoterealmauditlog",
+            index=models.Index(
+                condition=models.Q(("remote_realm__isnull", True)),
+                fields=["server"],
+                name="zilencer_remoterealmauditlog_server",
+            ),
+        ),
+        AddIndexConcurrently(
+            model_name="remoterealmcount",
+            index=models.Index(
+                condition=models.Q(("remote_realm__isnull", True)),
+                fields=["server", "realm_id"],
+                name="zilencer_remoterealmcount_server_realm",
+            ),
+        ),
+        AddIndexConcurrently(
+            model_name="remoterealmcount",
+            index=models.Index(
+                condition=models.Q(("remote_realm__isnull", True)),
+                fields=["server"],
+                name="zilencer_remoterealmcount_server",
+            ),
+        ),
+    ]

zilencer/migrations/0046_remotezulipserver_last_audit_log_update.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.7 on 2023-12-06 18:23
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0045_remoterealmauditlog_zilencer_remoterealmauditlog_server_realm_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="remotezulipserver",
+            name="last_audit_log_update",
+            field=models.DateTimeField(null=True),
+        ),
+    ]

zilencer/migrations/0047_preregistrationremoteserverbillinguser_and_more.py

@@ -0,0 +1,58 @@
+# Generated by Django 4.2.8 on 2023-12-08 19:13
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0046_remotezulipserver_last_audit_log_update"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="PreregistrationRemoteServerBillingUser",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                ("email", models.EmailField(max_length=254)),
+                ("status", models.IntegerField(default=0)),
+                ("next_page", models.TextField(null=True)),
+                (
+                    "remote_server",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="zilencer.remotezulipserver"
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+        migrations.CreateModel(
+            name="RemoteServerBillingUser",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                ("email", models.EmailField(max_length=254)),
+                ("full_name", models.TextField(default="")),
+                (
+                    "remote_server",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="zilencer.remotezulipserver"
+                    ),
+                ),
+            ],
+            options={
+                "unique_together": {("remote_server", "email")},
+            },
+        ),
+    ]

zilencer/migrations/0048_remotezulipserver_last_api_feature_level.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.8 on 2023-12-09 18:08
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0047_preregistrationremoteserverbillinguser_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="remotezulipserver",
+            name="last_api_feature_level",
+            field=models.PositiveIntegerField(null=True),
+        ),
+    ]

zilencer/migrations/0049_alter_remoterealmbillinguser_unique_together_and_more.py

@@ -0,0 +1,42 @@
+# Generated by Django 4.2.8 on 2023-12-10 01:01
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0048_remotezulipserver_last_api_feature_level"),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name="remoterealmbillinguser",
+            unique_together={("remote_realm", "user_uuid")},
+        ),
+        migrations.CreateModel(
+            name="PreregistrationRemoteRealmBillingUser",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                ("user_uuid", models.UUIDField()),
+                ("email", models.EmailField(max_length=254)),
+                ("status", models.IntegerField(default=0)),
+                ("next_page", models.TextField(null=True)),
+                ("uri_scheme", models.TextField()),
+                (
+                    "remote_realm",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="zilencer.remoterealm"
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+    ]

zilencer/migrations/0050_preregistrationremoterealmbillinguser_created_user_and_more.py

@@ -0,0 +1,72 @@
+# Generated by Django 4.2.8 on 2023-12-11 12:55
+
+import django.db.models.deletion
+import django.utils.timezone
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0049_alter_remoterealmbillinguser_unique_together_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="preregistrationremoterealmbillinguser",
+            name="created_user",
+            field=models.ForeignKey(
+                null=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                to="zilencer.remoterealmbillinguser",
+            ),
+        ),
+        migrations.AddField(
+            model_name="preregistrationremoterealmbillinguser",
+            name="date_joined",
+            field=models.DateTimeField(default=django.utils.timezone.now),
+        ),
+        migrations.AddField(
+            model_name="preregistrationremoteserverbillinguser",
+            name="created_user",
+            field=models.ForeignKey(
+                null=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                to="zilencer.remoteserverbillinguser",
+            ),
+        ),
+        migrations.AddField(
+            model_name="preregistrationremoteserverbillinguser",
+            name="date_joined",
+            field=models.DateTimeField(default=django.utils.timezone.now),
+        ),
+        migrations.AddField(
+            model_name="remoterealmbillinguser",
+            name="date_joined",
+            field=models.DateTimeField(default=django.utils.timezone.now),
+        ),
+        migrations.AddField(
+            model_name="remoterealmbillinguser",
+            name="is_active",
+            field=models.BooleanField(default=True),
+        ),
+        migrations.AddField(
+            model_name="remoterealmbillinguser",
+            name="last_login",
+            field=models.DateTimeField(null=True),
+        ),
+        migrations.AddField(
+            model_name="remoteserverbillinguser",
+            name="date_joined",
+            field=models.DateTimeField(default=django.utils.timezone.now),
+        ),
+        migrations.AddField(
+            model_name="remoteserverbillinguser",
+            name="is_active",
+            field=models.BooleanField(default=True),
+        ),
+        migrations.AddField(
+            model_name="remoteserverbillinguser",
+            name="last_login",
+            field=models.DateTimeField(null=True),
+        ),
+    ]

zilencer/migrations/0051_remoterealm_is_system_bot_realm.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.8 on 2023-12-11 20:38
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0050_preregistrationremoterealmbillinguser_created_user_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="remoterealm",
+            name="is_system_bot_realm",
+            field=models.BooleanField(default=False),
+        ),
+    ]

zilencer/migrations/0052_alter_remoterealm_plan_type_and_more.py

@@ -0,0 +1,41 @@
+# Generated by Django 4.2.8 on 2023-12-13 23:20
+
+from django.db import migrations, models
+from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+from django.db.migrations.state import StateApps
+
+PLAN_TYPE_SELF_HOSTED = 1
+PLAN_TYPE_SELF_MANAGED = 100
+
+
+def renumber_plan_types(apps: StateApps, schema_editor: BaseDatabaseSchemaEditor) -> None:
+    RemoteZulipServer = apps.get_model("zilencer", "RemoteZulipServer")
+    RemoteRealm = apps.get_model("zilencer", "RemoteRealm")
+    RemoteRealm.objects.filter(plan_type=PLAN_TYPE_SELF_HOSTED).update(
+        plan_type=PLAN_TYPE_SELF_MANAGED
+    )
+    RemoteZulipServer.objects.filter(plan_type=PLAN_TYPE_SELF_HOSTED).update(
+        plan_type=PLAN_TYPE_SELF_MANAGED
+    )
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0051_remoterealm_is_system_bot_realm"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="remoterealm",
+            name="plan_type",
+            field=models.PositiveSmallIntegerField(db_index=True, default=PLAN_TYPE_SELF_MANAGED),
+        ),
+        migrations.AlterField(
+            model_name="remotezulipserver",
+            name="plan_type",
+            field=models.PositiveSmallIntegerField(default=PLAN_TYPE_SELF_MANAGED),
+        ),
+        migrations.RunPython(
+            renumber_plan_types, reverse_code=migrations.RunPython.noop, elidable=True
+        ),
+    ]

zilencer/migrations/0053_remoterealmauditlog_acting_remote_user_and_more.py

@@ -0,0 +1,47 @@
+# Generated by Django 4.2.8 on 2023-12-14 14:18
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ("zilencer", "0052_alter_remoterealm_plan_type_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="remoterealmauditlog",
+            name="acting_remote_user",
+            field=models.ForeignKey(
+                null=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                to="zilencer.remoterealmbillinguser",
+            ),
+        ),
+        migrations.AddField(
+            model_name="remoterealmauditlog",
+            name="acting_support_user",
+            field=models.ForeignKey(
+                null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL
+            ),
+        ),
+        migrations.AddField(
+            model_name="remotezulipserverauditlog",
+            name="acting_remote_user",
+            field=models.ForeignKey(
+                null=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                to="zilencer.remoteserverbillinguser",
+            ),
+        ),
+        migrations.AddField(
+            model_name="remotezulipserverauditlog",
+            name="acting_support_user",
+            field=models.ForeignKey(
+                null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL
+            ),
+        ),
+    ]

zilencer/migrations/0054_remoterealmbillinguser_enable_maintenance_release_emails_and_more.py

@@ -0,0 +1,32 @@
+# Generated by Django 4.2.8 on 2023-12-14 17:16
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0053_remoterealmauditlog_acting_remote_user_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="remoterealmbillinguser",
+            name="enable_maintenance_release_emails",
+            field=models.BooleanField(default=True),
+        ),
+        migrations.AddField(
+            model_name="remoterealmbillinguser",
+            name="enable_major_release_emails",
+            field=models.BooleanField(default=True),
+        ),
+        migrations.AddField(
+            model_name="remoteserverbillinguser",
+            name="enable_maintenance_release_emails",
+            field=models.BooleanField(default=True),
+        ),
+        migrations.AddField(
+            model_name="remoteserverbillinguser",
+            name="enable_major_release_emails",
+            field=models.BooleanField(default=True),
+        ),
+    ]

zilencer/migrations/0055_remoteserverbillinguser_tos_version.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.8 on 2023-12-14 17:27
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0054_remoterealmbillinguser_enable_maintenance_release_emails_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="remoteserverbillinguser",
+            name="tos_version",
+            field=models.TextField(default="-1"),
+        ),
+    ]

zilencer/migrations/0056_remoterealm_realm_locally_deleted.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.8 on 2023-12-15 13:52
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0055_remoteserverbillinguser_tos_version"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="remoterealm",
+            name="realm_locally_deleted",
+            field=models.BooleanField(default=False),
+        ),
+    ]

zilencer/migrations/0057_remoterealm_last_request_timestamp_and_more.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.2.8 on 2023-12-25 00:54
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0056_remoterealm_realm_locally_deleted"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="remoterealm",
+            name="last_request_datetime",
+            field=models.DateTimeField(null=True),
+        ),
+        migrations.AddField(
+            model_name="remotezulipserver",
+            name="last_request_datetime",
+            field=models.DateTimeField(null=True),
+        ),
+    ]

zilencer/migrations/0058_remoteinstallationcount_add_mobile_pushes_forwarded_index.py

@@ -0,0 +1,20 @@
+# Generated by Django 4.2.9 on 2024-02-01 13:58
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0057_remoterealm_last_request_timestamp_and_more"),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="remoteinstallationcount",
+            index=models.Index(
+                condition=models.Q(("property", "mobile_pushes_forwarded::day")),
+                fields=["server_id", "end_time"],
+                name="zilencer_remoteinstallationcount_server_end_time_mobile_pushes_forwarded",
+            ),
+        ),
+    ]

zilencer/migrations/0059_remoterealmauditlog_add_synced_billing_event_type_index.py

@@ -0,0 +1,20 @@
+# Generated by Django 4.2.9 on 2024-02-20 12:04
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0058_remoteinstallationcount_add_mobile_pushes_forwarded_index"),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="remoterealmauditlog",
+            index=models.Index(
+                condition=models.Q(("event_type__in", [101, 102, 103, 104, 105, 201, 202, 229])),
+                fields=["remote_realm_id", "id"],
+                name="zilencer_remoterealmauditlog_synced_billing_events",
+            ),
+        ),
+    ]

zilencer/migrations/0060_remove_remoterealmcount_unique_remote_realm_installation_count_and_more.py

@@ -0,0 +1,81 @@
+# Generated by Django 5.0.5 on 2024-05-06 15:17
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0059_remoterealmauditlog_add_synced_billing_event_type_index"),
+    ]
+
+    operations = [
+        migrations.SeparateDatabaseAndState(
+            # Django does not provide a RenameConstraint operation.
+            # "Constraints" are created as indexes in PostgreSQL, so
+            # we rename the underlying indexes.
+            database_operations=[
+                migrations.RunSQL(
+                    sql=(
+                        "ALTER INDEX unique_remote_realm_installation_count "
+                        "RENAME TO unique_server_realm_installation_count"
+                    ),
+                    reverse_sql=(
+                        "ALTER INDEX unique_server_realm_installation_count "
+                        "RENAME TO unique_remote_realm_installation_count"
+                    ),
+                ),
+                migrations.RunSQL(
+                    sql=(
+                        "ALTER INDEX unique_remote_realm_installation_count_null_subgroup "
+                        "RENAME TO unique_server_realm_installation_count_null_subgroup"
+                    ),
+                    reverse_sql=(
+                        "ALTER INDEX unique_server_realm_installation_count_null_subgroup "
+                        "RENAME TO unique_remote_realm_installation_count_null_subgroup"
+                    ),
+                ),
+            ],
+            state_operations=[
+                migrations.RemoveConstraint(
+                    model_name="remoterealmcount",
+                    name="unique_remote_realm_installation_count",
+                ),
+                migrations.RemoveConstraint(
+                    model_name="remoterealmcount",
+                    name="unique_remote_realm_installation_count_null_subgroup",
+                ),
+                migrations.AddConstraint(
+                    model_name="remoterealmcount",
+                    constraint=models.UniqueConstraint(
+                        condition=models.Q(("subgroup__isnull", False)),
+                        fields=("server", "realm_id", "property", "subgroup", "end_time"),
+                        name="unique_server_realm_installation_count",
+                    ),
+                ),
+                migrations.AddConstraint(
+                    model_name="remoterealmcount",
+                    constraint=models.UniqueConstraint(
+                        condition=models.Q(("subgroup__isnull", True)),
+                        fields=("server", "realm_id", "property", "end_time"),
+                        name="unique_server_realm_installation_count_null_subgroup",
+                    ),
+                ),
+            ],
+        ),
+        migrations.AddConstraint(
+            model_name="remoterealmcount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(("subgroup__isnull", False)),
+                fields=("remote_realm_id", "property", "subgroup", "end_time"),
+                name="unique_remote_realm_installation_count",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="remoterealmcount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(("subgroup__isnull", True)),
+                fields=("remote_realm_id", "property", "end_time"),
+                name="unique_remote_realm_installation_count_null_subgroup",
+            ),
+        ),
+    ]

zilencer/migrations/0061_clean_count_tables.py

@@ -0,0 +1,30 @@
+from django.db import migrations
+
+# This matches analytics.lib.counts.LOGGING_COUNT_STAT_PROPERTIES_NOT_SENT_TO_BOUNCER
+IGNORED = (
+    "invites_sent::day",
+    "mobile_pushes_sent::day",
+    "active_users_log:is_bot:day",
+    "active_users:is_bot:day",
+)
+
+
+class Migration(migrations.Migration):
+    elidable = True
+
+    dependencies = [
+        (
+            "zilencer",
+            "0060_remove_remoterealmcount_unique_remote_realm_installation_count_and_more",
+        ),
+    ]
+
+    operations = [
+        migrations.RunSQL(
+            [
+                ("DELETE FROM zilencer_remoterealmcount WHERE property IN %s", (IGNORED,)),
+                ("DELETE FROM zilencer_remoteinstallationcount WHERE property IN %s", (IGNORED,)),
+            ],
+            elidable=True,
+        ),
+    ]

zilencer/migrations/0062_alter_remoteinstallationcount_id_and_more.py

@@ -0,0 +1,26 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    atomic = False
+
+    dependencies = [
+        ("zilencer", "0061_clean_count_tables"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="remoteinstallationcount",
+            name="id",
+            field=models.BigAutoField(
+                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="remoterealmcount",
+            name="id",
+            field=models.BigAutoField(
+                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+            ),
+        ),
+    ]

zilencer/migrations/0063_convert_ids_to_bigints.py

@@ -0,0 +1,75 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    atomic = False
+
+    dependencies = [
+        ("zilencer", "0062_alter_remoteinstallationcount_id_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="preregistrationremoterealmbillinguser",
+            name="id",
+            field=models.BigAutoField(
+                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="preregistrationremoteserverbillinguser",
+            name="id",
+            field=models.BigAutoField(
+                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="remotepushdevicetoken",
+            name="id",
+            field=models.BigAutoField(
+                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="remoterealm",
+            name="id",
+            field=models.BigAutoField(
+                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="remoterealmauditlog",
+            name="id",
+            field=models.BigAutoField(
+                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="remoterealmbillinguser",
+            name="id",
+            field=models.BigAutoField(
+                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="remoteserverbillinguser",
+            name="id",
+            field=models.BigAutoField(
+                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="remotezulipserver",
+            name="id",
+            field=models.BigAutoField(
+                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="remotezulipserverauditlog",
+            name="id",
+            field=models.BigAutoField(
+                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+            ),
+        ),
+    ]

zilencer/migrations/0064_remotezulipserver_last_merge_base.py

@@ -0,0 +1,17 @@
+# Generated by Django 5.0.6 on 2024-06-23 00:29
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0063_convert_ids_to_bigints"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="remotezulipserver",
+            name="last_merge_base",
+            field=models.CharField(max_length=128, null=True),
+        ),
+    ]

zilencer/migrations/0065_remotepushdevice.py

@@ -0,0 +1,47 @@
+# Generated by Django 5.2.3 on 2025-06-24 07:49
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zerver", "0725_realmuserdefault_web_left_sidebar_unreads_count_summary_and_more"),
+        ("zilencer", "0001_squashed_0064_remotezulipserver_last_merge_base"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="RemotePushDevice",
+            fields=[
+                ("device_id", models.BigAutoField(primary_key=True, serialize=False)),
+                ("token", models.CharField(max_length=4096)),
+                ("token_kind", models.PositiveSmallIntegerField(choices=[(1, "APNs"), (2, "FCM")])),
+                ("push_account_id", models.BigIntegerField()),
+                ("expired_time", models.DateTimeField(null=True)),
+                ("ios_app_id", models.TextField(null=True)),
+                (
+                    "realm",
+                    models.ForeignKey(
+                        null=True, on_delete=django.db.models.deletion.CASCADE, to="zerver.realm"
+                    ),
+                ),
+                (
+                    "remote_realm",
+                    models.ForeignKey(
+                        null=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="zilencer.remoterealm",
+                    ),
+                ),
+            ],
+            options={
+                "constraints": [
+                    models.UniqueConstraint(
+                        fields=("push_account_id", "token"),
+                        name="unique_remote_push_device_push_account_id_token",
+                    )
+                ],
+            },
+        ),
+    ]

zilencer/migrations/0066_alter_remotepushdevice_token_kind.py

@@ -0,0 +1,17 @@
+# Generated by Django 5.2.3 on 2025-07-09 13:11
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0065_remotepushdevice"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="remotepushdevice",
+            name="token_kind",
+            field=models.CharField(choices=[("apns", "APNs"), ("fcm", "FCM")], max_length=4),
+        ),
+    ]

zilencer/migrations/0067_remotepushdevicetoken_apns_case_insensitive.py

@@ -0,0 +1,141 @@
+import django.db.models.functions.text
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0066_alter_remotepushdevice_token_kind"),
+    ]
+
+    operations = [
+        # This parallels zerver/migrations/0740 but must account for
+        # the user_id / user_uuid split.
+        migrations.RunSQL(
+            """
+            WITH dups AS (
+              SELECT server_id, user_id, kind, LOWER(token) AS token, MAX(last_updated) AS max_last_updated
+                FROM zilencer_remotepushdevicetoken
+               WHERE kind = 1
+                 AND user_uuid IS NULL
+               GROUP BY server_id, user_id, kind, LOWER(token)
+              HAVING COUNT(*) > 1
+            )
+            UPDATE zilencer_remotepushdevicetoken
+               SET last_updated = dups.max_last_updated
+              FROM dups
+             WHERE zilencer_remotepushdevicetoken.server_id = dups.server_id
+               AND zilencer_remotepushdevicetoken.user_id = dups.user_id
+               AND zilencer_remotepushdevicetoken.user_uuid IS NULL
+               AND zilencer_remotepushdevicetoken.kind = dups.kind
+               AND LOWER(zilencer_remotepushdevicetoken.token) = dups.token
+            """
+        ),
+        migrations.RunSQL(
+            """
+            WITH dups AS (
+              SELECT server_id, user_id, kind, LOWER(token) AS token, MIN(id) AS min_id
+                FROM zilencer_remotepushdevicetoken
+               WHERE kind = 1
+                 AND user_uuid IS NULL
+               GROUP BY server_id, user_id, kind, LOWER(token)
+              HAVING COUNT(*) > 1
+            )
+            DELETE FROM zilencer_remotepushdevicetoken
+             USING dups
+             WHERE zilencer_remotepushdevicetoken.server_id = dups.server_id
+               AND zilencer_remotepushdevicetoken.user_id = dups.user_id
+               AND zilencer_remotepushdevicetoken.user_uuid IS NULL
+               AND zilencer_remotepushdevicetoken.kind = dups.kind
+               AND LOWER(zilencer_remotepushdevicetoken.token) = dups.token
+               AND zilencer_remotepushdevicetoken.id != dups.min_id
+            """
+        ),
+        migrations.AddConstraint(
+            model_name="remotepushdevicetoken",
+            constraint=models.UniqueConstraint(
+                models.F("server_id"),
+                models.F("user_id"),
+                models.F("kind"),
+                django.db.models.functions.text.Lower(models.F("token")),
+                condition=models.Q(("kind", 1)),
+                name="zilencer_remotepushdevicetoken_apns_server_user_id_kind_token",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="remotepushdevicetoken",
+            constraint=models.UniqueConstraint(
+                models.F("server_id"),
+                models.F("user_id"),
+                models.F("kind"),
+                models.F("token"),
+                condition=models.Q(("kind", 2)),
+                name="zilencer_remotepushdevicetoken_fcm_server_user_id_kind_token",
+            ),
+        ),
+        migrations.RunSQL(
+            """
+            WITH dups AS (
+              SELECT server_id, user_uuid, kind, LOWER(token) AS token, MAX(last_updated) AS max_last_updated
+                FROM zilencer_remotepushdevicetoken
+               WHERE kind = 1
+                 AND user_id IS NULL
+               GROUP BY server_id, user_uuid, kind, LOWER(token)
+              HAVING COUNT(*) > 1
+            )
+            UPDATE zilencer_remotepushdevicetoken
+               SET last_updated = dups.max_last_updated
+              FROM dups
+             WHERE zilencer_remotepushdevicetoken.server_id = dups.server_id
+               AND zilencer_remotepushdevicetoken.user_uuid = dups.user_uuid
+               AND zilencer_remotepushdevicetoken.user_id IS NULL
+               AND zilencer_remotepushdevicetoken.kind = dups.kind
+               AND LOWER(zilencer_remotepushdevicetoken.token) = dups.token
+            """
+        ),
+        migrations.RunSQL(
+            """
+            WITH dups AS (
+              SELECT server_id, user_uuid, kind, LOWER(token) AS token, MIN(id) AS min_id
+                FROM zilencer_remotepushdevicetoken
+               WHERE kind = 1
+                 AND user_id IS NULL
+               GROUP BY server_id, user_uuid, kind, LOWER(token)
+              HAVING COUNT(*) > 1
+            )
+            DELETE FROM zilencer_remotepushdevicetoken
+             USING dups
+             WHERE zilencer_remotepushdevicetoken.server_id = dups.server_id
+               AND zilencer_remotepushdevicetoken.user_uuid = dups.user_uuid
+               AND zilencer_remotepushdevicetoken.user_id IS NULL
+               AND zilencer_remotepushdevicetoken.kind = dups.kind
+               AND LOWER(zilencer_remotepushdevicetoken.token) = dups.token
+               AND zilencer_remotepushdevicetoken.id != dups.min_id
+            """
+        ),
+        migrations.AddConstraint(
+            model_name="remotepushdevicetoken",
+            constraint=models.UniqueConstraint(
+                models.F("server_id"),
+                models.F("user_uuid"),
+                models.F("kind"),
+                django.db.models.functions.text.Lower(models.F("token")),
+                condition=models.Q(("kind", 1)),
+                name="zilencer_remotepushdevicetoken_apns_server_uuid_kind_token",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="remotepushdevicetoken",
+            constraint=models.UniqueConstraint(
+                models.F("server_id"),
+                models.F("user_uuid"),
+                models.F("kind"),
+                models.F("token"),
+                condition=models.Q(("kind", 2)),
+                name="zilencer_remotepushdevicetoken_fcm_server_uuid_kind_token",
+            ),
+        ),
+        migrations.AlterUniqueTogether(
+            name="remotepushdevicetoken",
+            unique_together=set(),
+        ),
+    ]

zilencer/migrations/0068_add_remotepushdevice_token_id_and_more.py

@@ -0,0 +1,30 @@
+# Generated by Django 5.2.10 on 2026-02-02 17:15
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zerver", "0779_device"),
+        ("zilencer", "0067_remotepushdevicetoken_apns_case_insensitive"),
+    ]
+
+    operations = [
+        migrations.RemoveConstraint(
+            model_name="remotepushdevice",
+            name="unique_remote_push_device_push_account_id_token",
+        ),
+        migrations.AddField(
+            model_name="remotepushdevice",
+            name="token_id",
+            field=models.BigIntegerField(default=None),
+            preserve_default=False,
+        ),
+        migrations.AddConstraint(
+            model_name="remotepushdevice",
+            constraint=models.UniqueConstraint(
+                fields=("token_id", "realm", "remote_realm"),
+                name="unique_remote_push_device_token_id_realm_remote_realm",
+            ),
+        ),
+    ]

zilencer/migrations/0069_remove_remotepushdevice_push_account_id.py

@@ -0,0 +1,16 @@
+# Generated by Django 5.2.10 on 2026-02-03 16:35
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zilencer", "0068_add_remotepushdevice_token_id_and_more"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="remotepushdevice",
+            name="push_account_id",
+        ),
+    ]

zilencer/migrations/0070_remove_remoterealmauditlog_zilencer_remoterealmauditlog_synced_billing_events_and_more.py

@@ -0,0 +1,32 @@
+# Generated by Django 5.2.12 on 2026-04-08 16:02
+
+from django.contrib.postgres.operations import AddIndexConcurrently, RemoveIndexConcurrently
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    atomic = False
+
+    dependencies = [
+        ("zilencer", "0069_remove_remotepushdevice_push_account_id"),
+    ]
+
+    operations = [
+        RemoveIndexConcurrently(
+            model_name="remoterealmauditlog",
+            name="zilencer_remoterealmauditlog_synced_billing_events",
+        ),
+        AddIndexConcurrently(
+            model_name="remoterealmauditlog",
+            index=models.Index(
+                condition=models.Q(
+                    (
+                        "event_type__in",
+                        [101, 102, 103, 104, 105, 201, 202, 229, 725],
+                    )
+                ),
+                fields=["remote_realm_id", "id"],
+                name="zilencer_remoterealmauditlog_synced_billing_events",
+            ),
+        ),
+    ]

zilencer/models.py

@@ -0,0 +1,683 @@
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from typing import Literal
+
+from django.conf import settings
+from django.core.exceptions import ValidationError
+from django.db import models
+from django.db.models import F, Max, Q, QuerySet, UniqueConstraint
+from django.db.models.functions import Lower
+from django.utils.timezone import now as timezone_now
+from typing_extensions import override
+
+from analytics.models import BaseCount
+from zerver.lib.rate_limiter import RateLimitedObject
+from zerver.models import AbstractPushDeviceToken, AbstractRealmAuditLog, Realm, UserProfile
+from zerver.models.realm_audit_logs import AuditLogEventType
+
+
+def get_remote_server_by_uuid(uuid: str) -> "RemoteZulipServer":
+    try:
+        return RemoteZulipServer.objects.get(uuid=uuid)
+    except ValidationError:
+        raise RemoteZulipServer.DoesNotExist
+
+
+class RemoteZulipServer(models.Model):
+    """Each object corresponds to a single remote Zulip server that is
+    registered for the Mobile Push Notifications Service via
+    `manage.py register_server`.
+    """
+
+    UUID_LENGTH = 36
+    API_KEY_LENGTH = 64
+    HOSTNAME_MAX_LENGTH = 128
+    VERSION_MAX_LENGTH = 128
+    MERGE_BASE_MAX_LENGTH = 128
+
+    # The unique UUID (`zulip_org_id`) and API key (`zulip_org_key`)
+    # for this remote server registration.
+    uuid = models.UUIDField(unique=True)
+    api_key = models.CharField(max_length=API_KEY_LENGTH)
+
+    # The hostname and contact details are not verified/trusted. Thus,
+    # they primarily exist so that we can communicate with the
+    # maintainer of a server about abuse problems.
+    hostname = models.CharField(max_length=HOSTNAME_MAX_LENGTH)
+    contact_email = models.EmailField(blank=True, null=False)
+    last_updated = models.DateTimeField("last updated", auto_now=True)
+    last_request_datetime = models.DateTimeField(null=True)
+    last_version = models.CharField(max_length=VERSION_MAX_LENGTH, null=True)
+    last_merge_base = models.CharField(max_length=MERGE_BASE_MAX_LENGTH, null=True)
+    last_api_feature_level = models.PositiveIntegerField(null=True)
+
+    # Whether the server registration has been deactivated.
+    deactivated = models.BooleanField(default=False)
+
+    # Plan types for self-hosted customers
+    #
+    # We reserve PLAN_TYPE_SELF_HOSTED=Realm.PLAN_TYPE_SELF_HOSTED for
+    # self-hosted installations that aren't using the notifications
+    # service.
+    #
+    # The other values align with, e.g., CustomerPlan.TIER_SELF_HOSTED_BASE
+    PLAN_TYPE_SELF_HOSTED = 1
+    PLAN_TYPE_SELF_MANAGED = 100
+    PLAN_TYPE_SELF_MANAGED_LEGACY = 101
+    PLAN_TYPE_COMMUNITY = 102
+    PLAN_TYPE_BASIC = 103
+    PLAN_TYPE_BUSINESS = 104
+    PLAN_TYPE_ENTERPRISE = 105
+
+    # The current billing plan for the remote server, similar to Realm.plan_type.
+    plan_type = models.PositiveSmallIntegerField(default=PLAN_TYPE_SELF_MANAGED)
+
+    # This is not synced with the remote server, but only filled for sponsorship requests.
+    org_type = models.PositiveSmallIntegerField(
+        default=Realm.ORG_TYPES["unspecified"]["id"],
+        choices=[(t["id"], t["name"]) for t in Realm.ORG_TYPES.values()],
+    )
+
+    # The last time 'RemoteRealmAuditlog' was updated for this server.
+    last_audit_log_update = models.DateTimeField(null=True)
+
+    @override
+    def __str__(self) -> str:
+        return f"{self.hostname} {str(self.uuid)[0:12]}"
+
+    def format_requester_for_logs(self) -> str:
+        return "zulip-server:" + str(self.uuid)
+
+    def get_remote_server_billing_users(self) -> QuerySet["RemoteServerBillingUser"]:
+        return RemoteServerBillingUser.objects.filter(
+            remote_server=self,
+            is_active=True,
+        )
+
+
+class RemotePushDeviceToken(AbstractPushDeviceToken):
+    """Class for storing legacy non-E2EE mobile push
+    tokens. Deprecated in favor of RemotePushDevice, which is designed
+    for E2EE mobile notifications.
+
+    Like PushDeviceToken, but for a device connected to a remote server.
+    """
+
+    server = models.ForeignKey(RemoteZulipServer, on_delete=models.CASCADE)
+    # The user id on the remote server for this device
+    user_id = models.BigIntegerField(null=True)
+    user_uuid = models.UUIDField(null=True)
+
+    remote_realm = models.ForeignKey("RemoteRealm", on_delete=models.SET_NULL, null=True)
+
+    class Meta:
+        constraints = [
+            # These indexes rely on the property that in Postgres,
+            # NULL != NULL in the context of unique indexes, so multiple
+            # rows with the same values in these columns can exist
+            # if one of them is NULL.
+            models.UniqueConstraint(
+                "server_id",
+                "user_id",
+                "kind",
+                Lower(F("token")),
+                name="zilencer_remotepushdevicetoken_apns_server_user_id_kind_token",
+                condition=Q(kind=1),
+            ),
+            models.UniqueConstraint(
+                "server_id",
+                "user_id",
+                "kind",
+                "token",
+                name="zilencer_remotepushdevicetoken_fcm_server_user_id_kind_token",
+                condition=Q(kind=2),
+            ),
+            models.UniqueConstraint(
+                "server_id",
+                "user_uuid",
+                "kind",
+                Lower(F("token")),
+                name="zilencer_remotepushdevicetoken_apns_server_uuid_kind_token",
+                condition=Q(kind=1),
+            ),
+            models.UniqueConstraint(
+                "server_id",
+                "user_uuid",
+                "kind",
+                "token",
+                name="zilencer_remotepushdevicetoken_fcm_server_uuid_kind_token",
+                condition=Q(kind=2),
+            ),
+        ]
+
+    @override
+    def __str__(self) -> str:
+        return f"{self.server!r} {self.user_id}"
+
+
+class RemoteRealm(models.Model):
+    """
+    Each object corresponds to a single remote Realm that is using the
+    Mobile Push Notifications Service via `manage.py register_server`.
+    """
+
+    server = models.ForeignKey(RemoteZulipServer, on_delete=models.CASCADE)
+
+    # The unique UUID and secret for this realm.
+    uuid = models.UUIDField(unique=True)
+    uuid_owner_secret = models.TextField()
+
+    # Value obtained's from the remote server's realm.host.
+    host = models.TextField()
+
+    name = models.TextField(default="")
+
+    is_system_bot_realm = models.BooleanField(default=False)
+
+    authentication_methods = models.JSONField(default=dict)
+
+    org_type = models.PositiveSmallIntegerField(
+        default=Realm.ORG_TYPES["unspecified"]["id"],
+        choices=[(t["id"], t["name"]) for t in Realm.ORG_TYPES.values()],
+    )
+
+    # The fields below are analogous to RemoteZulipServer fields.
+
+    last_updated = models.DateTimeField("last updated", auto_now=True)
+    last_request_datetime = models.DateTimeField(null=True)
+
+    # Whether the realm registration has been deactivated.
+    registration_deactivated = models.BooleanField(default=False)
+    # Whether the realm has been deactivated on the remote server.
+    realm_deactivated = models.BooleanField(default=False)
+    # Whether we believe the remote server deleted this realm
+    # from the database.
+    realm_locally_deleted = models.BooleanField(default=False)
+
+    # When the realm was created on the remote server.
+    realm_date_created = models.DateTimeField()
+
+    # Plan types for self-hosted customers
+    #
+    # We reserve PLAN_TYPE_SELF_HOSTED=Realm.PLAN_TYPE_SELF_HOSTED for
+    # self-hosted installations that aren't using the notifications
+    # service.
+    #
+    # The other values align with, e.g., CustomerPlan.TIER_SELF_HOSTED_BASE
+    PLAN_TYPE_SELF_HOSTED = 1
+    PLAN_TYPE_SELF_MANAGED = 100
+    PLAN_TYPE_SELF_MANAGED_LEGACY = 101
+    PLAN_TYPE_COMMUNITY = 102
+    PLAN_TYPE_BASIC = 103
+    PLAN_TYPE_BUSINESS = 104
+    PLAN_TYPE_ENTERPRISE = 105
+    plan_type = models.PositiveSmallIntegerField(default=PLAN_TYPE_SELF_MANAGED, db_index=True)
+
+    @override
+    def __str__(self) -> str:
+        return f"{self.host} {str(self.uuid)[0:12]}"
+
+    def get_remote_realm_billing_users(self) -> QuerySet["RemoteRealmBillingUser"]:
+        return RemoteRealmBillingUser.objects.filter(
+            remote_realm=self,
+            is_active=True,
+        )
+
+
+class AbstractRemoteRealmBillingUser(models.Model):
+    remote_realm = models.ForeignKey(RemoteRealm, on_delete=models.CASCADE)
+
+    # The .uuid of the UserProfile on the remote server
+    user_uuid = models.UUIDField()
+    email = models.EmailField()
+
+    date_joined = models.DateTimeField(default=timezone_now)
+
+    class Meta:
+        abstract = True
+
+
+class RemoteRealmBillingUser(AbstractRemoteRealmBillingUser):
+    full_name = models.TextField(default="")
+
+    last_login = models.DateTimeField(null=True)
+
+    is_active = models.BooleanField(default=True)
+
+    TOS_VERSION_BEFORE_FIRST_LOGIN = UserProfile.TOS_VERSION_BEFORE_FIRST_LOGIN
+    tos_version = models.TextField(default=TOS_VERSION_BEFORE_FIRST_LOGIN)
+
+    enable_major_release_emails = models.BooleanField(default=True)
+    enable_maintenance_release_emails = models.BooleanField(default=True)
+
+    class Meta:
+        unique_together = [
+            ("remote_realm", "user_uuid"),
+        ]
+
+
+class PreregistrationRemoteRealmBillingUser(AbstractRemoteRealmBillingUser):
+    # status: whether an object has been confirmed.
+    #   if confirmed, set to confirmation.settings.STATUS_USED
+    status = models.IntegerField(default=0)
+
+    # These are for carrying certain information that's originally
+    # in an IdentityDict across the confirmation link flow. These
+    # values will be restored in the final, fully authenticated IdentityDict.
+    next_page = models.TextField(null=True)
+    uri_scheme = models.TextField()
+
+    created_user = models.ForeignKey(RemoteRealmBillingUser, null=True, on_delete=models.SET_NULL)
+
+
+class AbstractRemoteServerBillingUser(models.Model):
+    remote_server = models.ForeignKey(RemoteZulipServer, on_delete=models.CASCADE)
+
+    email = models.EmailField()
+
+    date_joined = models.DateTimeField(default=timezone_now)
+
+    class Meta:
+        abstract = True
+
+
+class RemoteServerBillingUser(AbstractRemoteServerBillingUser):
+    full_name = models.TextField(default="")
+
+    last_login = models.DateTimeField(null=True)
+
+    is_active = models.BooleanField(default=True)
+
+    TOS_VERSION_BEFORE_FIRST_LOGIN = UserProfile.TOS_VERSION_BEFORE_FIRST_LOGIN
+    tos_version = models.TextField(default=TOS_VERSION_BEFORE_FIRST_LOGIN)
+
+    enable_major_release_emails = models.BooleanField(default=True)
+    enable_maintenance_release_emails = models.BooleanField(default=True)
+
+    class Meta:
+        unique_together = [
+            ("remote_server", "email"),
+        ]
+
+
+class PreregistrationRemoteServerBillingUser(AbstractRemoteServerBillingUser):
+    # status: whether an object has been confirmed.
+    #   if confirmed, set to confirmation.settings.STATUS_USED
+    status = models.IntegerField(default=0)
+
+    next_page = models.TextField(null=True)
+
+    created_user = models.ForeignKey(RemoteServerBillingUser, null=True, on_delete=models.SET_NULL)
+
+
+RemoteServerDeactivationReasonType = Literal[
+    "owner_request",
+    "tos_violation",
+]
+
+
+class RemoteZulipServerAuditLog(AbstractRealmAuditLog):
+    """Audit data associated with a remote Zulip server (not specific to a
+    realm).  Used primarily for tracking registration and billing
+    changes for self-hosted customers.
+
+    In contrast with RemoteRealmAuditLog, which has a copy of data
+    that is generated on the client Zulip server, this table is the
+    authoritative storage location for the server's history.
+    """
+
+    server = models.ForeignKey(RemoteZulipServer, on_delete=models.CASCADE)
+
+    acting_remote_user = models.ForeignKey(
+        RemoteServerBillingUser, null=True, on_delete=models.SET_NULL
+    )
+    acting_support_user = models.ForeignKey(UserProfile, null=True, on_delete=models.SET_NULL)
+
+    @override
+    def __str__(self) -> str:
+        event_type_name = AuditLogEventType(self.event_type).name
+        return f"{event_type_name} {self.event_time} (id={self.id}): {self.server!r}"
+
+
+class RemoteRealmAuditLog(AbstractRealmAuditLog):
+    """Synced audit data from a remote Zulip server, used primarily for
+    billing.  See RealmAuditLog and AbstractRealmAuditLog for details.
+    """
+
+    server = models.ForeignKey(RemoteZulipServer, on_delete=models.CASCADE)
+
+    # With modern Zulip servers, we can link to the RemoteRealm object.
+    remote_realm = models.ForeignKey(RemoteRealm, on_delete=models.CASCADE, null=True)
+    # For pre-8.0 servers, we might only have the realm ID and thus no
+    # RemoteRealm object yet. We will eventually be able to drop this
+    # column once all self-hosted servers have upgraded in favor of
+    # just using the foreign key everywhere.
+    realm_id = models.IntegerField(null=True, blank=True)
+    # The remote_id field lets us deduplicate data from the remote server
+    remote_id = models.IntegerField(null=True)
+
+    acting_remote_user = models.ForeignKey(
+        RemoteRealmBillingUser, null=True, on_delete=models.SET_NULL
+    )
+    acting_support_user = models.ForeignKey(UserProfile, null=True, on_delete=models.SET_NULL)
+
+    @override
+    def __str__(self) -> str:
+        event_type_name = AuditLogEventType(self.event_type).name
+        return f"{event_type_name} {self.event_time} (id={self.id}): {self.server!r}"
+
+    class Meta:
+        constraints = [
+            models.UniqueConstraint(
+                fields=["server", "remote_id"],
+                name="zilencer_remoterealmauditlog_server_remote",
+            ),
+        ]
+        indexes = [
+            models.Index(
+                fields=["server", "realm_id", "remote_id"],
+                name="zilencer_remoterealmauditlog_server_realm_remote",
+            ),
+            models.Index(
+                fields=["server", "realm_id"],
+                condition=Q(remote_realm__isnull=True),
+                name="zilencer_remoterealmauditlog_server_realm",
+            ),
+            models.Index(
+                fields=["server"],
+                condition=Q(remote_realm__isnull=True),
+                name="zilencer_remoterealmauditlog_server",
+            ),
+            models.Index(
+                fields=["remote_realm_id", "id"],
+                condition=Q(event_type__in=AbstractRealmAuditLog.SYNCED_BILLING_EVENTS),
+                name="zilencer_remoterealmauditlog_synced_billing_events",
+            ),
+        ]
+
+
+class BaseRemoteCount(BaseCount):
+    server = models.ForeignKey(RemoteZulipServer, on_delete=models.CASCADE)
+    # The remote_id field is the id value of the corresponding *Count object
+    # on the remote server.
+    # It lets us deduplicate data from the remote server.
+    # Note: Some counts don't come from the remote server, but rather
+    # are stats we track on the bouncer server itself, pertaining to the remote server.
+    # E.g. mobile_pushes_received::day. Such counts will set this field to None.
+    remote_id = models.IntegerField(null=True)
+
+    class Meta:
+        abstract = True
+
+
+class RemoteInstallationCount(BaseRemoteCount):
+    class Meta:
+        constraints = [
+            UniqueConstraint(
+                fields=["server", "property", "subgroup", "end_time"],
+                condition=Q(subgroup__isnull=False),
+                name="unique_remote_installation_count",
+            ),
+            UniqueConstraint(
+                fields=["server", "property", "end_time"],
+                condition=Q(subgroup__isnull=True),
+                name="unique_remote_installation_count_null_subgroup",
+            ),
+            UniqueConstraint(
+                fields=["server", "remote_id"],
+                # As noted above, remote_id may be null, so we only
+                # enforce uniqueness if it isn't.  This is not
+                # technically necessary, since null != null, but it
+                # makes the property more explicit.
+                condition=Q(remote_id__isnull=False),
+                name="unique_remote_installation_count_server_id_remote_id",
+            ),
+        ]
+        indexes = [
+            models.Index(
+                fields=["server_id", "end_time"],
+                condition=Q(property="mobile_pushes_forwarded::day"),
+                name="zilencer_remoteinstallationcount_server_end_time_mobile_pushes_forwarded",
+            )
+        ]
+
+    @override
+    def __str__(self) -> str:
+        return f"{self.property} {self.subgroup} {self.value}"
+
+
+# We can't subclass RealmCount because we only have a realm_id here, not a foreign key.
+class RemoteRealmCount(BaseRemoteCount):
+    realm_id = models.IntegerField(null=True)
+    # Certain RemoteRealmCount will be counts tracked on the bouncer server directly, about
+    # stats pertaining to a realm on a remote server. For such objects, we will link to
+    # the corresponding RemoteRealm object that the remote server registered with us.
+    # In the future we may be able to link all RemoteRealmCount objects to a RemoteRealm,
+    # including the RemoteRealmCount objects are results of just syncing the RealmCount
+    # table from the remote server.
+    remote_realm = models.ForeignKey(RemoteRealm, on_delete=models.CASCADE, null=True)
+
+    class Meta:
+        constraints = [
+            # These two constraints come from the information as
+            # provided by the remote server, for rows they provide.
+            UniqueConstraint(
+                fields=["server", "realm_id", "property", "subgroup", "end_time"],
+                condition=Q(subgroup__isnull=False),
+                name="unique_server_realm_installation_count",
+            ),
+            UniqueConstraint(
+                fields=["server", "realm_id", "property", "end_time"],
+                condition=Q(subgroup__isnull=True),
+                name="unique_server_realm_installation_count_null_subgroup",
+            ),
+            # These two constraints come from our internal
+            # record-keeping, which has a RemoteRealm object.
+            UniqueConstraint(
+                fields=["remote_realm_id", "property", "subgroup", "end_time"],
+                condition=Q(subgroup__isnull=False),
+                name="unique_remote_realm_installation_count",
+            ),
+            UniqueConstraint(
+                fields=["remote_realm_id", "property", "end_time"],
+                condition=Q(subgroup__isnull=True),
+                name="unique_remote_realm_installation_count_null_subgroup",
+            ),
+            UniqueConstraint(
+                fields=["server", "remote_id"],
+                # As with RemoteInstallationCount above, remote_id may
+                # be null; since null != null, this condition is not
+                # strictly necessary, but serves to make the property
+                # more explicit.
+                condition=Q(remote_id__isnull=False),
+                name="unique_remote_realm_installation_count_server_id_remote_id",
+            ),
+        ]
+        indexes = [
+            models.Index(
+                fields=["property", "end_time"],
+                name="zilencer_remoterealmcount_property_end_time_506a0b38_idx",
+            ),
+            models.Index(
+                fields=["server", "realm_id"],
+                condition=Q(remote_realm__isnull=True),
+                name="zilencer_remoterealmcount_server_realm",
+            ),
+            models.Index(
+                fields=["server"],
+                condition=Q(remote_realm__isnull=True),
+                name="zilencer_remoterealmcount_server",
+            ),
+        ]
+
+    @override
+    def __str__(self) -> str:
+        return f"{self.server!r} {self.realm_id} {self.property} {self.subgroup} {self.value}"
+
+
+class RateLimitedRemoteZulipServer(RateLimitedObject):
+    def __init__(
+        self, remote_server: RemoteZulipServer, domain: str = "api_by_remote_server"
+    ) -> None:
+        # Remote servers can only make API requests regarding push notifications
+        # which requires ZILENCER_ENABLED and of course can't happen on API endpoints
+        # inside Tornado.
+        assert not settings.RUNNING_INSIDE_TORNADO
+        assert settings.ZILENCER_ENABLED
+
+        self.uuid = str(remote_server.uuid)
+        self.domain = domain
+        super().__init__()
+
+    @override
+    def key(self) -> str:
+        return f"{type(self).__name__}:<{self.uuid}>:{self.domain}"
+
+    @override
+    def rules(self) -> list[tuple[int, int]]:
+        return settings.RATE_LIMITING_RULES[self.domain]
+
+
+@dataclass
+class RemoteCustomerUserCount:
+    guest_user_count: int
+    non_guest_user_count: int
+
+
+def get_remote_customer_user_count(
+    audit_logs: list[RemoteRealmAuditLog],
+) -> RemoteCustomerUserCount:
+    guest_count = 0
+    non_guest_count = 0
+    for log in audit_logs:
+        humans_count_dict = log.extra_data[RemoteRealmAuditLog.ROLE_COUNT][
+            RemoteRealmAuditLog.ROLE_COUNT_HUMANS
+        ]
+        for role_type in UserProfile.ROLE_TYPES:
+            if role_type == UserProfile.ROLE_GUEST:
+                guest_count += humans_count_dict.get(str(role_type), 0)
+            else:
+                non_guest_count += humans_count_dict.get(str(role_type), 0)
+
+    return RemoteCustomerUserCount(
+        non_guest_user_count=non_guest_count, guest_user_count=guest_count
+    )
+
+
+def get_remote_server_guest_and_non_guest_count(
+    server_id: int, event_time: datetime | None = None
+) -> RemoteCustomerUserCount:
+    # For each realm hosted on the server, find the latest audit log
+    # entry indicating the number of active users in that realm.
+    realm_last_audit_log_ids = (
+        RemoteRealmAuditLog.objects.filter(
+            server_id=server_id,
+            event_type__in=RemoteRealmAuditLog.SYNCED_BILLING_EVENTS,
+            event_time__lte=timezone_now() if event_time is None else event_time,
+        )
+        # Important: extra_data is empty for some pre-2020 audit logs
+        # prior to the introduction of realm_user_count_by_role
+        # logging. Meanwhile, modern Zulip servers using
+        # bulk_create_users to create the users in the system bot
+        # realm also generate such audit logs. Such audit logs should
+        # never be the latest in a normal realm.
+        .exclude(extra_data={})
+        .values("realm_id")
+        .annotate(max_id=Max("id"))
+        .values_list("max_id", flat=True)
+    )
+
+    realm_audit_logs = RemoteRealmAuditLog.objects.filter(id__in=list(realm_last_audit_log_ids))
+
+    # Now we add up the user counts from the different realms.
+    user_count = get_remote_customer_user_count(list(realm_audit_logs))
+    return user_count
+
+
+def get_remote_realm_guest_and_non_guest_count(
+    remote_realm: RemoteRealm, event_time: datetime | None = None
+) -> RemoteCustomerUserCount:
+    latest_audit_log = (
+        RemoteRealmAuditLog.objects.filter(
+            remote_realm=remote_realm,
+            event_type__in=RemoteRealmAuditLog.SYNCED_BILLING_EVENTS,
+            event_time__lte=timezone_now() if event_time is None else event_time,
+        )
+        # Important: extra_data is empty for some pre-2020 audit logs
+        # prior to the introduction of realm_user_count_by_role
+        # logging. Meanwhile, modern Zulip servers using
+        # bulk_create_users to create the users in the system bot
+        # realm also generate such audit logs. Such audit logs should
+        # never be the latest in a normal realm.
+        .exclude(extra_data={})
+        .last()
+    )
+
+    if latest_audit_log is not None:
+        assert latest_audit_log is not None
+        user_count = get_remote_customer_user_count([latest_audit_log])
+    else:
+        user_count = RemoteCustomerUserCount(guest_user_count=0, non_guest_user_count=0)
+    return user_count
+
+
+def has_stale_audit_log(server: RemoteZulipServer) -> bool:
+    if server.last_audit_log_update is None:
+        return True
+
+    if timezone_now() - server.last_audit_log_update > timedelta(days=2):
+        return True
+
+    return False
+
+
+class RemotePushDevice(models.Model):
+    """Core bouncer server table storing registrations to receive
+    mobile push notifications via the bouncer server.
+
+    Each row corresponds to an install of the app registered
+    to receive mobile push notifications for a specific realm.
+    """
+
+    # Unique identifier assigned by the bouncer for this registration.
+    device_id = models.BigAutoField(primary_key=True)
+
+    # Realm that the account is associated with. Both Cloud and
+    # self-hosted are supported; exactly one of these will be null.
+    realm = models.ForeignKey(Realm, on_delete=models.CASCADE, null=True)
+    remote_realm = models.ForeignKey(RemoteRealm, on_delete=models.CASCADE, null=True)
+
+    # The token that uniquely identifies the app instance to the
+    # FCM/APNs servers. There will be duplicates in this table when a
+    # device is logged into multiple Zulip realms.
+    #
+    # FCM and APNs don't specify a maximum token length, so we only enforce
+    # that they're at most the maximum FCM / APNs payload size of 4096 bytes.
+    token = models.CharField(max_length=4096)
+    # ID to reference the `token`.
+    token_id = models.BigIntegerField()
+
+    class TokenKind(models.TextChoices):
+        APNS = "apns", "APNs"
+        FCM = "fcm", "FCM"
+
+    token_kind = models.CharField(max_length=4, choices=TokenKind.choices)
+
+    # If the token is expired, the date when the bouncer learned it
+    # was expired via an error from the FCM/APNs server. Used to
+    # support delayed deletion. Null if the bouncer believes this
+    # token is still valid.
+    expired_time = models.DateTimeField(null=True)
+
+    # Contains the app id of the device if and only if token_kind is APNs.
+    ios_app_id = models.TextField(null=True)
+
+    class Meta:
+        constraints = [
+            UniqueConstraint(
+                # The unique index created is used by a query in
+                # 'do_register_remote_push_device'.
+                fields=["token_id", "realm", "remote_realm"],
+                name="unique_remote_push_device_token_id_realm_remote_realm",
+            ),
+        ]

zilencer/urls.py

@@ -0,0 +1,71 @@
+from typing import Any
+
+from django.conf.urls import include
+from django.urls import path
+
+from corporate.views.remote_billing_page import remote_realm_billing_entry
+from zilencer.auth import remote_server_path
+from zilencer.views import (
+    deactivate_remote_server,
+    register_remote_push_device,
+    register_remote_push_device_for_e2ee_push_notification,
+    register_remote_server,
+    remote_server_check_analytics,
+    remote_server_notify_push,
+    remote_server_post_analytics,
+    remote_server_send_e2ee_push_notification,
+    remote_server_send_test_notification,
+    transfer_remote_server_registration,
+    unregister_all_remote_push_devices,
+    unregister_remote_push_device,
+    verify_registration_transfer_challenge_ack_endpoint,
+)
+
+i18n_urlpatterns: Any = []
+
+# Zilencer views following the REST API style
+# The endpoints marked "intentionally_undocumented" are part of the older system
+# for sending non-E2EE push notifications, and will be removed in the future.
+push_bouncer_patterns = [
+    remote_server_path(
+        "remotes/push/register", POST=(register_remote_push_device, {"intentionally_undocumented"})
+    ),
+    remote_server_path(
+        "remotes/push/e2ee/register", POST=register_remote_push_device_for_e2ee_push_notification
+    ),
+    remote_server_path(
+        "remotes/push/unregister",
+        POST=(unregister_remote_push_device, {"intentionally_undocumented"}),
+    ),
+    remote_server_path(
+        "remotes/push/unregister/all",
+        POST=(unregister_all_remote_push_devices, {"intentionally_undocumented"}),
+    ),
+    remote_server_path(
+        "remotes/push/notify", POST=(remote_server_notify_push, {"intentionally_undocumented"})
+    ),
+    remote_server_path("remotes/push/e2ee/notify", POST=remote_server_send_e2ee_push_notification),
+    remote_server_path(
+        "remotes/push/test_notification",
+        POST=(remote_server_send_test_notification, {"intentionally_undocumented"}),
+    ),
+    # Push signup doesn't use the REST API, since there's no auth.
+    path("remotes/server/register", register_remote_server),
+    path("remotes/server/register/transfer", transfer_remote_server_registration),
+    path(
+        "remotes/server/register/verify_challenge",
+        verify_registration_transfer_challenge_ack_endpoint,
+    ),
+    remote_server_path("remotes/server/deactivate", POST=deactivate_remote_server),
+    # For receiving table data used in analytics and billing
+    remote_server_path("remotes/server/analytics", POST=remote_server_post_analytics),
+    remote_server_path("remotes/server/analytics/status", GET=remote_server_check_analytics),
+]
+
+billing_patterns = [remote_server_path("remotes/server/billing", POST=remote_realm_billing_entry)]
+
+v1_api_bouncer_patterns = push_bouncer_patterns + billing_patterns
+
+urlpatterns = [
+    path("api/v1/", include(v1_api_bouncer_patterns)),
+]