snapshot of https://github.com/zulip/zulip.git @ 48c69911c7dd4e03b20552eb3bd77ad8151c1357 (2026-09-15)

zerver/management/commands/add_users_to_streams.py

@@ -0,0 +1,44 @@
+from typing import Any
+
+from django.core.management.base import CommandParser
+from typing_extensions import override
+
+from zerver.actions.streams import bulk_add_subscriptions
+from zerver.lib.management import ZulipBaseCommand
+from zerver.lib.streams import ensure_stream
+
+
+class Command(ZulipBaseCommand):
+    help = """Add some or all users in a realm to a set of streams."""
+
+    @override
+    def add_arguments(self, parser: CommandParser) -> None:
+        self.add_realm_args(parser, required=True)
+        self.add_user_list_args(parser, all_users_help="Add all users in realm to these streams.")
+
+        parser.add_argument(
+            "-s", "--streams", required=True, help="A comma-separated list of stream names."
+        )
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        realm = self.get_realm(options)
+        assert realm is not None  # Should be ensured by parser
+
+        user_profiles = self.get_users(options, realm)
+        stream_names = {stream.strip() for stream in options["streams"].split(",")}
+
+        for stream_name in stream_names:
+            for user_profile in user_profiles:
+                stream = ensure_stream(realm, stream_name, acting_user=None)
+                _ignore, already_subscribed = bulk_add_subscriptions(
+                    realm, [stream], [user_profile], acting_user=None
+                )
+                was_there_already = user_profile.id in (info.user.id for info in already_subscribed)
+                print(
+                    "{} {} to {}".format(
+                        "Already subscribed" if was_there_already else "Subscribed",
+                        user_profile.delivery_email,
+                        stream_name,
+                    )
+                )

zerver/management/commands/archive_messages.py

@@ -0,0 +1,27 @@
+from typing import Any
+
+from typing_extensions import override
+
+from zerver.actions.realm_settings import (
+    clean_deactivated_realm_data,
+    delete_expired_demo_organizations,
+)
+from zerver.lib.management import ZulipBaseCommand, abort_unless_locked
+from zerver.lib.retention import archive_messages, clean_archived_data
+
+
+class Command(ZulipBaseCommand):
+    @override
+    @abort_unless_locked
+    def handle(self, *args: Any, **options: str) -> None:
+        clean_archived_data()
+        archive_messages()
+        scrub_realms()
+
+
+def scrub_realms() -> None:
+    # First, scrub currently deactivated realms that have an expired
+    # scheduled deletion date. Then, deactivate and scrub realms with
+    # an expired scheduled demo organization deletion date.
+    clean_deactivated_realm_data()
+    delete_expired_demo_organizations()

zerver/management/commands/audit_fts_indexes.py

@@ -0,0 +1,23 @@
+from typing import Any
+
+from django.db import connection
+from typing_extensions import override
+
+from zerver.lib.management import ZulipBaseCommand
+
+
+class Command(ZulipBaseCommand):
+    @override
+    def handle(self, *args: Any, **kwargs: str) -> None:
+        with connection.cursor() as cursor:
+            cursor.execute(
+                """
+                UPDATE zerver_message
+                SET search_tsvector =
+                to_tsvector('zulip.english_us_search', subject || rendered_content)
+                WHERE to_tsvector('zulip.english_us_search', subject || rendered_content) != search_tsvector
+            """
+            )
+
+            fixed_message_count = cursor.rowcount
+            print(f"Fixed {fixed_message_count} messages.")

zerver/management/commands/backup.py

@@ -0,0 +1,152 @@
+import os
+import platform
+import re
+import tempfile
+from argparse import ArgumentParser, RawTextHelpFormatter
+from contextlib import ExitStack
+from typing import Any
+
+from django.conf import settings
+from django.core.management.base import CommandParser
+from django.db import connection
+from django.utils.timezone import now as timezone_now
+from typing_extensions import override
+
+from scripts.lib.zulip_tools import TIMESTAMP_FORMAT, run
+from version import ZULIP_VERSION
+from zerver.lib.management import ZulipBaseCommand
+from zerver.logging_handlers import try_git_describe
+
+
+class Command(ZulipBaseCommand):
+    # Fix support for multi-line usage strings
+    @override
+    def create_parser(self, prog_name: str, subcommand: str, **kwargs: Any) -> CommandParser:
+        parser = super().create_parser(prog_name, subcommand, **kwargs)
+        parser.formatter_class = RawTextHelpFormatter
+        return parser
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        parser.add_argument("--output", help="Filename of output tarball")
+        parser.add_argument("--skip-db", action="store_true", help="Skip database backup")
+        parser.add_argument("--skip-uploads", action="store_true", help="Skip uploads backup")
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        timestamp = timezone_now().strftime(TIMESTAMP_FORMAT)
+        with ExitStack() as stack:
+            tmp = stack.enter_context(
+                tempfile.TemporaryDirectory(prefix=f"zulip-backup-{timestamp}-")
+            )
+            os.mkdir(os.path.join(tmp, "zulip-backup"))
+            members = []
+            paths = []
+
+            with open(os.path.join(tmp, "zulip-backup", "zulip-version"), "w") as f:
+                print(ZULIP_VERSION, file=f)
+                git = try_git_describe()
+                if git:
+                    print(git, file=f)
+            members.append("zulip-backup/zulip-version")
+
+            with open(os.path.join(tmp, "zulip-backup", "os-version"), "w") as f:
+                print(
+                    "{ID} {VERSION_ID}".format(**platform.freedesktop_os_release()),
+                    file=f,
+                )
+            members.append("zulip-backup/os-version")
+
+            with open(os.path.join(tmp, "zulip-backup", "postgres-version"), "w") as f:
+                pg_server_version = connection.cursor().connection.server_version
+                major_pg_version = pg_server_version // 10000
+                print(pg_server_version, file=f)
+            members.append("zulip-backup/postgres-version")
+
+            if settings.DEVELOPMENT:
+                members.append(
+                    os.path.join(settings.DEPLOY_ROOT, "zproject", "dev-secrets.conf"),
+                )
+                paths.append(
+                    ("zproject", os.path.join(settings.DEPLOY_ROOT, "zproject")),
+                )
+            else:
+                members.append("/etc/zulip")
+                paths.append(("settings", "/etc/zulip"))
+
+            if not options["skip_db"]:
+                pg_dump_command = [
+                    f"/usr/lib/postgresql/{major_pg_version}/bin/pg_dump",
+                    "--format=directory",
+                    "--file=" + os.path.join(tmp, "zulip-backup", "database"),
+                    "--username=" + settings.DATABASES["default"]["USER"],
+                    "--dbname=" + settings.DATABASES["default"]["NAME"],
+                    "--no-password",
+                ]
+                if settings.DATABASES["default"].get("HOST"):
+                    pg_dump_command += ["--host=" + settings.DATABASES["default"]["HOST"]]
+                if settings.DATABASES["default"].get("PORT"):
+                    pg_dump_command += ["--port=" + str(settings.DATABASES["default"]["PORT"])]
+
+                os.environ["PGPASSWORD"] = settings.DATABASES["default"]["PASSWORD"]
+
+                run(
+                    pg_dump_command,
+                    cwd=tmp,
+                )
+                members.append("zulip-backup/database")
+
+            if (
+                not options["skip_uploads"]
+                and settings.LOCAL_UPLOADS_DIR is not None
+                and os.path.exists(
+                    os.path.join(settings.DEPLOY_ROOT, settings.LOCAL_UPLOADS_DIR),
+                )
+            ):
+                members.append(
+                    os.path.join(settings.DEPLOY_ROOT, settings.LOCAL_UPLOADS_DIR),
+                )
+                paths.append(
+                    (
+                        "uploads",
+                        os.path.join(settings.DEPLOY_ROOT, settings.LOCAL_UPLOADS_DIR),
+                    ),
+                )
+
+            assert not any("|" in name or "|" in path for name, path in paths)
+            transform_args = [
+                r"--transform=s|^{}(/.*)?$|zulip-backup/{}\1|x".format(
+                    re.escape(path),
+                    name.replace("\\", r"\\"),
+                )
+                for name, path in paths
+            ]
+
+            try:
+                if options["output"] is None:
+                    tarball_path = stack.enter_context(
+                        tempfile.NamedTemporaryFile(
+                            prefix=f"zulip-backup-{timestamp}-",
+                            suffix=".tar.gz",
+                            delete=False,
+                        )
+                    ).name
+                else:
+                    tarball_path = options["output"]
+
+                run(
+                    [
+                        "tar",
+                        f"--directory={tmp}",
+                        "-cPhzf",
+                        tarball_path,
+                        *transform_args,
+                        "--",
+                        *members,
+                    ]
+                )
+                print(f"Backup tarball written to {tarball_path}")
+            except BaseException:
+                if options["output"] is None:
+                    os.unlink(tarball_path)
+                raise

zerver/management/commands/bulk_change_user_name.py

@@ -0,0 +1,37 @@
+from argparse import ArgumentParser
+from typing import Any
+
+from django.core.management.base import CommandError
+from typing_extensions import override
+
+from zerver.actions.user_settings import do_change_full_name
+from zerver.lib.management import ZulipBaseCommand
+
+
+class Command(ZulipBaseCommand):
+    help = """Change the names for many users."""
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        parser.add_argument(
+            "data_file",
+            metavar="<data file>",
+            help="file containing rows of the form <email>,<desired name>",
+        )
+        self.add_realm_args(parser, required=True)
+
+    @override
+    def handle(self, *args: Any, **options: str) -> None:
+        data_file = options["data_file"]
+        realm = self.get_realm(options)
+        with open(data_file) as f:
+            for line in f:
+                email, new_name = line.strip().split(",", 1)
+
+                try:
+                    user_profile = self.get_user(email, realm)
+                    old_name = user_profile.full_name
+                    print(f"{email}: {old_name} -> {new_name}")
+                    do_change_full_name(user_profile, new_name, None, notify=True)
+                except CommandError:
+                    print(f"e-mail {email} doesn't exist in the realm {realm}, skipping")

zerver/management/commands/change_auth_backends.py

@@ -0,0 +1,83 @@
+from argparse import ArgumentParser
+from typing import Any
+
+from django.core.management.base import CommandError
+from typing_extensions import override
+
+from zerver.actions.realm_settings import do_set_realm_authentication_methods
+from zerver.lib.management import ZulipBaseCommand
+
+
+class Command(ZulipBaseCommand):
+    help = """Enable or disable an authentication backend for a realm"""
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        self.add_realm_args(parser, required=True)
+        group = parser.add_mutually_exclusive_group(required=True)
+        group.add_argument(
+            "--enable",
+            help="Name of the authentication backend to enable",
+        )
+        group.add_argument(
+            "--disable",
+            help="Name of the authentication backend to disable",
+        )
+        group.add_argument(
+            "--show",
+            action="store_true",
+            help="Show current authentication backends for the realm",
+        )
+
+    @override
+    def handle(self, *args: Any, **options: str) -> None:
+        realm = self.get_realm(options)
+        assert realm is not None  # Should be ensured by parser
+
+        auth_methods = realm.authentication_methods_dict()
+
+        if options["show"]:
+            print("Current authentication backends for the realm:")
+            print_auth_methods_dict(auth_methods)
+            return
+
+        if options["enable"]:
+            backend_name = options["enable"]
+            check_backend_name_valid(backend_name, auth_methods)
+
+            auth_methods[backend_name] = True
+            print(f"Enabling {backend_name} backend for realm {realm.name}")
+        elif options["disable"]:
+            backend_name = options["disable"]
+            check_backend_name_valid(backend_name, auth_methods)
+
+            auth_methods[backend_name] = False
+            print(f"Disabling {backend_name} backend for realm {realm.name}")
+
+        do_set_realm_authentication_methods(realm, auth_methods, acting_user=None)
+
+        print("Updated authentication backends for the realm:")
+        print_auth_methods_dict(realm.authentication_methods_dict())
+        print("Done!")
+
+
+def check_backend_name_valid(backend_name: str, auth_methods_dict: dict[str, bool]) -> None:
+    if backend_name not in auth_methods_dict:
+        raise CommandError(
+            f"Backend {backend_name} is not a valid authentication backend. Valid backends: {list(auth_methods_dict.keys())}"
+        )
+
+
+def print_auth_methods_dict(auth_methods: dict[str, bool]) -> None:
+    enabled_backends = [backend for backend, enabled in auth_methods.items() if enabled]
+    disabled_backends = [backend for backend, enabled in auth_methods.items() if not enabled]
+
+    if enabled_backends:
+        print("Enabled backends:")
+        for backend in enabled_backends:
+            print(f"  {backend}")
+
+    if disabled_backends:
+        print("Disabled backends:")
+        for backend in disabled_backends:
+            print(f"  {backend}")

zerver/management/commands/change_password.py

@@ -0,0 +1,72 @@
+import getpass
+from argparse import ArgumentParser
+from typing import Any
+
+from django.contrib.auth.password_validation import validate_password
+from django.core.exceptions import ValidationError
+from django.core.management.base import CommandError
+from typing_extensions import override
+
+from zerver.lib.management import ZulipBaseCommand
+
+
+class Command(ZulipBaseCommand):
+    # This is our version of the original Django changepassword command adjusted
+    # to be able to find UserProfiles by email+realm.
+    # We change the arguments the command takes to fit our
+    # model of username+realm and change accordingly the
+    # logic inside the handle method which fetches the user
+    # from the database. The rest of the logic remains unchanged.
+
+    help = "Change a user's password."
+    requires_migrations_checks = True
+    requires_system_checks: list[str] = []
+
+    def _get_pass(self, prompt: str = "Password: ") -> str:
+        p = getpass.getpass(prompt=prompt)
+        if not p:
+            raise CommandError("aborted")
+        return p
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        parser.add_argument("email", metavar="<email>", help="email of user to change role")
+        self.add_realm_args(parser, required=True)
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> str:
+        email = options["email"]
+        realm = self.get_realm(options)
+
+        u = self.get_user(email, realm)
+
+        # Code below is taken from the Django version of this command:
+        self.stdout.write(f"Changing password for user '{u}'")
+
+        MAX_TRIES = 3
+        count = 0
+        p1, p2 = "1", "2"  # To make them initially mismatch.
+        password_validated = False
+        while (p1 != p2 or not password_validated) and count < MAX_TRIES:
+            p1 = self._get_pass()
+            p2 = self._get_pass("Password (again): ")
+            if p1 != p2:
+                self.stdout.write("Passwords do not match. Please try again.")
+                count += 1
+                # Don't validate passwords that don't match.
+                continue
+            try:
+                validate_password(p2, u)
+            except ValidationError as err:
+                self.stderr.write("\n".join(err.messages))
+                count += 1
+            else:
+                password_validated = True
+
+        if count == MAX_TRIES:
+            raise CommandError(f"Aborting password change for user '{u}' after {count} attempts")
+
+        u.set_password(p1)
+        u.save()
+
+        return f"Password changed successfully for user '{u}'"

zerver/management/commands/change_realm_subdomain.py

@@ -0,0 +1,48 @@
+from argparse import ArgumentParser
+from typing import Any
+
+from django.core.exceptions import ValidationError
+from django.core.management.base import CommandError
+from typing_extensions import override
+
+from zerver.actions.create_realm import do_change_realm_subdomain
+from zerver.forms import check_subdomain_available
+from zerver.lib.management import ZulipBaseCommand
+
+
+class Command(ZulipBaseCommand):
+    help = """Change realm's subdomain."""
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        self.add_realm_args(parser, required=True)
+        parser.add_argument("new_subdomain", metavar="<new subdomain>", help="realm new subdomain")
+        parser.add_argument(
+            "--skip-redirect",
+            action="store_true",
+            help="Do not redirect the old name to the new one",
+        )
+        parser.add_argument(
+            "--allow-reserved-subdomain",
+            action="store_true",
+            help="Allow use of reserved subdomains",
+        )
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        realm = self.get_realm(options)
+        assert realm is not None  # Should be ensured by parser
+
+        new_subdomain = options["new_subdomain"]
+        allow_reserved_subdomain = options["allow_reserved_subdomain"]
+        try:
+            check_subdomain_available(new_subdomain, allow_reserved_subdomain)
+        except ValidationError as error:
+            raise CommandError(error.message)
+        do_change_realm_subdomain(
+            realm,
+            new_subdomain,
+            acting_user=None,
+            add_deactivated_redirect=not options["skip_redirect"],
+        )
+        print("Done!")

zerver/management/commands/change_user_email.py

@@ -0,0 +1,27 @@
+from argparse import ArgumentParser
+from typing import Any
+
+from typing_extensions import override
+
+from zerver.actions.user_settings import do_change_user_delivery_email
+from zerver.lib.management import ZulipBaseCommand
+
+
+class Command(ZulipBaseCommand):
+    help = """Change the email address for a user."""
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        self.add_realm_args(parser)
+        parser.add_argument("old_email", metavar="<old email>", help="email address to change")
+        parser.add_argument("new_email", metavar="<new email>", help="new email address")
+
+    @override
+    def handle(self, *args: Any, **options: str) -> None:
+        old_email = options["old_email"]
+        new_email = options["new_email"]
+
+        realm = self.get_realm(options)
+        user_profile = self.get_user(old_email, realm)
+
+        do_change_user_delivery_email(user_profile, new_email, acting_user=None)

zerver/management/commands/change_user_role.py

@@ -0,0 +1,121 @@
+from argparse import ArgumentParser
+from typing import Any
+
+from django.conf import settings
+from django.core.management.base import CommandError
+from typing_extensions import override
+
+from zerver.actions.users import (
+    do_change_can_change_user_emails,
+    do_change_can_create_users,
+    do_change_can_forge_sender,
+    do_change_user_role,
+)
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.management import ZulipBaseCommand
+from zerver.models import UserProfile
+
+ROLE_CHOICES = [
+    "owner",
+    "admin",
+    "moderator",
+    "member",
+    "guest",
+    "can_forge_sender",
+    "can_create_users",
+    "can_change_user_emails",
+]
+
+
+class Command(ZulipBaseCommand):
+    help = """Change role of an existing user in their (own) Realm.
+
+ONLY perform this on customer request from an authorized person.
+"""
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        parser.add_argument("email", metavar="<email>", help="email of user to change role")
+        parser.add_argument(
+            "new_role",
+            metavar="<new_role>",
+            choices=ROLE_CHOICES,
+            help="new role of the user; choose from " + ", ".join(ROLE_CHOICES),
+        )
+        parser.add_argument(
+            "--revoke",
+            dest="grant",
+            action="store_false",
+            help="Remove can_forge_sender or can_create_users permission.",
+        )
+        self.add_realm_args(parser, required=True)
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        email = options["email"]
+        realm = self.get_realm(options)
+        assert realm is not None
+        user = self.get_user(email, realm)
+
+        user_role_map = {
+            "owner": UserProfile.ROLE_REALM_OWNER,
+            "admin": UserProfile.ROLE_REALM_ADMINISTRATOR,
+            "moderator": UserProfile.ROLE_MODERATOR,
+            "member": UserProfile.ROLE_MEMBER,
+            "guest": UserProfile.ROLE_GUEST,
+        }
+
+        if options["new_role"] not in [
+            "can_forge_sender",
+            "can_create_users",
+            "can_change_user_emails",
+        ]:
+            new_role = user_role_map[options["new_role"]]
+            if not options["grant"]:
+                raise CommandError(
+                    "Revoke not supported with this permission; please specify new role."
+                )
+            if new_role == user.role:
+                raise CommandError("User already has this role.")
+            if settings.BILLING_ENABLED and user.is_guest:
+                from corporate.lib.registration import (
+                    check_spare_license_available_for_changing_guest_user_role,
+                )
+
+                try:
+                    check_spare_license_available_for_changing_guest_user_role(realm)
+                except JsonableError:
+                    raise CommandError(
+                        "This realm does not have enough licenses to change a guest user's role."
+                    )
+            old_role_name = UserProfile.ROLE_ID_TO_NAME_MAP[user.role]
+            do_change_user_role(user, new_role, acting_user=None, notify=True)
+            new_role_name = UserProfile.ROLE_ID_TO_NAME_MAP[user.role]
+            print(
+                f"Role for {user.delivery_email} changed from {old_role_name} to {new_role_name}."
+            )
+            return
+
+        if options["new_role"] == "can_forge_sender":
+            if user.can_forge_sender and options["grant"]:
+                raise CommandError("User can already forge messages for this realm.")
+            elif not user.can_forge_sender and not options["grant"]:
+                raise CommandError("User can't forge messages for this realm.")
+            do_change_can_forge_sender(user, options["grant"])
+
+            granted_text = "have" if options["grant"] else "not have"
+            print(
+                f"{user.delivery_email} changed to {granted_text} {options['new_role']} permission."
+            )
+        elif options["new_role"] == "can_create_users":
+            if user.can_create_users and options["grant"]:
+                raise CommandError("User can already create users for this realm.")
+            elif not user.can_create_users and not options["grant"]:
+                raise CommandError("User can't create users for this realm.")
+            do_change_can_create_users(user, options["grant"])
+        elif options["new_role"] == "can_change_user_emails":
+            if user.can_change_user_emails and options["grant"]:
+                raise CommandError("User can already change user emails for this realm.")
+            elif not user.can_change_user_emails and not options["grant"]:
+                raise CommandError("User can't change user emails for this realm.")
+            do_change_can_change_user_emails(user, options["grant"])

zerver/management/commands/check_redis.py

@@ -0,0 +1,82 @@
+import logging
+import time
+from collections.abc import Callable
+from typing import Any
+
+from django.conf import settings
+from django.core.management.base import CommandError, CommandParser
+from typing_extensions import override
+
+from zerver.lib.management import ZulipBaseCommand
+from zerver.lib.partial import partial
+from zerver.lib.rate_limiter import RateLimitedUser, client
+from zerver.models.users import get_user_profile_by_id
+
+
+class Command(ZulipBaseCommand):
+    help = """Checks Redis to make sure our rate limiting system hasn't grown a bug
+    and left Redis with a bunch of data
+
+    Usage: ./manage.py [--trim] check_redis"""
+
+    @override
+    def add_arguments(self, parser: CommandParser) -> None:
+        parser.add_argument("-t", "--trim", action="store_true", help="Actually trim excess")
+
+    def _check_within_range(
+        self,
+        key: bytes,
+        count_func: Callable[[], int],
+        trim_func: Callable[[bytes, int], object] | None = None,
+    ) -> None:
+        user_id = int(key.split(b":")[2])
+        user = get_user_profile_by_id(user_id)
+        entity = RateLimitedUser(user)
+        max_calls = entity.max_api_calls()
+
+        age = int(client.ttl(key))
+        if age < 0:
+            logging.error("Found key with age of %s, will never expire: %s", age, key)
+
+        count = count_func()
+        if count > max_calls:
+            logging.error(
+                "Redis health check found key with more elements \
+than max_api_calls! (trying to trim) %s %s",
+                key,
+                count,
+            )
+            if trim_func is not None:
+                client.expire(key, entity.max_api_window())
+                trim_func(key, max_calls)
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        if not settings.RATE_LIMITING:
+            raise CommandError("This machine is not using Redis or rate limiting, aborting")
+
+        # Find all keys, and make sure they're all within size constraints
+        wildcard_list = "ratelimit:*:*:*:list"
+        wildcard_zset = "ratelimit:*:*:*:zset"
+
+        trim_func: Callable[[bytes, int], object] | None = lambda key, max_calls: client.ltrim(
+            key, 0, max_calls - 1
+        )
+        if not options["trim"]:
+            trim_func = None
+
+        lists = client.keys(wildcard_list)
+        for list_name in lists:
+            self._check_within_range(list_name, partial(client.llen, list_name), trim_func)
+
+        zsets = client.keys(wildcard_zset)
+        for zset in zsets:
+            now = time.time()
+            # We can warn on our zset being too large, but we don't know what
+            # elements to trim. We'd have to go through every list item and take
+            # the intersection. The best we can do is expire it
+            self._check_within_range(
+                zset,
+                partial(client.zcount, zset, 0, now),
+                lambda key, max_calls: None,
+            )

zerver/management/commands/compilemessages.py

@@ -0,0 +1,160 @@
+import os
+import re
+import unicodedata
+from subprocess import CalledProcessError, check_output
+from typing import Any
+
+import orjson
+import polib
+from django.conf import settings
+from django.conf.locale import LANG_INFO
+from django.core.management.base import CommandParser
+from django.core.management.commands import compilemessages
+from django.utils.translation import gettext as _
+from django.utils.translation import override as override_language
+from django.utils.translation import to_language
+from pyuca import Collator
+from typing_extensions import override
+
+
+class Command(compilemessages.Command):
+    @override
+    def add_arguments(self, parser: CommandParser) -> None:
+        super().add_arguments(parser)
+
+        parser.add_argument(
+            "--strict", "-s", action="store_true", help="Stop execution in case of errors."
+        )
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        super().handle(*args, **options)
+        self.strict = options["strict"]
+        self.extract_language_options()
+        self.create_language_name_map()
+
+    def create_language_name_map(self) -> None:
+        join = os.path.join
+        deploy_root = settings.DEPLOY_ROOT
+        path = join(deploy_root, "locale", "language_options.json")
+        output_path = join(deploy_root, "locale", "language_name_map.json")
+
+        with open(path, "rb") as reader:
+            languages = orjson.loads(reader.read())
+            lang_list = []
+            for lang_info in languages["languages"]:
+                lang_info["name"] = lang_info["name_local"]
+                del lang_info["name_local"]
+                lang_list.append(lang_info)
+
+            collator = Collator()
+            lang_list.sort(key=lambda lang: collator.sort_key(lang["name"]))
+
+        with open(output_path, "wb") as output_file:
+            output_file.write(
+                orjson.dumps(
+                    {"name_map": lang_list},
+                    option=orjson.OPT_APPEND_NEWLINE | orjson.OPT_INDENT_2 | orjson.OPT_SORT_KEYS,
+                )
+            )
+
+    def get_po_filename(self, locale_path: str, locale: str) -> str:
+        po_template = "{}/{}/LC_MESSAGES/django.po"
+        return po_template.format(locale_path, locale)
+
+    def get_json_filename(self, locale_path: str, locale: str) -> str:
+        return f"{locale_path}/{locale}/translations.json"
+
+    def get_name_from_po_file(self, po_filename: str, locale: str) -> str:
+        try:
+            team = polib.pofile(po_filename).metadata["Language-Team"]
+            return team[: team.rindex(" <")]
+        except (KeyError, ValueError):
+            raise Exception(f"Unknown language {locale}")
+
+    def get_locales(self) -> list[str]:
+        output = check_output(["git", "ls-files", "locale"], text=True)
+        tracked_files = output.split()
+        regex = re.compile(r"locale/(\w+)/LC_MESSAGES/django.po")
+        locales = ["en"]
+        for tracked_file in tracked_files:
+            matched = regex.search(tracked_file)
+            if matched:
+                locales.append(matched.group(1))
+
+        return locales
+
+    def extract_language_options(self) -> None:
+        locale_path = f"{settings.DEPLOY_ROOT}/locale"
+        output_path = f"{locale_path}/language_options.json"
+
+        data: dict[str, list[dict[str, Any]]] = {"languages": []}
+
+        try:
+            locales = self.get_locales()
+        except CalledProcessError:
+            # In case we are not under a Git repo, fallback to getting the
+            # locales using listdir().
+            locales = os.listdir(locale_path)
+            locales.append("en")
+            locales = list(set(locales))
+
+        for locale in sorted(locales):
+            if locale == "en":
+                data["languages"].append(
+                    {
+                        "name": "English",
+                        "name_local": "English",
+                        "code": "en",
+                        "locale": "en",
+                    }
+                )
+                continue
+
+            lc_messages_path = os.path.join(locale_path, locale, "LC_MESSAGES")
+            if not os.path.exists(lc_messages_path):
+                # Not a locale.
+                continue
+
+            info: dict[str, Any] = {}
+            code = to_language(locale)
+            percentage = self.get_translation_percentage(locale_path, locale)
+            try:
+                name = LANG_INFO[code]["name"]
+                name_local = LANG_INFO[code]["name_local"]
+            except KeyError:
+                # Fallback to getting the name from PO file.
+                filename = self.get_po_filename(locale_path, locale)
+                name = self.get_name_from_po_file(filename, locale)
+                with override_language(code):
+                    name_local = _(name)
+
+            info["name"] = unicodedata.normalize("NFC", name)
+            info["name_local"] = unicodedata.normalize("NFC", name_local)
+            info["code"] = code
+            info["locale"] = locale
+            info["percent_translated"] = percentage
+            data["languages"].append(info)
+
+        with open(output_path, "wb") as writer:
+            writer.write(
+                orjson.dumps(
+                    data,
+                    option=orjson.OPT_APPEND_NEWLINE | orjson.OPT_INDENT_2 | orjson.OPT_SORT_KEYS,
+                )
+            )
+
+    def get_translation_percentage(self, locale_path: str, locale: str) -> int:
+        # backend stats
+        po = polib.pofile(self.get_po_filename(locale_path, locale))
+        not_translated = len(po.untranslated_entries())
+        total = len(po.translated_entries()) + not_translated
+
+        # frontend stats
+        with open(self.get_json_filename(locale_path, locale), "rb") as reader:
+            for value in orjson.loads(reader.read()).values():
+                total += 1
+                if value == "":
+                    not_translated += 1
+
+        return (total - not_translated) * 100 // total

zerver/management/commands/convert_mattermost_data.py

@@ -0,0 +1,83 @@
+import argparse
+import os
+from typing import Any
+
+from typing_extensions import override
+
+"""
+Example usage for testing purposes. For testing data see the mattermost_fixtures
+in zerver/tests/.
+
+    ./manage.py convert_mattermost_data mattermost_fixtures --output mm_export
+    ./manage.py import --destroy-rebuild-database mattermost mm_export/gryffindor
+
+Test out the realm:
+    ./tools/run-dev
+    go to browser and use your dev url
+"""
+
+from django.core.management.base import CommandError, CommandParser
+
+from zerver.data_import.mattermost import do_convert_data
+from zerver.lib.management import ZulipBaseCommand
+
+
+class Command(ZulipBaseCommand):
+    help = """Convert the mattermost data into Zulip data format."""
+
+    @override
+    def add_arguments(self, parser: CommandParser) -> None:
+        dir_help = (
+            "Directory containing exported JSON file and exported_emoji (optional) directory."
+        )
+        parser.add_argument(
+            "mattermost_data_dir", metavar="<mattermost data directory>", help=dir_help
+        )
+
+        parser.add_argument(
+            "--output", dest="output_dir", help="Directory to write converted data to."
+        )
+
+        parser.add_argument(
+            "--mask",
+            dest="masking_content",
+            action="store_true",
+            help="Mask the content for privacy during QA.",
+        )
+
+        parser.add_argument(
+            "--combine-teams-into-one-org",
+            dest="combine_into_one_realm",
+            action="store_true",
+            help="Combine all exported teams into one Zulip organization.",
+        )
+
+        parser.formatter_class = argparse.RawTextHelpFormatter
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        output_dir = options["output_dir"]
+        if output_dir is None:
+            raise CommandError("You need to specify --output <output directory>")
+
+        if os.path.exists(output_dir) and not os.path.isdir(output_dir):
+            raise CommandError(output_dir + " is not a directory")
+
+        os.makedirs(output_dir, exist_ok=True)
+
+        if os.listdir(output_dir):
+            raise CommandError("Output directory should be empty!")
+        output_dir = os.path.realpath(output_dir)
+
+        data_dir = options["mattermost_data_dir"]
+        if not os.path.exists(data_dir):
+            raise CommandError(f"Directory not found: '{data_dir}'")
+        data_dir = os.path.realpath(data_dir)
+
+        print("Converting data ...")
+        do_convert_data(
+            mattermost_data_dir=data_dir,
+            output_dir=output_dir,
+            masking_content=options.get("masking_content", False),
+            combine_into_one_realm=options.get("combine_into_one_realm", False),
+        )

zerver/management/commands/convert_microsoft_teams_data.py

@@ -0,0 +1,78 @@
+import argparse
+import os
+import tempfile
+from typing import Any
+
+from django.conf import settings
+from django.core.management.base import CommandError, CommandParser
+from typing_extensions import override
+
+from zerver.data_import.microsoft_teams import do_convert_directory
+from zerver.lib.management import ZulipBaseCommand
+
+
+class Command(ZulipBaseCommand):
+    help = """Convert the Microsoft Teams data into Zulip data format."""
+
+    @override
+    def add_arguments(self, parser: CommandParser) -> None:
+        parser.add_argument(
+            "microsoft_teams_data_path",
+            nargs="+",
+            metavar="<Microsoft Teams data path>",
+            help="Zipped Microsoft Teams data or directory",
+        )
+
+        parser.add_argument(
+            "--output", dest="output_dir", help="Directory to write exported data to."
+        )
+
+        parser.add_argument(
+            "--token",
+            metavar="<microsoft_graph_api_token>",
+            help="Microsoft Graph API token, see https://learn.microsoft.com/en-us/graph/auth-v2-service?tabs=http",
+        )
+
+        parser.add_argument(
+            "--processes",
+            default=settings.DEFAULT_DATA_EXPORT_IMPORT_PARALLELISM,
+            help="Processes to use in exporting UserMessage objects in parallel",
+        )
+
+        parser.formatter_class = argparse.RawTextHelpFormatter
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        output_dir = options["output_dir"]
+        if output_dir is None:
+            output_dir = tempfile.mkdtemp(prefix="converted-ms-teams-data-")
+        else:
+            output_dir = os.path.realpath(output_dir)
+
+        token = options["token"]
+        if token is None:
+            raise CommandError("Enter Microsoft Graph API token!")
+
+        num_processes = int(options["processes"])
+        if num_processes < 1:
+            raise CommandError("You must have at least one thread.")
+
+        for path in options["microsoft_teams_data_path"]:
+            if not os.path.exists(path):
+                raise CommandError(f"Microsoft Teams data file or directory not found: '{path}'")
+
+            print("Converting data ...")
+            if os.path.isdir(path):
+                print(path)
+                do_convert_directory(
+                    path,
+                    output_dir,
+                    token,
+                    processes=num_processes,
+                )
+            elif os.path.isfile(path) and path.endswith(".zip"):
+                raise ValueError(
+                    "Importing .zip Microsoft Teams data is not yet supported, please try again with the extracted data."
+                )
+            else:
+                raise ValueError(f"Don't know how to import Microsoft Teams data from {path}")

zerver/management/commands/convert_rocketchat_data.py

@@ -0,0 +1,49 @@
+import argparse
+import os
+from typing import Any
+
+from django.core.management.base import CommandError, CommandParser
+from typing_extensions import override
+
+from zerver.data_import.rocketchat import do_convert_data
+from zerver.lib.management import ZulipBaseCommand
+
+
+class Command(ZulipBaseCommand):
+    help = """Convert the Rocketchat data into Zulip data format."""
+
+    @override
+    def add_arguments(self, parser: CommandParser) -> None:
+        dir_help = "Directory containing all the `bson` files from mongodb dump of rocketchat."
+        parser.add_argument(
+            "rocketchat_data_dir", metavar="<rocketchat data directory>", help=dir_help
+        )
+
+        parser.add_argument(
+            "--output", dest="output_dir", help="Directory to write converted data to."
+        )
+
+        parser.formatter_class = argparse.RawTextHelpFormatter
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        output_dir = options["output_dir"]
+        if output_dir is None:
+            raise CommandError("You need to specify --output <output directory>")
+
+        if os.path.exists(output_dir) and not os.path.isdir(output_dir):
+            raise CommandError(output_dir + " is not a directory")
+
+        os.makedirs(output_dir, exist_ok=True)
+
+        if os.listdir(output_dir):
+            raise CommandError("Output directory should be empty!")
+        output_dir = os.path.realpath(output_dir)
+
+        data_dir = options["rocketchat_data_dir"]
+        if not os.path.exists(data_dir):
+            raise CommandError(f"Directory not found: '{data_dir}'")
+        data_dir = os.path.realpath(data_dir)
+
+        print("Converting Data ...")
+        do_convert_data(rocketchat_data_dir=data_dir, output_dir=output_dir)

zerver/management/commands/convert_slack_data.py

@@ -0,0 +1,87 @@
+import argparse
+import os
+import tempfile
+from typing import Any
+
+from django.conf import settings
+from django.core.management.base import CommandError, CommandParser
+from typing_extensions import override
+
+from zerver.data_import.slack import do_convert_directory, do_convert_zipfile
+from zerver.lib.management import ZulipBaseCommand
+
+
+class Command(ZulipBaseCommand):
+    help = """Convert the Slack data into Zulip data format."""
+
+    @override
+    def add_arguments(self, parser: CommandParser) -> None:
+        parser.add_argument(
+            "slack_data_path",
+            nargs="+",
+            metavar="<Slack data path>",
+            help="Zipped Slack data or directory",
+        )
+
+        parser.add_argument(
+            "--token", metavar="<slack_token>", help="Bot user OAuth token, starting xoxb-"
+        )
+
+        parser.add_argument(
+            "--output", dest="output_dir", help="Directory to write exported data to."
+        )
+
+        parser.add_argument(
+            "--processes",
+            default=settings.DEFAULT_DATA_EXPORT_IMPORT_PARALLELISM,
+            help="Processes to use in exporting UserMessage objects in parallel",
+        )
+
+        parser.add_argument(
+            "--no-convert-slack-threads",
+            action="store_true",
+            help="If specified, do not convert Slack threads to separate Zulip topics",
+        )
+
+        parser.formatter_class = argparse.RawTextHelpFormatter
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        output_dir = options["output_dir"]
+        if output_dir is None:
+            output_dir = tempfile.mkdtemp(prefix="converted-slack-data-")
+        else:
+            output_dir = os.path.realpath(output_dir)
+
+        token = options["token"]
+        if token is None:
+            raise CommandError("Enter Slack legacy token!")
+
+        num_processes = int(options["processes"])
+        if num_processes < 1:
+            raise CommandError("You must have at least one process.")
+
+        for path in options["slack_data_path"]:
+            if not os.path.exists(path):
+                raise CommandError(f"Slack data file or directory not found: '{path}'")
+
+            print("Converting data ...")
+            convert_slack_threads = not options["no_convert_slack_threads"]
+            if os.path.isdir(path):
+                do_convert_directory(
+                    path,
+                    output_dir,
+                    token,
+                    processes=num_processes,
+                    convert_slack_threads=convert_slack_threads,
+                )
+            elif os.path.isfile(path) and path.endswith(".zip"):
+                do_convert_zipfile(
+                    path,
+                    output_dir,
+                    token,
+                    processes=num_processes,
+                    convert_slack_threads=convert_slack_threads,
+                )
+            else:
+                raise ValueError(f"Don't know how to import Slack data from {path}")

zerver/management/commands/create_default_stream_groups.py

@@ -0,0 +1,67 @@
+from argparse import ArgumentParser
+from typing import Any
+
+from typing_extensions import override
+
+from zerver.lib.management import ZulipBaseCommand
+from zerver.lib.streams import ensure_stream
+from zerver.models import DefaultStreamGroup
+
+
+class Command(ZulipBaseCommand):
+    help = """
+Create default stream groups which the users can choose during sign up.
+
+./manage.py create_default_stream_groups -s gsoc-1,gsoc-2,gsoc-3 -d "Google summer of code"  -r zulip
+"""
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        self.add_realm_args(parser, required=True)
+
+        parser.add_argument(
+            "-n",
+            "--name",
+            required=True,
+            help="Name of the group you want to create.",
+        )
+
+        parser.add_argument(
+            "-d",
+            "--description",
+            required=True,
+            help="Description of the group.",
+        )
+
+        parser.add_argument(
+            "-s", "--streams", required=True, help="A comma-separated list of stream names."
+        )
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        realm = self.get_realm(options)
+        assert realm is not None  # Should be ensured by parser
+
+        streams = []
+        stream_names = {stream.strip() for stream in options["streams"].split(",")}
+        for stream_name in stream_names:
+            stream = ensure_stream(realm, stream_name, acting_user=None)
+            streams.append(stream)
+
+        try:
+            default_stream_group = DefaultStreamGroup.objects.get(
+                name=options["name"], realm=realm, description=options["description"]
+            )
+        except DefaultStreamGroup.DoesNotExist:
+            default_stream_group = DefaultStreamGroup.objects.create(
+                name=options["name"], realm=realm, description=options["description"]
+            )
+        default_stream_group.streams.set(streams)
+
+        default_stream_groups = DefaultStreamGroup.objects.all()
+        for default_stream_group in default_stream_groups:
+            print(default_stream_group.name)
+            print(default_stream_group.description)
+            for stream in default_stream_group.streams.all():
+                print(stream.name)
+            print()

zerver/management/commands/create_realm.py

@@ -0,0 +1,82 @@
+import argparse
+from typing import Any
+
+from django.core.exceptions import ValidationError
+from django.core.management.base import CommandError
+from typing_extensions import override
+
+from zerver.actions.create_realm import do_create_realm
+from zerver.actions.create_user import do_create_user
+from zerver.forms import check_subdomain_available
+from zerver.lib.management import ZulipBaseCommand
+from zerver.models import UserProfile
+
+
+class Command(ZulipBaseCommand):
+    help = """\
+Create a new Zulip organization (realm) via the command line.
+
+We recommend `./manage.py generate_realm_creation_link` for most
+users, for several reasons:
+
+* Has a more user-friendly web flow for account creation.
+* Manages passwords in a more natural way.
+* Automatically logs the user in during account creation.
+
+This management command is available as an alternative for situations
+where one wants to script the realm creation process.
+
+Since every Zulip realm must have an owner, this command creates the
+initial organization owner user for the new realm, using the same
+workflow as `./manage.py create_user`.
+"""
+
+    @override
+    def add_arguments(self, parser: argparse.ArgumentParser) -> None:
+        parser.add_argument("realm_name", help="Name for the new organization")
+        parser.add_argument(
+            "--string-id",
+            help="Subdomain for the new organization. Empty if root domain.",
+            default="",
+        )
+        parser.add_argument(
+            "--allow-reserved-subdomain",
+            action="store_true",
+            help="Allow use of reserved subdomains",
+        )
+        self.add_create_user_args(parser)
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        realm_name = options["realm_name"]
+        string_id = options["string_id"]
+        allow_reserved_subdomain = options["allow_reserved_subdomain"]
+
+        try:
+            check_subdomain_available(string_id, allow_reserved_subdomain)
+        except ValidationError as error:
+            raise CommandError(error.message)
+
+        create_user_params = self.get_create_user_params(options)
+
+        try:
+            realm = do_create_realm(string_id=string_id, name=realm_name)
+        except AssertionError as e:
+            raise CommandError(str(e))
+
+        do_create_user(
+            create_user_params.email,
+            create_user_params.password,
+            realm,
+            create_user_params.full_name,
+            # Explicitly set tos_version=-1. This means that users
+            # created via this mechanism would be prompted to set
+            # the email_address_visibility setting on first login.
+            # For servers that have configured Terms of Service,
+            # users will also be prompted to accept the Terms of
+            # Service on first login.
+            role=UserProfile.ROLE_REALM_OWNER,
+            realm_creation=True,
+            tos_version=UserProfile.TOS_VERSION_BEFORE_FIRST_LOGIN,
+            acting_user=None,
+        )

zerver/management/commands/create_realm_internal_bots.py

@@ -0,0 +1,21 @@
+from typing import Any
+
+from typing_extensions import override
+
+from zerver.lib.management import ZulipBaseCommand
+from zerver.lib.onboarding import create_if_missing_realm_internal_bots
+
+
+class Command(ZulipBaseCommand):
+    help = """\
+Create realm internal bots if absent, in all realms.
+
+These are normally created when the realm is, so this should be a no-op
+except when upgrading to a version that adds a new realm internal bot.
+"""
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        create_if_missing_realm_internal_bots()
+        # create_users is idempotent -- it's a no-op when a given email
+        # already has a user in a given realm.

zerver/management/commands/create_stream.py

@@ -0,0 +1,27 @@
+from argparse import ArgumentParser
+from typing import Any
+
+from typing_extensions import override
+
+from zerver.lib.management import ZulipBaseCommand
+from zerver.lib.streams import create_stream_if_needed
+
+
+class Command(ZulipBaseCommand):
+    help = """Create a stream, and subscribe all active users (excluding bots).
+
+This should be used for TESTING only, unless you understand the limitations of
+the command."""
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        self.add_realm_args(parser, required=True, help="realm in which to create the stream")
+        parser.add_argument("stream_name", metavar="<stream name>", help="name of stream to create")
+
+    @override
+    def handle(self, *args: Any, **options: str) -> None:
+        realm = self.get_realm(options)
+        assert realm is not None  # Should be ensured by parser
+
+        stream_name = options["stream_name"]
+        create_stream_if_needed(realm, stream_name, acting_user=None)

zerver/management/commands/create_user.py

@@ -0,0 +1,55 @@
+import argparse
+from typing import Any
+
+from django.core.management.base import CommandError
+from django.db.utils import IntegrityError
+from typing_extensions import override
+
+from zerver.actions.create_user import do_create_user
+from zerver.lib.management import ZulipBaseCommand
+from zerver.models import UserProfile
+
+
+class Command(ZulipBaseCommand):
+    help = """Create a new Zulip user via the command line.
+
+Prompts the user for <email> and <full name> if not specified.
+
+We recommend the Zulip API (https://zulip.com/api/create-user) instead
+of this tool for most use cases.
+
+If the server has Terms of Service configured, the user will be
+prompted to accept the Terms of Service the first time they login.
+"""
+
+    @override
+    def add_arguments(self, parser: argparse.ArgumentParser) -> None:
+        self.add_create_user_args(parser)
+        self.add_realm_args(
+            parser, required=True, help="The name of the existing realm to which to add the user."
+        )
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        realm = self.get_realm(options)
+        assert realm is not None  # Should be ensured by parser
+
+        create_user_params = self.get_create_user_params(options)
+
+        try:
+            do_create_user(
+                create_user_params.email,
+                create_user_params.password,
+                realm,
+                create_user_params.full_name,
+                # Explicitly set tos_version=-1. This means that users
+                # created via this mechanism would be prompted to set
+                # the email_address_visibility setting on first login.
+                # For servers that have configured Terms of Service,
+                # users will also be prompted to accept the Terms of
+                # Service on first login.
+                tos_version=UserProfile.TOS_VERSION_BEFORE_FIRST_LOGIN,
+                acting_user=None,
+            )
+        except IntegrityError:
+            raise CommandError("User already exists.")

zerver/management/commands/deactivate_realm.py

@@ -0,0 +1,54 @@
+from argparse import ArgumentParser
+from typing import Any, cast
+
+from typing_extensions import override
+
+from zerver.actions.realm_settings import do_add_deactivated_redirect, do_deactivate_realm
+from zerver.lib.management import ZulipBaseCommand
+
+
+class Command(ZulipBaseCommand):
+    help = """Script to deactivate a realm."""
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        parser.add_argument(
+            "--redirect_url", metavar="<redirect_url>", help="URL to which the realm has moved"
+        )
+        parser.add_argument(
+            "--deactivation_reason",
+            metavar="<deactivation_reason>",
+            help="Reason for deactivation",
+            required=True,
+        )
+        parser.add_argument(
+            "--email_owners",
+            action="store_true",
+            help="Whether to email organization owners about realm deactivation",
+        )
+        self.add_realm_args(parser, required=True)
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        realm = self.get_realm(options)
+        deactivation_reason = options["deactivation_reason"]
+
+        assert realm is not None  # Should be ensured by parser
+
+        if options["redirect_url"]:
+            print("Setting the redirect URL to", options["redirect_url"])
+            do_add_deactivated_redirect(realm, options["redirect_url"])
+
+        if realm.deactivated:
+            print("The realm", options["realm_id"], "is already deactivated.")
+            return
+
+        send_realm_deactivation_email = options["email_owners"]
+        print("Deactivating", options["realm_id"])
+        do_deactivate_realm(
+            realm,
+            acting_user=None,
+            deactivation_reason=cast(Any, deactivation_reason),
+            email_owners=send_realm_deactivation_email,
+        )
+        print("Done!")

zerver/management/commands/deactivate_user.py

@@ -0,0 +1,47 @@
+from argparse import ArgumentParser
+from typing import Any
+
+from django.core.management.base import CommandError
+from typing_extensions import override
+
+from zerver.actions.users import do_deactivate_user
+from zerver.lib.management import ZulipBaseCommand
+from zerver.lib.sessions import user_sessions
+from zerver.lib.users import get_active_bots_owned_by_user
+
+
+class Command(ZulipBaseCommand):
+    help = "Deactivate a user, including forcibly logging them out."
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        parser.add_argument(
+            "-f",
+            "--for-real",
+            action="store_true",
+            help="Actually deactivate the user. Default is a dry run.",
+        )
+        parser.add_argument("email", metavar="<email>", help="email of user to deactivate")
+        self.add_realm_args(parser)
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        realm = self.get_realm(options)
+        user_profile = self.get_user(options["email"], realm)
+
+        print(
+            f"Deactivating {user_profile.full_name} ({user_profile.delivery_email}) - {user_profile.realm.string_id}"
+        )
+        print(f"{user_profile.delivery_email} has the following active sessions:")
+        for session in user_sessions(user_profile):
+            print(session.expire_date, session.get_decoded())
+        print()
+        print(
+            f"{user_profile.delivery_email} has {get_active_bots_owned_by_user(user_profile).count()} active bots that will also be deactivated."
+        )
+
+        if not options["for_real"]:
+            raise CommandError("This was a dry run. Pass -f to actually deactivate.")
+
+        do_deactivate_user(user_profile, acting_user=None)
+        print("Sessions deleted, user deactivated.")

zerver/management/commands/delete_old_unclaimed_attachments.py

@@ -0,0 +1,100 @@
+from argparse import ArgumentParser
+from datetime import timedelta
+from typing import Any
+
+from django.core.management.base import CommandError
+from django.utils.timezone import now as timezone_now
+from typing_extensions import override
+
+from zerver.actions.uploads import do_delete_old_unclaimed_attachments
+from zerver.lib.attachments import get_old_unclaimed_attachments
+from zerver.lib.management import ZulipBaseCommand, abort_unless_locked
+from zerver.lib.thumbnail import split_thumbnail_path
+from zerver.lib.upload import all_message_attachments, delete_message_attachments
+from zerver.models import ArchivedAttachment, Attachment
+
+
+class Command(ZulipBaseCommand):
+    help = """Remove unclaimed attachments from storage older than a supplied
+              numerical value indicating the limit of how old the attachment can be.
+              The default is five weeks."""
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        parser.add_argument(
+            "-w",
+            "--weeks",
+            dest="delta_weeks",
+            default=5,
+            type=int,
+            help="How long unattached attachments are preserved; defaults to 5 weeks.",
+        )
+
+        parser.add_argument(
+            "-f",
+            "--for-real",
+            action="store_true",
+            help="Actually remove the files from the storage.",
+        )
+
+        parser.add_argument(
+            "-C",
+            "--clean-up-storage",
+            action="store_true",
+            help="Examine all attachments in storage (local disk or S3) and remove "
+            "any files which are not in the database. This may take a very long time!",
+        )
+
+    @override
+    @abort_unless_locked
+    def handle(self, *args: Any, **options: Any) -> None:
+        delta_weeks = options["delta_weeks"]
+        print(f"Deleting unclaimed attached files older than {delta_weeks} weeks")
+
+        # print the list of files that are going to be removed
+        old_attachments, old_archived_attachments = get_old_unclaimed_attachments(delta_weeks)
+        for old_attachment in old_attachments:
+            print(f"* {old_attachment.file_name} created at {old_attachment.create_time}")
+        for old_archived_attachment in old_archived_attachments:
+            print(
+                f"* {old_archived_attachment.file_name} created at {old_archived_attachment.create_time}"
+            )
+
+        if options["for_real"]:
+            do_delete_old_unclaimed_attachments(delta_weeks)
+            print()
+            print("Unclaimed files deleted.")
+
+        if options["clean_up_storage"]:
+            print()
+            self.clean_attachment_upload_backend(dry_run=not options["for_real"])
+
+        if not options["for_real"]:
+            print()
+            raise CommandError("This was a dry run. Pass -f to actually delete.")
+
+    def clean_attachment_upload_backend(self, dry_run: bool = True) -> None:
+        cutoff = timezone_now() - timedelta(minutes=5)
+        print(f"Removing extra files in storage black-end older than {cutoff.isoformat()}")
+        with delete_message_attachments(raw_paths=True) as delete_one:
+            for file_path, modified_at in all_message_attachments(include_thumbnails=True):
+                if file_path.startswith("thumbnail/"):
+                    path_id = split_thumbnail_path(file_path)[0]
+                else:
+                    path_id = file_path
+                if Attachment.objects.filter(path_id=path_id).exists():
+                    continue
+                if ArchivedAttachment.objects.filter(path_id=path_id).exists():
+                    continue
+                if modified_at > cutoff:
+                    # We upload files to the backend storage and _then_
+                    # make the database entry, so must give some leeway to
+                    # recently-added files which do not have DB rows.
+                    continue
+                print(f"* {file_path} modified at {modified_at}")
+                if dry_run:
+                    continue
+
+                # This operates on raw paths due to the raw_paths=True
+                # on the contextmanager above.
+                delete_one(file_path)

zerver/management/commands/delete_realm.py

@@ -0,0 +1,61 @@
+from argparse import ArgumentParser
+from typing import Any
+
+from django.conf import settings
+from django.core.management.base import CommandError
+from typing_extensions import override
+
+from zerver.actions.realm_settings import do_delete_realm
+from zerver.lib.management import ZulipBaseCommand
+from zerver.models import Message, UserProfile
+
+
+class Command(ZulipBaseCommand):
+    help = """Script to permanently delete a realm. Recommended only for removing
+realms used for testing; consider using deactivate_realm instead."""
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        self.add_realm_args(parser, required=True)
+
+    @override
+    def handle(self, *args: Any, **options: str) -> None:
+        realm = self.get_realm(options)
+        assert realm is not None  # Should be ensured by parser
+
+        user_count = UserProfile.objects.filter(
+            realm_id=realm.id,
+            is_active=True,
+            is_bot=False,
+        ).count()
+
+        message_count = Message.objects.filter(realm=realm).count()
+
+        print(f"This realm has {user_count} users and {message_count} messages.\n")
+
+        if settings.BILLING_ENABLED:
+            # Deleting a Realm object also deletes associating billing
+            # metadata in an invariant-violating way, so we should
+            # never use this tool for a realm with billing set up.
+            from corporate.models.customers import get_customer_by_realm
+            from corporate.models.plans import CustomerPlan
+
+            customer = get_customer_by_realm(realm)
+            if customer and (
+                customer.stripe_customer_id
+                or CustomerPlan.objects.filter(customer=customer).exists()
+            ):
+                raise CommandError("This realm has had a billing relationship associated with it!")
+
+        print(
+            "This command will \033[91mPERMANENTLY DELETE\033[0m all data for this realm.  "
+            "Most use cases will be better served by scrub_realm and/or deactivate_realm."
+        )
+
+        confirmation = input("Type the name of the realm to confirm: ")
+        if confirmation != realm.string_id:
+            raise CommandError("Aborting!")
+
+        do_delete_realm(realm)
+
+        print("Realm has been successfully permanently deleted.")

zerver/management/commands/delete_user.py

@@ -0,0 +1,68 @@
+from argparse import ArgumentParser
+from typing import Any
+
+from django.core.management.base import CommandError
+from typing_extensions import override
+
+from zerver.actions.users import do_delete_user
+from zerver.lib.management import ZulipBaseCommand
+from zerver.lib.users import get_active_bots_owned_by_user
+
+
+class Command(ZulipBaseCommand):
+    help = """
+
+Delete a user or users, including all messages sent by them and
+personal messages received by them, and audit records, like what
+streams they had been subscribed to. Deactivating users is generally
+recommended over this tool, but deletion can be useful if you
+specifically to completely delete an account created for testing.
+This will:
+
+* Delete the user's account, including metadata like name, email
+  address, custom profile fields, historical subscriptions, etc.
+
+* Delete any messages they've sent and any non-group direct messages
+  they've received.
+
+* Group direct messages in which the user participated won't be
+  deleted (with the exceptions of those message the deleted user
+  sent). An inactive, inaccessible dummy user account named "Deleted
+  User <id>" is created to replace the deleted user as a recipient in
+  group direct message conversations, in order to somewhat preserve
+  their integrity.
+
+* Delete other records of the user's activity, such as emoji reactions.
+
+* Deactivate all bots owned by the user, without deleting them or
+  their data.  If you want to delete the bots and the message
+  sent/received by them, you can use the command on them individually.
+"""
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        parser.add_argument(
+            "-f",
+            "--for-real",
+            action="store_true",
+            help="Actually delete the user(s). Default is a dry run.",
+        )
+        self.add_realm_args(parser)
+        self.add_user_list_args(parser)
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        realm = self.get_realm(options)
+        user_profiles = self.get_users(options, realm)
+
+        for user_profile in user_profiles:
+            print(
+                f"{user_profile.delivery_email} has {get_active_bots_owned_by_user(user_profile).count()} active bots that will be deactivated as a result of the user's deletion."
+            )
+
+        if not options["for_real"]:
+            raise CommandError("This was a dry run. Pass -f to actually delete.")
+
+        for user_profile in user_profiles:
+            do_delete_user(user_profile, acting_user=None)
+            print(f"Successfully deleted user {user_profile.delivery_email}.")

zerver/management/commands/deliver_scheduled_emails.py

@@ -0,0 +1,61 @@
+"""\
+Send email messages that have been queued for later delivery by
+various things (e.g. invitation reminders and welcome emails).
+
+This management command is run via supervisor.
+"""
+
+import logging
+import time
+from typing import Any
+
+from django.conf import settings
+from django.db import transaction
+from django.utils.timezone import now as timezone_now
+from typing_extensions import override
+
+from zerver.lib.logging_util import log_to_file
+from zerver.lib.management import ZulipBaseCommand
+from zerver.lib.send_email import EmailNotDeliveredError, queue_scheduled_emails
+from zerver.models import ScheduledEmail
+
+## Setup ##
+logger = logging.getLogger(__name__)
+log_to_file(logger, settings.EMAIL_DELIVERER_LOG_PATH)
+
+
+class Command(ZulipBaseCommand):
+    help = """Send emails queued by various parts of Zulip
+for later delivery.
+
+Run this command under supervisor.
+
+Usage: ./manage.py deliver_scheduled_emails
+"""
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        try:
+            while True:
+                with transaction.atomic(durable=True):
+                    job = (
+                        ScheduledEmail.objects.filter(scheduled_timestamp__lte=timezone_now())
+                        .prefetch_related("users")
+                        .select_for_update(
+                            skip_locked=True,
+                            # These rows will later be deleted, so take a FOR UPDATE
+                            # lock.
+                            no_key=False,
+                        )
+                        .order_by("scheduled_timestamp")
+                        .first()
+                    )
+                    if job:
+                        try:
+                            queue_scheduled_emails(job)
+                        except EmailNotDeliveredError:
+                            logger.warning("%r not delivered", job)
+                    else:
+                        time.sleep(10)
+        except KeyboardInterrupt:
+            pass

zerver/management/commands/deliver_scheduled_messages.py

@@ -0,0 +1,35 @@
+import time
+from datetime import timedelta
+from typing import Any
+
+from django.utils.timezone import now as timezone_now
+from typing_extensions import override
+
+from zerver.actions.scheduled_messages import try_deliver_one_scheduled_message
+from zerver.lib.management import ZulipBaseCommand
+
+
+## Setup ##
+class Command(ZulipBaseCommand):
+    help = """Deliver scheduled messages from the ScheduledMessage table.
+Run this command under supervisor.
+
+This management command is run via supervisor.
+
+Usage: ./manage.py deliver_scheduled_messages
+"""
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        try:
+            while True:
+                if try_deliver_one_scheduled_message():
+                    continue
+
+                # If there's no overdue scheduled messages, go to sleep until the next minute.
+                cur_time = timezone_now()
+                time_next_min = (cur_time + timedelta(minutes=1)).replace(second=0, microsecond=0)
+                sleep_time = (time_next_min - cur_time).total_seconds()
+                time.sleep(sleep_time)
+        except KeyboardInterrupt:
+            pass

zerver/management/commands/edit_linkifiers.py

@@ -0,0 +1,69 @@
+import sys
+from argparse import ArgumentParser
+from typing import Any
+
+from django.core.management.base import CommandError
+from typing_extensions import override
+
+from zerver.actions.realm_linkifiers import do_add_linkifier, do_remove_linkifier
+from zerver.lib.management import ZulipBaseCommand
+from zerver.models.linkifiers import linkifiers_for_realm
+
+
+class Command(ZulipBaseCommand):
+    help = """Create a link filter rule for the specified realm.
+
+NOTE: Regexes must be simple enough that they can be easily translated to JavaScript
+      RegExp syntax. In addition to JS-compatible syntax, the following features are available:
+
+      * Named groups will be converted to numbered groups automatically
+      * Inline-regex flags will be stripped, and where possible translated to RegExp-wide flags
+
+Example: ./manage.py edit_linkifiers --realm=zulip --op=add '#(?P<id>[0-9]{2,8})' \
+    'https://support.example.com/ticket/{id}'
+Example: ./manage.py edit_linkifiers --realm=zulip --op=remove '#(?P<id>[0-9]{2,8})'
+Example: ./manage.py edit_linkifiers --realm=zulip --op=show
+"""
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        parser.add_argument(
+            "--op", default="show", help="What operation to do (add, show, remove)."
+        )
+        parser.add_argument(
+            "pattern", metavar="<pattern>", nargs="?", help="regular expression to match"
+        )
+        parser.add_argument(
+            "url_template",
+            metavar="<URL template>",
+            nargs="?",
+            help="URL template to expand",
+        )
+        self.add_realm_args(parser, required=True)
+
+    @override
+    def handle(self, *args: Any, **options: str) -> None:
+        realm = self.get_realm(options)
+        assert realm is not None  # Should be ensured by parser
+        if options["op"] == "show":
+            print(f"{realm.string_id}: {linkifiers_for_realm(realm.id)}")
+            sys.exit(0)
+
+        pattern = options["pattern"]
+        if not pattern:
+            self.print_help("./manage.py", "edit_linkifiers")
+            raise CommandError
+
+        if options["op"] == "add":
+            url_template = options["url_template"]
+            if not url_template:
+                self.print_help("./manage.py", "edit_linkifiers")
+                raise CommandError
+            do_add_linkifier(realm, pattern, url_template, acting_user=None)
+            sys.exit(0)
+        elif options["op"] == "remove":
+            do_remove_linkifier(realm, pattern=pattern, acting_user=None)
+            sys.exit(0)
+        else:
+            self.print_help("./manage.py", "edit_linkifiers")
+            raise CommandError

zerver/management/commands/email_mirror.py

@@ -0,0 +1,89 @@
+"""Cron job implementation of Zulip's incoming email gateway's helper
+for forwarding emails into Zulip.
+
+https://zulip.readthedocs.io/en/latest/production/email-gateway.html
+
+The email gateway supports two major modes of operation: An email
+server where the email address configured in EMAIL_GATEWAY_PATTERN
+delivers emails directly to Zulip, and this, a cron job that connects
+to an IMAP inbox (which receives the emails) periodically.
+
+Run this in a cron job every N minutes if you have configured Zulip to
+poll an external IMAP mailbox for messages. The script will then
+connect to your IMAP server and batch-process all messages.
+
+We extract and validate the target stream from information in the
+recipient address and retrieve, forward, and archive the message.
+
+"""
+
+import email.parser
+import email.policy
+import logging
+from collections.abc import Generator
+from email.message import EmailMessage
+from imaplib import IMAP4_SSL
+from typing import Any
+
+from django.conf import settings
+from django.core.management.base import CommandError
+from typing_extensions import override
+
+from zerver.lib.email_mirror import logger, process_message
+from zerver.lib.management import ZulipBaseCommand, skip_unless_locked
+
+## Setup ##
+
+log_format = "%(asctime)s: %(message)s"
+logging.basicConfig(format=log_format)
+
+formatter = logging.Formatter(log_format)
+file_handler = logging.FileHandler(settings.EMAIL_MIRROR_LOG_PATH)
+file_handler.setFormatter(formatter)
+logger.setLevel(logging.DEBUG)
+logger.addHandler(file_handler)
+
+
+def get_imap_messages() -> Generator[EmailMessage, None, None]:
+    # We're probably running from cron, try to batch-process mail
+    if (
+        not settings.EMAIL_GATEWAY_BOT
+        or not settings.EMAIL_GATEWAY_LOGIN
+        or not settings.EMAIL_GATEWAY_PASSWORD
+        or not settings.EMAIL_GATEWAY_IMAP_SERVER
+        or not settings.EMAIL_GATEWAY_IMAP_PORT
+        or not settings.EMAIL_GATEWAY_IMAP_FOLDER
+    ):
+        raise CommandError(
+            "Please configure the email mirror gateway in /etc/zulip/, "
+            "or specify $ORIGINAL_RECIPIENT if piping a single mail."
+        )
+    mbox = IMAP4_SSL(settings.EMAIL_GATEWAY_IMAP_SERVER, settings.EMAIL_GATEWAY_IMAP_PORT)
+    mbox.login(settings.EMAIL_GATEWAY_LOGIN, settings.EMAIL_GATEWAY_PASSWORD)
+    try:
+        mbox.select(settings.EMAIL_GATEWAY_IMAP_FOLDER)
+        try:
+            _status, num_ids_data = mbox.search(None, "ALL")
+            for message_id in num_ids_data[0].split():
+                _status, msg_data = mbox.fetch(message_id, "(RFC822)")
+                assert isinstance(msg_data[0], tuple)
+                msg_as_bytes = msg_data[0][1]
+                yield email.parser.BytesParser(
+                    _class=EmailMessage, policy=email.policy.default
+                ).parsebytes(msg_as_bytes)
+                mbox.store(message_id, "+FLAGS", "\\Deleted")
+            mbox.expunge()
+        finally:
+            mbox.close()
+    finally:
+        mbox.logout()
+
+
+class Command(ZulipBaseCommand):
+    help = __doc__
+
+    @override
+    @skip_unless_locked
+    def handle(self, *args: Any, **options: str) -> None:
+        for message in get_imap_messages():
+            process_message(message)

zerver/management/commands/email_server.py

@@ -0,0 +1,64 @@
+import os
+import ssl
+from typing import Any
+
+from django.core.management.base import BaseCommand, CommandParser
+from typing_extensions import override
+
+from zerver.lib.email_mirror_server import parse_listen_address, run_smtp_server
+
+
+class Command(BaseCommand):
+    help = """SMTP server to ingest incoming emails"""
+
+    @override
+    def add_arguments(self, parser: CommandParser) -> None:
+        parser.add_argument(
+            "--listen",
+            help="[Port, or address:port, to bind the SMTP server to; "
+            "enclose IPv6 addresses in square brackets]",
+            default="25",
+        )
+        parser.add_argument(
+            "--user",
+            help="User to drop privileges to, if started as root.",
+            required=(os.geteuid() == 0),
+        )
+        parser.add_argument(
+            "--group",
+            help="Group to drop privileges to, if started as root.",
+            required=(os.geteuid() == 0),
+        )
+        tls_cert: str | None = None
+        tls_key: str | None = None
+        if os.access("/etc/ssl/certs/zulip.combined-chain.crt", os.R_OK) and os.access(
+            "/etc/ssl/private/zulip.key", os.R_OK
+        ):
+            tls_cert = "/etc/ssl/certs/zulip.combined-chain.crt"
+            tls_key = "/etc/ssl/private/zulip.key"
+        elif os.access("/etc/ssl/certs/ssl-cert-snakeoil.pem", os.R_OK) and os.access(
+            "/etc/ssl/private/ssl-cert-snakeoil.key", os.R_OK
+        ):
+            tls_cert = "/etc/ssl/certs/ssl-cert-snakeoil.pem"
+            tls_key = "/etc/ssl/private/ssl-cert-snakeoil.key"
+        parser.add_argument(
+            "--tls-cert",
+            help="Path to TLS certificate chain file",
+            default=tls_cert,
+        )
+        parser.add_argument(
+            "--tls-key",
+            help="Path to TLS private key file",
+            default=tls_key,
+        )
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        host, port = parse_listen_address(options["listen"])
+        if options["tls_cert"] and options["tls_key"]:
+            tls_context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
+            tls_context.load_cert_chain(options["tls_cert"], options["tls_key"])
+        else:
+            tls_context = None
+
+        run_smtp_server(options["user"], options["group"], host, port, tls_context)

zerver/management/commands/enqueue_digest_emails.py

@@ -0,0 +1,27 @@
+import logging
+from datetime import timedelta
+from typing import Any
+
+from django.conf import settings
+from django.utils.timezone import now as timezone_now
+from typing_extensions import override
+
+from zerver.lib.digest import DIGEST_CUTOFF, enqueue_emails
+from zerver.lib.logging_util import log_to_file
+from zerver.lib.management import ZulipBaseCommand, abort_unless_locked
+
+## Logging setup ##
+logger = logging.getLogger(__name__)
+log_to_file(logger, settings.DIGEST_LOG_PATH)
+
+
+class Command(ZulipBaseCommand):
+    help = """Enqueue digest emails for users that haven't checked the app
+in a while.
+"""
+
+    @override
+    @abort_unless_locked
+    def handle(self, *args: Any, **options: Any) -> None:
+        cutoff = timezone_now() - timedelta(days=DIGEST_CUTOFF)
+        enqueue_emails(cutoff)

zerver/management/commands/enqueue_file.py

@@ -0,0 +1,63 @@
+import sys
+from argparse import ArgumentParser
+from typing import IO, Any
+
+import orjson
+from typing_extensions import override
+
+from zerver.lib.management import ZulipBaseCommand
+from zerver.lib.queue import queue_json_publish_rollback_unsafe
+
+
+def error(*args: Any) -> None:
+    raise Exception("We cannot enqueue because settings.USING_RABBITMQ is False.")
+
+
+def enqueue_file(queue_name: str, f: IO[str]) -> None:
+    for line in f:
+        line = line.strip()
+        try:
+            payload = line.split("\t")[1]
+        except IndexError:
+            payload = line
+
+        print(f"Queueing to queue {queue_name}: {payload}")
+
+        # Verify that payload is valid json.
+        data = orjson.loads(payload)
+
+        # This is designed to use the `error` method rather than
+        # the call_consume_in_tests flow.
+        queue_json_publish_rollback_unsafe(queue_name, data, error)
+
+
+class Command(ZulipBaseCommand):
+    help = """Read JSON lines from a file and enqueue them to a worker queue.
+
+Each line in the file should either be a JSON payload or two tab-separated
+fields, the second of which is a JSON payload.  (The latter is to accommodate
+the format of error files written by queue workers that catch exceptions--their
+first field is a timestamp that we ignore.)
+
+You can use "-" to represent stdin.
+"""
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        parser.add_argument(
+            "queue_name", metavar="<queue>", help="name of worker queue to enqueue to"
+        )
+        parser.add_argument(
+            "file_name", metavar="<file>", help="name of file containing JSON lines"
+        )
+
+    @override
+    def handle(self, *args: Any, **options: str) -> None:
+        queue_name = options["queue_name"]
+        file_name = options["file_name"]
+
+        if file_name == "-":
+            enqueue_file(queue_name, sys.stdin)
+        else:
+            with open(file_name) as f:
+                enqueue_file(queue_name, f)

zerver/management/commands/export.py

@@ -0,0 +1,190 @@
+import os
+import tempfile
+from argparse import ArgumentParser
+from typing import Any
+
+from django.conf import settings
+from django.core.management.base import CommandError
+from django.utils.timezone import now as timezone_now
+from typing_extensions import override
+
+from zerver.actions.realm_settings import do_deactivate_realm
+from zerver.lib.export import export_realm_wrapper, export_tarball_prefix
+from zerver.lib.management import ZulipBaseCommand
+from zerver.models import RealmExport
+
+
+class Command(ZulipBaseCommand):
+    help = """Exports all data from a Zulip realm
+
+    This command exports all significant data from a Zulip realm.  The
+    result can be imported using the `./manage.py import` command.
+
+    Things that are exported:
+    * All user-accessible data in the Zulip database (Messages,
+      Streams, UserMessages, RealmEmoji, etc.)
+    * Copies of all uploaded files and avatar images along with
+      metadata needed to restore them even in the ab
+
+    Things that are not exported:
+    * Confirmation and PreregistrationUser (transient tables)
+    * Sessions (everyone will need to log in again post-export)
+    * Users' passwords and API keys (users will need to use SSO or reset password)
+    * Mobile tokens for APNS/GCM (users will need to reconnect their mobile devices)
+    * ScheduledEmail (not relevant on a new server)
+    * RemoteZulipServer (unlikely to be migrated)
+    * third_party_api_results cache (this means rerendering all old
+      messages could be expensive)
+
+    Things that will break as a result of the export:
+    * Passwords will not be transferred.  They will all need to go
+      through the password reset flow to obtain a new password (unless
+      they intend to only use e.g. Google auth).
+    * Users will need to log out and re-log in to the Zulip desktop and
+      mobile apps.  The apps now all have an option on the login page
+      where you can specify which Zulip server to use; your users
+      should enter <domain name>.
+    * All bots will stop working since they will be pointing to the
+      wrong server URL, and all users' API keys have been rotated as
+      part of the migration.  So to re-enable your integrations, you
+      will need to direct your integrations at the new server.
+      Usually this means updating the URL and the bots' API keys.  You
+      can see a list of all the bots that have been configured for
+      your realm on the `/#organization` page, and use that list to
+      make sure you migrate them all.
+
+    The proper procedure for using this to export a realm is as follows:
+
+    * Use `./manage.py export --deactivate` to deactivate and export
+      the realm, producing a data tarball.
+
+    * Transfer the tarball to the new server and unpack it.
+
+    * Use `./manage.py import` to import the realm
+
+    * Inform the users about the things broken above.
+
+    We recommend testing by exporting without `--deactivate` first, to
+    make sure you have the procedure right and minimize downtime.
+
+    Performance: In one test, the tool exported a realm with hundreds
+    of users and ~1M messages of history with --parallel=1 in about 3
+    hours of serial runtime (goes down to ~50m with --parallel=6 on a
+    machine with 8 CPUs).  Importing that same data set took about 30
+    minutes.  But this will vary a lot depending on the average number
+    of recipients of messages in the realm, hardware, etc."""
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        parser.add_argument(
+            "--output", dest="output_dir", help="Directory to write exported data to."
+        )
+        parser.add_argument(
+            "--parallel",
+            default=settings.DEFAULT_DATA_EXPORT_IMPORT_PARALLELISM,
+            help="Processes to use in exporting UserMessage objects in parallel",
+        )
+        parser.add_argument(
+            "--public-only",
+            action="store_true",
+            help="Export only public stream messages and associated attachments",
+        )
+        parser.add_argument(
+            "--deactivate-realm",
+            action="store_true",
+            help=(
+                "Deactivate the realm immediately before exporting; the exported data "
+                "will show the realm as active"
+            ),
+        )
+        parser.add_argument(
+            "--export-full-with-consent",
+            action="store_true",
+            help="Whether to export private data of users who consented",
+        )
+        parser.add_argument(
+            "--upload",
+            action="store_true",
+            help="Whether to upload resulting tarball to s3 or LOCAL_UPLOADS_DIR",
+        )
+        self.add_realm_args(parser, required=True)
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        realm = self.get_realm(options)
+        assert realm is not None  # Should be ensured by parser
+
+        output_dir = options["output_dir"]
+        public_only = options["public_only"]
+        export_full_with_consent = options["export_full_with_consent"]
+        assert not (public_only and export_full_with_consent)
+
+        print(f"\033[94mExporting realm\033[0m: {realm.string_id}")
+
+        processes = int(options["parallel"])
+        if processes < 1:
+            raise CommandError("You must have at least one process.")
+
+        if public_only and export_full_with_consent:
+            raise CommandError("Please pass either --public-only or --export-full-with-consennt")
+
+        if options["deactivate_realm"] and realm.deactivated:
+            raise CommandError(f"The realm {realm.string_id} is already deactivated.  Aborting...")
+
+        if output_dir is None:
+            output_dir = tempfile.mkdtemp(prefix=export_tarball_prefix(realm))
+        else:
+            output_dir = os.path.realpath(os.path.expanduser(output_dir))
+            if os.path.exists(output_dir):
+                if os.listdir(output_dir):
+                    raise CommandError(
+                        f"Refusing to overwrite nonempty directory: {output_dir}. Aborting...",
+                    )
+            else:
+                os.makedirs(output_dir)
+
+        tarball_path = output_dir.rstrip("/") + ".tar.gz"
+        try:
+            with open(tarball_path, "x"):
+                pass
+        except FileExistsError:
+            raise CommandError(
+                f"Refusing to overwrite existing tarball: {tarball_path}. Aborting..."
+            )
+
+        if options["deactivate_realm"]:
+            print(f"\033[94mDeactivating realm\033[0m: {realm.string_id}")
+            do_deactivate_realm(
+                realm,
+                acting_user=None,
+                deactivation_reason="self_hosting_migration",
+                email_owners=False,
+            )
+
+        def percent_callback(bytes_transferred: Any) -> None:
+            print(end=".", flush=True)
+
+        if public_only:
+            export_type = RealmExport.EXPORT_PUBLIC
+        elif export_full_with_consent:
+            export_type = RealmExport.EXPORT_FULL_WITH_CONSENT
+        else:
+            export_type = RealmExport.EXPORT_FULL_WITHOUT_CONSENT
+
+        export_row = RealmExport.objects.create(
+            realm=realm,
+            type=export_type,
+            acting_user=None,
+            status=RealmExport.REQUESTED,
+            date_requested=timezone_now(),
+        )
+
+        # Allows us to trigger exports separately from command line argument parsing
+        export_realm_wrapper(
+            export_row=export_row,
+            output_dir=output_dir,
+            processes=processes,
+            upload=options["upload"],
+            percent_callback=percent_callback,
+            export_as_active=True if options["deactivate_realm"] else None,
+        )

zerver/management/commands/export_search.py

@@ -0,0 +1,339 @@
+import csv
+import os
+import queue
+import shutil
+from argparse import ArgumentParser
+from collections.abc import Iterator
+from datetime import datetime, timezone
+from email.headerregistry import Address
+from functools import lru_cache, reduce
+from operator import or_
+from threading import Lock, Thread
+from typing import Any, NoReturn, Union
+
+from django.core.management.base import CommandError
+from django.db.models import Q
+from typing_extensions import override
+
+from zerver.lib.export import orjson_stream
+from zerver.lib.management import ZulipBaseCommand
+from zerver.lib.soft_deactivation import reactivate_user_if_soft_deactivated
+from zerver.lib.upload import save_attachment_contents
+from zerver.models import AbstractUserMessage, Message, Recipient, Stream, UserProfile
+from zerver.models.recipients import get_or_create_direct_message_group
+from zerver.models.streams import get_stream
+from zerver.models.users import get_user_by_delivery_email
+
+check_lock = Lock()
+download_queue: queue.Queue[str] = queue.Queue()
+BATCH_SIZE = 1000
+
+
+def write_attachment(base_path: str, path_id: str, file_lock: Union["Lock", None] = None) -> None:
+    dir_path_id = os.path.dirname(path_id)
+    assert "../" not in dir_path_id
+    os.makedirs(base_path + "/" + dir_path_id, exist_ok=True)
+    with open(base_path + "/" + path_id, "wb") as attachment_file:
+        if file_lock:
+            file_lock.release()
+        save_attachment_contents(path_id, attachment_file)
+
+
+def download_worker(base_path: str) -> NoReturn:
+    while True:
+        path_id = download_queue.get()
+
+        check_lock.acquire()
+        if os.path.exists(base_path + "/" + path_id):
+            check_lock.release()
+            download_queue.task_done()
+            continue
+
+        print(f"Downloading {path_id} ({download_queue.qsize()} remaining in queue)")
+        write_attachment(base_path, path_id, check_lock)
+        download_queue.task_done()
+
+
+class Command(ZulipBaseCommand):
+    help = """Exports the messages matching certain search terms, or from
+senders/recipients.
+
+This is most often used for legal compliance.
+"""
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        self.add_realm_args(parser, required=True)
+        parser.add_argument(
+            "--output",
+            metavar="<path>",
+            help="File to output JSON/CSV results to; it must not exist, unless --force is given",
+            required=True,
+        )
+        parser.add_argument(
+            "--write-attachments",
+            metavar="<directory>",
+            help="If provided, export all referenced attachments into the directory",
+        )
+        parser.add_argument(
+            "--force", action="store_true", help="Overwrite the output file if it exists already"
+        )
+        parser.add_argument("--threads", default=5, type=int)
+
+        parser.add_argument(
+            "--file",
+            metavar="<path>",
+            help="Read search terms from the named file, one per line",
+        )
+        parser.add_argument(
+            "search_terms",
+            nargs="*",
+            metavar="<search term>",
+            help="Terms to search for in message body or topic",
+        )
+        parser.add_argument(
+            "--after",
+            metavar="<datetime>",
+            help="Limit to messages on or after this ISO datetime, treated as UTC",
+            type=lambda s: datetime.fromisoformat(s).astimezone(timezone.utc),
+        )
+        parser.add_argument(
+            "--before",
+            metavar="<datetime>",
+            help="Limit to messages on or before this ISO datetime, treated as UTC",
+            type=lambda s: datetime.fromisoformat(s).astimezone(timezone.utc),
+        )
+        source_dest = parser.add_mutually_exclusive_group()
+
+        source_dest.add_argument(
+            "--sender",
+            action="append",
+            metavar="<email>",
+            help="Limit to messages sent by users with any of these emails (may be specified more than once)",
+        )
+        source_dest.add_argument(
+            "--recipient",
+            action="append",
+            metavar="<email>",
+            help="Limit to messages received by users with any of these emails (may be specified more than once).  This is a superset of --sender, since senders receive every message they send.",
+        )
+        source_dest.add_argument(
+            "--dm",
+            action="append",
+            metavar="<email>",
+            help="Limit to messages in a DM between all of the users provided.",
+        )
+        source_dest.add_argument(
+            "--channel",
+            action="append",
+            metavar="<channel-name>",
+            help="Limit to messages in the given channel.",
+        )
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        terms = set()
+        if options["file"]:
+            with open(options["file"]) as f:
+                terms.update(f.read().splitlines())
+        terms.update(options["search_terms"])
+
+        if (
+            not terms
+            and not options["before"]
+            and not options["after"]
+            and not options["sender"]
+            and not options["recipient"]
+            and not options["dm"]
+            and not options["channel"]
+        ):
+            raise CommandError("One or more limits are required!")
+
+        if not options["output"].endswith((".json", ".csv")):
+            raise CommandError(
+                "Unknown file format: {options['output']}  Only .csv and .json are supported"
+            )
+
+        if os.path.exists(options["output"]) and not options["force"]:
+            raise CommandError(
+                f"Output path '{options['output']}' already exists; use --force to overwrite"
+            )
+
+        if options["write_attachments"] and os.path.exists(options["write_attachments"]):
+            if not options["force"]:
+                raise CommandError(
+                    f"Attachments output path '{options['write_attachments']}' already exists; use --force to overwrite"
+                )
+            shutil.rmtree(options["write_attachments"])
+
+        realm = self.get_realm(options)
+        assert realm is not None
+        need_distinct = False
+        usermessage_joined = False
+        limits = Q()
+
+        limits = reduce(
+            or_,
+            [
+                Q(content__icontains=term) | Q(is_channel_message=True, subject__icontains=term)
+                for term in terms
+            ],
+            limits,
+        )
+
+        if options["after"]:
+            limits &= Q(date_sent__gt=options["after"])
+        if options["before"]:
+            limits &= Q(date_sent__lt=options["before"])
+        if options["recipient"]:
+            user_profiles = [get_user_by_delivery_email(e, realm) for e in options["recipient"]]
+            for user_profile in user_profiles:
+                # Users need to not be long-term idle for the
+                # UserMessages to be a judge of which messages they
+                # received.
+                reactivate_user_if_soft_deactivated(user_profile)
+            limits &= Q(
+                usermessage__user_profile_id__in=[user_profile.id for user_profile in user_profiles]
+            )
+            need_distinct = len(user_profiles) > 1
+            usermessage_joined = True
+        elif options["sender"]:
+            limits &= reduce(
+                or_,
+                [Q(sender__delivery_email__iexact=e) for e in options["sender"]],
+            )
+        elif options["dm"]:
+            user_profiles = [get_user_by_delivery_email(e, realm) for e in options["dm"]]
+            for user_profile in user_profiles:
+                reactivate_user_if_soft_deactivated(user_profile)
+            if len(user_profiles) == 1:
+                limits &= Q(
+                    usermessage__user_profile_id=user_profiles[0].id,
+                    usermessage__flags__andnz=AbstractUserMessage.flags.is_private.mask,
+                )
+                usermessage_joined = True
+            elif len(user_profiles) == 2:
+                user_a, user_b = user_profiles
+                direct_message_group = get_or_create_direct_message_group(
+                    id_list=[user_a.id, user_b.id]
+                )
+                limits &= Q(recipient=direct_message_group.recipient)
+            else:
+                direct_message_group = get_or_create_direct_message_group(
+                    [user.id for user in user_profiles]
+                )
+                limits &= Q(recipient=direct_message_group.recipient)
+        elif options["channel"]:
+            channels = [get_stream(n.lstrip("#"), realm) for n in options["channel"]]
+            limits &= Q(recipient__in=[s.recipient_id for s in channels])
+
+        messages_query = (
+            Message.objects.filter(limits, realm=realm)
+            .select_related("sender")
+            .only(
+                "id",
+                "date_sent",
+                "sender__full_name",
+                "sender__delivery_email",
+                "recipient_id",
+                "subject",
+                "content",
+                "edit_history",
+                "has_attachment",
+            )
+            .order_by("id")
+        )
+        if need_distinct:
+            messages_query = messages_query.distinct("id")
+
+        if options["write_attachments"]:
+            for i in range(options["threads"]):
+                Thread(
+                    target=download_worker, daemon=True, args=(options["write_attachments"],)
+                ).start()
+
+        @lru_cache(maxsize=1000)
+        def format_sender(full_name: str, delivery_email: str) -> str:
+            return str(Address(display_name=full_name, addr_spec=delivery_email))
+
+        def format_full_recipient(recipient_id: int, subject: str) -> str:
+            recip_str, has_subject = format_recipient(recipient_id)
+            if not has_subject:
+                return recip_str
+            return f"{recip_str} > {subject}"
+
+        @lru_cache(maxsize=1000)
+        def format_recipient(recipient_id: int) -> tuple[str, bool]:
+            recipient = Recipient.objects.get(id=recipient_id)
+
+            if recipient.type == Recipient.STREAM:
+                stream = Stream.objects.values("name").get(id=recipient.type_id)
+                return "#" + stream["name"], True
+
+            users = (
+                UserProfile.objects.filter(
+                    subscription__recipient_id=recipient.id,
+                )
+                .order_by("full_name")
+                .values_list("full_name", "delivery_email")
+            )
+
+            return ", ".join(format_sender(e[0], e[1]) for e in users), False
+
+        def transform_message(message: Message) -> dict[str, str]:
+            row = {
+                "id": str(message.id),
+                "timestamp (UTC)": message.date_sent.astimezone(timezone.utc)
+                .replace(tzinfo=None)
+                .isoformat(" ", "seconds"),
+                "sender": format_sender(message.sender.full_name, message.sender.delivery_email),
+                "recipient": format_full_recipient(message.recipient_id, message.subject),
+                "content": message.content,
+                "edit history": message.edit_history if message.edit_history is not None else "",
+            }
+            if options["write_attachments"]:
+                if message.has_attachment:
+                    attachments = message.attachment_set.all()
+                    row["attachments"] = " ".join(a.path_id for a in attachments)
+                    for attachment in attachments:
+                        download_queue.put(attachment.path_id)
+                else:
+                    row["attachments"] = ""
+            return row
+
+        def chunked_results() -> Iterator[dict[str, str]]:
+            min_id = 0
+            while True:
+                batch_query = messages_query.filter(id__gt=min_id)
+                if usermessage_joined:
+                    batch_query = batch_query.extra(
+                        where=["zerver_usermessage.message_id > %s"], params=[min_id]
+                    )
+                batch = [transform_message(m) for m in batch_query[:BATCH_SIZE]]
+                if len(batch) == 0:
+                    break
+                min_id = int(batch[-1]["id"])
+                print(".")
+                yield from batch
+
+        print("Exporting messages...")
+        if options["output"].endswith(".json"):
+            with open(options["output"], "wb") as json_file:
+                json_file.writelines(orjson_stream(chunked_results()))
+        elif options["output"].endswith(".csv"):
+            with open(options["output"], "w") as csv_file:
+                columns = [
+                    "id",
+                    "timestamp (UTC)",
+                    "sender",
+                    "recipient",
+                    "content",
+                    "edit history",
+                ]
+
+                if options["write_attachments"]:
+                    columns += ["attachments"]
+                csvwriter = csv.DictWriter(csv_file, columns)
+                csvwriter.writeheader()
+                csvwriter.writerows(chunked_results())
+        download_queue.join()

zerver/management/commands/export_single_user.py

@@ -0,0 +1,60 @@
+import os
+import subprocess
+import tempfile
+from argparse import ArgumentParser
+from typing import Any
+
+from django.core.management.base import CommandError
+from typing_extensions import override
+
+from zerver.lib.export import do_export_user, export_tarball_prefix
+from zerver.lib.management import ZulipBaseCommand
+
+
+class Command(ZulipBaseCommand):
+    help = """Exports message data from a Zulip user
+
+    This command exports the message history for a single Zulip user.
+
+    Note that this only exports the user's message history and
+    realm-public metadata needed to understand it; it does nothing
+    with (for example) any bots owned by the user."""
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        parser.add_argument("email", metavar="<email>", help="email of user to export")
+        parser.add_argument(
+            "--output", dest="output_dir", help="Directory to write exported data to."
+        )
+        self.add_realm_args(parser)
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        realm = self.get_realm(options)
+        user_profile = self.get_user(options["email"], realm)
+
+        output_dir = options["output_dir"]
+        if output_dir is None:
+            output_dir = tempfile.mkdtemp(prefix=export_tarball_prefix(user_profile.realm))
+        else:
+            output_dir = os.path.abspath(output_dir)
+            if os.path.exists(output_dir) and os.listdir(output_dir):
+                raise CommandError(
+                    f"Refusing to overwrite nonempty directory: {output_dir}. Aborting...",
+                )
+            else:
+                os.makedirs(output_dir)
+
+        print(f"Exporting user {user_profile.delivery_email}")
+        do_export_user(user_profile, output_dir)
+        print(f"Finished exporting to {output_dir}; tarring")
+        tarball_path = output_dir.rstrip("/") + ".tar.gz"
+        subprocess.check_call(
+            [
+                "tar",
+                f"-czf{tarball_path}",
+                f"-C{os.path.dirname(output_dir)}",
+                os.path.basename(output_dir),
+            ]
+        )
+        print(f"Tarball written to {tarball_path}")

zerver/management/commands/fetch_tor_exit_nodes.py

@@ -0,0 +1,75 @@
+import os
+from argparse import ArgumentParser
+from typing import Any
+
+import orjson
+from django.conf import settings
+from typing_extensions import override
+from urllib3.util import Retry
+
+from zerver.lib.management import ZulipBaseCommand, abort_cron_during_deploy, abort_unless_locked
+from zerver.lib.outgoing_http import OutgoingSession
+
+
+class TorDataSession(OutgoingSession):
+    def __init__(self, max_retries: int) -> None:
+        Retry.DEFAULT_BACKOFF_MAX = 64
+        retry = Retry(
+            total=max_retries,
+            backoff_factor=2.0,
+            status_forcelist={  # Retry on these
+                429,  # The formal rate-limiting response code
+                500,  # Server error
+                502,  # Bad gateway
+                503,  # Service unavailable
+            },
+        )
+        super().__init__(role="tor_data", timeout=3, max_retries=retry)
+
+
+class Command(ZulipBaseCommand):
+    help = """Fetch the list of TOR exit nodes, and write the list of IP addresses
+to a file for access from Django for rate-limiting purposes.
+
+Does nothing unless RATE_LIMIT_TOR_TOGETHER is enabled.
+"""
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        parser.add_argument(
+            "--max-retries",
+            type=int,
+            default=10,
+            help="Number of times to retry fetching data from TOR",
+        )
+
+    @override
+    @abort_cron_during_deploy
+    @abort_unless_locked
+    def handle(self, *args: Any, **options: Any) -> None:
+        if not settings.RATE_LIMIT_TOR_TOGETHER:
+            return
+
+        session = TorDataSession(max_retries=options["max_retries"])
+        response = session.get("https://check.torproject.org/exit-addresses")
+        response.raise_for_status()
+
+        # Format:
+        #     ExitNode 4273E6D162ED2717A1CF4207A254004CD3F5307B
+        #     Published 2021-11-02 11:01:07
+        #     LastStatus 2021-11-02 23:00:00
+        #     ExitAddress 176.10.99.200 2021-11-02 23:17:02
+        exit_nodes: set[str] = set()
+        for line in response.text.splitlines():
+            if line.startswith("ExitAddress "):
+                exit_nodes.add(line.split()[1])
+
+        # Write to a tmpfile to ensure we can't read a partially-written file
+        with open(settings.TOR_EXIT_NODE_FILE_PATH + ".tmp", "wb") as f:
+            f.write(orjson.dumps(list(exit_nodes)))
+
+        # Do an atomic rename into place
+        os.rename(
+            settings.TOR_EXIT_NODE_FILE_PATH + ".tmp",
+            settings.TOR_EXIT_NODE_FILE_PATH,
+        )

zerver/management/commands/fill_memcached_caches.py

@@ -0,0 +1,24 @@
+from argparse import ArgumentParser
+from typing import Any
+
+from typing_extensions import override
+
+from zerver.lib.cache_helpers import cache_fillers, fill_remote_cache
+from zerver.lib.management import ZulipBaseCommand
+
+
+class Command(ZulipBaseCommand):
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        parser.add_argument(
+            "--cache", help="Populate one specific cache", choices=cache_fillers.keys()
+        )
+
+    @override
+    def handle(self, *args: Any, **options: str | None) -> None:
+        if options["cache"] is not None:
+            fill_remote_cache(options["cache"])
+            return
+
+        for cache in cache_fillers:
+            fill_remote_cache(cache)

zerver/management/commands/generate_realm_creation_link.py

@@ -0,0 +1,40 @@
+from typing import Any
+
+from django.core.management.base import CommandError
+from django.db import ProgrammingError
+from typing_extensions import override
+
+from confirmation.models import generate_realm_creation_url
+from zerver.lib.management import ZulipBaseCommand
+from zerver.models import Realm
+
+
+class Command(ZulipBaseCommand):
+    help = """
+    Outputs a randomly generated, 1-time-use link for Organization creation.
+    Whoever visits the link can create a new organization on this server, regardless of whether
+    settings.OPEN_REALM_CREATION is enabled. The link would expire automatically after
+    settings.CAN_CREATE_REALM_LINK_VALIDITY_DAYS.
+
+    Usage: ./manage.py generate_realm_creation_link """
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        try:
+            # first check if the db has been initialized
+            Realm.objects.first()
+        except ProgrammingError:
+            raise CommandError(
+                "The Zulip database does not appear to exist. Have you run initialize-database?"
+            )
+
+        url = generate_realm_creation_url(by_admin=True)
+        self.stdout.write(
+            self.style.SUCCESS(
+                "Please visit the following secure single-use link to register your "
+            )
+        )
+        self.stdout.write(self.style.SUCCESS("new Zulip organization:\033[0m"))
+        self.stdout.write("")
+        self.stdout.write(self.style.SUCCESS(f"    \033[1;92m{url}\033[0m"))
+        self.stdout.write("")

zerver/management/commands/get_migration_status.py

@@ -0,0 +1,39 @@
+import argparse
+import os
+from typing import Any
+
+from django.db import DEFAULT_DB_ALIAS
+from typing_extensions import override
+
+from scripts.lib.zulip_tools import get_dev_uuid_var_path
+from zerver.lib.management import ZulipBaseCommand
+from zerver.lib.migration_status import get_migration_status
+
+
+class Command(ZulipBaseCommand):
+    help = "Get status of migrations."
+
+    @override
+    def add_arguments(self, parser: argparse.ArgumentParser) -> None:
+        parser.add_argument(
+            "app_label", nargs="?", help="App label of an application to synchronize the state."
+        )
+
+        parser.add_argument(
+            "--database",
+            default=DEFAULT_DB_ALIAS,
+            help='Nominates a database to synchronize. Defaults to the "default" database.',
+        )
+
+        parser.add_argument("--output", help="Path to store the status to (default to stdout).")
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        result = get_migration_status(**options)
+        if options["output"] is not None:
+            uuid_var_path = get_dev_uuid_var_path()
+            path = os.path.join(uuid_var_path, options["output"])
+            with open(path, "w") as f:
+                f.write(result)
+        else:
+            self.stdout.write(result)

zerver/management/commands/import.py

@@ -0,0 +1,107 @@
+# noqa: N999
+
+import argparse
+import os
+import subprocess
+from typing import Any
+
+from django.conf import settings
+from django.core.exceptions import ValidationError
+from django.core.management import call_command
+from django.core.management.base import CommandError, CommandParser
+from typing_extensions import override
+
+from zerver.forms import OverridableValidationError, check_subdomain_available
+from zerver.lib.import_realm import do_import_realm
+from zerver.lib.management import ZulipBaseCommand
+
+
+class Command(ZulipBaseCommand):
+    help = """Import extracted Zulip database dump directories into a fresh Zulip instance.
+
+This command should be used only on a newly created, empty Zulip instance to
+import a database dump from one or more JSON files."""
+
+    @override
+    def add_arguments(self, parser: CommandParser) -> None:
+        parser.add_argument(
+            "--destroy-rebuild-database",
+            action="store_true",
+            help="Destroys and rebuilds the databases prior to import.",
+        )
+
+        parser.add_argument(
+            "--import-into-nonempty",
+            action="store_true",
+            help="Import into an existing nonempty database.",
+        )
+
+        parser.add_argument(
+            "--allow-reserved-subdomain",
+            action="store_true",
+            help="Allow use of reserved subdomains",
+        )
+
+        parser.add_argument("subdomain", metavar="<subdomain>", help="Subdomain")
+
+        parser.add_argument(
+            "export_paths",
+            nargs="+",
+            metavar="<export path>",
+            help="list of export directories to import",
+        )
+        parser.add_argument(
+            "--processes",
+            default=settings.DEFAULT_DATA_EXPORT_IMPORT_PARALLELISM,
+            help="Number of processes to use for uploading Avatars to S3 in parallel",
+        )
+        parser.formatter_class = argparse.RawTextHelpFormatter
+
+    def do_destroy_and_rebuild_database(self, db_name: str) -> None:
+        call_command("flush", verbosity=0, skip_checks=True, interactive=False)
+        subprocess.check_call([os.path.join(settings.DEPLOY_ROOT, "scripts/setup/flush-memcached")])
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        num_processes = int(options["processes"])
+        if num_processes < 1:
+            raise CommandError("You must have at least one process.")
+
+        subdomain = options["subdomain"]
+
+        if options["destroy_rebuild_database"]:
+            print("Rebuilding the database!")
+            db_name = settings.DATABASES["default"]["NAME"]
+            self.do_destroy_and_rebuild_database(db_name)
+        elif options["import_into_nonempty"]:
+            print("NOTE: The argument 'import_into_nonempty' is now the default behavior.")
+
+        allow_reserved_subdomain = False
+
+        if options["allow_reserved_subdomain"]:
+            allow_reserved_subdomain = True
+
+        try:
+            check_subdomain_available(subdomain, allow_reserved_subdomain)
+        except OverridableValidationError as e:
+            raise CommandError(
+                e.messages[0]
+                + "\nPass --allow-reserved-subdomain to override subdomain restrictions."
+            )
+        except ValidationError as e:
+            raise CommandError(e.messages[0])
+
+        paths = []
+        for path in options["export_paths"]:
+            path = os.path.realpath(os.path.expanduser(path))
+            if not os.path.exists(path):
+                raise CommandError(f"Directory not found: '{path}'")
+            if not os.path.isdir(path):
+                raise CommandError(
+                    "Export file should be folder; if it's a tarball, please unpack it first."
+                )
+            paths.append(path)
+
+        for path in paths:
+            print(f"Processing dump: {path} ...")
+            do_import_realm(path, subdomain, num_processes)

zerver/management/commands/list_realms.py

@@ -0,0 +1,80 @@
+import sys
+from argparse import ArgumentParser
+from typing import Any
+
+from typing_extensions import override
+
+from zerver.lib.management import ZulipBaseCommand
+from zerver.models import Realm
+
+
+class Command(ZulipBaseCommand):
+    help = """List realms in the server and it's configuration settings(optional).
+
+Usage examples:
+
+./manage.py list_realms
+./manage.py list_realms --all"""
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        parser.add_argument(
+            "--all", action="store_true", help="Print all the configuration settings of the realms."
+        )
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        realms = Realm.objects.all()
+
+        outer_format = "{:<5} {:<20} {!s:<30} {:<50}"
+        inner_format = "{:<40} {}"
+        deactivated = False
+
+        if not options["all"]:
+            print(outer_format.format("id", "string_id", "name", "domain"))
+            print(outer_format.format("--", "---------", "----", "------"))
+
+            for realm in realms:
+                display_string_id = realm.string_id if realm.string_id != "" else "''"
+                if realm.deactivated:
+                    print(
+                        self.style.ERROR(
+                            outer_format.format(realm.id, display_string_id, realm.name, realm.url)
+                        )
+                    )
+                    deactivated = True
+                else:
+                    print(outer_format.format(realm.id, display_string_id, realm.name, realm.url))
+            if deactivated:
+                print(self.style.WARNING("\nRed rows represent deactivated realms."))
+            sys.exit(0)
+
+        # The remaining code path is the --all case.
+        identifier_attributes = ["id", "name", "string_id"]
+        for realm in realms:
+            # Start with just all the fields on the object, which is
+            # hacky but doesn't require any work to maintain.
+            realm_dict = vars(realm).copy()
+            # Remove a field that is confusingly useless
+            del realm_dict["_state"]
+
+            # This is not an attribute of realm strictly speaking, but valuable info to include.
+            realm_dict["authentication_methods"] = str(realm.authentication_methods_dict())
+
+            for key in identifier_attributes:
+                if realm.deactivated:
+                    print(self.style.ERROR(inner_format.format(key, realm_dict[key])))
+                    deactivated = True
+                else:
+                    print(inner_format.format(key, realm_dict[key]))
+
+            for key, value in sorted(realm_dict.items()):
+                if key not in identifier_attributes:
+                    if realm.deactivated:
+                        print(self.style.ERROR(inner_format.format(key, value)))
+                    else:
+                        print(inner_format.format(key, value))
+            print("-" * 80)
+
+        if deactivated:
+            print(self.style.WARNING("\nRed is used to highlight deactivated realms."))

zerver/management/commands/logout_all_users.py

@@ -0,0 +1,58 @@
+from argparse import ArgumentParser
+from typing import Any
+
+from django.db.models import Q
+from typing_extensions import override
+
+from zerver.actions.user_settings import bulk_regenerate_api_keys
+from zerver.lib.management import ZulipBaseCommand
+from zerver.lib.sessions import (
+    delete_all_deactivated_user_sessions,
+    delete_all_user_sessions,
+    delete_realm_user_sessions,
+)
+from zerver.models import UserProfile
+
+
+class Command(ZulipBaseCommand):
+    help = """\
+Log out all users from active browser sessions.
+
+Does not disable API keys, and thus will not log users out of the
+mobile apps.
+"""
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        parser.add_argument(
+            "--deactivated-only",
+            action="store_true",
+            help="Only log out all users who are deactivated",
+        )
+        parser.add_argument(
+            "--rotate-api-keys",
+            action="store_true",
+            help="Also rotate API keys of the affected users",
+        )
+        self.add_realm_args(parser, help="Only log out all users in a particular realm")
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        realm = self.get_realm(options)
+        rotate_api_keys = options["rotate_api_keys"]
+        if realm:
+            delete_realm_user_sessions(realm)
+            regenerate_api_key_queryset = UserProfile.objects.filter(realm=realm).values_list(
+                "id", flat=True
+            )
+        elif options["deactivated_only"]:
+            delete_all_deactivated_user_sessions()
+            regenerate_api_key_queryset = UserProfile.objects.filter(
+                Q(is_active=False) | Q(realm__deactivated=True)
+            ).values_list("id", flat=True)
+        else:
+            delete_all_user_sessions()
+            regenerate_api_key_queryset = UserProfile.objects.values_list("id", flat=True)
+
+        if rotate_api_keys:
+            bulk_regenerate_api_keys(regenerate_api_key_queryset)

zerver/management/commands/makemessages.py

@@ -0,0 +1,308 @@
+"""
+See https://zulip.readthedocs.io/en/latest/translating/internationalization.html
+for background.
+
+The contents of this file are taken from
+https://github.com/niwinz/django-jinja/blob/master/django_jinja/management/commands/makemessages.py
+
+Jinja2's i18n functionality is not exactly the same as Django's.
+In particular, the tags names and their syntax are different:
+
+  1. The Django ``trans`` tag is replaced by a _() global.
+  2. The Django ``blocktrans`` tag is called ``trans``.
+
+(1) isn't an issue, since the whole ``makemessages`` process is based on
+converting the template tags to ``_()`` calls. However, (2) means that
+those Jinja2 ``trans`` tags will not be picked up by Django's
+``makemessages`` command.
+
+There aren't any nice solutions here. While Jinja2's i18n extension does
+come with extraction capabilities built in, the code behind ``makemessages``
+unfortunately isn't extensible, so we can:
+
+  * Duplicate the command + code behind it.
+  * Offer a separate command for Jinja2 extraction.
+  * Try to get Django to offer hooks into makemessages().
+  * Monkey-patch.
+
+We are currently doing that last thing. It turns out there we are lucky
+for once: It's simply a matter of extending two regular expressions.
+Credit for the approach goes to:
+https://stackoverflow.com/questions/2090717
+
+"""
+
+import glob
+import os
+import re
+import subprocess
+from collections.abc import Collection, Iterator, Mapping
+from typing import Any
+
+import orjson
+from django.core.management.base import CommandParser
+from django.core.management.commands import makemessages
+from django.template import engines
+from django.template.backends.jinja2 import Jinja2
+from django.template.base import BLOCK_TAG_END, BLOCK_TAG_START
+from django.utils.translation import template
+from jinja2.environment import Environment
+from typing_extensions import override
+
+strip_whitespace_right = re.compile(
+    rf"({BLOCK_TAG_START}-?\s*(trans|pluralize).*?-{BLOCK_TAG_END})\s+"
+)
+strip_whitespace_left = re.compile(
+    rf"\s+({BLOCK_TAG_START}-\s*(endtrans|pluralize).*?-?{BLOCK_TAG_END})"
+)
+trim_blocks = re.compile(rf"({BLOCK_TAG_START}[-+]?\s*(trans|pluralize)[^+]*?{BLOCK_TAG_END})\n")
+lstrip_blocks = re.compile(
+    rf"^[ \t]+({BLOCK_TAG_START}\s*(endtrans|pluralize).*?[-+]?{BLOCK_TAG_END})",
+    re.MULTILINE,
+)
+
+regexes = [
+    r"{{~?#tr}}([\s\S]*?)(?:~?{{/tr}}|{{~?#\*inline )",  # '.' doesn't match '\n' by default
+    r'{{~?\s*t "([\s\S]*?)"\W*~?}}',
+    r"{{~?\s*t '([\s\S]*?)'\W*~?}}",
+    r'\(t "([\s\S]*?)"\)',
+    r'=\(t "([\s\S]*?)"\)(?=[^{]*}})',
+    r"=\(t '([\s\S]*?)'\)(?=[^{]*}})",
+]
+tags = [
+    ("err_", "error"),
+]
+
+frontend_compiled_regexes = [re.compile(regex) for regex in regexes]
+
+
+def strip_whitespaces(src: str, env: Environment) -> str:
+    src = strip_whitespace_left.sub(r"\1", src)
+    src = strip_whitespace_right.sub(r"\1", src)
+    if env.trim_blocks:
+        src = trim_blocks.sub(r"\1", src)
+    if env.lstrip_blocks:
+        src = lstrip_blocks.sub(r"\1", src)
+    return src
+
+
+# this regex looks for {% trans %} blocks that don't have 'trimmed' or 'notrimmed' set.
+# capturing {% endtrans %} ensures this doesn't affect DTL {% trans %} tags.
+trans_block_re = re.compile(
+    rf"({BLOCK_TAG_START}[-+]?\s*trans)(?!\s+(?:no)?trimmed)"
+    rf"(.*?{BLOCK_TAG_END}.*?{BLOCK_TAG_START}[-+]?\s*?endtrans\s*?[-+]?{BLOCK_TAG_END})",
+    re.DOTALL,
+)
+
+
+def apply_i18n_trimmed_policy(src: str, env: Environment) -> str:
+    # if env.policies["ext.i18n.trimmed"]: insert 'trimmed' flag on jinja {% trans %} blocks.
+    if not env.policies.get("ext.i18n.trimmed", False):
+        return src
+    return trans_block_re.sub(r"\1 trimmed \2", src)
+
+
+class Command(makemessages.Command):
+    xgettext_options = makemessages.Command.xgettext_options
+    for func, tag in tags:
+        xgettext_options += [f'--keyword={func}:1,"{tag}"']
+
+    @override
+    def add_arguments(self, parser: CommandParser) -> None:
+        super().add_arguments(parser)
+        parser.add_argument(
+            "--frontend-source",
+            default="web/templates",
+            help="Name of the Handlebars template directory",
+        )
+        parser.add_argument(
+            "--frontend-output",
+            default="locale",
+            help="Name of the frontend messages output directory",
+        )
+        parser.add_argument(
+            "--frontend-namespace",
+            default="translations.json",
+            help="Namespace of the frontend locale file",
+        )
+        parser.set_defaults(add_location="file")
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        self.handle_django_locales(*args, **options)
+        self.handle_frontend_locales(**options)
+
+    def handle_frontend_locales(
+        self,
+        *,
+        frontend_source: str,
+        frontend_output: str,
+        frontend_namespace: str,
+        locale: list[str],
+        exclude: list[str],
+        all: bool,
+        **options: Any,
+    ) -> None:
+        self.frontend_source = frontend_source
+        self.frontend_output = frontend_output
+        self.frontend_namespace = frontend_namespace
+        self.frontend_locale = locale
+        self.frontend_exclude = exclude
+        self.frontend_all = all
+
+        translation_strings = self.get_translation_strings()
+        self.write_translation_strings(translation_strings)
+
+    def handle_django_locales(self, *args: Any, **options: Any) -> None:
+        old_endblock_re = template.endblock_re
+        old_block_re = template.block_re
+        old_constant_re = template.constant_re
+
+        old_templatize = template.templatize
+        # Extend the regular expressions that are used to detect
+        # translation blocks with an "OR jinja-syntax" clause.
+        template.endblock_re = re.compile(
+            template.endblock_re.pattern + "|" + r"""^[-+]?\s*endtrans\s*[-+]?$"""
+        )
+        template.block_re = re.compile(
+            template.block_re.pattern
+            + "|"
+            + r"""^[-+]?\s*trans(?:\s+(?:no)?trimmed)?(?:\s+(?!'|")(?=.*?=.*?)|\s*[-+]?$)"""
+        )
+        template.plural_re = re.compile(
+            template.plural_re.pattern + "|" + r"""^[-+]?\s*pluralize(?:\s+.+|[-+]?$)"""
+        )
+        template.constant_re = re.compile(r""".*?_\(((?:".*?(?<!\\)")|(?:'.*?(?<!\\)')).*?\)""")
+
+        jinja_engine = engines["Jinja2"]
+        assert isinstance(jinja_engine, Jinja2)
+
+        def my_templatize(src: str, *args: Any, **kwargs: Any) -> str:
+            new_src = strip_whitespaces(src, jinja_engine.env)
+            new_src = apply_i18n_trimmed_policy(new_src, jinja_engine.env)
+            return old_templatize(new_src, *args, **kwargs)
+
+        template.templatize = my_templatize
+
+        try:
+            ignore_patterns = options.get("ignore_patterns", [])
+            ignore_patterns.append("docs/*")
+            ignore_patterns.append("templates/zerver/emails/custom/*")
+            ignore_patterns.append("var/*")
+            options["ignore_patterns"] = ignore_patterns
+            super().handle(*args, **options)
+        finally:
+            template.endblock_re = old_endblock_re
+            template.block_re = old_block_re
+            template.templatize = old_templatize
+            template.constant_re = old_constant_re
+
+    def extract_strings(self, data: str) -> list[str]:
+        translation_strings: list[str] = []
+        for regex in frontend_compiled_regexes:
+            for match in regex.findall(data):
+                match = match.strip()
+                match = " ".join(line.strip() for line in match.splitlines())
+                translation_strings.append(match)
+
+        return translation_strings
+
+    def get_translation_strings(self) -> list[str]:
+        translation_strings: list[str] = []
+        dirname = self.get_template_dir()
+
+        for dirpath, dirnames, filenames in os.walk(dirname):
+            for filename in [f for f in filenames if f.endswith(".hbs")]:
+                if filename.startswith("."):
+                    continue
+                with open(os.path.join(dirpath, filename)) as reader:
+                    data = reader.read()
+                    translation_strings.extend(self.extract_strings(data))
+
+        extracted = subprocess.check_output(
+            [
+                "node_modules/.bin/formatjs",
+                "extract",
+                "--additional-function-names=$t,$t_html",
+                "--format=simple",
+                "--ignore=**/*.d.ts",
+                "web/src/**/*.js",
+                "web/src/**/*.ts",
+            ]
+        )
+        translation_strings.extend(orjson.loads(extracted).values())
+
+        return list(set(translation_strings))
+
+    def get_template_dir(self) -> str:
+        return self.frontend_source
+
+    def get_namespace(self) -> str:
+        return self.frontend_namespace
+
+    def get_locales(self) -> Collection[str]:
+        locale = self.frontend_locale
+        exclude = self.frontend_exclude
+        process_all = self.frontend_all
+
+        # After calling super().handle(), default_locale_path gets set on self
+        # so that we can reuse it here.
+        default_locale_path = self.default_locale_path
+        paths = glob.glob(f"{default_locale_path}/*")
+        all_locales = [os.path.basename(path) for path in paths if os.path.isdir(path)]
+
+        # Account for excluded locales
+        if process_all:
+            return all_locales
+        else:
+            locales = locale or all_locales
+            return set(locales) - set(exclude)
+
+    def get_base_path(self) -> str:
+        return self.frontend_output
+
+    def get_output_paths(self) -> Iterator[str]:
+        base_path = self.get_base_path()
+        locales = self.get_locales()
+        for path in [os.path.join(base_path, locale) for locale in locales]:
+            if not os.path.exists(path):
+                os.makedirs(path)
+
+            yield os.path.join(path, self.get_namespace())
+
+    def get_new_strings(
+        self, old_strings: Mapping[str, str], translation_strings: list[str], locale: str
+    ) -> dict[str, str]:
+        """
+        Missing strings are removed, new strings are added and already
+        translated strings are not touched.
+        """
+        new_strings = {}  # Dict[str, str]
+        for k in translation_strings:
+            if locale == "en":
+                # For English language, translation is equal to the key.
+                new_strings[k] = old_strings.get(k, k)
+            else:
+                new_strings[k] = old_strings.get(k, "")
+
+        return new_strings
+
+    def write_translation_strings(self, translation_strings: list[str]) -> None:
+        for locale, output_path in zip(self.get_locales(), self.get_output_paths(), strict=False):
+            self.stdout.write(f"[frontend] processing locale {locale}")
+            try:
+                with open(output_path, "rb") as reader:
+                    old_strings = orjson.loads(reader.read())
+            except (OSError, ValueError):
+                old_strings = {}
+
+            new_strings = self.get_new_strings(old_strings, translation_strings, locale)
+            with open(output_path, "wb") as writer:
+                writer.write(
+                    orjson.dumps(
+                        new_strings,
+                        option=orjson.OPT_APPEND_NEWLINE
+                        | orjson.OPT_INDENT_2
+                        | orjson.OPT_SORT_KEYS,
+                    )
+                )

zerver/management/commands/merge_streams.py

@@ -0,0 +1,31 @@
+from argparse import ArgumentParser
+from typing import Any
+
+from typing_extensions import override
+
+from zerver.actions.streams import merge_streams
+from zerver.lib.management import ZulipBaseCommand
+from zerver.models.streams import get_stream
+
+
+class Command(ZulipBaseCommand):
+    help = """Merge two streams."""
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        parser.add_argument("stream_to_keep", help="name of stream to keep")
+        parser.add_argument(
+            "stream_to_destroy", help="name of stream to merge into the stream being kept"
+        )
+        self.add_realm_args(parser, required=True)
+
+    @override
+    def handle(self, *args: Any, **options: str) -> None:
+        realm = self.get_realm(options)
+        assert realm is not None  # Should be ensured by parser
+        stream_to_keep = get_stream(options["stream_to_keep"], realm)
+        stream_to_destroy = get_stream(options["stream_to_destroy"], realm)
+        stats = merge_streams(realm, stream_to_keep, stream_to_destroy)
+        print(f"Added {stats[0]} subscriptions")
+        print(f"Moved {stats[1]} messages")
+        print(f"Deactivated {stats[2]} subscriptions")

zerver/management/commands/process_queue.py

@@ -0,0 +1,134 @@
+import logging
+import os
+import signal
+import sys
+import threading
+from argparse import ArgumentParser
+from collections.abc import Iterator
+from contextlib import contextmanager
+from types import FrameType
+from typing import Any
+
+import sentry_sdk
+from django.conf import settings
+from django.core.management.base import CommandError
+from django.utils import autoreload
+from typing_extensions import override
+
+from zerver.lib.management import ZulipBaseCommand
+from zerver.worker.queue_processors import get_active_worker_queues, get_worker
+
+
+@contextmanager
+def log_and_exit_if_exception(
+    logger: logging.Logger, queue_name: str, threaded: bool
+) -> Iterator[None]:
+    try:
+        yield
+    except Exception:
+        logger.exception("Unhandled exception from queue: %s", queue_name, stack_info=True)
+        if threaded:
+            # Sending SIGUSR1 is the right way to exit - triggering the
+            # exit_with_three signal handler, causing exit and reload.
+            os.kill(os.getpid(), signal.SIGUSR1)
+        else:
+            sys.exit(1)
+
+
+class Command(ZulipBaseCommand):
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        parser.add_argument("--queue_name", metavar="<queue name>", help="queue to process")
+        parser.add_argument(
+            "--worker_num", metavar="<worker number>", type=int, default=0, help="worker label"
+        )
+        parser.add_argument("--all", action="store_true", help="run all queues")
+        parser.add_argument(
+            "--multi_threaded",
+            nargs="+",
+            metavar="<list of queue name>",
+            required=False,
+            help="list of queue to process",
+        )
+
+    help = "Runs a queue processing worker"
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        logging.basicConfig()
+        logger = logging.getLogger("process_queue")
+
+        def exit_with_three(signal: int, frame: FrameType | None) -> None:
+            """
+            This process is watched by Django's autoreload, so exiting
+            with status code 3 will cause this process to restart.
+            """
+            logger.warning("SIGUSR1 received. Restarting this queue processor.")
+            sys.exit(3)
+
+        if not settings.USING_RABBITMQ:
+            # Make the warning silent when running the tests
+            if settings.TEST_SUITE:
+                logger.info("Not using RabbitMQ queue workers in the test suite.")
+            else:
+                logger.error("Cannot run a queue processor when USING_RABBITMQ is False!")
+            raise CommandError
+
+        def run_threaded_workers(queues: list[str], logger: logging.Logger) -> None:
+            cnt = 0
+            for queue_name in queues:
+                if not settings.DEVELOPMENT:
+                    logger.info("launching queue worker thread %s", queue_name)
+                cnt += 1
+                td = ThreadedWorker(queue_name, logger)
+                td.start()
+            assert len(queues) == cnt
+            logger.info("%d queue worker threads were launched", cnt)
+
+        if options["all"]:
+            signal.signal(signal.SIGUSR1, exit_with_three)
+            autoreload.run_with_reloader(run_threaded_workers, get_active_worker_queues(), logger)
+        elif options["multi_threaded"]:
+            signal.signal(signal.SIGUSR1, exit_with_three)
+            queues = options["multi_threaded"]
+            autoreload.run_with_reloader(run_threaded_workers, queues, logger)
+        else:
+            queue_name = options["queue_name"]
+            worker_num = options["worker_num"]
+
+            def signal_handler(signal: int, frame: FrameType | None) -> None:
+                logger.info("Worker %d disconnecting from queue %s", worker_num, queue_name)
+                worker.stop()
+                sys.exit(0)
+
+            logger.info("Worker %d connecting to queue %s", worker_num, queue_name)
+            with log_and_exit_if_exception(logger, queue_name, threaded=False):
+                worker = get_worker(queue_name, worker_num=worker_num)
+                with sentry_sdk.isolation_scope() as scope:
+                    scope.set_tag("queue_worker", queue_name)
+                    scope.set_tag("worker_num", worker_num)
+
+                    worker.setup()
+                    signal.signal(signal.SIGTERM, signal_handler)
+                    signal.signal(signal.SIGINT, signal_handler)
+                    signal.signal(signal.SIGUSR1, signal_handler)
+                    worker.start()
+
+
+class ThreadedWorker(threading.Thread):
+    def __init__(self, queue_name: str, logger: logging.Logger) -> None:
+        threading.Thread.__init__(self)
+        self.logger = logger
+        self.queue_name = queue_name
+
+    @override
+    def run(self) -> None:
+        with (
+            sentry_sdk.isolation_scope() as scope,
+            log_and_exit_if_exception(self.logger, self.queue_name, threaded=True),
+        ):
+            scope.set_tag("queue_worker", self.queue_name)
+            worker = get_worker(self.queue_name, threaded=True)
+            worker.setup()
+            logging.debug("starting consuming %s", self.queue_name)
+            worker.start()

zerver/management/commands/promote_new_full_members.py

@@ -0,0 +1,16 @@
+from typing import Any
+
+from typing_extensions import override
+
+from zerver.actions.user_groups import promote_new_full_members
+from zerver.lib.management import ZulipBaseCommand, abort_cron_during_deploy, abort_unless_locked
+
+
+class Command(ZulipBaseCommand):
+    help = """Add users to full members system group."""
+
+    @override
+    @abort_cron_during_deploy
+    @abort_unless_locked
+    def handle(self, *args: Any, **options: Any) -> None:
+        promote_new_full_members()

zerver/management/commands/purge_queue.py

@@ -0,0 +1,42 @@
+from argparse import ArgumentParser
+from typing import Any
+
+from django.core.management import CommandError
+from typing_extensions import override
+
+from zerver.lib.management import ZulipBaseCommand
+from zerver.lib.queue import SimpleQueueClient
+from zerver.worker.queue_processors import get_active_worker_queues
+
+
+class Command(ZulipBaseCommand):
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        parser.add_argument(dest="queue_name", nargs="?", help="queue to purge")
+        parser.add_argument("--all", action="store_true", help="purge all queues")
+
+    help = "Discards all messages from the given queue"
+
+    @override
+    def handle(self, *args: Any, **options: str) -> None:
+        def purge_queue(queue_name: str) -> None:
+            queue = SimpleQueueClient()
+            queue.ensure_queue(queue_name, lambda channel: channel.queue_purge(queue_name))
+
+        if options["all"]:
+            for queue_name in get_active_worker_queues():
+                purge_queue(queue_name)
+            print("All queues purged")
+        elif not options["queue_name"]:
+            raise CommandError("Missing queue_name argument!")
+        else:
+            queue_name = options["queue_name"]
+            if not (
+                queue_name in get_active_worker_queues() or queue_name.startswith("notify_tornado")
+            ):
+                raise CommandError(f"Unknown queue {queue_name}")
+
+            print(f"Purging queue {queue_name}")
+            purge_queue(queue_name)
+
+        print("Done")

zerver/management/commands/query_ldap.py

@@ -0,0 +1,20 @@
+from argparse import ArgumentParser
+from typing import Any
+
+from typing_extensions import override
+
+from zerver.lib.management import ZulipBaseCommand
+from zproject.backends import query_ldap
+
+
+class Command(ZulipBaseCommand):
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        parser.add_argument("email", metavar="<email>", help="email of user to query")
+
+    @override
+    def handle(self, *args: Any, **options: str) -> None:
+        email = options["email"]
+        values = query_ldap(email)
+        for value in values:
+            print(value)

zerver/management/commands/rate_limit.py

@@ -0,0 +1,73 @@
+from argparse import ArgumentParser
+from typing import Any
+
+from django.core.management.base import CommandError
+from typing_extensions import override
+
+from zerver.lib.management import ZulipBaseCommand
+from zerver.lib.rate_limiter import RateLimitedUser
+from zerver.models import UserProfile
+from zerver.models.users import get_user_profile_by_api_key
+
+
+class Command(ZulipBaseCommand):
+    help = """Manually block or unblock a user from accessing the API"""
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        parser.add_argument("-e", "--email", help="Email account of user.")
+        parser.add_argument("-a", "--api-key", help="API key of user.")
+        parser.add_argument("-s", "--seconds", default=60, type=int, help="Seconds to block for.")
+        parser.add_argument(
+            "-d",
+            "--domain",
+            default="api_by_user",
+            help="Rate-limiting domain. Defaults to 'api_by_user'.",
+        )
+        parser.add_argument(
+            "-b",
+            "--all-bots",
+            dest="bots",
+            action="store_true",
+            help="Whether or not to also block all bots for this user.",
+        )
+        parser.add_argument(
+            "operation",
+            metavar="<operation>",
+            choices=["block", "unblock"],
+            help="operation to perform (block or unblock)",
+        )
+        self.add_realm_args(parser)
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        if (not options["api_key"] and not options["email"]) or (
+            options["api_key"] and options["email"]
+        ):
+            raise CommandError("Please enter either an email or API key to manage")
+
+        realm = self.get_realm(options)
+        if options["email"]:
+            user_profile = self.get_user(options["email"], realm)
+        else:
+            try:
+                user_profile = get_user_profile_by_api_key(options["api_key"])
+            except UserProfile.DoesNotExist:
+                raise CommandError(
+                    "Unable to get user profile for API key {}".format(options["api_key"])
+                )
+
+        users = [user_profile]
+        if options["bots"]:
+            users.extend(
+                bot for bot in UserProfile.objects.filter(is_bot=True, bot_owner=user_profile)
+            )
+
+        operation = options["operation"]
+        for user in users:
+            print(f"Applying operation to User ID: {user.id}: {operation}")
+
+            if operation == "block":
+                RateLimitedUser(user, domain=options["domain"]).block_access(options["seconds"])
+            elif operation == "unblock":
+                RateLimitedUser(user, domain=options["domain"]).unblock_access()

zerver/management/commands/reactivate_realm.py

@@ -0,0 +1,26 @@
+from argparse import ArgumentParser
+from typing import Any
+
+from typing_extensions import override
+
+from zerver.actions.realm_settings import do_reactivate_realm
+from zerver.lib.management import ZulipBaseCommand
+
+
+class Command(ZulipBaseCommand):
+    help = """Script to reactivate a deactivated realm."""
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        self.add_realm_args(parser, required=True)
+
+    @override
+    def handle(self, *args: Any, **options: str) -> None:
+        realm = self.get_realm(options)
+        assert realm is not None  # Should be ensured by parser
+        if not realm.deactivated:
+            print("Realm", options["realm_id"], "is already active.")
+            return
+        print("Reactivating", options["realm_id"])
+        do_reactivate_realm(realm)
+        print("Done!")

zerver/management/commands/realm_domain.py

@@ -0,0 +1,67 @@
+import sys
+from argparse import ArgumentParser
+from typing import Any
+
+from django.core.exceptions import ValidationError
+from django.core.management.base import CommandError
+from django.db.utils import IntegrityError
+from typing_extensions import override
+
+from zerver.lib.domains import validate_domain
+from zerver.lib.management import ZulipBaseCommand
+from zerver.models import RealmDomain
+from zerver.models.realms import get_realm_domains
+
+
+class Command(ZulipBaseCommand):
+    help = """Manage domains for the specified realm"""
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        parser.add_argument(
+            "--op", default="show", help="What operation to do (add, show, remove)."
+        )
+        parser.add_argument(
+            "--allow-subdomains", action="store_true", help="Whether subdomains are allowed or not."
+        )
+        parser.add_argument("domain", metavar="<domain>", nargs="?", help="domain to add or remove")
+        self.add_realm_args(parser, required=True)
+
+    @override
+    def handle(self, *args: Any, **options: str | bool) -> None:
+        realm = self.get_realm(options)
+        assert realm is not None  # Should be ensured by parser
+        if options["op"] == "show":
+            print(f"Domains for {realm.string_id}:")
+            for realm_domain in get_realm_domains(realm):
+                assert isinstance(realm_domain["domain"], str)
+                if realm_domain["allow_subdomains"]:
+                    print(realm_domain["domain"] + " (subdomains allowed)")
+                else:
+                    print(realm_domain["domain"] + " (subdomains not allowed)")
+            sys.exit(0)
+
+        assert isinstance(options["domain"], str)
+        domain = options["domain"].strip().lower()
+        try:
+            validate_domain(domain)
+        except ValidationError as e:
+            raise CommandError(e.messages[0])
+        if options["op"] == "add":
+            assert isinstance(options["allow_subdomains"], bool)
+            try:
+                RealmDomain.objects.create(
+                    realm=realm, domain=domain, allow_subdomains=options["allow_subdomains"]
+                )
+                sys.exit(0)
+            except IntegrityError:
+                raise CommandError(f"The domain {domain} is already a part of your organization.")
+        elif options["op"] == "remove":
+            try:
+                RealmDomain.objects.get(realm=realm, domain=domain).delete()
+                sys.exit(0)
+            except RealmDomain.DoesNotExist:
+                raise CommandError("No such entry found!")
+        else:
+            self.print_help("./manage.py", "realm_domain")
+            raise CommandError

zerver/management/commands/register_server.py

@@ -0,0 +1,255 @@
+import os
+import subprocess
+from argparse import ArgumentParser
+from typing import Any
+
+import requests
+from django.conf import settings
+from django.core.management.base import CommandError
+from django.utils.crypto import get_random_string
+from requests.models import Response
+from typing_extensions import override
+
+from zerver.lib.management import ZulipBaseCommand
+from zerver.lib.remote_server import (
+    PushBouncerSession,
+    prepare_for_registration_transfer_challenge,
+    send_json_to_push_bouncer,
+    send_server_data_to_push_bouncer,
+)
+
+if settings.DEVELOPMENT:
+    SECRETS_FILENAME = "zproject/dev-secrets.conf"
+else:
+    SECRETS_FILENAME = "/etc/zulip/zulip-secrets.conf"
+
+
+class Command(ZulipBaseCommand):
+    help = """Register a remote Zulip server for push notifications."""
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        parser.add_argument(
+            "--agree-to-terms-of-service",
+            action="store_true",
+            help="Agree to the Zulipchat Terms of Service: https://zulip.com/policies/terms.",
+        )
+        action = parser.add_mutually_exclusive_group()
+        action.add_argument(
+            "--rotate-key",
+            action="store_true",
+            help="Automatically rotate your server's zulip_org_key",
+        )
+        action.add_argument(
+            "--registration-transfer",
+            action="store_true",
+            help="""\
+If your server uses a publicly verifiable SSL certificate for a
+hostname that is already registered for Zulip services, transfers the
+registration to this server by changing the zulip_org_key secret for
+that registration and saving the updated secret in
+/etc/zulip/zulip-secrets.conf.""",
+        )
+        action.add_argument(
+            "--deactivate",
+            action="store_true",
+            help="Deregister the server; this will stop mobile push notifications",
+        )
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        if not settings.ZULIP_ORG_ID:
+            raise CommandError(
+                "Missing zulip_org_id; run scripts/setup/generate_secrets.py to generate."
+            )
+        if not settings.ZULIP_ORG_KEY:
+            raise CommandError(
+                "Missing zulip_org_key; run scripts/setup/generate_secrets.py to generate."
+            )
+        if not settings.ZULIP_SERVICES_URL:
+            raise CommandError(
+                "ZULIP_SERVICES_URL is not set; was the default incorrectly overridden in /etc/zulip/settings.py?"
+            )
+        if not settings.ZULIP_SERVICE_PUSH_NOTIFICATIONS:
+            raise CommandError(
+                "Please set ZULIP_SERVICE_PUSH_NOTIFICATIONS to True in /etc/zulip/settings.py"
+            )
+
+        if options["deactivate"]:
+            send_json_to_push_bouncer("POST", "server/deactivate", {})
+            print("Mobile Push Notification Service registration successfully deactivated!")
+            return
+
+        hostname = settings.EXTERNAL_HOST
+        request: dict[str, object] = {
+            "zulip_org_id": settings.ZULIP_ORG_ID,
+            "zulip_org_key": settings.ZULIP_ORG_KEY,
+            "hostname": hostname,
+            "contact_email": settings.ZULIP_ADMINISTRATOR,
+        }
+        if options["rotate_key"]:
+            if not os.access(SECRETS_FILENAME, os.W_OK):
+                raise CommandError(f"{SECRETS_FILENAME} is not writable by the current user.")
+            request["new_org_key"] = get_random_string(64)
+
+        print(
+            "This command registers your server for the Mobile Push Notifications Service.\n"
+            "Doing so will share basic metadata with the service's maintainers:\n\n"
+            f"* This server's configured hostname: {request['hostname']}\n"
+            f"* This server's configured contact email address: {request['contact_email']}\n"
+            "* Metadata about each organization hosted by the server; see:\n\n"
+            "    <https://zulip.com/doc-permalinks/basic-metadata>\n\n"
+            "Use of this service is governed by the Zulip Terms of Service:\n\n"
+            "    <https://zulip.com/policies/terms>\n"
+        )
+
+        if not options["agree_to_terms_of_service"] and not options["rotate_key"]:
+            tos_prompt = input(
+                "Do you want to agree to the Zulip Terms of Service and proceed? [Y/n] "
+            )
+            print()
+            if not (
+                tos_prompt.lower() == "y" or tos_prompt.lower() == "" or tos_prompt.lower() == "yes"
+            ):
+                # Exit without registering; no need to print anything
+                # special, as the "n" reply to the query is clear
+                # enough about what happened.
+                return
+
+        if options["registration_transfer"]:
+            org_id, org_key = self.do_registration_transfer_flow(hostname)
+            # We still want to proceed with a regular request to the registration endpoint,
+            # as it'll update the registration with new information such as the contact email.
+            request["zulip_org_id"] = org_id
+            request["zulip_org_key"] = org_key
+            settings.ZULIP_ORG_ID = org_id
+            settings.ZULIP_ORG_KEY = org_key
+            print()
+            print("Proceeding to update the registration with current metadata...")
+
+        response = self._request_push_notification_bouncer_url(
+            "/api/v1/remotes/server/register", request
+        )
+
+        send_server_data_to_push_bouncer(consider_usage_statistics=False, raise_on_error=True)
+
+        if response.json()["created"]:
+            print(
+                "Your server is now registered for the Mobile Push Notification Service!\n"
+                "Return to the documentation for next steps."
+            )
+        else:
+            if options["rotate_key"]:
+                print(f"Success! Updating {SECRETS_FILENAME} with the new key...")
+                new_org_key = request["new_org_key"]
+                assert isinstance(new_org_key, str)
+                subprocess.check_call(
+                    [
+                        "crudini",
+                        "--inplace",
+                        "--set",
+                        SECRETS_FILENAME,
+                        "secrets",
+                        "zulip_org_key",
+                        new_org_key,
+                    ]
+                )
+            print("Mobile Push Notification Service registration successfully updated!")
+
+        if options["registration_transfer"]:
+            print()
+            print(
+                "Make sure to restart the server next by running /home/zulip/deployments/current/scripts/restart-server "
+                "so that the new credentials are reloaded."
+            )
+
+    def do_registration_transfer_flow(self, hostname: str) -> tuple[str, str]:
+        params = {"hostname": hostname}
+        response = self._request_push_notification_bouncer_url(
+            "/api/v1/remotes/server/register/transfer", params
+        )
+        verification_secret = response.json()["verification_secret"]
+
+        print(
+            "Received a verification secret from the service. Preparing to serve it at the verification URL."
+        )
+        token_for_push_bouncer = prepare_for_registration_transfer_challenge(verification_secret)
+
+        print("Sending ACK to the service and awaiting completion of verification...")
+        response = self._request_push_notification_bouncer_url(
+            "/api/v1/remotes/server/register/verify_challenge",
+            dict(hostname=params["hostname"], access_token=token_for_push_bouncer),
+        )
+
+        org_id = response.json()["zulip_org_id"]
+        org_key = response.json()["zulip_org_key"]
+        # Update the secrets file.
+        print("Success! Updating secrets file with received credentials.")
+        subprocess.check_call(
+            [
+                "crudini",
+                "--inplace",
+                "--set",
+                SECRETS_FILENAME,
+                "secrets",
+                "zulip_org_id",
+                org_id,
+            ]
+        )
+        subprocess.check_call(
+            [
+                "crudini",
+                "--inplace",
+                "--set",
+                SECRETS_FILENAME,
+                "secrets",
+                "zulip_org_key",
+                org_key,
+            ]
+        )
+        print("Mobile Push Notification Service registration successfully transferred.")
+        return org_id, org_key
+
+    def _request_push_notification_bouncer_url(self, url: str, params: dict[str, Any]) -> Response:
+        assert settings.ZULIP_SERVICES_URL is not None
+        request_url = settings.ZULIP_SERVICES_URL + url
+        session = PushBouncerSession()
+        try:
+            response = session.post(request_url, data=params)
+        except requests.RequestException:
+            raise CommandError(
+                "Network error connecting to push notifications service "
+                f"({settings.ZULIP_SERVICES_URL})",
+            )
+        try:
+            response.raise_for_status()
+        except requests.HTTPError as e:
+            # Report nice errors from the Zulip API if possible.
+            try:
+                content_dict = response.json()
+            except Exception:
+                raise e
+
+            if (
+                "code" in content_dict
+                and content_dict["code"] == "HOSTNAME_ALREADY_IN_USE_BOUNCER_ERROR"
+            ):
+                print(
+                    "--------------------------------\n"
+                    "The hostname is already in use by another server. If you control the hostname \n"
+                    "and want to transfer the registration to this server, you can run manage.py register_server \n"
+                    "with the --registration-transfer flag.\n"
+                    "Note that this will invalidate old credentials if another server is still using them.\n"
+                    "\n"
+                    "For more information, see: \n"
+                    "\n"
+                    "<https://zulip.com/doc-permalinks/registration-transfer>"
+                )
+                raise CommandError
+
+            error_message = content_dict["msg"]
+            raise CommandError(
+                f'Error received from the push notification service: "{error_message}"'
+            )
+
+        return response

zerver/management/commands/remove_users_from_stream.py

@@ -0,0 +1,39 @@
+from typing import Any
+
+from django.core.management.base import CommandParser
+from typing_extensions import override
+
+from zerver.actions.streams import bulk_remove_subscriptions
+from zerver.lib.management import ZulipBaseCommand
+from zerver.models.streams import get_stream
+
+
+class Command(ZulipBaseCommand):
+    help = """Remove some or all users in a realm from a stream."""
+
+    @override
+    def add_arguments(self, parser: CommandParser) -> None:
+        parser.add_argument("-s", "--stream", required=True, help="A stream name.")
+
+        self.add_realm_args(parser, required=True)
+        self.add_user_list_args(
+            parser, all_users_help="Remove all users in realm from this stream."
+        )
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        realm = self.get_realm(options)
+        assert realm is not None  # Should be ensured by parser
+        user_profiles = self.get_users(options, realm)
+        stream_name = options["stream"].strip()
+        stream = get_stream(stream_name, realm)
+
+        result = bulk_remove_subscriptions(realm, user_profiles, [stream], acting_user=None)
+        not_subscribed = result[1]
+        not_subscribed_users = {tup[0] for tup in not_subscribed}
+
+        for user_profile in user_profiles:
+            if user_profile in not_subscribed_users:
+                print(f"{user_profile.delivery_email} was not subscribed")
+            else:
+                print(f"Removed {user_profile.delivery_email} from {stream_name}")

zerver/management/commands/reset_authentication_attempt_count.py

@@ -0,0 +1,25 @@
+from argparse import ArgumentParser
+from typing import Any
+
+from django.core.management.base import CommandError
+from typing_extensions import override
+
+from zerver.lib.management import ZulipBaseCommand
+from zproject.backends import RateLimitedAuthenticationByUsername
+
+
+class Command(ZulipBaseCommand):
+    help = """Reset the rate limit for authentication attempts for username."""
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        parser.add_argument("-u", "--username", help="Username to reset the rate limit for.")
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        if not options["username"]:
+            self.print_help("./manage.py", "reset_authentication_attempt_count")
+            raise CommandError("Please enter a username")
+
+        username = options["username"]
+        RateLimitedAuthenticationByUsername(username).clear_history()

zerver/management/commands/restore_messages.py

@@ -0,0 +1,62 @@
+from typing import Any
+
+from django.core.management.base import CommandParser
+from typing_extensions import override
+
+from zerver.lib.management import ZulipBaseCommand
+from zerver.lib.retention import (
+    restore_all_data_from_archive,
+    restore_data_from_archive,
+    restore_data_from_archive_by_realm,
+)
+from zerver.models import ArchiveTransaction
+
+
+class Command(ZulipBaseCommand):
+    help = """
+Restore recently deleted messages from the archive, that
+have not been vacuumed (because the time limit of
+ARCHIVED_DATA_VACUUMING_DELAY_DAYS has not passed).
+
+Intended primarily for use after against potential bugs in
+Zulip's message retention and deletion features.
+
+Examples:
+To restore all recently deleted messages:
+  ./manage.py restore_messages --all --restore-deleted
+To restore a specific ArchiveTransaction:
+  ./manage.py restore_messages --transaction-id=1
+"""
+
+    @override
+    def add_arguments(self, parser: CommandParser) -> None:
+        target = parser.add_mutually_exclusive_group(required=True)
+        target.add_argument(
+            "--all",
+            action="store_true",
+            help="Restore archived messages from all realms. "
+            "(Does not restore manually deleted messages.)",
+        )
+        parser.add_argument(
+            "--restore-deleted",
+            action="store_true",
+            help="With --all, also restores manually deleted messages.",
+        )
+        target.add_argument(
+            "-t", "--transaction-id", type=int, help="Restore a specific ArchiveTransaction."
+        )
+        self.add_realm_args(
+            target,
+            help="Restore archived messages from the specified realm. "
+            "(Does not restore manually deleted messages.)",
+        )
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        realm = self.get_realm(options)
+        if realm:
+            restore_data_from_archive_by_realm(realm)
+        elif options["transaction_id"]:
+            restore_data_from_archive(ArchiveTransaction.objects.get(id=options["transaction_id"]))
+        elif options["all"]:
+            restore_all_data_from_archive(restore_manual_transactions=options["restore_deleted"])

zerver/management/commands/runtornado.py

@@ -0,0 +1,147 @@
+import asyncio
+import logging
+import signal
+import sys
+from contextlib import AsyncExitStack
+from typing import Any
+from urllib.parse import SplitResult
+
+from asgiref.sync import async_to_sync, sync_to_async
+from django.conf import settings
+from django.core.management.base import CommandError, CommandParser
+from typing_extensions import override
+
+from zerver.lib.management import ZulipBaseCommand
+
+if settings.PRODUCTION:
+    settings.SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
+
+from zerver.lib.debug import interactive_debug_listen
+from zerver.tornado.application import create_tornado_application, setup_tornado_rabbitmq
+from zerver.tornado.descriptors import set_current_port
+from zerver.tornado.event_queue import (
+    add_client_gc_hook,
+    dump_event_queues,
+    get_wrapped_process_notification,
+    missedmessage_hook,
+    setup_event_queue,
+)
+from zerver.tornado.sharding import notify_tornado_queue_name
+
+if settings.USING_RABBITMQ:
+    from zerver.lib.queue import TornadoQueueClient, set_queue_client
+
+if sys.version_info < (3, 14):
+    from zerver.lib.async_utils import NoAutoCreateEventLoopPolicy
+
+    asyncio.set_event_loop_policy(NoAutoCreateEventLoopPolicy())  # type: ignore[deprecated] # Python < 3.14 compatibility
+
+
+class Command(ZulipBaseCommand):
+    help = "Starts a Tornado Web server wrapping Django."
+
+    @override
+    def add_arguments(self, parser: CommandParser) -> None:
+        parser.add_argument("--autoreload", action="store_true", help="Enable Tornado autoreload")
+        parser.add_argument(
+            "--immediate-reloads",
+            action="store_true",
+            help="Tell web app clients to immediately reload after Tornado starts",
+        )
+        parser.add_argument(
+            "addrport",
+            help="[port number or ipaddr:port]",
+        )
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        interactive_debug_listen()
+        addrport = options["addrport"]
+        assert isinstance(addrport, str)
+
+        from tornado import httpserver
+
+        if addrport.isdigit():
+            addr, port = "", int(addrport)
+        else:
+            r = SplitResult("", addrport, "", "", "")
+            if r.port is None:
+                raise CommandError(f"{addrport!r} does not have a valid port number.")
+            addr, port = r.hostname or "", r.port
+
+        if not addr:
+            addr = "127.0.0.1"
+
+        if settings.DEBUG:
+            logging.basicConfig(
+                level=logging.INFO, format="%(asctime)s %(levelname)-8s %(message)s"
+            )
+
+        async def inner_run() -> None:
+            from django.utils import translation
+
+            loop = asyncio.get_running_loop()
+            stop_fut = loop.create_future()
+
+            def stop() -> None:
+                if not stop_fut.done():
+                    stop_fut.set_result(None)
+
+            def add_signal_handlers() -> None:
+                loop.add_signal_handler(signal.SIGINT, stop)
+                loop.add_signal_handler(signal.SIGTERM, stop)
+
+            def remove_signal_handlers() -> None:
+                loop.remove_signal_handler(signal.SIGINT)
+                loop.remove_signal_handler(signal.SIGTERM)
+
+            async with AsyncExitStack() as stack:
+                stack.push_async_callback(
+                    sync_to_async(remove_signal_handlers, thread_sensitive=True)
+                )
+                await sync_to_async(add_signal_handlers, thread_sensitive=True)()
+
+                set_current_port(port)
+                translation.activate(settings.LANGUAGE_CODE)
+
+                if settings.CUSTOM_DEVELOPMENT_SETTINGS:
+                    print("Using custom settings from zproject/custom_dev_settings.py.")
+
+                # We pass display_num_errors=False, since Django will
+                # likely display similar output anyway.
+                if not options["skip_checks"]:
+                    self.check(display_num_errors=False)
+                print(f"Tornado server (re)started on port {port}")
+
+                if settings.USING_RABBITMQ:
+                    queue_client = TornadoQueueClient()
+                    set_queue_client(queue_client)
+                    # Process notifications received via RabbitMQ
+                    queue_name = notify_tornado_queue_name(port)
+                    stack.callback(queue_client.close)
+                    queue_client.start_json_consumer(
+                        queue_name, get_wrapped_process_notification(queue_name)
+                    )
+
+                # Application is an instance of Django's standard wsgi handler.
+                application = create_tornado_application(autoreload=options["autoreload"])
+
+                # start tornado web server in single-threaded mode
+                http_server = httpserver.HTTPServer(application, xheaders=True)
+                stack.push_async_callback(http_server.close_all_connections)
+                stack.callback(http_server.stop)
+                http_server.listen(port, address=addr)
+
+                from zerver.tornado.ioloop_logging import logging_data
+
+                logging_data["port"] = str(port)
+                send_reloads = options.get("immediate_reloads", False)
+                await setup_event_queue(http_server, port, send_reloads)
+                stack.callback(dump_event_queues, port)
+                add_client_gc_hook(missedmessage_hook)
+                if settings.USING_RABBITMQ:
+                    setup_tornado_rabbitmq(queue_client)
+
+                await stop_fut
+
+        async_to_sync(inner_run, force_new_loop=True)()

zerver/management/commands/runtusd.py

@@ -0,0 +1,82 @@
+import os
+from typing import Any
+from urllib.parse import SplitResult
+
+from django.conf import settings
+from django.core.management.base import BaseCommand, CommandError, CommandParser
+from typing_extensions import override
+
+from scripts.lib.zulip_tools import get_config, get_config_file
+
+
+class Command(BaseCommand):
+    help = """Starts the tusd server"""
+
+    @override
+    def add_arguments(self, parser: CommandParser) -> None:
+        parser.add_argument("listen", help="[Port, or address:port, to bind HTTP server to]")
+        local_port = 80
+        config_file = get_config_file()
+        if get_config(config_file, "application_server", "http_only", False):
+            local_port = int(
+                get_config(config_file, "application_server", "nginx_listen_port", "80")
+            )
+        parser.add_argument(
+            "hooks_http",
+            help="[An HTTP endpoint to which hook events will be sent to]",
+            default=f"http://127.0.0.1:{local_port}/api/internal/tusd",
+            nargs="?",
+        )
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        listen = options["listen"]
+        if listen.isdigit():
+            addr, port = "127.0.0.1", int(listen)
+        else:
+            r = SplitResult("", listen, "", "", "")
+            if r.port is None:
+                raise CommandError(f"{listen!r} does not have a valid port number.")
+            addr, port = r.hostname or "127.0.0.1", r.port
+
+        hooks_http = options["hooks_http"]
+
+        # https://tus.github.io/tusd/getting-started/configuration/
+        # We do not set a maximum upload size, as the pre-create hooks
+        # will set the max size that they want, based on the intended
+        # use of the uploaded file.
+        tusd_args = [
+            "tusd",
+            "-base-path=/api/v1/tus/",
+            f"-port={port}",
+            f"-host={addr}",
+            "-behind-proxy",
+            f"-hooks-http={hooks_http}",
+            "-hooks-http-forward-headers=Cookie,Authorization",
+            "--hooks-enabled-events=pre-create,pre-finish,pre-terminate",
+            "-disable-download",
+            "--show-startup-logs=false",
+        ]
+        env_vars = os.environ.copy()
+        if settings.LOCAL_UPLOADS_DIR is not None:
+            assert settings.LOCAL_FILES_DIR is not None
+            tusd_args.append(f"-upload-dir={settings.LOCAL_FILES_DIR}")
+        elif settings.S3_ENDPOINT_URL in (
+            "https://storage.googleapis.com",
+            "https://storage.googleapis.com/",
+        ):
+            tusd_args.append(f"-gcs-bucket={settings.S3_AUTH_UPLOADS_BUCKET}")
+            env_vars["GCS_SERVICE_ACCOUNT_FILE"] = "/etc/zulip/gcp_key.json"
+        else:
+            tusd_args.append(f"-s3-bucket={settings.S3_AUTH_UPLOADS_BUCKET}")
+            if settings.S3_ENDPOINT_URL is not None:
+                tusd_args.append(f"-s3-endpoint={settings.S3_ENDPOINT_URL}")
+            if settings.S3_KEY is not None:
+                env_vars["AWS_ACCESS_KEY_ID"] = settings.S3_KEY
+            if settings.S3_SECRET_KEY is not None:
+                env_vars["AWS_SECRET_ACCESS_KEY"] = settings.S3_SECRET_KEY
+            if settings.S3_REGION is not None:
+                env_vars["AWS_REGION"] = settings.S3_REGION
+            if settings.S3_SKIP_CHECKSUM:
+                env_vars["AWS_REQUEST_CHECKSUM_CALCULATION"] = "when_required"
+        os.execvpe("tusd", tusd_args, env_vars)

zerver/management/commands/scrub_realm.py

@@ -0,0 +1,28 @@
+from argparse import ArgumentParser
+from typing import Any
+
+from django.core.management.base import CommandError
+from typing_extensions import override
+
+from zerver.actions.realm_settings import do_scrub_realm
+from zerver.lib.management import ZulipBaseCommand
+
+
+class Command(ZulipBaseCommand):
+    help = """Script to scrub a deactivated realm."""
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        self.add_realm_args(parser, required=True)
+
+    @override
+    def handle(self, *args: Any, **options: str) -> None:
+        realm = self.get_realm(options)
+        assert realm is not None  # Should be ensured by parser
+        if not realm.deactivated:
+            raise CommandError(
+                f"Realm {options['realm_id']} is active. Please deactivate the realm the first."
+            )
+        print("Scrubbing", options["realm_id"])
+        do_scrub_realm(realm, acting_user=None)
+        print("Done!")

zerver/management/commands/send_custom_email.py

@@ -0,0 +1,268 @@
+from argparse import ArgumentParser
+from collections.abc import Callable
+from typing import Any
+
+import orjson
+from django.conf import settings
+from django.db.models import Q, QuerySet
+from typing_extensions import override
+
+from confirmation.models import one_click_unsubscribe_link
+from zerver.lib.management import ZulipBaseCommand
+from zerver.lib.send_email import custom_email_sender, send_custom_email, send_custom_server_email
+from zerver.models import Realm, UserProfile
+
+if settings.ZILENCER_ENABLED:
+    from corporate.lib.stripe import BILLING_SUPPORT_EMAIL
+    from zilencer.models import RemoteRealmBillingUser, RemoteServerBillingUser, RemoteZulipServer
+
+
+class Command(ZulipBaseCommand):
+    help = """
+    Send a custom email with Zulip branding to the specified users.
+
+    Useful to send a notice to all users of a realm or server.
+
+    The From and Subject headers can be provided in the body of the Markdown
+    document used to generate the email, or on the command line."""
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        targets = parser.add_mutually_exclusive_group(required=True)
+        targets.add_argument(
+            "--entire-server", action="store_true", help="Send to every user on the server."
+        )
+        targets.add_argument(
+            "--marketing",
+            action="store_true",
+            help="Send to active users and realm owners with the enable_marketing_emails setting enabled.",
+        )
+        targets.add_argument(
+            "--remote-servers",
+            action="store_true",
+            help="Send to registered contact email addresses for remote Zulip servers.",
+        )
+        targets.add_argument(
+            "--announce-release",
+            metavar="VERSION",
+            help="Announce a major or minor release to remote servers.",
+        )
+        targets.add_argument(
+            "--all-sponsored-org-admins",
+            action="store_true",
+            help="Send to all organization administrators of sponsored organizations.",
+        )
+        self.add_user_list_args(
+            targets,
+            help="Email addresses of user(s) to send emails to.",
+            all_users_help="Send to every user on the realm.",
+        )
+        # Realm is only required for --users and --all-users, so it is
+        # not mutually exclusive with the rest of the above.
+        self.add_realm_args(parser)
+
+        # This is an additional filter on the above.  It ideally would
+        # be in the mutually-exclusive set, but we would like to reuse
+        # it with --remote-servers
+        parser.add_argument(
+            "--json-file",
+            help="Load the JSON file, and send to the users whose ids are the keys in that dict; "
+            "the context for each email will be extended by each value in the dict.",
+        )
+
+        # This is an additional filter which is composed with the above
+        parser.add_argument(
+            "--admins-only",
+            help="Filter recipients selected via other options to only organization administrators",
+            action="store_true",
+        )
+
+        parser.add_argument(
+            "--markdown-template-path",
+            "--path",
+            required=True,
+            help="Path to a Markdown-format body for the email.",
+        )
+        parser.add_argument(
+            "--subject",
+            help="Subject for the email. It can be declared in Markdown file in headers",
+        )
+        parser.add_argument(
+            "--campaign-name",
+            required=True,
+            help="Unique identifier for this email campaign (e.g., 'q4_2025_update') to prevent duplicate sends.",
+        )
+        parser.add_argument(
+            "--from-name",
+            help="From line for the email. It can be declared in Markdown file in headers",
+        )
+        parser.add_argument(
+            "--from-address",
+            help="From email address",
+        )
+        parser.add_argument("--reply-to", help="Optional reply-to line for the email")
+
+        parser.add_argument(
+            "--dry-run",
+            action="store_true",
+            help="Prints emails of the recipients and text of the email.",
+        )
+
+    @override
+    def handle(
+        self, *args: Any, dry_run: bool = False, admins_only: bool = False, **options: str
+    ) -> None:
+        users: QuerySet[UserProfile] = UserProfile.objects.none()
+        add_context: Callable[[dict[str, object], UserProfile], None] | None = None
+        distinct_email = False
+
+        if options["remote_servers"]:
+            servers = RemoteZulipServer.objects.filter(deactivated=False)
+            add_server_context = None
+            if options["json_file"]:
+                with open(options["json_file"]) as f:
+                    server_data: dict[str, dict[str, object]] = orjson.loads(f.read())
+                servers = RemoteZulipServer.objects.filter(
+                    id__in=[int(server_id) for server_id in server_data]
+                )
+
+                def add_server_context_from_dict(
+                    context: dict[str, object], server: RemoteZulipServer
+                ) -> None:
+                    context.update(server_data.get(str(server.id), {}))
+
+                add_server_context = add_server_context_from_dict
+
+            send_custom_server_email(
+                servers,
+                dry_run=dry_run,
+                options=options,
+                add_context=add_server_context,
+            )
+            if dry_run:
+                print("Would send the above email to:")
+                for server in servers:
+                    print(f"  {server.contact_email} ({server.hostname})")
+            return
+        elif options["announce_release"]:
+            server_users = RemoteServerBillingUser.objects.filter(
+                is_active=True,
+                remote_server__deactivated=False,
+            )
+            realm_users = RemoteRealmBillingUser.objects.filter(
+                is_active=True,
+                remote_realm__server__deactivated=False,
+                remote_realm__is_system_bot_realm=False,
+                remote_realm__registration_deactivated=False,
+                remote_realm__realm_deactivated=False,
+                remote_realm__realm_locally_deleted=False,
+            )
+            if options["announce_release"].endswith(".0"):
+                server_users = server_users.filter(enable_major_release_emails=True)
+                realm_users = realm_users.filter(enable_major_release_emails=True)
+            else:
+                server_users = server_users.filter(enable_maintenance_release_emails=True)
+                realm_users = realm_users.filter(enable_maintenance_release_emails=True)
+            # This does an implicit "distinct"
+            all_emails = server_users.union(realm_users).values_list("email", flat=True)
+            del options["from_address"]
+            email_sender = custom_email_sender(
+                dry_run=dry_run, from_address=BILLING_SUPPORT_EMAIL, **options
+            )
+
+            for email in all_emails:
+                email_sender(
+                    to_email=email,
+                    context={
+                        "remote_server_email": True,
+                        "released_version": options["announce_release"],
+                    },
+                )
+            if dry_run:
+                print("Would send the above email to:")
+                for email in all_emails:
+                    print(f"  {email}")
+            return
+
+        if options["entire_server"]:
+            users = UserProfile.objects.filter(
+                is_active=True, is_bot=False, is_mirror_dummy=False, realm__deactivated=False
+            )
+        elif options["marketing"]:
+            # Marketing email sent at most once to each email address
+            # for users who are recently active (!long_term_idle)
+            # users of the product, or who are admins/owners.
+            users = UserProfile.objects.filter(
+                is_active=True,
+                is_bot=False,
+                is_mirror_dummy=False,
+                realm__deactivated=False,
+                enable_marketing_emails=True,
+            ).filter(
+                Q(long_term_idle=False)
+                | Q(
+                    role__in=[
+                        UserProfile.ROLE_REALM_OWNER,
+                        UserProfile.ROLE_REALM_ADMINISTRATOR,
+                    ]
+                )
+            )
+            distinct_email = True
+
+            def add_marketing_unsubscribe(context: dict[str, object], user: UserProfile) -> None:
+                context["unsubscribe_link"] = one_click_unsubscribe_link(user, "marketing")
+
+            add_context = add_marketing_unsubscribe
+
+        elif options["all_sponsored_org_admins"]:
+            # Sends at most one copy to each email address, even if it
+            # is an administrator in several organizations.
+            sponsored_realms = Realm.objects.filter(
+                plan_type=Realm.PLAN_TYPE_STANDARD_FREE, deactivated=False
+            )
+            admin_roles = [UserProfile.ROLE_REALM_ADMINISTRATOR, UserProfile.ROLE_REALM_OWNER]
+            users = UserProfile.objects.filter(
+                is_active=True,
+                is_bot=False,
+                is_mirror_dummy=False,
+                role__in=admin_roles,
+                realm__deactivated=False,
+                realm__in=sponsored_realms,
+            )
+            distinct_email = True
+        else:
+            realm = self.get_realm(options)
+            users = self.get_users(options, realm, is_bot=False)
+
+        if options["json_file"]:
+            with open(options["json_file"]) as f:
+                user_data: dict[str, dict[str, object]] = orjson.loads(f.read())
+            users = users.filter(id__in=[int(user_id) for user_id in user_data])
+
+            def add_context_from_dict(context: dict[str, object], user: UserProfile) -> None:
+                context.update(user_data.get(str(user.id), {}))
+
+            add_context = add_context_from_dict
+
+        if admins_only:
+            users = users.filter(
+                role__in=[UserProfile.ROLE_REALM_ADMINISTRATOR, UserProfile.ROLE_REALM_OWNER]
+            )
+
+        # Only email users who've agreed to the terms of service.
+        if settings.TERMS_OF_SERVICE_VERSION is not None:
+            users = users.exclude(
+                Q(tos_version=None) | Q(tos_version=UserProfile.TOS_VERSION_BEFORE_FIRST_LOGIN)
+            )
+        users = send_custom_email(
+            users,
+            dry_run=dry_run,
+            options=options,
+            add_context=add_context,
+            distinct_email=distinct_email,
+        )
+
+        if dry_run:
+            print("Would send the above email to:")
+            for user in users:
+                print(f"  {user.delivery_email} ({user.realm.string_id})")

zerver/management/commands/send_password_reset_email.py

@@ -0,0 +1,62 @@
+from argparse import ArgumentParser
+from typing import Any
+
+from django.core.management.base import CommandError
+from django.db.models import QuerySet
+from typing_extensions import override
+
+from zerver.actions.users import do_send_password_reset_email
+from zerver.lib.management import ZulipBaseCommand
+from zerver.models import UserProfile
+
+
+class Command(ZulipBaseCommand):
+    help = """Send email to specified email address."""
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        parser.add_argument(
+            "--only-never-logged-in",
+            action="store_true",
+            help="Filter to only users which have not accepted the TOS.",
+        )
+        parser.add_argument(
+            "--entire-server", action="store_true", help="Send to every user on the server. "
+        )
+        self.add_user_list_args(
+            parser,
+            help="Email addresses of user(s) to send password reset emails to.",
+            all_users_help="Send to every user on the realm.",
+        )
+        self.add_realm_args(parser)
+
+    @override
+    def handle(self, *args: Any, **options: str) -> None:
+        if options["entire_server"]:
+            users: QuerySet[UserProfile] = UserProfile.objects.filter(
+                is_active=True, is_bot=False, is_mirror_dummy=False
+            )
+        else:
+            realm = self.get_realm(options)
+            try:
+                users = self.get_users(options, realm, is_bot=False)
+            except CommandError as error:
+                if str(error) == "You have to pass either -u/--users or -a/--all-users.":
+                    raise CommandError(
+                        "You have to pass -u/--users or -a/--all-users or --entire-server."
+                    )
+                raise
+        if options["only_never_logged_in"]:
+            users = users.filter(tos_version=-1)
+
+        if not users.exists():
+            print("No matching users!")
+
+        self.send(users)
+
+    def send(self, users: QuerySet[UserProfile]) -> None:
+        """Sends one-use only links for resetting password to target users"""
+        for user_profile in users:
+            do_send_password_reset_email(
+                user_profile.delivery_email, user_profile.realm, user_profile
+            )

zerver/management/commands/send_realm_reactivation_email.py

@@ -0,0 +1,26 @@
+from argparse import ArgumentParser
+from typing import Any
+
+from django.core.management.base import CommandError
+from typing_extensions import override
+
+from zerver.actions.realm_settings import do_send_realm_reactivation_email
+from zerver.lib.management import ZulipBaseCommand
+
+
+class Command(ZulipBaseCommand):
+    help = """Sends realm reactivation email to admins"""
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        self.add_realm_args(parser, required=True)
+
+    @override
+    def handle(self, *args: Any, **options: str) -> None:
+        realm = self.get_realm(options)
+        assert realm is not None
+        if not realm.deactivated:
+            raise CommandError(f"The realm {realm.name} is already active.")
+        print("Sending email to admins")
+        do_send_realm_reactivation_email(realm, acting_user=None)
+        print("Done!")

zerver/management/commands/send_test_email.py

@@ -0,0 +1,81 @@
+import io
+import smtplib
+from contextlib import redirect_stderr
+from typing import Any
+
+from django.conf import settings
+from django.core.mail import mail_admins, mail_managers, send_mail
+from django.core.management import CommandError
+from django.core.management.commands import sendtestemail
+from typing_extensions import override
+
+from zerver.lib.send_email import FromAddress, log_email_config_errors
+
+
+class Command(sendtestemail.Command):
+    @override
+    def handle(self, *args: Any, **kwargs: str) -> None:
+        if settings.WARN_NO_EMAIL:
+            raise CommandError(
+                "Outgoing email not yet configured, see\n  "
+                "https://zulip.readthedocs.io/en/latest/production/email.html"
+            )
+
+        log_email_config_errors()
+
+        if len(kwargs["email"]) == 0:
+            raise CommandError(
+                "Usage: /home/zulip/deployments/current/manage.py "
+                "send_test_email username@example.com"
+            )
+
+        print("If you run into any trouble, read:")
+        print()
+        print("  https://zulip.readthedocs.io/en/latest/production/email.html#troubleshooting")
+        print()
+        print("The most common error is not setting `ADD_TOKENS_TO_NOREPLY_ADDRESS=False` when")
+        print("using an email provider that doesn't support that feature.")
+        print()
+        print("Sending 2 test emails from:")
+
+        message = (
+            "Success!  If you receive this message (and a second with a different subject), "
+            "you've successfully configured sending emails from your Zulip server.  "
+            "Remember that you need to restart "
+            "the Zulip server with /home/zulip/deployments/current/scripts/restart-server "
+            "after changing the settings in /etc/zulip before your changes will take effect."
+        )
+        with redirect_stderr(io.StringIO()) as f:
+            smtplib.SMTP.debuglevel = 1
+            try:
+                sender = FromAddress.SUPPORT
+                print(f"  * {sender}")
+                send_mail("Zulip email test", message, sender, kwargs["email"])
+
+                noreply_sender = FromAddress.tokenized_no_reply_address()
+                print(f"  * {noreply_sender}")
+                send_mail("Zulip noreply email test", message, noreply_sender, kwargs["email"])
+            except smtplib.SMTPException as e:
+                print(f"Failed to send mails: {e}")
+                print()
+                print("Full SMTP log follows:")
+                print(f.getvalue())
+                raise CommandError("Email sending failed!")
+            except OSError as e:
+                print(f"Failed to send mails: {e}")
+                print()
+                print("Full SMTP log follows:")
+                print(f.getvalue())
+                print()
+                print(
+                    "This usually means that your host or firewall is blocking outgoing SMTP traffic."
+                )
+                raise CommandError("Email sending failed!")
+        print()
+        print("Successfully sent 2 emails to {}!".format(", ".join(kwargs["email"])))
+
+        if kwargs["managers"]:
+            mail_managers("Zulip manager email test", "This email was sent to the site managers.")
+
+        if kwargs["admins"]:
+            mail_admins("Zulip admins email test", "This email was sent to the site admins.")

zerver/management/commands/send_to_email_mirror.py

@@ -0,0 +1,167 @@
+import base64
+import email.parser
+import email.policy
+import os
+from email.message import EmailMessage
+from typing import Any
+
+import orjson
+from django.conf import settings
+from django.core.management.base import CommandError, CommandParser
+from typing_extensions import override
+
+from zerver.lib.email_mirror import validate_to_address
+from zerver.lib.email_mirror_helpers import encode_email_address, get_channel_email_token
+from zerver.lib.management import ZulipBaseCommand
+from zerver.lib.queue import queue_json_publish_rollback_unsafe
+from zerver.models import Realm, UserProfile
+from zerver.models.realms import get_realm
+from zerver.models.streams import get_stream
+from zerver.models.users import get_system_bot, get_user_profile_by_email, get_user_profile_by_id
+
+# This command loads an email from a specified file and sends it
+# to the email mirror. Simple emails can be passed in a JSON file,
+# Look at zerver/tests/fixtures/email/1.json for an example of how
+# it should look. You can also pass a file which has the raw email,
+# for example by writing an email.message.EmailMessage type object
+# to a file using as_string() or as_bytes() methods, or copy-pasting
+# the content of "Show original" on an email in Gmail.
+# See zerver/tests/fixtures/email/1.txt for a very simple example,
+# but anything that the message_from_binary_file function
+# from the email library can parse should work.
+# Value of the TO: header doesn't matter, as it is overridden
+# by the command in order for the email to be sent to the correct stream.
+
+
+class Command(ZulipBaseCommand):
+    help = """
+Send specified email from a fixture file to the email mirror
+Example:
+./manage.py send_to_email_mirror --fixture=zerver/tests/fixtures/emails/filename
+
+"""
+
+    @override
+    def add_arguments(self, parser: CommandParser) -> None:
+        parser.add_argument(
+            "-f",
+            "--fixture",
+            help="The path to the email message you'd like to send "
+            "to the email mirror.\n"
+            "Accepted formats: json or raw email file. "
+            "See zerver/tests/fixtures/email/ for examples",
+        )
+        parser.add_argument(
+            "-s",
+            "--stream",
+            help="The name of the stream to which you'd like to send the message. Default: Denmark",
+        )
+        parser.add_argument(
+            "--sender-id",
+            type=int,
+            help="The ID of a user or bot which should appear as the sender; "
+            "Default: ID of Email gateway bot",
+        )
+
+        self.add_realm_args(parser, help="Specify which realm to connect to; default is zulip")
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        if options["fixture"] is None:
+            self.print_help("./manage.py", "send_to_email_mirror")
+            raise CommandError
+
+        if options["stream"] is None:
+            stream = "Denmark"
+        else:
+            stream = options["stream"]
+
+        realm = self.get_realm(options)
+        if realm is None:
+            realm = get_realm("zulip")
+
+        email_gateway_bot = get_system_bot(settings.EMAIL_GATEWAY_BOT, realm.id)
+        if options["sender_id"] is None:
+            sender = email_gateway_bot
+        else:
+            sender = get_user_profile_by_id(options["sender_id"])
+
+        full_fixture_path = os.path.join(settings.DEPLOY_ROOT, options["fixture"])
+
+        # parse the input email into EmailMessage type and prepare to process_message() it
+        message = self._parse_email_fixture(full_fixture_path)
+        creator = get_user_profile_by_email(message["From"])
+        if (
+            sender.id not in [creator.id, email_gateway_bot.id]
+            and sender.bot_owner_id != creator.id
+        ):
+            raise CommandError(
+                "The sender ID must be either the current user's ID, the email gateway bot's ID, or the ID of a bot owned by the user."
+            )
+        self._prepare_message(message, realm, stream, creator, sender)
+
+        rcpt_to = message["To"].addresses[0].addr_spec
+        validate_to_address(rcpt_to, rate_limit=False)
+
+        queue_json_publish_rollback_unsafe(
+            "email_mirror",
+            {
+                "rcpt_to": rcpt_to,
+                "msg_base64": base64.b64encode(message.as_bytes()).decode(),
+            },
+        )
+
+    def _does_fixture_path_exist(self, fixture_path: str) -> bool:
+        return os.path.exists(fixture_path)
+
+    def _parse_email_json_fixture(self, fixture_path: str) -> EmailMessage:
+        with open(fixture_path, "rb") as fp:
+            json_content = orjson.loads(fp.read())[0]
+
+        message = EmailMessage()
+        message["From"] = json_content["from"]
+        message["Subject"] = json_content["subject"]
+        message.set_content(json_content["body"])
+        return message
+
+    def _parse_email_fixture(self, fixture_path: str) -> EmailMessage:
+        if not self._does_fixture_path_exist(fixture_path):
+            raise CommandError(f"Fixture {fixture_path} does not exist")
+
+        if fixture_path.endswith(".json"):
+            return self._parse_email_json_fixture(fixture_path)
+        else:
+            with open(fixture_path, "rb") as fp:
+                return email.parser.BytesParser(
+                    _class=EmailMessage, policy=email.policy.default
+                ).parse(fp)
+
+    def _prepare_message(
+        self,
+        message: EmailMessage,
+        realm: Realm,
+        stream_name: str,
+        creator: UserProfile,
+        sender: UserProfile,
+    ) -> None:
+        stream = get_stream(stream_name, realm)
+        email_token = get_channel_email_token(stream, creator=creator, sender=sender)
+
+        # The block below ensures that the imported email message doesn't have any recipient-like
+        # headers that are inconsistent with the recipient we want (the stream address).
+        recipient_headers = [
+            "X-Gm-Original-To",
+            "Delivered-To",
+            "Envelope-To",
+            "Resent-To",
+            "Resent-CC",
+            "CC",
+        ]
+        for header in recipient_headers:
+            if header in message:
+                del message[header]
+                message[header] = encode_email_address(stream.name, email_token)
+
+        if "To" in message:
+            del message["To"]
+        message["To"] = encode_email_address(stream.name, email_token)

zerver/management/commands/send_webhook_fixture_message.py

@@ -0,0 +1,104 @@
+import os
+from typing import Any
+
+import orjson
+from django.conf import settings
+from django.core.management.base import CommandError, CommandParser
+from django.test import Client
+from typing_extensions import override
+
+from zerver.lib.management import ZulipBaseCommand
+from zerver.lib.webhooks.common import standardize_headers
+from zerver.models.realms import get_realm
+
+
+class Command(ZulipBaseCommand):
+    help = """
+Create webhook message based on given fixture
+Example:
+./manage.py send_webhook_fixture_message \
+    [--realm=zulip] \
+    --fixture=zerver/webhooks/integration/fixtures/name.json \
+    '--url=/api/v1/external/integration?stream=stream_name&api_key=api_key'
+
+To pass custom headers along with the webhook message use the --custom-headers
+command line option.
+Example:
+    --custom-headers='{"X-Custom-Header": "value"}'
+
+The format is a JSON object, so make sure that the header names do
+not contain any spaces in them and that you use the precise quoting
+approach shown above.
+"""
+
+    @override
+    def add_arguments(self, parser: CommandParser) -> None:
+        parser.add_argument(
+            "-f", "--fixture", help="The path to the fixture you'd like to send into Zulip"
+        )
+
+        parser.add_argument(
+            "-u", "--url", help="The URL on your Zulip server that you want to post the fixture to"
+        )
+
+        parser.add_argument(
+            "-H",
+            "--custom-headers",
+            help="The headers you want to provide along with your mock request to Zulip.",
+        )
+
+        self.add_realm_args(
+            parser, help="Specify which realm/subdomain to connect to; default is zulip"
+        )
+
+    def parse_headers(self, custom_headers: str | None) -> dict[str, str] | None:
+        if not custom_headers:
+            return {}
+        try:
+            custom_headers_dict = orjson.loads(custom_headers)
+        except orjson.JSONDecodeError as ve:
+            raise CommandError(
+                f"Encountered an error while attempting to parse custom headers: {ve}\n"
+                "Note: all strings must be enclosed within \"\" instead of ''"
+            )
+        return standardize_headers(custom_headers_dict)
+
+    @override
+    def handle(self, *args: Any, **options: str | None) -> None:
+        if options["fixture"] is None or options["url"] is None:
+            self.print_help("./manage.py", "send_webhook_fixture_message")
+            raise CommandError
+
+        full_fixture_path = os.path.join(settings.DEPLOY_ROOT, options["fixture"])
+
+        if not self._does_fixture_path_exist(full_fixture_path):
+            raise CommandError("Fixture {} does not exist".format(options["fixture"]))
+
+        headers = self.parse_headers(options["custom_headers"])
+        json = self._get_fixture_as_json(full_fixture_path)
+        realm = self.get_realm(options)
+        if realm is None:
+            realm = get_realm("zulip")
+
+        client = Client()
+        if headers:
+            result = client.post(
+                options["url"],
+                json,
+                content_type="application/json",
+                HTTP_HOST=realm.host,
+                extra=headers,
+            )
+        else:
+            result = client.post(
+                options["url"], json, content_type="application/json", HTTP_HOST=realm.host
+            )
+        if result.status_code != 200:
+            raise CommandError(f"Error status {result.status_code}: {result.content!r}")
+
+    def _does_fixture_path_exist(self, fixture_path: str) -> bool:
+        return os.path.exists(fixture_path)
+
+    def _get_fixture_as_json(self, fixture_path: str) -> bytes:
+        with open(fixture_path, "rb") as f:
+            return orjson.dumps(orjson.loads(f.read()))

zerver/management/commands/send_welcome_bot_message.py

@@ -0,0 +1,25 @@
+from argparse import ArgumentParser
+from typing import Any
+
+from typing_extensions import override
+
+from zerver.lib.management import ZulipBaseCommand
+from zerver.lib.onboarding import send_initial_direct_messages_to_user
+
+
+class Command(ZulipBaseCommand):
+    help = """Sends the initial welcome bot message."""
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        self.add_user_list_args(
+            parser,
+            help="Email addresses of user(s) to send welcome bot messages to.",
+            all_users_help="Send to every user on the realm.",
+        )
+        self.add_realm_args(parser)
+
+    @override
+    def handle(self, *args: Any, **options: str) -> None:
+        for user_profile in self.get_users(options, self.get_realm(options), is_bot=False):
+            send_initial_direct_messages_to_user(user_profile)

zerver/management/commands/send_zulip_update_announcements.py

@@ -0,0 +1,41 @@
+from argparse import ArgumentParser
+from typing import Any
+
+from django.conf import settings
+from typing_extensions import override
+
+from zerver.lib.management import ZulipBaseCommand, abort_cron_during_deploy, abort_unless_locked
+from zerver.lib.zulip_update_announcements import send_zulip_update_announcements
+from zerver.models import Realm
+
+
+class Command(ZulipBaseCommand):
+    help = """Script to send zulip update announcements to realms."""
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        parser.add_argument("--progress", action="store_true", help="Log every 50 completed realms")
+        parser.add_argument(
+            "--skip-delay",
+            action="store_true",
+            help="Immediately send updates if 'zulip_update_announcements_stream' is configured.",
+        )
+        parser.add_argument(
+            "--reset-level",
+            type=int,
+            help="The level to reset all active realms to.",
+        )
+
+    @override
+    @abort_cron_during_deploy
+    @abort_unless_locked
+    def handle(self, *args: Any, **options: Any) -> None:
+        if options["reset_level"] is not None:
+            Realm.objects.filter(deactivated=False).exclude(
+                string_id=settings.SYSTEM_BOT_REALM
+            ).update(zulip_update_announcements_level=options["reset_level"])
+            return
+
+        send_zulip_update_announcements(
+            skip_delay=options["skip_delay"], progress=options["progress"]
+        )

zerver/management/commands/set_owner_full_content_access.py

@@ -0,0 +1,42 @@
+from argparse import ArgumentParser
+from typing import Any
+
+from typing_extensions import override
+
+from zerver.lib.management import ZulipBaseCommand
+
+
+class Command(ZulipBaseCommand):
+    help = (
+        "Set the owner_full_content_access flag for a realm. The flag determines "
+        "whether the organization's owner will have the ability to access private "
+        "content in the organization in the Zulip UI."
+    )
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        toggle_group = parser.add_mutually_exclusive_group(required=True)
+        toggle_group.add_argument(
+            "--enable",
+            action="store_true",
+            help="Set owner_full_content_access to True.",
+        )
+        toggle_group.add_argument(
+            "--disable",
+            action="store_false",
+            help="Set owner_full_content_access to False.",
+        )
+        self.add_realm_args(parser, required=True)
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        realm = self.get_realm(options)
+        assert realm is not None  # Should be ensured by parser
+
+        owner_full_content_access = options["enable"]
+        realm.owner_full_content_access = owner_full_content_access
+        realm.save(update_fields=["owner_full_content_access"])
+
+        print(
+            f"owner_full_content_access set to {owner_full_content_access} for realm {realm.name} (id={realm.id})."
+        )

zerver/management/commands/show_admins.py

@@ -0,0 +1,36 @@
+from argparse import ArgumentParser
+from typing import Any
+
+from django.core.management.base import CommandError
+from typing_extensions import override
+
+from zerver.lib.management import ZulipBaseCommand
+
+
+class Command(ZulipBaseCommand):
+    help = """Show the owners and administrators in an organization."""
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        self.add_realm_args(parser, required=True)
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        realm = self.get_realm(options)
+        assert realm is not None  # True because of required=True above
+
+        admin_users = realm.get_admin_users_and_bots()
+        owner_user_ids = set(realm.get_human_owner_users().values_list("id", flat=True))
+
+        if admin_users:
+            print("Administrators:\n")
+            for user in admin_users:
+                owner_detail = ""
+                if user.id in owner_user_ids:
+                    owner_detail = " [owner]"
+                print(f"  {user.delivery_email} ({user.full_name}){owner_detail}")
+
+        else:
+            raise CommandError("There are no admins for this realm!")
+
+        print('\nYou can use the "change_user_role" management command to adjust roles.')

zerver/management/commands/soft_deactivate_users.py

@@ -0,0 +1,98 @@
+import sys
+from argparse import ArgumentParser
+from typing import Any
+
+from django.conf import settings
+from django.core.management.base import CommandError
+from typing_extensions import override
+
+from zerver.lib.management import ZulipBaseCommand, abort_unless_locked
+from zerver.lib.soft_deactivation import (
+    do_auto_soft_deactivate_users,
+    do_soft_activate_users,
+    do_soft_deactivate_users,
+    logger,
+)
+from zerver.models import Realm, UserProfile
+
+
+def get_users_from_emails(emails: list[str], filter_kwargs: dict[str, Realm]) -> list[UserProfile]:
+    # Bug: Ideally, this would be case-insensitive like our other email queries.
+    users = list(UserProfile.objects.filter(delivery_email__in=emails, **filter_kwargs))
+
+    if len(users) != len(emails):
+        user_emails_found = {user.delivery_email for user in users}
+        user_emails_not_found = "\n".join(set(emails) - user_emails_found)
+        raise CommandError(
+            "Users with the following emails were not found:\n\n"
+            f"{user_emails_not_found}\n\n"
+            "Check if they are correct.",
+        )
+    return users
+
+
+class Command(ZulipBaseCommand):
+    help = """Soft activate/deactivate users. Users are recognised by their emails here."""
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        self.add_realm_args(parser)
+        parser.add_argument(
+            "-d", "--deactivate", action="store_true", help="Used to deactivate user/users."
+        )
+        parser.add_argument(
+            "-a", "--activate", action="store_true", help="Used to activate user/users."
+        )
+        parser.add_argument(
+            "--inactive-for",
+            type=int,
+            default=28,
+            help="Number of days of inactivity before soft-deactivation",
+        )
+        parser.add_argument(
+            "users",
+            metavar="<users>",
+            nargs="*",
+            help="A list of user emails to soft activate/deactivate.",
+        )
+
+    @override
+    @abort_unless_locked
+    def handle(self, *args: Any, **options: Any) -> None:
+        if settings.STAGING:
+            print("This is a Staging server. Suppressing management command.")
+            sys.exit(0)
+
+        realm = self.get_realm(options)
+        user_emails = options["users"]
+        activate = options["activate"]
+        deactivate = options["deactivate"]
+
+        filter_kwargs: dict[str, Realm] = {}
+        if realm is not None:
+            filter_kwargs = dict(realm=realm)
+
+        if activate:
+            if not user_emails:
+                print("You need to specify at least one user to use the activate option.")
+                self.print_help("./manage.py", "soft_deactivate_users")
+                raise CommandError
+
+            users_to_activate = get_users_from_emails(user_emails, filter_kwargs)
+            users_activated = do_soft_activate_users(users_to_activate)
+            logger.info("Soft reactivated %d user(s)", len(users_activated))
+
+        elif deactivate:
+            if user_emails:
+                users_to_deactivate = get_users_from_emails(user_emails, filter_kwargs)
+                print("Soft deactivating forcefully...")
+                users_deactivated = do_soft_deactivate_users(users_to_deactivate)
+            else:
+                users_deactivated = do_auto_soft_deactivate_users(
+                    int(options["inactive_for"]), realm
+                )
+            logger.info("Soft deactivated %d user(s)", len(users_deactivated))
+
+        else:
+            self.print_help("./manage.py", "soft_deactivate_users")
+            raise CommandError

zerver/management/commands/sync_ldap_user_data.py

@@ -0,0 +1,98 @@
+import logging
+from argparse import ArgumentParser
+from typing import Any
+
+from django.conf import settings
+from django.core.management.base import CommandError
+from django.db import transaction
+from django.db.models import QuerySet
+from typing_extensions import override
+
+from zerver.lib.logging_util import log_to_file
+from zerver.lib.management import ZulipBaseCommand
+from zerver.models import UserProfile
+from zproject.backends import ZulipLDAPError, sync_user_from_ldap
+
+## Setup ##
+logger = logging.getLogger("zulip.sync_ldap_user_data")
+log_to_file(logger, settings.LDAP_SYNC_LOG_PATH)
+
+
+# Run this on a cron job to pick up on name changes.
+@transaction.atomic(durable=True)
+def sync_ldap_user_data(
+    user_profiles: QuerySet[UserProfile], deactivation_protection: bool = True
+) -> None:
+    logger.info("Starting update.")
+    try:
+        realms = {u.realm.string_id for u in user_profiles}
+
+        for u in user_profiles:
+            # This will save the user if relevant, and will do nothing if the user
+            # does not exist.
+            try:
+                sync_user_from_ldap(u, logger)
+            except ZulipLDAPError as e:
+                logger.error("Error attempting to update user %s:", u.delivery_email)
+                logger.error(e.args[0])
+
+        if deactivation_protection:
+            if not UserProfile.objects.filter(is_bot=False, is_active=True).exists():
+                raise Exception(
+                    "LDAP sync would have deactivated all users. This is most likely due "
+                    "to a misconfiguration of LDAP settings. Rolling back...\n"
+                    "Use the --force option if the mass deactivation is intended."
+                )
+            for string_id in realms:
+                if not UserProfile.objects.filter(
+                    is_bot=False,
+                    is_active=True,
+                    realm__string_id=string_id,
+                    role=UserProfile.ROLE_REALM_OWNER,
+                ).exists():
+                    raise Exception(
+                        f"LDAP sync would have deactivated all owners of realm {string_id}. "
+                        "This is most likely due "
+                        "to a misconfiguration of LDAP settings. Rolling back...\n"
+                        "Use the --force option if the mass deactivation is intended."
+                    )
+    except Exception:
+        logger.exception("LDAP sync failed")
+        raise
+
+    logger.info("Finished update.")
+
+
+class Command(ZulipBaseCommand):
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        parser.add_argument(
+            "-f",
+            "--force",
+            action="store_true",
+            help="Disable the protection against deactivating all users.",
+        )
+
+        self.add_realm_args(parser)
+        self.add_user_list_args(parser)
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        if options.get("realm_id") is not None:
+            realm = self.get_realm(options)
+            user_profiles = self.get_users(options, realm, is_bot=False, include_deactivated=True)
+        else:
+            user_profiles = UserProfile.objects.select_related("realm").filter(is_bot=False)
+
+            if not user_profiles.exists():
+                # This case provides a special error message if one
+                # tries setting up LDAP sync before creating a realm.
+                raise CommandError("Zulip server contains no users. Have you created a realm?")
+
+        if len(user_profiles) == 0:
+            # We emphasize that this error is purely about the
+            # command-line parameters, since this has nothing to do
+            # with your LDAP configuration.
+            raise CommandError("Zulip server contains no users matching command-line parameters.")
+
+        sync_ldap_user_data(user_profiles, not options["force"])

zerver/management/commands/thumbnail.py

@@ -0,0 +1,107 @@
+from datetime import timedelta
+from typing import Any
+
+from django.core.management.base import CommandParser
+from django.db.models import Exists, OuterRef
+from django.utils.timezone import now as timezone_now
+from typing_extensions import override
+
+from zerver.actions.message_edit import re_thumbnail
+from zerver.lib.management import ZulipBaseCommand
+from zerver.lib.queue import queue_event_on_commit
+from zerver.lib.thumbnail import StoredThumbnailFormat, get_image_thumbnail_path
+from zerver.lib.upload import all_message_attachments
+from zerver.models import ArchivedMessage, Attachment, ImageAttachment, Message
+
+
+class Command(ZulipBaseCommand):
+    help = """Manages thumbnailing in messages."""
+
+    @override
+    def add_arguments(self, parser: CommandParser) -> None:
+        self.add_realm_args(parser, required=True)
+        mode = parser.add_mutually_exclusive_group(required=True)
+        mode.add_argument(
+            "--stuck-spinners",
+            action="store_true",
+            help="Attempt to re-render messages with stuck spinners",
+        )
+        mode.add_argument(
+            "--old-images",
+            action="store_true",
+            help="Generate thumbnails of old images",
+        )
+        mode.add_argument(
+            "--missing-files",
+            action="store_true",
+            help="Regenerate thumbnails when their files are missing",
+        )
+        parser.add_argument(
+            "--cutoff",
+            help="Only process messages sent less than this many days ago",
+            type=int,
+            default=100,
+        )
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        realm = self.get_realm(options)
+        assert realm is not None
+
+        if options.get("missing_files"):
+            realm_imageattachments = (
+                ImageAttachment.objects.alias(
+                    in_realm=Exists(
+                        Attachment.objects.filter(path_id=OuterRef("path_id"), realm_id=realm.id)
+                    )
+                )
+                .filter(in_realm=True)
+                .exclude(thumbnail_metadata=[])
+            )
+            for image_attachment in realm_imageattachments:
+                found = []
+                changed = False
+                thumb_dir = f"thumbnail/{image_attachment.path_id}"
+                found_thumbs = [
+                    e[0] for e in all_message_attachments(include_thumbnails=True, prefix=thumb_dir)
+                ]
+                for existing_thumbnail in image_attachment.thumbnail_metadata:
+                    thumb = StoredThumbnailFormat(**existing_thumbnail)
+                    if get_image_thumbnail_path(image_attachment, thumb) in found_thumbs:
+                        found.append(existing_thumbnail)
+                    else:
+                        changed = True
+                if changed:
+                    image_attachment.thumbnail_metadata = found
+                    image_attachment.save(update_fields=["thumbnail_metadata"])
+                    queue_event_on_commit(
+                        "thumbnail",
+                        {"id": image_attachment.id, "path_id": image_attachment.path_id},
+                    )
+            return
+
+        for message_class in (Message, ArchivedMessage):
+            messages = message_class.objects.filter(
+                realm_id=realm.id,
+                has_image=True,
+                date_sent__gt=timezone_now() - timedelta(days=options["cutoff"]),
+            )
+            if options.get("stuck_spinners"):
+                messages = messages.filter(
+                    rendered_content__contains='class="image-loading-placeholder"',
+                    date_sent__lt=timezone_now() - timedelta(seconds=60),
+                )
+            elif options.get("old_images"):
+                messages = messages.filter(
+                    rendered_content__contains='<img src="/user_uploads/',
+                ).exclude(
+                    rendered_content__contains='<img src="/user_uploads/thumbnail/',
+                )
+
+            message_ids = list(messages.values_list("id", flat=True))
+            print(f"Processing {len(message_ids)} {message_class.__name__} objects")
+            for i, message_id in enumerate(message_ids):
+                re_thumbnail(message_class, message_id, enqueue=options["old_images"])
+
+                if (i + 1) % 100 == 0:
+                    print(f"Processed {i + 1}/{len(message_ids)} {message_class.__name__} objects")

zerver/management/commands/transfer_uploads_to_s3.py

@@ -0,0 +1,32 @@
+from typing import Any
+
+from django.conf import settings
+from django.core.management.base import CommandError, CommandParser
+from typing_extensions import override
+
+from zerver.lib.management import ZulipBaseCommand
+from zerver.lib.transfer import transfer_uploads_to_s3
+
+
+class Command(ZulipBaseCommand):
+    help = """Transfer uploads to S3 """
+
+    @override
+    def add_arguments(self, parser: CommandParser) -> None:
+        parser.add_argument(
+            "--processes",
+            default=settings.DEFAULT_DATA_EXPORT_IMPORT_PARALLELISM,
+            help="Processes to use for exporting uploads in parallel",
+        )
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        num_processes = int(options["processes"])
+        if num_processes < 1:
+            raise CommandError("You must have at least one process.")
+
+        if not settings.LOCAL_UPLOADS_DIR:
+            raise CommandError("Please set the value of LOCAL_UPLOADS_DIR.")
+
+        transfer_uploads_to_s3(num_processes)
+        print("Transfer to S3 completed successfully.")

zerver/management/commands/unarchive_channel.py

@@ -0,0 +1,103 @@
+from argparse import ArgumentParser
+from typing import Any
+
+from django.core.management.base import CommandError
+from typing_extensions import override
+
+from zerver.actions.streams import deactivated_streams_by_old_name, do_unarchive_stream
+from zerver.lib.management import ZulipBaseCommand
+from zerver.models import RealmAuditLog, Stream
+from zerver.models.realm_audit_logs import AuditLogEventType
+
+
+class Command(ZulipBaseCommand):
+    help = """Reactivate a channel that was deactivated."""
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        specify_channel = parser.add_mutually_exclusive_group(required=True)
+        specify_channel.add_argument(
+            "-c",
+            "--channel",
+            help="Name of a deactivated channel in the realm.",
+        )
+        specify_channel.add_argument(
+            "--channel-id",
+            help="ID of a deactivated channel in the realm.",
+        )
+        parser.add_argument(
+            "-n",
+            "--new-name",
+            required=False,
+            help="Name to reactivate as; defaults to the old name.",
+        )
+        self.add_realm_args(parser, required=True)
+
+    @override
+    def handle(self, *args: Any, **options: str | None) -> None:
+        realm = self.get_realm(options)
+        assert realm is not None  # Should be ensured by parser
+
+        # Looking up the channel is complicated, since they get renamed
+        # when they are deactivated, in a transformation which may be
+        # lossy.
+
+        if options["channel_id"] is not None:
+            channel = Stream.objects.get(id=options["channel_id"])
+            if channel.realm_id != realm.id:
+                raise CommandError(
+                    f"Channel id {channel.id}, named '{channel.name}', is in realm '{channel.realm.string_id}', not '{realm.string_id}'"
+                )
+            if not channel.deactivated:
+                raise CommandError(
+                    f"Channel id {channel.id}, named '{channel.name}', is not deactivated"
+                )
+            if options["new_name"] is None:
+                raise CommandError("--new-name flag is required with --channel-id")
+            new_name = options["new_name"]
+        else:
+            channel_name = options["channel"]
+            assert channel_name is not None
+
+            possible_channels = deactivated_streams_by_old_name(realm, channel_name)
+            if len(possible_channels) == 0:
+                raise CommandError("No matching deactivated channels found!")
+
+            if len(possible_channels) > 1:
+                # Print ids of all possible channels, support passing by id
+                print("Matching channels:")
+                for channel in possible_channels:
+                    last_deactivation = (
+                        RealmAuditLog.objects.filter(
+                            realm=realm,
+                            modified_stream=channel,
+                            event_type=AuditLogEventType.CHANNEL_DEACTIVATED,
+                        )
+                        .order_by("-id")
+                        .first()
+                    )
+                    assert last_deactivation is not None
+                    print(
+                        f"  ({channel.id}) {channel.name}, deactivated on {last_deactivation.event_time}"
+                    )
+                raise CommandError(
+                    "More than one matching channel found!  Specify which with --channel-id"
+                )
+
+            channel = possible_channels[0]
+            if options["new_name"] is not None:
+                new_name = options["new_name"]
+            else:
+                new_name = channel_name
+
+        if (
+            Stream.objects.filter(realm=realm, name__iexact=new_name)
+            .exclude(id=channel.id)
+            .exists()
+        ):
+            raise CommandError(
+                f"Channel with name '{new_name}' already exists; pass a different --new-name"
+            )
+
+        assert channel is not None
+        do_unarchive_stream(channel, new_name, acting_user=None)

zerver/management/commands/update_channel_recently_active_status.py

@@ -0,0 +1,23 @@
+import logging
+from typing import Any
+
+from django.conf import settings
+from typing_extensions import override
+
+from zerver.lib.logging_util import log_to_file
+from zerver.lib.management import ZulipBaseCommand, abort_unless_locked
+from zerver.lib.streams import check_update_all_streams_active_status
+
+## Logging setup ##
+logger = logging.getLogger(__name__)
+log_to_file(logger, settings.DIGEST_LOG_PATH)
+
+
+class Command(ZulipBaseCommand):
+    help = """Update the `Stream.is_recently_active` field to False for channels whose message history has aged to the point where it is no longer recently active."""
+
+    @override
+    @abort_unless_locked
+    def handle(self, *args: Any, **options: Any) -> None:
+        count = check_update_all_streams_active_status()
+        logger.info("Marked %s channels as not recently active.", count)

zerver/management/commands/update_subscriber_counts.py

@@ -0,0 +1,121 @@
+import argparse
+import logging
+from datetime import timedelta
+from typing import Any
+
+from django.conf import settings
+from django.db import transaction
+from django.db.models import F, QuerySet
+from django.utils.timezone import now as timezone_now
+from typing_extensions import override
+
+from zerver.lib.logging_util import log_to_file
+from zerver.lib.management import ZulipBaseCommand, abort_unless_locked
+from zerver.models import RealmAuditLog, Stream, Subscription
+from zerver.models.realm_audit_logs import AuditLogEventType
+
+## Logging setup ##
+logger = logging.getLogger(__name__)
+log_to_file(logger, settings.DIGEST_LOG_PATH)
+
+
+class Command(ZulipBaseCommand):
+    help = """Update the `Stream.subscriber_count` field based on current subscribers.
+
+There may be race conditions with keeping the cached subscriber count
+accurate; this command is run as a daily cron job to ensure the number is accurate.
+"""
+
+    @override
+    def add_arguments(self, parser: argparse.ArgumentParser) -> None:
+        parser.add_argument(
+            "--since",
+            type=int,
+            help="Only examine channels with changed subscribers in this many hours",
+        )
+        self.add_realm_args(parser, help="The optional name of the realm to limit to")
+
+    @override
+    @abort_unless_locked
+    def handle(self, *args: Any, **options: Any) -> None:
+        realm = self.get_realm(options)
+        streams = Stream.objects.all()
+        if options["since"]:
+            since_time = timezone_now() - timedelta(hours=options["since"])
+            # Two ways the count can change -- via a subscription
+            # being changed, or via a user being (de)activated.
+            changed_subs = RealmAuditLog.objects.filter(
+                event_type__in=(
+                    AuditLogEventType.SUBSCRIPTION_CREATED,
+                    AuditLogEventType.SUBSCRIPTION_ACTIVATED,
+                    AuditLogEventType.SUBSCRIPTION_DEACTIVATED,
+                ),
+                event_time__gte=since_time,
+            )
+            if realm:
+                changed_subs = changed_subs.filter(realm=realm)
+
+            # Find all users changed in the time period, join those to
+            # their subscriptions and distinct recipients, and thence
+            # to streams.
+            changed_users = RealmAuditLog.objects.filter(
+                event_type__in=(
+                    AuditLogEventType.USER_CREATED,
+                    AuditLogEventType.USER_DEACTIVATED,
+                    AuditLogEventType.USER_ACTIVATED,
+                    AuditLogEventType.USER_REACTIVATED,
+                ),
+                event_time__gte=since_time,
+            )
+            if realm:
+                changed_users = changed_users.filter(realm=realm)
+            changed_user_ids = (
+                changed_users.values_list("modified_user_id", flat=True)
+                .distinct()
+                .order_by("modified_user_id")
+            )
+
+            changed_user_subs = (
+                Subscription.objects.filter(user_profile_id__in=changed_user_ids)
+                .values_list("recipient_id", flat=True)
+                .distinct()
+                .order_by("recipient_id")
+            )
+            streams_from_users = Stream.objects.filter(recipient_id__in=changed_user_subs)
+            if realm:
+                streams_from_users = streams_from_users.filter(realm=realm)
+
+            stream_ids: QuerySet[Any, int] = (
+                changed_subs.distinct("modified_stream_id")
+                .order_by("modified_stream_id")
+                .annotate(stream_id=F("modified_stream_id"))
+                .union(streams_from_users.annotate(stream_id=F("id")))
+                .values_list("stream_id", flat=True)
+            )
+        elif realm := self.get_realm(options):
+            stream_ids = streams.filter(realm=realm).values_list("id", flat=True)
+        else:
+            stream_ids = streams.all().values_list("id", flat=True)
+
+        for stream_id in stream_ids.iterator():
+            with transaction.atomic(durable=True):
+                stream = Stream.objects.select_for_update(no_key=True).get(id=stream_id)
+                actual_subscriber_count = Subscription.objects.filter(
+                    active=True,
+                    recipient__type=2,
+                    recipient__type_id=stream_id,
+                    is_user_active=True,
+                ).count()
+                db_count = stream.subscriber_count
+                if actual_subscriber_count == db_count:
+                    continue
+                stream.subscriber_count = actual_subscriber_count
+                stream.save(update_fields=["subscriber_count"])
+
+            logging.info(
+                "Updated subscriber count of %s, #%s: from %d to %d",
+                stream.realm.string_id,
+                stream.name,
+                db_count,
+                actual_subscriber_count,
+            )