snapshot of https://github.com/zulip/zulip.git @ 48c69911c7dd4e03b20552eb3bd77ad8151c1357 (2026-09-15)

tools/test-backend

@@ -0,0 +1,561 @@
+#!/usr/bin/env python3
+
+import argparse
+import contextlib
+import glob
+import multiprocessing
+import os
+import shlex
+import subprocess
+import sys
+import tempfile
+from collections.abc import Iterator
+from typing import TYPE_CHECKING, cast
+from unittest import mock
+
+if TYPE_CHECKING:
+    # This script does not have access to the zerver module during runtime.
+    # We can only import this when type checking.
+    from zerver.lib.test_runner import Runner
+
+multiprocessing.set_start_method("fork")
+
+TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
+os.chdir(os.path.dirname(TOOLS_DIR))
+sys.path.insert(0, os.path.dirname(TOOLS_DIR))
+
+# check for the venv
+from tools.lib import sanity_check
+
+sanity_check.check_venv(__file__)
+
+import django
+import orjson
+import responses
+from django.conf import settings
+from django.test.utils import get_runner
+
+source_files = [
+    "analytics/**/*.py",
+    "confirmation/**/*.py",
+    "corporate/**/*.py",
+    "pgroonga/**/*.py",
+    "zerver/**/*.py",
+    "zilencer/**/*.py",
+    "zproject/**/*.py",
+]
+
+not_yet_fully_covered = [
+    "*/migrations/*.py",
+    "*/management/commands/*.py",
+    # Analytics fixtures library is used to generate test fixtures;
+    # isn't properly accounted for in test coverage analysis since it
+    # runs before tests.
+    "analytics/lib/fixtures.py",
+    # We have 100% coverage on the new stuff; need to refactor old stuff.
+    "analytics/views/stats.py",
+    # TODO: This is a work in progress and therefore without
+    # tests yet.
+    "corporate/views/installation_activity.py",
+    "corporate/views/plan_activity.py",
+    "corporate/views/realm_activity.py",
+    "corporate/views/remote_billing_page.py",
+    "corporate/views/audit_logs.py",
+    "corporate/views/support.py",
+    "corporate/lib/activity.py",
+    "corporate/lib/remote_billing_util.py",
+    "corporate/lib/billing_management.py",
+    # Major lib files should have 100% coverage
+    "zerver/lib/addressee.py",
+    "zerver/lib/markdown/__init__.py",
+    "zerver/lib/cache.py",
+    "zerver/lib/cache_helpers.py",
+    "zerver/lib/i18n.py",
+    "zerver/lib/send_email.py",
+    "zerver/lib/url_preview/preview.py",
+    # Markdown sub-libs should have full coverage too; a lot are really close
+    "zerver/lib/markdown/api_arguments_table_generator.py",
+    "zerver/lib/markdown/fenced_code.py",
+    "zerver/lib/markdown/nested_code_blocks.py",
+    # Workers should get full coverage; many have it already
+    "zerver/worker/deferred_work.py",
+    "zerver/worker/missedmessage_emails.py",
+    # Worker-associated files; lower-priority to get testing on
+    "zerver/worker/base.py",
+    "zerver/worker/queue_processors.py",
+    "zerver/worker/test.py",
+    # Other lib files that ideally would coverage, but aren't sorted
+    "zerver/middleware.py",
+    "zerver/lib/bot_lib.py",
+    "zerver/lib/camo.py",
+    "zerver/lib/debug.py",
+    "zerver/lib/export.py",
+    "zerver/lib/fix_unreads.py",
+    "zerver/lib/import_realm.py",
+    "zerver/lib/logging_util.py",
+    "zerver/lib/profile.py",
+    "zerver/lib/queue.py",
+    "zerver/lib/sqlalchemy_utils.py",
+    "zerver/lib/storage.py",
+    "zerver/lib/templates.py",
+    # Low priority for coverage
+    "zerver/lib/generate_test_data.py",
+    "zerver/lib/server_initialization.py",
+    "zerver/lib/test_fixtures.py",
+    "zerver/lib/test_runner.py",
+    "zerver/lib/test_console_output.py",
+    "zerver/lib/zstd_level9.py",
+    "zerver/openapi/curl_param_value_generators.py",
+    "zerver/openapi/javascript_examples.py",
+    "zerver/openapi/python_examples.py",
+    "zerver/openapi/test_curl_examples.py",
+    # Helper for tooling; doesn't need coverage
+    "zerver/openapi/merge_api_changelogs.py",
+    # Tornado should ideally have full coverage, but we're not there.
+    "zerver/tornado/descriptors.py",
+    "zerver/tornado/django_api.py",
+    "zerver/tornado/event_queue.py",
+    "zerver/tornado/exceptions.py",
+    "zerver/tornado/handlers.py",
+    "zerver/tornado/ioloop_logging.py",
+    "zerver/tornado/sharding.py",
+    "zerver/tornado/views.py",
+    # Data import files; relatively low priority
+    "zerver/data_import/slack.py",
+    "zerver/data_import/import_util.py",
+    # Webhook integrations with incomplete coverage
+    "zerver/webhooks/greenhouse/view.py",
+    "zerver/webhooks/teamcity/view.py",
+    "zerver/webhooks/zapier/view.py",
+    # This is hard to get test coverage for, and low value to do so
+    "zerver/views/sentry.py",
+    # Cannot have coverage, as tests run in a transaction
+    "zerver/lib/safe_session_cached_db.py",
+    "zerver/lib/singleton_bmemcached.py",
+    # Only covered when we run migrations tests, which are not
+    # guaranteed to exist or be runnable
+    "zerver/lib/migrate.py",
+    # Branches in Django settings files are hard to test
+    "zproject/computed_settings.py",
+    # Used to override settings
+    "zproject/custom_dev_settings.py",
+    "zproject/dev_settings.py",
+    "zproject/test_extra_settings.py",
+    # Better tested in a full deployment
+    "zproject/sentry.py",
+    "zproject/wsgi.py",
+]
+
+enforce_fully_covered = sorted(
+    {path for target in source_files for path in glob.glob(target, recursive=True)}
+    - {path for target in not_yet_fully_covered for path in glob.glob(target, recursive=True)}
+)
+
+FAILED_TEST_PATH = "var/last_test_failure.json"
+
+
+def get_failed_tests() -> list[str]:
+    try:
+        with open(FAILED_TEST_PATH, "rb") as f:
+            return orjson.loads(f.read())
+    except OSError:
+        print("var/last_test_failure.json doesn't exist; running all tests.")
+        return []
+
+
+@contextlib.contextmanager
+def block_internet() -> Iterator[None]:
+    # Monkey-patching - responses library raises requests.ConnectionError when access to an unregistered URL
+    # is attempted. We want to replace that with our own exception, so that it propagates all the way:
+    with (
+        mock.patch.object(responses, "ConnectionError", new=ZulipInternetBlockedError),
+        # We'll run all tests in this context manager. It'll cause an error to be raised (see above comment),
+        # if any code attempts to access the internet.
+        responses.RequestsMock(),
+    ):
+        yield
+
+
+class ZulipInternetBlockedError(Exception):
+    def __init__(self, original_msg: str) -> None:
+        zulip_msg = (
+            "Outgoing network requests are not allowed in the Zulip tests. "
+            "More details and advice are available here:"
+            "https://zulip.readthedocs.io/en/latest/testing/testing.html#internet-access-inside-test-suites"
+        )
+        msg = f"{zulip_msg}\nResponses library error message: {original_msg}"
+        super().__init__(msg)
+
+
+def main() -> None:
+    default_parallel = os.cpu_count()
+
+    # Remove proxy settings for running backend tests
+    os.environ.pop("http_proxy", "")
+    os.environ.pop("https_proxy", "")
+
+    from tools.lib.test_script import (
+        add_provision_check_override_param,
+        assert_provisioning_status_ok,
+    )
+    from zerver.lib.test_fixtures import (
+        remove_test_run_directories,
+        update_test_databases_if_required,
+    )
+
+    os.environ["DJANGO_SETTINGS_MODULE"] = "zproject.test_settings"
+    # "-u" uses unbuffered IO, which is important when wrapping it in subprocess
+    os.environ["PYTHONUNBUFFERED"] = "y"
+
+    usage = """test-backend [options]
+    test-backend # Runs all backend tests
+    test-backend zerver.tests.test_markdown # run all tests in a test module
+    test-backend zerver/tests/test_markdown.py # run all tests in a test module
+    test-backend test_markdown # run all tests in a test module
+    test-backend zerver.tests.test_markdown.MarkdownEmbedsTest # run all tests in a test class
+    test-backend MarkdownEmbedsTest # run all tests in a test class
+    test-backend zerver.tests.test_markdown.MarkdownEmbedsTest.test_inline_youtube # run a single test
+    test-backend MarkdownEmbedsTest.test_inline_youtube # run a single test"""
+
+    parser = argparse.ArgumentParser(
+        description=usage, formatter_class=argparse.RawTextHelpFormatter
+    )
+
+    parser.add_argument(
+        "-x",
+        "--stop",
+        action="store_true",
+        dest="fatal_errors",
+        help="Stop running tests after the first failure.",
+    )
+    parser.add_argument("--coverage", action="store_true", help="Compute test coverage.")
+    parser.add_argument(
+        "--verbose-coverage", action="store_true", help="Enable verbose print of coverage report."
+    )
+    parser.add_argument(
+        "--xml-report", action="store_true", help="Enable (slow) XML coverage report."
+    )
+    parser.add_argument(
+        "--no-html-report", action="store_true", help="Disable (slow) HTML coverage report."
+    )
+    parser.add_argument(
+        "--no-cov-cleanup", action="store_true", help="Do not clean generated coverage files."
+    )
+
+    parser.add_argument(
+        "--parallel",
+        dest="processes",
+        type=int,
+        default=None,
+        help="Specify the number of processes to run the "
+        "tests in. Default is the number of logical CPUs",
+    )
+    parser.add_argument("--profile", action="store_true", help="Profile test runtime.")
+    add_provision_check_override_param(parser)
+    parser.add_argument(
+        "--no-shallow",
+        action="store_true",
+        help="Don't allow shallow testing of templates (deprecated)",
+    )
+    parser.add_argument("--verbose", action="store_true", help="Show detailed output")
+    parser.add_argument("--reverse", action="store_true", help="Run tests in reverse order.")
+    parser.add_argument(
+        "--rerun",
+        action="store_true",
+        help="Run the tests which failed the last time test-backend was run.  Implies not --stop.",
+    )
+    parser.add_argument(
+        "--include-webhooks",
+        action="store_true",
+        help="Include webhook tests.  By default, they are skipped for performance.",
+    )
+    parser.add_argument(
+        "--include-transaction-tests",
+        action="store_true",
+        help="Include transaction tests.  By default, they are skipped for performance.",
+    )
+    parser.add_argument(
+        "--generate-stripe-fixtures",
+        action="store_true",
+        help="Generate Stripe test fixtures by making requests to Stripe test network",
+    )
+    parser.add_argument(
+        "--generate-llm-fixtures",
+        action="store_true",
+        help="Generate LLM test fixtures using credentials in zproject/dev-secrets.conf",
+    )
+    parser.add_argument("args", nargs="*")
+    parser.add_argument(
+        "--ban-console-output",
+        action="store_true",
+        help="Require stdout and stderr to be clean of unexpected output.",
+    )
+
+    options = parser.parse_args()
+    if options.ban_console_output:
+        os.environ["BAN_CONSOLE_OUTPUT"] = "1"
+
+    args = options.args
+    include_webhooks = options.coverage or options.include_webhooks
+    include_transaction_tests = options.coverage or options.include_transaction_tests
+
+    if options.processes is not None and options.processes < 1:
+        raise argparse.ArgumentTypeError("option processes: Only positive integers are allowed.")
+
+    test_dirs = ["zerver/tests/", "corporate/tests", "analytics/tests"]
+
+    # While running --rerun, we read var/last_test_failure.json to get
+    # the list of tests that failed on the last run, and then pretend
+    # those tests were passed explicitly.  --rerun implies
+    # !fatal_errors, so that we don't end up removing tests from
+    # the list that weren't run.
+    if options.rerun:
+        default_parallel = 1
+        options.fatal_errors = False
+        failed_tests = get_failed_tests()
+        if failed_tests:
+            args = failed_tests
+    if len(args) > 0:
+        # If we passed a specific set of tests, run in serial mode.
+        default_parallel = 1
+
+        # to transform forward slashes '/' present in the argument into dots '.'
+        for i, suite in enumerate(args):
+            args[i] = suite.rstrip("/").replace("/", ".")
+
+        def rewrite_arguments(search_key: str) -> None:
+            for test_dir in test_dirs:
+                for root, dirs, files_names in os.walk(test_dir, topdown=False):
+                    for file_name in files_names:
+                        # Check for files starting with alphanumeric characters and ending with '.py'
+                        # Ignore backup files if any
+                        if not file_name[0].isalnum() or not file_name.endswith(".py"):
+                            continue
+                        filepath = os.path.join(root, file_name)
+                        with open(filepath) as f:
+                            for line in f:
+                                if search_key not in line:
+                                    continue
+                                new_suite = filepath.replace(".py", ".") + suite
+                                args[i] = new_suite
+                                return
+
+        for suite in args:
+            if suite[0].isupper() and "test_" in suite:
+                classname = suite.rsplit(".", 1)[0]
+                rewrite_arguments(classname)
+            elif suite[0].isupper():
+                rewrite_arguments(f"class {suite}(")
+
+        for i, suite in enumerate(args):
+            if suite.startswith("test"):
+                for test_dir in test_dirs:
+                    for root, dirs, files_names in os.walk(test_dir):
+                        for file_name in files_names:
+                            if file_name in (suite, f"{suite}.py"):
+                                new_suite = os.path.join(root, file_name)
+                                args[i] = new_suite
+                                break
+
+        for i, suite in enumerate(args):
+            args[i] = suite.replace(".py", "")
+
+        # to transform forward slashes '/' introduced by the zerver_test_dir into dots '.'
+        # taking care of any forward slashes that might be present
+        for i, suite in enumerate(args):
+            args[i] = suite.replace("/", ".")
+
+    full_suite = len(args) == 0
+
+    if full_suite:
+        suites = [
+            "zerver.tests",
+            "analytics.tests",
+            "corporate.tests",
+        ]
+    else:
+        suites = args
+
+    if full_suite and include_webhooks:
+        suites.append("zerver.webhooks")
+
+    if full_suite and include_transaction_tests:
+        suites.append("zerver.transaction_tests")
+
+    if options.generate_stripe_fixtures:
+        if full_suite:
+            suites = [
+                "corporate.tests.test_billing_lib",
+                "corporate.tests.test_sponsorship",
+                "corporate.tests.test_stripe",
+                "corporate.tests.test_stripe_remote_realm",
+                "corporate.tests.test_stripe_remote_server",
+            ]
+            full_suite = False
+        print("-- Forcing serial mode for generating stripe fixtures.", flush=True)
+        default_parallel = 1
+        os.environ["GENERATE_STRIPE_FIXTURES"] = "1"
+
+    if options.generate_llm_fixtures:
+        if full_suite:
+            suites = [
+                "zerver.tests.test_message_summary",
+            ]
+            full_suite = False
+        print("-- Forcing serial mode for generating LLM fixtures.", flush=True)
+        default_parallel = 1
+        os.environ["GENERATE_LLM_FIXTURES"] = "1"
+
+    assert_provisioning_status_ok(options.skip_provision_check)
+
+    if options.coverage:
+        import coverage
+
+        cov = coverage.Coverage(
+            data_suffix="", config_file="tools/coveragerc", concurrency="multiprocessing"
+        )
+        # Do not clean .coverage file in continuous integration job so that coverage data can be uploaded.
+        if not options.no_cov_cleanup:
+            import atexit
+
+            atexit.register(cov.erase)  # Ensure the data file gets cleaned up at the end.
+        cov.start()
+    if options.profile:
+        import cProfile
+
+        prof = cProfile.Profile()
+        prof.enable()
+
+    # This is kind of hacky, but it's the most reliable way
+    # to make sure instrumentation decorators know the
+    # setting when they run.
+    os.environ["TEST_INSTRUMENT_URL_COVERAGE"] = "TRUE"
+
+    # setup() needs to be called after coverage is started to get proper coverage reports of model
+    # files, since part of setup is importing the models for all applications in INSTALLED_APPS.
+    django.setup()
+
+    update_test_databases_if_required()
+
+    subprocess.check_call(["tools/webpack", "--test"])
+
+    # isinstance check cannot be used with types. This can potentially improved by supporting
+    # dynamic resolution of the test runner type with the django-stubs mypy plugin.
+    TestRunner = cast("type[Runner]", get_runner(settings))
+
+    if options.processes:
+        parallel = options.processes
+    elif options.profile:
+        parallel = 1
+    else:
+        parallel = default_parallel
+    if parallel > 1:
+        print(f"-- Running tests in parallel mode with {parallel} processes.", flush=True)
+    else:
+        print("-- Running tests in serial mode.", flush=True)
+
+    with block_internet():
+        test_runner = TestRunner(
+            failfast=options.fatal_errors,
+            verbosity=2,
+            parallel=parallel,
+            reverse=options.reverse,
+            keepdb=True,
+        )
+        failures = test_runner.run_tests(
+            suites,
+            failed_tests_path=FAILED_TEST_PATH,
+            full_suite=full_suite,
+            include_webhooks=include_webhooks,
+        )
+
+    templates_not_rendered = test_runner.get_shallow_tested_templates()
+    # We only check the templates if all the tests ran and passed
+    if not failures and full_suite and templates_not_rendered:
+        missed_count = len(templates_not_rendered)
+        print(f"\nError: {missed_count} templates have no tests!")
+        for template in templates_not_rendered:
+            print(f"  {template}")
+        print("See zerver/tests/test_templates.py for the exclude list.")
+        failures = True
+
+    if options.coverage:
+        cov.stop()
+        cov.save()
+        cov.combine()
+        cov.save()
+        if options.verbose_coverage:
+            print("Printing coverage data")
+            cov.report(show_missing=False)
+        if options.xml_report:
+            print("Writing XML report")
+            cov.xml_report(outfile="var/coverage.xml")
+            print("XML report saved; see var/coverage.xml")
+
+    if full_suite and not failures and options.coverage:
+        # Assert that various files have full coverage
+        for path in enforce_fully_covered:
+            missing_lines = cov.analysis2(path)[3]
+            if len(missing_lines) > 0:
+                print(f"ERROR: {path} no longer has complete backend test coverage")
+                print(f"  Lines missing coverage: {missing_lines}")
+                print()
+                failures = True
+        if failures:
+            print("It looks like your changes lost 100% test coverage in one or more files")
+            print("Usually, the right fix for this is to add some tests.")
+            print("But also check out the include/exclude lists in tools/test-backend.")
+            print("If this line intentionally is not tested, you can use a # nocoverage comment.")
+            print("To run this check locally, use `test-backend --coverage`.")
+        ok = True
+        for path in not_yet_fully_covered:
+            try:
+                missing_lines = cov.analysis2(path)[3]
+                if not missing_lines:
+                    print(
+                        f"ERROR: {path} has complete backend test coverage but is still in not_yet_fully_covered."
+                    )
+                    ok = False
+            except coverage.misc.NoSource:
+                continue
+        if ok:
+            print("Coverage checks pass!")
+        else:
+            print()
+            print(
+                "There are one or more fully covered files that are still in not_yet_fully_covered."
+            )
+            print("Remove the file(s) from not_yet_fully_covered in `tools/test-backend`.")
+            failures = True
+
+    if options.coverage and not options.no_html_report:
+        # We do this late, because it can take quite a while.
+        print("Writing HTML report")
+        cov.html_report(directory="var/coverage", show_contexts=True)
+        print("HTML report saved; visit at http://127.0.0.1:9991/coverage/index.html")
+
+    if options.profile:
+        prof.disable()
+        with tempfile.NamedTemporaryFile(prefix="profile.data.", delete=False) as stats_file:
+            prof.dump_stats(stats_file.name)
+            print(f"Profile data saved to {stats_file.name}")
+            print(f"You can visualize it using e.g. `snakeviz {shlex.quote(stats_file.name)}`")
+            print("Note: If you are using vagrant for development environment you will need to do:")
+            print("1.) `vagrant ssh -- -L 8080:127.0.0.1:8080`")
+            print(f"2.) `snakeviz -s {shlex.quote(stats_file.name)}`")
+
+    # Ideally, we'd check for any leaked test databases here;
+    # but that needs some hackery with database names.
+    #
+    # destroy_leaked_test_databases()
+
+    removed = remove_test_run_directories()
+    if removed:
+        print(f"Removed {removed} stale test run directories!")
+
+    # We'll have printed whether tests passed or failed above
+    sys.exit(bool(failures))
+
+
+if __name__ == "__main__":
+    main()