upstream-zulip/zerver/tests/test_send_email.py

from datetime import timedelta
from smtplib import (
    SMTP,
    SMTPDataError,
    SMTPException,
    SMTPRecipientsRefused,
    SMTPServerDisconnected,
)
from unittest import mock

import time_machine
from django.core.mail.backends.locmem import EmailBackend
from django.core.mail.message import sanitize_address
from django.test import override_settings
from django.utils.timezone import now as timezone_now

from zerver.lib.send_email import (
    EmailNotDeliveredError,
    FromAddress,
    _send_messages,
    build_email,
    logger,
    send_email,
    send_immediate_email,
)
from zerver.lib.test_classes import ZulipTestCase
from zerver.worker.email_senders import ImmediateEmailSenderWorker
from zproject.email_backends import PersistentSMTPEmailBackend


class TestBuildEmail(ZulipTestCase):
    def test_limited_from_length(self) -> None:
        hamlet = self.example_user("hamlet")
        # This is exactly the max length
        limit_length_name = "a" * (320 - len(sanitize_address(FromAddress.NOREPLY, "utf-8")) - 3)
        mail = build_email(
            "zerver/emails/password_reset",
            to_emails=[hamlet.email],
            from_name=limit_length_name,
            from_address=FromAddress.NOREPLY,
            language="en",
        )
        self.assertEqual(mail.extra_headers["From"], f"{limit_length_name} <{FromAddress.NOREPLY}>")

        # One more character makes it flip to just the address, with no name
        mail = build_email(
            "zerver/emails/password_reset",
            to_emails=[hamlet.email],
            from_name=limit_length_name + "a",
            from_address=FromAddress.NOREPLY,
            language="en",
        )
        self.assertEqual(mail.extra_headers["From"], FromAddress.NOREPLY)

    def test_limited_to_length(self) -> None:
        hamlet = self.example_user("hamlet")
        # This is exactly the max length
        limit_length_name = "澳" * 61
        hamlet.full_name = limit_length_name
        hamlet.save()

        mail = build_email(
            "zerver/emails/password_reset",
            to_user_ids=[hamlet.id],
            from_name="Noreply",
            from_address=FromAddress.NOREPLY,
            language="en",
        )
        self.assertEqual(mail.to[0], f"{hamlet.full_name} <{hamlet.delivery_email}>")

        # One more character makes it flip to just the address, with no name
        hamlet.full_name += "澳"
        hamlet.save()
        mail = build_email(
            "zerver/emails/password_reset",
            to_user_ids=[hamlet.id],
            from_name="Noreply",
            from_address=FromAddress.NOREPLY,
            language="en",
        )
        self.assertEqual(mail.to[0], hamlet.delivery_email)

    def test_email_subject_strips_crlf(self) -> None:
        # Email header injection would require a CR or LF making it
        # through email-subject rendering.  Templates are trusted, but
        # we strip both CR and LF defensively so any future template
        # that interpolates a user-controlled value can't smuggle a
        # header-terminating sequence into the email header.
        hamlet = self.example_user("hamlet")
        with mock.patch(
            "zerver.lib.send_email.loader.render_to_string",
            side_effect=lambda template_name, *args, **kwargs: (
                "Subject\r\nBcc: attacker@example.com"
                if template_name == "zerver/emails/password_reset.subject.txt"
                else ""
            ),
        ):
            email = build_email(
                "zerver/emails/password_reset",
                to_user_ids=[hamlet.id],
                from_name="Noreply",
                from_address=FromAddress.NOREPLY,
                language="en",
            )
        self.assertNotIn("\r", email.subject)
        self.assertNotIn("\n", email.subject)
        self.assertEqual(email.subject, "SubjectBcc: attacker@example.com")


class TestSendEmail(ZulipTestCase):
    @override_settings(
        EMAIL_MAX_CONNECTION_LIFETIME_IN_MINUTES=5,
    )
    @time_machine.travel(timezone_now(), tick=False)
    def test_ensure_connected_noop_validation(self) -> None:
        backend = PersistentSMTPEmailBackend()
        with mock.patch.object(EmailBackend, "open", return_value=True):
            backend.ensure_connected()
            self.assertEqual(backend.opened_at, timezone_now())

        with (
            mock.patch.object(backend, "open", wraps=backend.open) as open_call,
            mock.patch.object(backend, "close", wraps=backend.close) as close_call,
        ):
            # Noop raises an exception — _validate_or_reconnect closes
            # and reopens. open() is called twice: once from
            # ensure_connected, once from the reconnect path.
            backend.connection = mock.MagicMock(spec=SMTP)
            backend.connection.noop.side_effect = AssertionError
            with mock.patch(
                "django.core.mail.backends.smtp.EmailBackend.open", side_effect=[False, True]
            ):
                backend.ensure_connected()
            self.assertEqual(open_call.call_count, 2)
            self.assertEqual(close_call.call_count, 0)
            close_call.reset_mock()
            open_call.reset_mock()

            # Noop succeeds — no reconnect needed.
            backend.connection = mock.MagicMock(spec=SMTP)
            backend.connection.noop.return_value = [250]
            backend.ensure_connected()

            self.assertEqual(open_call.call_count, 1)
            self.assertEqual(backend.connection.noop.call_count, 1)
            close_call.reset_mock()
            open_call.reset_mock()

            # Noop returns 404 — _validate_or_reconnect closes and reopens.
            backend.connection.noop.return_value = [404]
            with mock.patch(
                "django.core.mail.backends.smtp.EmailBackend.open", side_effect=[False, True]
            ):
                backend.ensure_connected()

            self.assertEqual(open_call.call_count, 2)
            self.assertEqual(close_call.call_count, 0)
            close_call.reset_mock()
            open_call.reset_mock()

            # Test backoff procedure
            open_call.side_effect = OSError
            with self.assertRaises(OSError):
                backend.ensure_connected()
            # 3 calls to open as we try 3 times before giving up
            self.assertEqual(open_call.call_count, 3)

    def test_send_messages_backoff(self) -> None:
        """_send_messages retries on connection-level errors but not
        on SMTP protocol errors."""
        connection = mock.MagicMock()

        # ConnectionError: retries 3 times then raises
        connection.send_messages.side_effect = ConnectionError
        with self.assertRaises(ConnectionError):
            _send_messages(connection, [])
        self.assertEqual(connection.send_messages.call_count, 3)
        connection.reset_mock()

        # SMTPServerDisconnected: retries (it's a dropped connection)
        connection.send_messages.side_effect = SMTPServerDisconnected
        with self.assertRaises(SMTPServerDisconnected):
            _send_messages(connection, [])
        self.assertEqual(connection.send_messages.call_count, 3)
        connection.reset_mock()

        # SMTPRecipientsRefused: no retry (protocol error)
        connection.send_messages.side_effect = SMTPRecipientsRefused(
            recipients={"a@b.com": (550, b"Unknown")}
        )
        with self.assertRaises(SMTPRecipientsRefused):
            _send_messages(connection, [])
        self.assertEqual(connection.send_messages.call_count, 1)
        connection.reset_mock()

        # SMTPDataError: no retry (protocol error)
        connection.send_messages.side_effect = SMTPDataError(550, b"Rejected")
        with self.assertRaises(SMTPDataError):
            _send_messages(connection, [])
        self.assertEqual(connection.send_messages.call_count, 1)

    def test_send_immediate_email_backoff_only_without_connection(self) -> None:
        """send_immediate_email uses backoff when it creates the
        connection (no caller-provided one), but sends directly
        when a connection is provided."""
        hamlet = self.example_user("hamlet")

        # Without a connection: _send_messages is called (with backoff)
        with (
            mock.patch("zerver.lib.send_email._send_messages", return_value=1) as patched,
            mock.patch.object(EmailBackend, "send_messages", return_value=1) as direct,
        ):
            send_immediate_email(
                "zerver/emails/password_reset",
                to_emails=[hamlet.email],
                from_name="Test",
                from_address=FromAddress.NOREPLY,
                language="en",
            )
            self.assertEqual(patched.call_count, 1)
            self.assertEqual(direct.call_count, 0)

        # With a connection: send_messages is called directly (no backoff)
        connection = mock.MagicMock()
        connection.send_messages.return_value = 1
        with mock.patch("zerver.lib.send_email._send_messages") as patched:
            send_immediate_email(
                "zerver/emails/password_reset",
                to_emails=[hamlet.email],
                from_name="Test",
                from_address=FromAddress.NOREPLY,
                language="en",
                connection=connection,
            )
            self.assertEqual(patched.call_count, 0)
            self.assertEqual(connection.send_messages.call_count, 1)

    @override_settings(
        EMAIL_BACKEND="django.core.mail.backends.smtp.EmailBackend",
        EMAIL_MAX_CONNECTION_LIFETIME_IN_MINUTES=0,
    )
    def test_worker_no_persistent_connection(self) -> None:
        """With EMAIL_MAX_CONNECTION_LIFETIME_IN_MINUTES=0, the worker
        does not hold a persistent SMTP connection, so connections
        are not left open between sends."""
        worker = ImmediateEmailSenderWorker()
        self.assertIsNone(worker.connection)

    @override_settings(
        EMAIL_BACKEND="zproject.email_backends.PersistentSMTPEmailBackend",
        EMAIL_MAX_CONNECTION_LIFETIME_IN_MINUTES=5,
    )
    def test_worker_persistent_connection(self) -> None:
        """With EMAIL_MAX_CONNECTION_LIFETIME_IN_MINUTES > 0, the
        worker holds a PersistentSMTPEmailBackend and calls
        ensure_connected() before each send."""
        hamlet = self.example_user("hamlet")
        worker = ImmediateEmailSenderWorker()
        self.assertIsInstance(worker.connection, PersistentSMTPEmailBackend)

        event = {
            "template_prefix": "zerver/emails/password_reset",
            "to_emails": [hamlet.email],
            "from_name": "Test",
            "from_address": FromAddress.NOREPLY,
            "language": "en",
            "context": {},
        }
        with (
            mock.patch.object(worker.connection, "ensure_connected") as mock_ensure,
            mock.patch("zerver.worker.email_senders_base.send_immediate_email") as mock_send,
        ):
            worker.send_email(event)
            mock_ensure.assert_called_once()
            mock_send.assert_called_once()
            # Verify the persistent connection was passed through
            self.assertIs(mock_send.call_args[1]["connection"], worker.connection)

    @time_machine.travel(timezone_now(), tick=False)
    @override_settings(EMAIL_MAX_CONNECTION_LIFETIME_IN_MINUTES=None)
    def test_max_connection_lifetime_none(self) -> None:
        backend = PersistentSMTPEmailBackend()

        with (
            mock.patch.object(backend, "open", wraps=backend.open) as open_call,
            mock.patch.object(backend, "close", wraps=backend.close) as close_call,
        ):
            backend.open()
            backend.connection = mock.MagicMock(spec=SMTP)
            backend.connection.noop.return_value = [250]

            self.assertEqual(close_call.call_count, 0)
            self.assertEqual(open_call.call_count, 1)
            self.assertEqual(backend.connection.noop.call_count, 0)
            self.assertEqual(backend.opened_at, timezone_now())
            close_call.reset_mock()
            open_call.reset_mock()

            # Old connection is open, we will not open a new connection because max connection lifetime is None.
            with time_machine.travel(
                timezone_now() + timedelta(minutes=10),
                tick=False,
            ):
                backend.ensure_connected()
            self.assertEqual(close_call.call_count, 0)
            # open is called once from ensure_connected; no reconnect.
            self.assertEqual(open_call.call_count, 1)
            self.assertEqual(backend.connection.noop.call_count, 1)

    @time_machine.travel(timezone_now(), tick=False)
    @override_settings(EMAIL_MAX_CONNECTION_LIFETIME_IN_MINUTES=5)
    def test_max_connection_lifetime_5_minutes(self) -> None:
        backend = PersistentSMTPEmailBackend()

        with (
            mock.patch.object(backend, "open", wraps=backend.open) as open_call,
            mock.patch.object(backend, "close", wraps=backend.close) as close_call,
        ):
            backend.open()
            self.assertEqual(close_call.call_count, 0)
            self.assertEqual(open_call.call_count, 1)
            self.assertEqual(backend.opened_at, timezone_now())
            close_call.reset_mock()
            open_call.reset_mock()

            backend.connection = mock.MagicMock(spec=SMTP)
            backend.connection.noop.return_value = [250]
            # Within lifetime — noop check, no reconnect.
            with time_machine.travel(
                timezone_now() + timedelta(minutes=3),
                tick=False,
            ):
                backend.ensure_connected()

            self.assertEqual(close_call.call_count, 0)
            # open is called once from ensure_connected; no reconnect.
            self.assertEqual(open_call.call_count, 1)
            self.assertEqual(backend.connection.noop.call_count, 1)
            close_call.reset_mock()
            open_call.reset_mock()
            backend.connection.noop.reset_mock()

            # Past lifetime — reconnects without noop.
            with time_machine.travel(
                timezone_now() + timedelta(minutes=6),
                tick=False,
            ):
                self.assertEqual(backend.opened_at, timezone_now() - timedelta(minutes=6))
                old_connection_mock = mock.MagicMock(spec=SMTP)
                backend.connection = old_connection_mock
                with mock.patch(
                    "django.core.mail.backends.smtp.EmailBackend.open", side_effect=[False, True]
                ):
                    backend.ensure_connected()
                self.assertEqual(backend.opened_at, timezone_now())

            self.assertEqual(close_call.call_count, 0)
            self.assertEqual(open_call.call_count, 2)
            self.assertEqual(old_connection_mock.noop.call_count, 0)
            close_call.reset_mock()
            open_call.reset_mock()

            # Connections older than 24 hours are properly
            # recycled: timedelta.seconds wraps at 24 hours, but
            # total_seconds() does not.
            with time_machine.travel(
                timezone_now() + timedelta(hours=25),
                tick=False,
            ):
                old_connection_mock = mock.MagicMock(spec=SMTP)
                backend.connection = old_connection_mock
                with mock.patch(
                    "django.core.mail.backends.smtp.EmailBackend.open", side_effect=[False, True]
                ):
                    backend.ensure_connected()
                self.assertEqual(backend.opened_at, timezone_now())

            # Noop should not have been attempted on the old connection.
            self.assertEqual(old_connection_mock.noop.call_count, 0)
            self.assertEqual(open_call.call_count, 2)

    def test_open_returns_false_for_new_connection(self) -> None:
        """open() returns False even for new connections, which
        prevents Django's send_messages from auto-closing the
        persistent connection."""
        backend = PersistentSMTPEmailBackend()
        with mock.patch.object(EmailBackend, "open", return_value=True):
            result = backend.open()
        self.assertIs(result, False)

    def test_close_clears_connection(self) -> None:
        """close() actually closes the connection, enabling error
        recovery in send_immediate_email."""
        backend = PersistentSMTPEmailBackend()
        backend.connection = mock.MagicMock(spec=SMTP)
        backend.close()
        self.assertIsNone(backend.connection)

    def test_send_email_exceptions(self) -> None:
        hamlet = self.example_user("hamlet")
        from_name = FromAddress.security_email_from_name(language="en")
        address = FromAddress.NOREPLY
        # Used to check the output
        mail = build_email(
            "zerver/emails/password_reset",
            to_emails=[hamlet.email],
            from_name=from_name,
            from_address=address,
            language="en",
        )
        self.assertEqual(mail.extra_headers["From"], f"{from_name} <{FromAddress.NOREPLY}>")

        # We test the cases that should raise an EmailNotDeliveredError
        errors = [
            (f"Unknown error sending password_reset email to {mail.to}", None, [0]),
            (f"Error sending password_reset email to {mail.to}", "", [SMTPException()]),
            (
                f"Error sending password_reset email to {mail.to}",
                f"{{{address!r}: (550, b'User unknown')}}",
                [SMTPRecipientsRefused(recipients={address: (550, b"User unknown")})],
            ),
            (
                f"Error sending password_reset email to {mail.to} with error code 242: From field too long.",
                "(242, 'From field too long.')",
                [SMTPDataError(242, "From field too long.")],
            ),
        ]

        for message, exc_message, side_effect in errors:
            with mock.patch.object(EmailBackend, "send_messages", side_effect=side_effect):
                with (
                    self.assertLogs(logger=logger) as info_log,
                    self.assertRaises(EmailNotDeliveredError),
                ):
                    send_email(
                        "zerver/emails/password_reset",
                        to_emails=[hamlet.email],
                        from_name=from_name,
                        from_address=FromAddress.NOREPLY,
                        language="en",
                    )
                self.assert_length(info_log.records, 2)
                self.assertEqual(info_log.records[0].levelname, "INFO")
                self.assertEqual(
                    info_log.records[0].getMessage(), f"Sending password_reset email to {mail.to}"
                )
                self.assertEqual(info_log.records[1].levelname, "ERROR")
                self.assertEqual(info_log.records[1].getMessage(), message)
                self.assertEqual(
                    info_log.records[1].exc_info and str(info_log.records[1].exc_info[1]),
                    exc_message,
                )

    def test_send_email_config_error_logging(self) -> None:
        hamlet = self.example_user("hamlet")

        with (
            self.settings(EMAIL_HOST_USER="test", EMAIL_HOST_PASSWORD=None),
            self.assertLogs(logger=logger, level="ERROR") as error_log,
        ):
            send_email(
                "zerver/emails/password_reset",
                to_emails=[hamlet.email],
                from_name="From Name",
                from_address=FromAddress.NOREPLY,
                language="en",
            )

        self.assertEqual(
            error_log.output,
            [
                (
                    "ERROR:zulip.send_email:"
                    "An SMTP username was set (EMAIL_HOST_USER), but password is unset (EMAIL_HOST_PASSWORD).  "
                    "To disable SMTP authentication, set EMAIL_HOST_USER to an empty string."
                )
            ],
        )

        # Empty string is OK
        with self.settings(EMAIL_HOST_USER="test", EMAIL_HOST_PASSWORD=""):
            send_email(
                "zerver/emails/password_reset",
                to_emails=[hamlet.email],
                from_name="From Name",
                from_address=FromAddress.NOREPLY,
                language="en",
            )