snapshot of https://github.com/zulip/zulip.git @ 48c69911c7dd4e03b20552eb3bd77ad8151c1357 (2026-09-15)

zerver/migrations/0423_fix_email_gateway_attachment_owner.py

@@ -0,0 +1,176 @@
+from datetime import timedelta
+
+from django.conf import settings
+from django.db import connection, migrations
+from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+from django.db.migrations.state import StateApps
+from psycopg2.sql import SQL, Identifier, Literal
+
+
+def fix_email_gateway_attachment_owner(
+    apps: StateApps, schema_editor: BaseDatabaseSchemaEditor
+) -> None:
+    Realm = apps.get_model("zerver", "Realm")
+    UserProfile = apps.get_model("zerver", "UserProfile")
+    Client = apps.get_model("zerver", "Client")
+    Message = apps.get_model("zerver", "Message")
+    ArchivedMessage = apps.get_model("zerver", "ArchivedMessage")
+    Stream = apps.get_model("zerver", "Stream")
+    Attachment = apps.get_model("zerver", "Attachment")
+    ArchivedAttachment = apps.get_model("zerver", "ArchivedAttachment")
+
+    if not Realm.objects.exists():
+        return
+
+    mail_gateway_bot = UserProfile.objects.get(email__iexact=settings.EMAIL_GATEWAY_BOT)
+
+    # "Internal" is the client-id of all mail gateway posts
+    internal_client, _ = Client.objects.get_or_create(name="Internal")
+
+    # We only look in Attachment and not ArchivedAttachment because,
+    # never having been associated with a message, there is no way for
+    # the attachments to have been archived.
+    orphan_attachments = Attachment.objects.filter(
+        messages=None,
+        owner_id=mail_gateway_bot.id,
+    )
+    if len(orphan_attachments) == 0:
+        return
+
+    print()
+    print(f"Found {len(orphan_attachments)} email gateway attachments to reattach")
+    for attachment in orphan_attachments:
+        # We look for the message posted by "Internal" at the same
+        # time, in the same realm, which has a link to the attachment
+        # but no "has_attachments".  There are potentially other,
+        # later, messages (possibly from other users, to other
+        # places!) which tried to link to the attachment; we do not
+        # fix those references, because finding them efficiently is
+        # quite hard, as is calculating if they "should" have had
+        # access to the attachment at the time.
+        print(
+            f"Looking for a message to attach {attachment.path_id}, created {attachment.create_time}"
+        )
+        possible_matches = []
+        for model_class in (Message, ArchivedMessage):
+            possible_matches.extend(
+                # All messages with this bug will have
+                # `has_attachment=False`, since they failed to attach
+                # the contents.  However, we cannot limit to
+                # sender=mail_gateway_bot because they were sent "as"
+                # some other user.
+                model_class.objects.filter(
+                    has_attachment=False,
+                    realm_id=attachment.realm_id,
+                    sending_client_id=internal_client.id,
+                    date_sent__gte=attachment.create_time,
+                    date_sent__lte=attachment.create_time + timedelta(minutes=5),
+                    content__contains="/user_uploads/" + attachment.path_id,
+                ).order_by("date_sent")
+            )
+        if len(possible_matches) == 0:
+            print("  No matches!")
+            continue
+
+        # If there are 1 or more matches, we assume the earliest is
+        # the correct one, since it's ~impossible to have predicted
+        # the URL before it was first sent.
+        message = possible_matches[0]
+        print(f"  Found {message.id} @ {message.date_sent} by {message.sender.delivery_email})")
+
+        # If this is an ArchivedMessage, then we have to move the
+        # Attachment into an ArchivedAttachment.  We also have to
+        # generate an zerver_archivedattachment_message row with an id
+        # based on the next free from zerver_attachment_message, since
+        # those are one id space.
+        if isinstance(message, ArchivedMessage):
+            # move_rows
+            fields = list(Attachment._meta.fields)
+            src_fields = [Identifier("zerver_attachment", field.column) for field in fields]
+            dst_fields = [Identifier(field.column) for field in fields]
+            with connection.cursor() as cursor:
+                raw_query = SQL(
+                    """
+                    INSERT INTO zerver_archivedattachment ({dst_fields})
+                        SELECT {src_fields}
+                        FROM zerver_attachment
+                        WHERE id = {id}
+                    ON CONFLICT (id) DO NOTHING
+                    RETURNING id
+                    """
+                )
+                cursor.execute(
+                    raw_query.format(
+                        src_fields=SQL(",").join(src_fields),
+                        dst_fields=SQL(",").join(dst_fields),
+                        id=Literal(attachment.id),
+                    )
+                )
+                archived_ids = [id for (id,) in cursor.fetchall()]
+                if len(archived_ids) != 1:
+                    print("!!! Did not create one archived attachment row!")
+            attachment.delete()
+            attachment = ArchivedAttachment.objects.get(id=archived_ids[0])
+
+        # Determine message (and thus attachment) properties; this is
+        # from do_claim_attachments
+        is_message_realm_public = False
+        is_message_web_public = False
+        if message.recipient.type == 2:  # Recipient.STREAM
+            stream = Stream.objects.get(id=message.recipient.type_id)
+            is_message_realm_public = not stream.invite_only and not stream.is_in_zephyr_realm
+            is_message_web_public = stream.is_web_public
+
+        attachment.owner_id = message.sender_id
+        attachment.is_web_public = is_message_web_public
+        attachment.is_realm_public = is_message_realm_public
+        attachment.save(update_fields=["owner_id", "is_web_public", "is_realm_public"])
+
+        if isinstance(attachment, ArchivedAttachment):
+            assert isinstance(message, ArchivedMessage)
+            # We need to use the sequence from
+            # zerver_attachment_messages, since that id is reused when
+            # restoring the message.
+            with connection.cursor() as cursor:
+                raw_query = SQL(
+                    """
+                    INSERT INTO zerver_archivedattachment_messages
+                           (id, archivedattachment_id, archivedmessage_id)
+                    VALUES (nextval(pg_get_serial_sequence('zerver_attachment_messages', 'id')),
+                            {attachment_id}, {message_id})
+                    """
+                )
+                cursor.execute(
+                    raw_query.format(
+                        attachment_id=Literal(attachment.id),
+                        message_id=Literal(message.id),
+                    )
+                )
+        else:
+            assert isinstance(message, Message)
+            attachment.messages.add(message)
+
+        message.has_attachment = True
+        message.save(update_fields=["has_attachment"])
+
+
+class Migration(migrations.Migration):
+    """
+    Messages sent "as" a user via the email gateway had their
+    attachments left orphan, accidentally owned by the email gateway
+    bot.  Find each such orphaned attachment, and re-own it and attach
+    it to the appropriate message.
+
+    """
+
+    dependencies = [
+        ("zerver", "0422_multiuseinvite_status"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            fix_email_gateway_attachment_owner,
+            reverse_code=migrations.RunPython.noop,
+            elidable=True,
+        )
+    ]