snapshot of https://github.com/zulip/zulip.git @ 48c69911c7dd4e03b20552eb3bd77ad8151c1357 (2026-09-15)

zerver/data_import/microsoft_teams.py

@@ -0,0 +1,877 @@
+import logging
+import os
+import re
+from collections import defaultdict
+from collections.abc import Callable, Iterable
+from dataclasses import dataclass
+from datetime import datetime
+from email.headerregistry import Address
+from multiprocessing import Manager
+from multiprocessing.managers import ListProxy
+from typing import Any, Literal, TypeAlias
+from urllib.parse import SplitResult
+
+import regex
+import requests
+from django.conf import settings
+from django.utils.timezone import now as timezone_now
+
+from zerver.data_import.import_util import (
+    AttachmentRecordData,
+    UploadFileRequest,
+    UploadRecordData,
+    ZerverFieldsT,
+    build_message,
+    build_realm,
+    build_recipient,
+    build_stream,
+    build_subscription,
+    build_user_profile,
+    build_usermessages,
+    build_zerver_realm,
+    create_converted_data_files,
+    get_attachment_path_and_content,
+    get_data_file,
+    make_subscriber_map,
+    request_file_stream,
+    scrub_missing_upload_records_after_download,
+    validate_user_emails_for_import,
+    write_response_file_stream_to_path,
+)
+from zerver.data_import.microsoft_teams_message_conversion import (
+    HOSTED_CONTENT_MARKDOWN_IMAGE_SYNTAX_REGEX,
+    convert_microsoft_teams_html_to_markdown,
+)
+from zerver.data_import.sequencer import NEXT_ID
+from zerver.lib.export import MESSAGE_BATCH_CHUNK_SIZE, do_common_export_processes
+from zerver.lib.markdown import get_markdown_image_for_url
+from zerver.lib.mime_types import bare_content_type
+from zerver.lib.parallel import run_parallel_queue
+from zerver.lib.partial import partial
+from zerver.models.recipients import Recipient
+from zerver.models.users import UserProfile
+
+
+@dataclass
+class TeamMetadata:
+    """
+    "team" is equivalent to a Zulip channel
+    """
+
+    description: str
+    display_name: str
+    visibility: Literal["public", "private"]
+    is_archived: bool
+    zulip_channel_id: int
+    zulip_recipient_id: int
+
+
+@dataclass
+class ChannelMetadata:
+    """
+    "channel" is equivalent to Zulip topics.
+    """
+
+    display_name: str
+    is_favourite_by_default: bool
+    is_archived: bool
+    is_favorite_by_default: bool
+    membership_type: str
+    team_id: str
+
+
+@dataclass
+class MessageConversionResult:
+    zerver_messages: list[ZerverFieldsT]
+    zerver_usermessages: list[ZerverFieldsT]
+    upload_records: list[UploadRecordData]
+
+
+@dataclass
+class AttachmentConversionResult:
+    updated_content: str
+    upload_records: list[UploadRecordData]
+
+
+@dataclass
+class ExportMessageAttachmentParameter:
+    realm_id: int
+    message_id: int
+    sender_id: int
+    upload_file_request: UploadFileRequest
+
+
+AddedTeamsT: TypeAlias = dict[str, TeamMetadata]
+TeamIdToZulipRecipientIdT: TypeAlias = dict[str, int]
+MicrosoftTeamsUserIdToZulipUserIdT: TypeAlias = dict[str, int]
+MicrosoftTeamsFieldsT: TypeAlias = dict[str, Any]
+
+MICROSOFT_TEAMS_DEFAULT_ANNOUNCEMENTS_CHANNEL_NAME = "All company"
+
+
+def convert_teams_to_channels(
+    microsoft_teams_user_id_to_zulip_user_id: MicrosoftTeamsUserIdToZulipUserIdT,
+    realm: dict[str, Any],
+    realm_id: int,
+    teams_data_dir: str,
+) -> AddedTeamsT:
+    team_data_folders = []
+    for f in os.listdir(teams_data_dir):
+        path = os.path.join(teams_data_dir, f)
+        if os.path.isdir(path):
+            team_data_folders.append(f)
+
+    logging.info("######### IMPORTING TEAMS STARTED #########\n")
+
+    # Build teams subscription map
+    team_id_to_zulip_subscriber_ids: dict[str, set[int]] = defaultdict(set)
+    for team_id in team_data_folders:
+        team_members_file_name = f"teamMembers_{team_id}.json"
+        team_members_file_path = os.path.join(teams_data_dir, team_id, team_members_file_name)
+        team_members: list[MicrosoftTeamsFieldsT] = get_data_file(team_members_file_path)
+
+        for member in team_members:
+            zulip_user_id = microsoft_teams_user_id_to_zulip_user_id[member["UserId"]]
+            team_id_to_zulip_subscriber_ids[team_id].add(zulip_user_id)
+
+    # Compile teamsSettings.json and teamsList.json and convert
+    # teams to Zulip channels.
+    teams_list = get_data_file(os.path.join(teams_data_dir, "teamsList.json"))
+    teams_settings = get_data_file(os.path.join(teams_data_dir, "teamsSettings.json"))
+    team_dict: dict[str, Any] = {team["GroupsId"]: team for team in teams_list}
+    teams_metadata: AddedTeamsT = {}
+    for team_settings in teams_settings:
+        team_id = team_settings.get("Id")
+
+        assert team_id and team_id in team_dict, (
+            f"Team {team_id} appears in teamsSettings.json but not teamsList.json!"
+        )
+
+        compiled_team_data: MicrosoftTeamsFieldsT = {**team_dict[team_id], **team_settings}
+
+        channel_id = NEXT_ID("channel")
+        recipient_id = NEXT_ID("recipient")
+
+        channel = build_stream(
+            # Microsoft Teams export doesn't include teams creation date.
+            date_created=float(timezone_now().timestamp()),
+            realm_id=realm_id,
+            name=compiled_team_data["Name"],
+            description=compiled_team_data["Description"] or "",
+            stream_id=channel_id,
+            deactivated=compiled_team_data["IsArchived"],
+            invite_only=compiled_team_data["Visibility"] == "private",
+        )
+        realm["zerver_stream"].append(channel)
+
+        recipient = build_recipient(channel_id, recipient_id, Recipient.STREAM)
+        realm["zerver_recipient"].append(recipient)
+
+        for zulip_user_id in team_id_to_zulip_subscriber_ids[team_id]:
+            sub = build_subscription(
+                recipient_id=recipient_id,
+                user_id=zulip_user_id,
+                subscription_id=NEXT_ID("subscription"),
+            )
+            realm["zerver_subscription"].append(sub)
+
+        # If the org uses the "All company" team, set it as the announcements channel.
+        if (
+            compiled_team_data["Name"] == MICROSOFT_TEAMS_DEFAULT_ANNOUNCEMENTS_CHANNEL_NAME
+        ):  # nocoverage
+            realm["zerver_realm"][0]["new_stream_announcements_stream"] = channel_id
+            realm["zerver_realm"][0]["zulip_update_announcements_stream"] = channel_id
+            realm["zerver_realm"][0]["signup_announcements_stream"] = channel_id
+            logging.info("Using the channel 'All company' as default announcements channel.")
+
+        teams_metadata[team_id] = TeamMetadata(
+            description=compiled_team_data["Description"],
+            display_name=compiled_team_data["DisplayName"],
+            visibility=compiled_team_data["Visibility"],
+            is_archived=compiled_team_data["IsArchived"],
+            zulip_channel_id=channel_id,
+            zulip_recipient_id=recipient_id,
+        )
+
+    return teams_metadata
+
+
+@dataclass
+class MicrosoftTeamsUserRoleData:
+    global_administrator_user_ids: set[str]
+    guest_user_ids: set[str]
+
+
+@dataclass
+class ODataQueryParameter:
+    # This is used to compute a request's OData query. It specifies the
+    # amount, type, and order of the data returned for the resource
+    # identified by the URL.
+    # https://learn.microsoft.com/en-us/graph/query-parameters?tabs=http
+    parameter: Literal["$filter", "$search", "$select", "$top"]
+    expression: str
+
+
+MICROSOFT_GRAPH_API_URL = "https://graph.microsoft.com/v1.0{endpoint}"
+
+
+def get_microsoft_graph_api_data(
+    api_url: str,
+    odata_parameters: list[ODataQueryParameter] | None = None,
+    **kwargs: Any,
+) -> Any:
+    if not kwargs.get("token"):
+        raise AssertionError("Microsoft authorization token missing in kwargs")
+    token = kwargs.pop("token")
+    accumulated_result = []
+    parameters = {}
+    if odata_parameters is not None:
+        for parameter in odata_parameters:
+            assert parameter.parameter not in parameters
+            parameters[parameter.parameter] = parameter.expression
+
+    # If a request is paginated, "@odata.nextLink" will be included in the response,
+    # it points to the next page of result. Even if the `$top` query is not specified,
+    # depending on the endpoint and the result size, it may be paged by the server.
+    # https://learn.microsoft.com/en-us/graph/paging?tabs=http#server-side-paging
+    next_link: str | None = api_url
+    while next_link is not None:
+        response = requests.get(
+            next_link, headers={"Authorization": f"Bearer {token}"}, params=parameters
+        )
+
+        if response.status_code != requests.codes.ok:
+            logging.info("HTTP error: %s, Response: %s", response.status_code, response.text)
+            raise Exception("HTTP error accessing the Microsoft Graph API.")
+
+        response_data = response.json()
+        accumulated_result.extend(response_data["value"])
+        next_link = response_data.get("@odata.nextLink")
+
+        # Don't apply initial request's parameters to subsequent paginated requests.
+        if next_link is not None:
+            parameters = {}
+
+    return accumulated_result
+
+
+def get_directory_roles(api_token: str) -> list[MicrosoftTeamsFieldsT]:
+    """
+    https://learn.microsoft.com/en-us/graph/api/directoryrole-list?view=graph-rest-1.0
+    """
+    return get_microsoft_graph_api_data(
+        MICROSOFT_GRAPH_API_URL.format(endpoint="/directoryRoles"), token=api_token
+    )
+
+
+def get_users_with_directory_role_id(
+    directory_role_id: str, api_token: str
+) -> list[MicrosoftTeamsFieldsT]:
+    """
+    https://learn.microsoft.com/en-us/graph/api/directoryrole-list-members?view=graph-rest-1.0
+    """
+    return get_microsoft_graph_api_data(
+        MICROSOFT_GRAPH_API_URL.format(endpoint=f"/directoryRoles/{directory_role_id}/members"),
+        token=api_token,
+    )
+
+
+def get_user_ids_with_member_type(
+    member_type: Literal["Member", "Guest"], api_token: str
+) -> list[MicrosoftTeamsFieldsT]:
+    """
+    https://learn.microsoft.com/en-us/graph/api/user-list?view=graph-rest-1.0
+    """
+    odata_parameter = [
+        ODataQueryParameter(parameter="$filter", expression=f"userType eq '{member_type}'"),
+        ODataQueryParameter(parameter="$select", expression="id"),
+    ]
+    return get_microsoft_graph_api_data(
+        MICROSOFT_GRAPH_API_URL.format(endpoint="/users"),
+        odata_parameter,
+        token=api_token,
+    )
+
+
+def get_user_roles(api_token: str) -> MicrosoftTeamsUserRoleData:
+    """
+    User roles are not included in the export file, so this calls
+    to Microsoft Graph API endpoints for that data. We mainly
+    want to find out who the admins and guests are.
+    """
+    directory_roles: list[MicrosoftTeamsFieldsT] = get_directory_roles(api_token)
+    global_administrator_role_id = None
+    for role in directory_roles:
+        if role["displayName"] == "Global Administrator":
+            global_administrator_role_id = role["id"]
+
+    if global_administrator_role_id is None:
+        raise AssertionError("Could not find Microsoft Teams organization owners/administrators.")
+
+    admin_users_data = get_users_with_directory_role_id(global_administrator_role_id, api_token)
+    guest_users_data = get_user_ids_with_member_type("Guest", api_token)
+
+    return MicrosoftTeamsUserRoleData(
+        global_administrator_user_ids={user_data["id"] for user_data in admin_users_data},
+        guest_user_ids={user_data["id"] for user_data in guest_users_data},
+    )
+
+
+def get_user_email(user: MicrosoftTeamsFieldsT) -> str:
+    if user["Mail"]:
+        return user["Mail"]
+    else:
+        raise AssertionError(f"Could not find email address for Microsoft Teams user {user}")
+
+
+def create_is_mirror_dummy_user(
+    microsoft_team_user_id: str,
+    microsoft_teams_user_id_to_zulip_user_id: MicrosoftTeamsUserIdToZulipUserIdT,
+    realm: dict[str, Any],
+    realm_id: int,
+    domain_name: str,
+) -> None:
+    zulip_user_id = NEXT_ID("user")
+    user_full_name = f"Deleted Teams user {microsoft_team_user_id}"
+    email = Address(username=microsoft_team_user_id, domain=domain_name).addr_spec
+    user_profile_dict = build_user_profile(
+        avatar_source=UserProfile.DEFAULT_AVATAR_SOURCE,
+        date_joined=int(timezone_now().timestamp()),
+        delivery_email=email,
+        email=email,
+        full_name=user_full_name,
+        id=zulip_user_id,
+        is_active=False,
+        role=UserProfile.ROLE_MEMBER,
+        is_mirror_dummy=True,
+        realm_id=realm_id,
+        short_name=user_full_name,
+        timezone="UTC",
+    )
+    realm["zerver_userprofile"].append(user_profile_dict)
+    microsoft_teams_user_id_to_zulip_user_id[microsoft_team_user_id] = zulip_user_id
+
+
+def convert_users(
+    microsoft_teams_user_role_data: MicrosoftTeamsUserRoleData,
+    realm: dict[str, Any],
+    realm_id: int,
+    timestamp: int,
+    users_list: list[MicrosoftTeamsFieldsT],
+) -> MicrosoftTeamsUserIdToZulipUserIdT:
+    zerver_user_profile: list[ZerverFieldsT] = []
+    microsoft_teams_user_id_to_zulip_user_id: MicrosoftTeamsUserIdToZulipUserIdT = defaultdict(int)
+    found_emails: dict[str, int] = {}
+    has_owner = False
+
+    logging.info("######### IMPORTING USERS STARTED #########\n")
+    for user in users_list:
+        microsoft_teams_user_id = user["Id"]
+        user_full_name = user["DisplayName"]
+
+        if microsoft_teams_user_id in microsoft_teams_user_role_data.global_administrator_user_ids:
+            user_role = UserProfile.ROLE_REALM_OWNER
+        elif microsoft_teams_user_id in microsoft_teams_user_role_data.guest_user_ids:
+            user_role = UserProfile.ROLE_GUEST
+        else:
+            user_role = UserProfile.ROLE_MEMBER
+
+        microsoft_teams_user_email = get_user_email(user)
+
+        zulip_user_id = NEXT_ID("user")
+        found_emails[microsoft_teams_user_email.lower()] = zulip_user_id
+
+        user_profile_dict = build_user_profile(
+            avatar_source=UserProfile.DEFAULT_AVATAR_SOURCE,
+            date_joined=timestamp,
+            delivery_email=microsoft_teams_user_email,
+            email=microsoft_teams_user_email,
+            full_name=user_full_name,
+            id=zulip_user_id,
+            # This function only processes user data from `users/usersList.json` which only
+            # lists active users -- no bot or deleted user accounts.
+            is_active=True,
+            role=user_role,
+            is_mirror_dummy=False,
+            realm_id=realm_id,
+            short_name=user_full_name,
+            timezone="UTC",
+        )
+
+        user_profile_dict["realm"] = realm_id
+        zerver_user_profile.append(user_profile_dict)
+        microsoft_teams_user_id_to_zulip_user_id[microsoft_teams_user_id] = zulip_user_id
+
+        if user_role == UserProfile.ROLE_REALM_OWNER:
+            has_owner = True
+
+        logging.info(
+            "%s: %s -> %s",
+            microsoft_teams_user_id,
+            user_full_name,
+            microsoft_teams_user_email,
+        )
+
+    if not has_owner:
+        logging.warning("Converted realm has no owners!")
+
+    validate_user_emails_for_import(list(found_emails))
+    realm["zerver_userprofile"] = zerver_user_profile
+    logging.info("######### IMPORTING USERS FINISHED #########\n")
+    return microsoft_teams_user_id_to_zulip_user_id
+
+
+def get_timestamp_from_message(message: MicrosoftTeamsFieldsT) -> float:
+    return datetime.fromisoformat(message["CreatedDateTime"]).timestamp()
+
+
+def get_microsoft_teams_sender_id_from_message(message: MicrosoftTeamsFieldsT) -> str:
+    return message["From"]["User"]["Id"]
+
+
+def is_microsoft_teams_event_message(message: MicrosoftTeamsFieldsT) -> bool:
+    return message["MessageType"] == "unknownFutureValue" and message["From"] is None
+
+
+def download_and_export_microsoft_teams_upload_file(
+    attachment_records: ListProxy | list[AttachmentRecordData],  # type: ignore[type-arg] # ListProxy is not subscriptable
+    output_dir: str,
+    args: ExportMessageAttachmentParameter,
+) -> None:
+    upload_file_request = args.upload_file_request
+    response = request_file_stream(
+        upload_file_request.request_url,
+        upload_file_request.params,
+        upload_file_request.headers,
+        **upload_file_request.kwargs,
+    )
+
+    # Don't add the file type extension here. The file path will be remapped
+    # during import and must match the updated URL in the message content.
+    # The actual file's content type will be included in its attachment
+    # record.
+    file_output_path = os.path.join(
+        output_dir, "uploads", f"{upload_file_request.output_file_path_id}"
+    )
+
+    os.makedirs(os.path.dirname(file_output_path), exist_ok=True)
+    write_response_file_stream_to_path(response, file_output_path)
+
+    raw_content_type = response.headers.get("Content-Type")
+    assert raw_content_type is not None
+
+    # Can't use build_attachment here since Django doesn't support parallel
+    # connection to the db.
+    attachment_records.append(
+        AttachmentRecordData(
+            content_type=bare_content_type(raw_content_type),
+            create_time=os.path.getmtime(file_output_path),
+            file_name=os.path.basename(file_output_path),
+            # IDs will be assigned later after all uploads have been downloaded.
+            id=0,
+            is_realm_public=True,
+            messages=[args.message_id],
+            owner=args.sender_id,
+            path_id=upload_file_request.output_file_path_id,
+            realm=args.realm_id,
+            scheduled_messages=[],
+            size=os.path.getsize(file_output_path),
+            is_web_public=False,
+        )
+    )
+
+
+def process_hosted_content_attachments(
+    content: str,
+    do_download_and_export_upload_file: Callable[[ExportMessageAttachmentParameter], None],
+    is_direct_message_type: bool,
+    microsoft_graph_api_token: str,
+    realm_id: int,
+    teams_message_id: str,
+    zulip_message_id: int,
+    zulip_user_id: int,
+) -> AttachmentConversionResult:
+    """
+    "Hosted contents" are message file attachments that can be fetched straight
+    from Microsoft Graph API. As of April 2026, there are only 3 known types of
+    items that are categorized as "hosted contents":
+     * images
+     * custom emojis
+     * code snippets
+
+    Since custom emoji and code snippet are formatted as custom HTML blocks,
+    convert_microsoft_teams_html_to_markdown will turn them into empty strings,
+    so this currently only supports processing image hosted contents.
+
+    It's worth checking the API documentation for the hosted content once in a
+    while to verify what file types "hosted content" includes, since the code path
+    for this relies on that being only images:
+        https://learn.microsoft.com/en-us/graph/api/chatmessagehostedcontent-get
+    """
+    upload_records: list[UploadRecordData] = []
+
+    def export_file_and_get_zulip_url(match: regex.Match[str]) -> str:
+        file_name = match.group("file_name")
+        hosted_content_api_url = match.group("api_url")
+        attachment_data = get_attachment_path_and_content(file_name, teams_message_id, realm_id)
+
+        upload_records.append(
+            UploadRecordData(
+                content_type=None,
+                last_modified=float(timezone_now().timestamp()),
+                path=attachment_data.path_id,
+                realm_id=realm_id,
+                s3_path=attachment_data.path_id,
+                size=0,
+                user_profile_id=zulip_user_id,
+            )
+        )
+
+        do_download_and_export_upload_file(
+            ExportMessageAttachmentParameter(
+                realm_id=realm_id,
+                message_id=zulip_message_id,
+                sender_id=zulip_user_id,
+                upload_file_request=UploadFileRequest(
+                    output_file_path_id=attachment_data.path_id,
+                    request_url=hosted_content_api_url,
+                    params=None,
+                    headers={"Authorization": f"Bearer {microsoft_graph_api_token}"},
+                    kwargs={},
+                ),
+            )
+        )
+        # Use inline thumbnail here since MS Teams only supports customizing the hosted
+        # content image's alt tag.
+        return get_markdown_image_for_url(file_name, attachment_data.url)
+
+    if is_direct_message_type:
+        # TODO: hosted content URL for private chats have a different format, we
+        # can process it once we support converting direct messages.
+        return AttachmentConversionResult(updated_content=content, upload_records=upload_records)
+    else:
+        updated_content = regex.sub(
+            HOSTED_CONTENT_MARKDOWN_IMAGE_SYNTAX_REGEX,
+            export_file_and_get_zulip_url,
+            content,
+            flags=re.VERBOSE,
+        )
+        return AttachmentConversionResult(
+            updated_content=updated_content,
+            upload_records=upload_records,
+        )
+
+
+def process_messages(
+    added_teams: dict[str, TeamMetadata],
+    channel_metadata: dict[str, ChannelMetadata] | None,
+    do_download_and_export_upload_file: Callable[[ExportMessageAttachmentParameter], None],
+    domain_name: str,
+    messages: list[MicrosoftTeamsFieldsT],
+    microsoft_graph_api_token: str,
+    microsoft_teams_user_id_to_zulip_user_id: MicrosoftTeamsUserIdToZulipUserIdT,
+    realm: dict[str, Any],
+    realm_id: int,
+    subscriber_map: dict[int, set[int]],
+) -> MessageConversionResult:
+    zerver_usermessage: list[ZerverFieldsT] = []
+    zerver_messages: list[ZerverFieldsT] = []
+    accumulated_batch_upload_records: list[UploadRecordData] = []
+
+    for message in messages:
+        if is_microsoft_teams_event_message(message):
+            continue
+
+        message_content_type = message["Body"]["ContentType"]
+        if message_content_type == "html":
+            try:
+                content = convert_microsoft_teams_html_to_markdown(message["Body"]["Content"])
+            except Exception:  # nocoverage
+                logging.warning(
+                    "Error converting HTML to text for message: '%s'; continuing", content
+                )
+                logging.warning(str(message))
+                continue
+        else:  # nocoverage
+            logging.warning("Unable to convert this message content type: %s", message_content_type)
+            continue
+
+        # Determine message type, private or channel.
+        if message["ChannelIdentity"] is not None:
+            if channel_metadata is None:
+                raise AssertionError("Failed to build channel data.")
+            current_channel = channel_metadata[message["ChannelIdentity"]["ChannelId"]]
+            if current_channel.membership_type == "private":
+                # Don't include private channel messages.
+                continue
+            topic_name = current_channel.display_name
+            is_direct_message_type = False
+            recipient_id = added_teams[message["ChannelIdentity"]["TeamId"]].zulip_recipient_id
+        else:  # nocoverage
+            assert message["ChatId"] is not None
+            # TODO: Converting direct messages is not yet supported. Since
+            # subscription list and recipient map of direct message conversations
+            # are not listed, we have to manually build them as we iterate over
+            # the user messages.
+            continue
+
+        microsoft_teams_sender_id: str = get_microsoft_teams_sender_id_from_message(message)
+
+        if microsoft_teams_sender_id not in microsoft_teams_user_id_to_zulip_user_id:
+            create_is_mirror_dummy_user(
+                microsoft_teams_sender_id,
+                microsoft_teams_user_id_to_zulip_user_id,
+                realm,
+                realm_id,
+                domain_name,
+            )
+
+        message_id = NEXT_ID("message")
+        user_id = microsoft_teams_user_id_to_zulip_user_id[microsoft_teams_sender_id]
+        attachment_result = process_hosted_content_attachments(
+            content=content,
+            do_download_and_export_upload_file=do_download_and_export_upload_file,
+            is_direct_message_type=is_direct_message_type,
+            microsoft_graph_api_token=microsoft_graph_api_token,
+            realm_id=realm_id,
+            teams_message_id=message["Id"],
+            zulip_user_id=user_id,
+            zulip_message_id=message_id,
+        )
+        content = attachment_result.updated_content
+        accumulated_batch_upload_records += attachment_result.upload_records
+
+        zulip_message = build_message(
+            topic_name=topic_name,
+            date_sent=get_timestamp_from_message(message),
+            message_id=message_id,
+            content=content,
+            rendered_content=None,
+            user_id=user_id,
+            recipient_id=recipient_id,
+            realm_id=realm_id,
+            is_channel_message=not is_direct_message_type,
+            # Import will correct the message's has_image and has_link attributes.
+            has_image=False,
+            has_link=False,
+            has_attachment=attachment_result.upload_records != [],
+            is_direct_message_type=is_direct_message_type,
+        )
+        zerver_messages.append(zulip_message)
+
+        (num_created, num_skipped) = build_usermessages(
+            zerver_usermessage=zerver_usermessage,
+            subscriber_map=subscriber_map,
+            recipient_id=recipient_id,
+            mentioned_user_ids=[],
+            message_id=message_id,
+            is_private=is_direct_message_type,
+        )
+
+        logging.debug(
+            "Created %s UserMessages; deferred %s due to long-term idle",
+            num_created,
+            num_skipped,
+        )
+
+    return MessageConversionResult(
+        zerver_messages=zerver_messages,
+        zerver_usermessages=zerver_usermessage,
+        upload_records=accumulated_batch_upload_records,
+    )
+
+
+def get_batched_export_message_data(
+    message_data_paths: list[str], chunk_size: int = MESSAGE_BATCH_CHUNK_SIZE
+) -> Iterable[list[MicrosoftTeamsFieldsT]]:
+    batched_messages: list[MicrosoftTeamsFieldsT] = []
+    for path in message_data_paths:
+        messages = get_data_file(path)
+        # Teams export tool doesn't sort messages in chronological order.
+        # Sort Microsoft Teams messages by their ID, which is their date
+        # sent in unix time.
+        for message in sorted(messages, key=lambda m: int(m["Id"])):
+            if len(batched_messages) == chunk_size:
+                yield batched_messages
+                batched_messages.clear()
+            batched_messages.append(message)
+
+    if batched_messages:
+        yield batched_messages
+
+
+def convert_messages(
+    added_teams: dict[str, TeamMetadata],
+    domain_name: str,
+    microsoft_graph_api_token: str,
+    microsoft_teams_user_id_to_zulip_user_id: MicrosoftTeamsUserIdToZulipUserIdT,
+    output_dir: str,
+    processes: int,
+    realm_id: int,
+    realm: dict[str, Any],
+    teams_data_dir: str,
+    chunk_size: int = MESSAGE_BATCH_CHUNK_SIZE,
+) -> tuple[list[UploadRecordData], list[AttachmentRecordData]]:
+    microsoft_teams_channel_metadata: dict[str, ChannelMetadata] = {}
+    subscriber_map = make_subscriber_map(
+        zerver_subscription=realm["zerver_subscription"],
+    )
+    team_data_folders = []
+    for f in os.listdir(teams_data_dir):
+        path = os.path.join(teams_data_dir, f)
+        if os.path.isdir(path):
+            team_data_folders.append(f)
+
+    message_file_paths = []
+    for team_id in team_data_folders:
+        team_data_folder = os.path.join(teams_data_dir, team_id)
+        team_messages_file_path = os.path.join(team_data_folder, f"messages_{team_id}.json")
+        message_file_paths.append(team_messages_file_path)
+
+        team_channels_list = get_data_file(
+            os.path.join(team_data_folder, f"channels_{team_id}.json")
+        )
+        for team_channel in team_channels_list:
+            microsoft_teams_channel_metadata[team_channel["Id"]] = ChannelMetadata(
+                display_name=team_channel["DisplayName"],
+                is_favourite_by_default=team_channel["IsFavoriteByDefault"],
+                is_archived=team_channel["IsArchived"],
+                is_favorite_by_default=team_channel["IsFavoriteByDefault"],
+                membership_type=team_channel["MembershipType"],
+                team_id=team_id,
+            )
+
+    # Attachment record data is only fully known after the file is
+    # downloaded, so records must be built inside the download
+    # workers. When processes > 1, workers run in separate processes
+    # and can't share memory with the parent, so we use a
+    # multiprocessing-shared list to gather their results.
+    #
+    # A nicer approach worth considering in the future would be to
+    # have run_parallel_queue return futures that the parent can
+    # wait on and collect results from, avoiding this shared-state
+    # coordination entirely.
+    total_attachment_records: list[AttachmentRecordData] | ListProxy = (  # type: ignore[type-arg] # ListProxy is not subscriptable
+        Manager().list() if processes > 1 else []
+    )
+
+    total_accumulated_upload_records: list[UploadRecordData] = []
+    with run_parallel_queue(
+        partial(
+            download_and_export_microsoft_teams_upload_file, total_attachment_records, output_dir
+        ),
+        processes,
+        catch=True,
+        report_every=100,
+        report=lambda count: logging.info("Downloaded %s attachments", count),
+    ) as do_download_and_export_upload_file:
+        dump_file_id = 1
+        for message_chunk in get_batched_export_message_data(message_file_paths, chunk_size):
+            conversion_result = process_messages(
+                added_teams=added_teams,
+                channel_metadata=microsoft_teams_channel_metadata,
+                do_download_and_export_upload_file=do_download_and_export_upload_file,
+                domain_name=domain_name,
+                messages=message_chunk,
+                microsoft_graph_api_token=microsoft_graph_api_token,
+                microsoft_teams_user_id_to_zulip_user_id=microsoft_teams_user_id_to_zulip_user_id,
+                realm=realm,
+                realm_id=realm_id,
+                subscriber_map=subscriber_map,
+            )
+            total_accumulated_upload_records += conversion_result.upload_records
+            create_converted_data_files(
+                dict(
+                    zerver_message=conversion_result.zerver_messages,
+                    zerver_usermessage=conversion_result.zerver_usermessages,
+                ),
+                output_dir,
+                f"/messages-{dump_file_id:06}.json",
+            )
+            dump_file_id += 1
+
+    scrubbed_data = scrub_missing_upload_records_after_download(
+        output_dir,
+        # Attachment record data is built inside download workers only
+        # on successful downloads, so there should be no stale records.
+        zerver_attachments=[],
+        upload_records=total_accumulated_upload_records,
+    )
+    total_accumulated_upload_records = scrubbed_data.upload_records
+
+    def fix_attachment_id(attachment: AttachmentRecordData) -> AttachmentRecordData:
+        attachment.id = NEXT_ID("attachment")
+        return attachment
+
+    return (
+        total_accumulated_upload_records,
+        list(map(fix_attachment_id, total_attachment_records)),
+    )
+
+
+def do_convert_directory(
+    microsoft_teams_dir: str,
+    output_dir: str,
+    microsoft_graph_api_token: str,
+    processes: int = 6,
+) -> None:
+    os.makedirs(output_dir, exist_ok=True)
+    if os.listdir(output_dir):  # nocoverage
+        raise Exception("Output directory should be empty!")
+    users_data_dir = os.path.join(microsoft_teams_dir, "users")
+    if not os.path.isdir(users_data_dir):  # nocoverage
+        raise ValueError("Import does not have the layout we expect from a Microsoft Teams export!")
+
+    realm_id = 0
+    domain_name = SplitResult("", settings.EXTERNAL_HOST, "", "", "").hostname
+    assert isinstance(domain_name, str)
+
+    NOW = float(timezone_now().timestamp())
+    zerver_realm: list[ZerverFieldsT] = build_zerver_realm(realm_id, "", NOW, "Microsoft Teams")
+    realm = build_realm(zerver_realm, realm_id, domain_name, import_source="microsoft_teams")
+    realm["zerver_stream"] = []
+    realm["zerver_defaultstream"] = []
+    realm["zerver_recipient"] = []
+    realm["zerver_subscription"] = []
+
+    microsoft_teams_user_id_to_zulip_user_id = convert_users(
+        microsoft_teams_user_role_data=get_user_roles(microsoft_graph_api_token),
+        realm=realm,
+        realm_id=realm_id,
+        timestamp=int(NOW),
+        users_list=get_data_file(os.path.join(users_data_dir, "usersList.json")),
+    )
+
+    teams_data_dir = os.path.join(microsoft_teams_dir, "teams")
+
+    added_teams = convert_teams_to_channels(
+        microsoft_teams_user_id_to_zulip_user_id=microsoft_teams_user_id_to_zulip_user_id,
+        realm=realm,
+        realm_id=realm_id,
+        teams_data_dir=teams_data_dir,
+    )
+
+    upload_records, attachment_records = convert_messages(
+        added_teams=added_teams,
+        domain_name=domain_name,
+        microsoft_graph_api_token=microsoft_graph_api_token,
+        microsoft_teams_user_id_to_zulip_user_id=microsoft_teams_user_id_to_zulip_user_id,
+        output_dir=output_dir,
+        processes=processes,
+        realm_id=realm_id,
+        realm=realm,
+        teams_data_dir=teams_data_dir,
+    )
+
+    create_converted_data_files(realm, output_dir, "/realm.json")
+    create_converted_data_files(upload_records, output_dir, "/uploads/records.json")
+    attachment: dict[str, list[Any]] = {"zerver_attachment": attachment_records}
+    create_converted_data_files(attachment, output_dir, "/attachment.json")
+    # TODO:
+    create_converted_data_files([], output_dir, "/emoji/records.json")
+    create_converted_data_files([], output_dir, "/avatars/records.json")
+    create_converted_data_files([], output_dir, "/realm_icons/records.json")
+    do_common_export_processes(output_dir)
+
+    logging.info("######### DATA CONVERSION FINISHED #########\n")
+    logging.info("Zulip data dump created at %s", output_dir)

zerver/data_import/microsoft_teams_message_conversion.py

@@ -0,0 +1,41 @@
+import re
+
+from bs4 import Tag
+from typing_extensions import override
+
+from zerver.lib.markdown import get_markdown_image_for_url
+from zerver.lib.markdown.from_html import ZulipMarkdownConverter, convert_html_to_markdown
+
+# https://learn.microsoft.com/en-us/graph/api/chatmessagehostedcontent-get
+HOSTED_CONTENT_GRAPH_API_URL_REGEX = r"https://graph\.microsoft\.com/v1\.0/teams/[^/]+/channels/[^/]+/messages/[^/]+/hostedContents/[^/]+/\$value"
+HOSTED_CONTENT_MARKDOWN_IMAGE_SYNTAX_REGEX = rf"""
+            !\[
+               (?P<file_name>[^\]]+)
+            \]\(
+               (?P<api_url>{HOSTED_CONTENT_GRAPH_API_URL_REGEX})
+            \)
+            """
+
+
+class MicrosoftTeamsToZulipMarkdownConverter(ZulipMarkdownConverter):
+    """HTML-to-Markdown converter for Microsoft Teams exports.
+
+    Hosted-content images are kept in inline `![alt](src)` form, which
+    process_hosted_content_attachments matches to download each image
+    and rewrite it into a Zulip upload link.  Other images get the base
+    class's external-link treatment.
+    """
+
+    @override
+    def convert_img(self, el: Tag, text: str, parent_tags: set[str]) -> str:
+        src = el.get("src", "")
+        alt = el.get("alt", "")
+        assert isinstance(src, str)
+        assert isinstance(alt, str)
+        if re.fullmatch(HOSTED_CONTENT_GRAPH_API_URL_REGEX, src):
+            return get_markdown_image_for_url(alt, src)
+        return super().convert_img(el, text, parent_tags)
+
+
+def convert_microsoft_teams_html_to_markdown(html: str) -> str:
+    return convert_html_to_markdown(html, converter_class=MicrosoftTeamsToZulipMarkdownConverter)

zerver/data_import/sequencer.py

@@ -0,0 +1,74 @@
+from collections.abc import Callable
+from typing import Generic, TypeVar
+
+"""
+This module helps you set up a bunch
+of sequences, similar to how database
+sequences work.
+
+You need to be a bit careful here, since
+you're dealing with a big singleton, but
+for data imports that's usually easy to
+manage.
+"""
+
+T = TypeVar("T")
+
+
+def _seq() -> Callable[[], int]:
+    i = 0
+
+    def next_one() -> int:
+        nonlocal i
+        i += 1
+        return i
+
+    return next_one
+
+
+def sequencer() -> Callable[[str], int]:
+    """
+    Use like this:
+
+    NEXT_ID = sequencer()
+    message_id = NEXT_ID('message')
+    """
+    seq_dict: dict[str, Callable[[], int]] = {}
+
+    def next_one(name: str) -> int:
+        if name not in seq_dict:
+            seq_dict[name] = _seq()
+        seq = seq_dict[name]
+        return seq()
+
+    return next_one
+
+
+"""
+NEXT_ID is a singleton used by an entire process, which is
+almost always reasonable.  If you want to have two parallel
+sequences, just use different `name` values.
+
+This object gets created once and only once during the first
+import of the file.
+"""
+
+NEXT_ID = sequencer()
+
+
+class IdMapper(Generic[T]):
+    def __init__(self) -> None:
+        self.map: dict[T, int] = {}
+        self.cnt = 0
+
+    def has(self, their_id: T) -> bool:
+        return their_id in self.map
+
+    def get(self, their_id: T) -> int:
+        if their_id in self.map:
+            return self.map[their_id]
+
+        self.cnt += 1
+        our_id = self.cnt
+        self.map[their_id] = our_id
+        return our_id

zerver/data_import/slack_message_conversion.py

@@ -0,0 +1,501 @@
+import contextlib
+import re
+from datetime import datetime, timezone
+from itertools import zip_longest
+from typing import Any, Literal, TypeAlias, TypedDict, cast
+
+import regex
+from django.core.exceptions import ValidationError
+from requests.utils import requote_uri
+
+from zerver.lib.timestamp import datetime_to_global_time
+from zerver.lib.types import Validator
+from zerver.lib.validator import (
+    WildValue,
+    check_dict,
+    check_int,
+    check_list,
+    check_string,
+    check_string_in,
+    check_url,
+)
+
+# stubs
+ZerverFieldsT: TypeAlias = dict[str, Any]
+SlackToZulipUserIDT: TypeAlias = dict[str, int]
+AddedChannelsT: TypeAlias = dict[str, tuple[str, int]]
+SlackFieldsT: TypeAlias = dict[str, Any]
+
+# Slack link can be in the format <http://www.foo.com|www.foo.com> and <http://foo.com/>
+LINK_REGEX = r"""
+              (<)                                                              # match '>'
+              (http:\/\/www\.|https:\/\/www\.|http:\/\/|https:\/\/|ftp:\/\/)?  # protocol and www
+                  ([a-z0-9]+([\-\.]{1}[a-z0-9]+)*)(\.)                         # domain name
+                      ([a-z]{2,63}(:[0-9]{1,5})?)                              # domain
+                  (\/[^>]*)?                                                   # path
+              (\|)?(?:\|([^>]+))?                                # char after pipe (for Slack links)
+              (>)
+              """
+
+SLACK_MAILTO_REGEX = r"""
+                      <((mailto:)?                     # match  `<mailto:`
+                      ([\w\.-]+@[\w\.-]+(\.[\w]+)+))   # match email
+                          (\|)?                        # match pipe
+                      ([\w\.-]+@[\w\.-]+(\.[\w]+)+)?>  # match email
+                      """
+
+SLACK_USERMENTION_REGEX = r"""
+                           (<@)                  # Start with '<@'
+                               ([a-zA-Z0-9]+)    # Here we have the Slack id
+                           (\|)?                 # We not always have a vertical line in mention
+                               ([a-zA-Z0-9]+)?   # If vertical line is present, this is short name
+                           (>)                   # ends with '>'
+                           """
+# Slack doesn't have mid-word message-formatting like Zulip.
+# Hence, ~stri~ke doesn't format the word in Slack, but ~~stri~~ke
+# formats the word in Zulip
+SLACK_STRIKETHROUGH_REGEX = r"""
+                             (
+                                # Capture punctuation (\p{P}), white space (\p{Zs}),
+                                # symbols (\p{S}) or newline.
+                                # Skip ~ to not reformat the same string twice
+                                # Skip @ and \
+                                # Skip closing brackets & closing quote (\p{Pf}\p{Pe})
+                                (?![~`@\\\p{Pf}\p{Pe}])
+                                [\p{P}\p{Zs}\p{S}]|^
+                             )
+                             (\~)                                  # followed by a ~
+                                 ([^~]+)                           # any character except ~
+                             (\~)                                  # followed by a ~
+                             (
+                                # Capture punctuation, white space, symbols or end of
+                                # line.
+                                # Skip ~ to not reformat the same string twice
+                                # Skip @ and \
+                                # Skip opening brackets & opening quote (\p{Pi}\p{Ps})
+                                (?![~`@\\\p{Pi}\p{Ps}])
+                                (?=[\p{P}\p{Zs}\p{S}]|$)
+                             )
+                             """
+SLACK_ITALIC_REGEX = r"""
+                      # Same as `SLACK_STRIKETHROUGH_REGEX`s. The difference
+                      # being, this skips _ instead of ~
+                      (
+                        (?![_`@\\\p{Pf}\p{Pe}])
+                        [\p{P}\p{Zs}\p{S}]|^
+                      )
+                      (\_)
+                          ([^_]+)                    # any character except _
+                      (\_)
+                      (
+                        (?![_`@\\\p{Pi}\p{Ps}])
+                        (?=[\p{P}\p{Zs}\p{S}]|$)
+                      )
+                      """
+SLACK_BOLD_REGEX = r"""
+                    # Same as `SLACK_STRIKETHROUGH_REGEX`s. The difference
+                    # being, this skips * instead of ~
+                    (
+                        (?![*`@\\\p{Pf}\p{Pe}])
+                        [\p{P}\p{Zs}\p{S}]|^
+                    )
+                    (\*)
+                        ([^*]+)                       # any character except *
+                    (\*)
+                    (
+                        (?![*`@\\\p{Pi}\p{Ps}])
+                        (?=[\p{P}\p{Zs}\p{S}]|$)
+                    )
+                    """
+
+FALLBACK_SLACK_USER_FULL_NAME = "Slack user {id}"
+
+
+def get_user_full_name(user: ZerverFieldsT) -> str:
+    if "deleted" in user and user["deleted"] is False:
+        return user["real_name"] or user["name"]
+    elif user["is_mirror_dummy"]:
+        return user["profile"].get("real_name", user["name"])
+    else:
+        return user["name"]
+
+
+def get_zulip_mention_for_slack_user(
+    slack_user_id: str | None,
+    slack_user_shortname: str | None,
+    users: list[ZerverFieldsT],
+    silent: bool = False,
+) -> str | None:
+    if slack_user_id:
+        for user in users:
+            if user["id"] == slack_user_id and (
+                slack_user_shortname is None or user["name"] == slack_user_shortname
+            ):
+                return ("@_**" if silent else "@**") + get_user_full_name(user) + "**"
+    return None
+
+
+def get_user_mentions(
+    token: str,
+    users: list[ZerverFieldsT],
+    slack_user_id_to_zulip_user_id: SlackToZulipUserIDT,
+) -> tuple[str, int | None]:
+    slack_usermention_match = re.search(SLACK_USERMENTION_REGEX, token, re.VERBOSE)
+    assert slack_usermention_match is not None
+    short_name = slack_usermention_match.group(4)
+    slack_id = slack_usermention_match.group(2)
+    zulip_mention = get_zulip_mention_for_slack_user(slack_id, short_name, users)
+    if zulip_mention is not None:
+        token = re.sub(SLACK_USERMENTION_REGEX, zulip_mention, token, flags=re.VERBOSE)
+        user_id = slack_user_id_to_zulip_user_id[slack_id]
+        return token, user_id
+    return token, None
+
+
+def convert_link_format(text: str) -> tuple[str, bool]:
+    """
+    1. Converts '<https://foo.com>' to 'https://foo.com'
+    2. Converts '<https://foo.com|foo>' to '[foo](https://foo.com)'
+    """
+    has_link = False
+    for match in re.finditer(LINK_REGEX, text, re.VERBOSE):
+        slack_url = match.group(0)
+        url_parts = slack_url[1:-1].split("|", maxsplit=1)
+        # Check if there's a pipe with text after it
+        if len(url_parts) == 2:
+            converted_url = f"[{url_parts[1]}]({url_parts[0]})"
+        else:
+            converted_url = url_parts[0]
+
+        has_link = True
+        text = text.replace(slack_url, converted_url)
+    return text, has_link
+
+
+def convert_mailto_format(text: str) -> tuple[str, bool]:
+    """
+    1. Converts '<mailto:foo@foo.com>' to 'mailto:foo@foo.com'
+    2. Converts '<mailto:foo@foo.com|foo@foo.com>' to 'mailto:foo@foo.com'
+    """
+    has_link = False
+    for match in re.finditer(SLACK_MAILTO_REGEX, text, re.VERBOSE):
+        has_link = True
+        text = text.replace(match.group(0), match.group(1))
+    return text, has_link
+
+
+# Map italic, bold and strikethrough Markdown
+def convert_markdown_syntax(text: str, pattern: str, zulip_keyword: str) -> str:
+    """
+    Returns:
+    1. For strikethrough formatting: This maps Slack's '~strike~' to Zulip's '~~strike~~'
+    2. For bold formatting: This maps Slack's '*bold*' to Zulip's '**bold**'
+    3. For italic formatting: This maps Slack's '_italic_' to Zulip's '*italic*'
+    """
+
+    def replace_slack_format(match: regex.Match[str]) -> str:
+        return match.group(1) + zulip_keyword + match.group(3) + zulip_keyword
+
+    return regex.sub(pattern, replace_slack_format, text, flags=re.VERBOSE | re.MULTILINE)
+
+
+def convert_slack_workspace_mentions(text: str) -> str:
+    # Map Slack's '<!everyone>', '<!channel>' and '<!here>'
+    # mentions to Zulip's '@**all**' wildcard mention.
+    # No regex for these as they can be present anywhere
+    # in the sentence.
+    text = text.replace("<!everyone>", "@**all**")
+    text = text.replace("<!channel>", "@**all**")
+    text = text.replace("<!here>", "@**all**")
+    return text
+
+
+def convert_slack_formatting(text: str) -> str:
+    text = convert_markdown_syntax(text, SLACK_BOLD_REGEX, "**")
+    text = convert_markdown_syntax(text, SLACK_STRIKETHROUGH_REGEX, "~~")
+    text = convert_markdown_syntax(text, SLACK_ITALIC_REGEX, "*")
+    return text
+
+
+# Markdown mapping
+def convert_to_zulip_markdown(
+    text: str,
+    users: list[ZerverFieldsT],
+    added_channels: AddedChannelsT,
+    slack_user_id_to_zulip_user_id: SlackToZulipUserIDT,
+) -> tuple[str, list[int], bool]:
+    mentioned_users_id = []
+    text = convert_slack_formatting(text)
+    text = convert_slack_workspace_mentions(text)
+
+    # Map Slack channel mention: '<#C5Z73A7RA|general>' to '#**general**'
+    for cname, ids in added_channels.items():
+        cid = ids[0]
+        text = text.replace(f"<#{cid}|{cname}>", "#**" + cname + "**")
+
+    tokens = text.split(" ")
+    for iterator in range(len(tokens)):
+        # Check user mentions and change mention format from
+        # '<@slack_id|short_name>' to '@**full_name**'
+        if re.findall(SLACK_USERMENTION_REGEX, tokens[iterator], re.VERBOSE):
+            tokens[iterator], user_id = get_user_mentions(
+                tokens[iterator], users, slack_user_id_to_zulip_user_id
+            )
+            if user_id is not None:
+                mentioned_users_id.append(user_id)
+
+    text = " ".join(tokens)
+
+    # Check and convert link format
+    text, has_link = convert_link_format(text)
+    # convert `<mailto:foo@foo.com>` to `mailto:foo@foo.com`
+    text, has_mailto_link = convert_mailto_format(text)
+
+    message_has_link = has_link or has_mailto_link
+
+    return text, mentioned_users_id, message_has_link
+
+
+class LossyConversionError(Exception):
+    pass
+
+
+def render_block(block: WildValue) -> str:
+    """
+    Raises `LossyConversionError` if it's a rich_text block type.
+    """
+    # https://api.slack.com/reference/block-kit/blocks
+    supported_types = {
+        "context",
+        "divider",
+        "header",
+        "image",
+        "section",
+    }
+    unhandled_types = {
+        # `call` is a block type we've observed in the wild in a Slack export,
+        # despite not being documented in
+        # https://docs.slack.dev/reference/block-kit/blocks/
+        # It likes maps to a request for a Slack call. If we can verify that,
+        # probably it would be worth replacing with a string indicating a Slack
+        # call occurred.
+        "call",
+        "contact_card",
+        "file",
+        "table",
+        "video",
+        # The "actions" block is used to format literal in-message clickable
+        # buttons and similar elements, which Zulip currently doesn't support.
+        # https://docs.slack.dev/reference/block-kit/blocks/actions-block
+        "actions",
+        "input",
+        "condition",
+    }
+    known_types = {
+        *supported_types,
+        *unhandled_types,
+        # rich_text is a special case. It probably composes the bulk of a message
+        # body since it's responsible for very basic text formatting--such as
+        # plain text, bold, italic, etc. Our current Slack text conversion module
+        # can't process this block since this is not formatted in Markdown.
+        "rich_text",
+    }
+    block_type = block["type"].tame(check_string_in(known_types))
+
+    if block_type in unhandled_types:
+        return ""
+    elif block_type == "rich_text":
+        raise LossyConversionError
+    elif block_type == "context" and block.get("elements"):
+        pieces = []
+        # Slack renders these pieces left-to-right, packed in as
+        # closely as possible.  We just render them above each other,
+        # for simplicity.
+        for element in block["elements"]:
+            element_type = element["type"].tame(check_string_in(["image", "plain_text", "mrkdwn"]))
+            if element_type == "image":
+                pieces.append(render_block_element(element))
+            else:
+                pieces.append(element.tame(check_text_block())["text"])
+        return "\n\n".join(piece.strip() for piece in pieces if piece.strip() != "")
+    elif block_type == "divider":
+        return "----"
+    elif block_type == "header":
+        return "## " + block["text"].tame(check_text_block(plain_text_only=True))["text"]
+    elif block_type == "image":
+        image_url = block["image_url"].tame(check_url)
+        alt_text = block["alt_text"].tame(check_string)
+        if "title" in block:
+            alt_text = block["title"].tame(check_text_block(plain_text_only=True))["text"]
+        return f"[{alt_text}]({image_url})"
+    elif block_type == "section":
+        pieces = []
+        if "text" in block:
+            pieces.append(block["text"].tame(check_text_block())["text"])
+
+        if "accessory" in block:
+            pieces.append(render_block_element(block["accessory"]))
+
+        if "fields" in block:
+            fields = block["fields"].tame(check_list(check_text_block()))
+            if len(fields) == 1:
+                # Special-case a single field to display a bit more
+                # nicely, without extraneous borders and limitations
+                # on its contents.
+                pieces.append(fields[0]["text"])
+            else:
+                # It is not possible to have newlines in a table, nor
+                # escape the pipes that make it up; replace them with
+                # whitespace.
+                field_text = [f["text"].replace("\n", " ").replace("|", " ") for f in fields]
+                # Because Slack formats this as two columns, but not
+                # necessarily a table with a bold header, we emit a
+                # blank header row first.
+                table = "| | |\n|-|-|\n"
+                # Then take the fields two-at-a-time to make the table
+                iters = [iter(field_text)] * 2
+                for left, right in zip_longest(*iters, fillvalue=""):
+                    table += f"| {left} | {right} |\n"
+                pieces.append(table)
+
+        return "\n\n".join(piece.strip() for piece in pieces if piece.strip() != "")
+
+    return ""  # nocoverage
+
+
+class TextField(TypedDict):
+    text: str
+    type: Literal["plain_text", "mrkdwn"]
+
+
+def check_text_block(plain_text_only: bool = False) -> Validator[TextField]:
+    if plain_text_only:
+        type_validator = check_string_in(["plain_text"])
+    else:
+        type_validator = check_string_in(["plain_text", "mrkdwn"])
+
+    def f(var_name: str, val: object) -> TextField:
+        block = check_dict(
+            [
+                ("type", type_validator),
+                ("text", check_string),
+            ],
+        )(var_name, val)
+
+        return cast(TextField, block)
+
+    return f
+
+
+def render_block_element(element: WildValue) -> str:
+    # https://api.slack.com/reference/block-kit/block-elements
+    # Zulip doesn't support interactive elements, so we only render images here
+    element_type = element["type"].tame(check_string)
+    if element_type == "image":
+        try:
+            image_url = element["image_url"].tame(check_url)
+            alt_text = element["alt_text"].tame(check_string)
+            return f"[{alt_text}]({image_url})"
+        except ValidationError:
+            # Just drop invalid image URLs, rather than drop the whole message.
+            return ""
+    else:
+        # Unsupported
+        return ""
+
+
+def render_attachment(attachment: WildValue) -> str:
+    # https://api.slack.com/reference/messaging/attachments
+    # Slack recommends the usage of "blocks" even within attachments; the
+    # rest of the fields we handle here are legacy fields. These fields are
+    # optional and may contain null values.
+    pieces = []
+    if attachment.get("title"):
+        title = attachment["title"].tame(check_string)
+        if attachment.get("title_link"):
+            title_link = attachment["title_link"].tame(check_url)
+            pieces.append(f"## [{title}]({title_link})")
+        else:
+            pieces.append(f"## {title}")
+    if attachment.get("pretext"):
+        pieces.append(attachment["pretext"].tame(check_string))
+    if attachment.get("text"):
+        pieces.append(attachment["text"].tame(check_string))
+    if "fields" in attachment:
+        fields = []
+        for field in attachment["fields"]:
+            if "title" in field and "value" in field and field["title"] and field["value"]:
+                title = field["title"].tame(check_string)
+                value = field["value"].tame(check_string)
+                fields.append(f"*{title}*: {value}")
+            elif field.get("title"):
+                title = field["title"].tame(check_string)
+                fields.append(f"*{title}*")
+            elif field.get("value"):
+                value = field["value"].tame(check_string)
+                fields.append(f"{value}")
+        pieces.append("\n".join(fields))
+    if attachment.get("blocks"):
+        for block in attachment["blocks"]:
+            # rich_text blocks in "attachments" elements don't have a
+            # corresponding raw value in the "text" field, so we can't
+            # fallback to the "text" field in this case like we can do
+            # with rich_text in message "blocks".
+            # TODO: We should use the raw rich_text strings instead.
+            with contextlib.suppress(LossyConversionError):  # nocoverage
+                pieces.append(render_block(block))
+    if image_url_wv := attachment.get("image_url"):
+        try:
+            image_url = image_url_wv.tame(check_url)
+        except ValidationError:  # nocoverage
+            image_url = image_url_wv.tame(check_string)
+            image_url = requote_uri(image_url)
+        pieces.append(f"[]({image_url})")
+    if attachment.get("footer"):
+        pieces.append(attachment["footer"].tame(check_string))
+    if attachment.get("ts"):
+        try:
+            time = attachment["ts"].tame(check_int)
+        except ValidationError as e:  # nocoverage
+            # In some cases Slack has the ts as a string with a float
+            # number. The reason is unknown, but we've observed it
+            # in the wild several times.
+            ts = attachment["ts"].tame(check_string)
+            try:
+                ts_float = float(ts)
+            except ValueError:
+                raise e
+
+            time = int(ts_float)
+        pieces.append(datetime_to_global_time(datetime.fromtimestamp(time, timezone.utc)))
+
+    return "\n\n".join(piece.strip() for piece in pieces if piece.strip() != "")
+
+
+def replace_links(text: str) -> str:
+    text, _ = convert_link_format(text)
+    text, _ = convert_mailto_format(text)
+    return text
+
+
+def process_slack_block_and_attachment(message: WildValue) -> str:
+    pieces: list[str] = []
+    raw_text = message["text"].tame(check_string)
+
+    if message.get("blocks"):
+        try:
+            pieces += map(render_block, message["blocks"])
+        except LossyConversionError:
+            # render_block doesn't yet handle the "rich_text" block, which is used
+            # for very basic text formatting. To avoid message content loss, process
+            # the message["text"] field instead if rich_text is used.
+            pieces.append(raw_text)
+
+    if set(pieces) in ({""}, set()):
+        pieces.append(raw_text)
+
+    # The attachments typically don't have a corresponding raw value in
+    # message["text"] and only add to it, so the output is appended unconditionally.
+    if message.get("attachments"):
+        pieces += map(render_attachment, message["attachments"])
+    return "\n".join(piece.strip() for piece in pieces if piece.strip() != "")

zerver/data_import/user_handler.py

@@ -0,0 +1,33 @@
+from typing import Any
+
+from zerver.data_import.import_util import validate_user_emails_for_import
+
+
+class UserHandler:
+    """
+    Our UserHandler class is a glorified wrapper
+    around the data that eventually goes into
+    zerver_userprofile.
+
+    The class helps us do things like map ids
+    to names for mentions.
+    """
+
+    def __init__(self) -> None:
+        self.id_to_user_map: dict[int, dict[str, Any]] = {}
+
+    def add_user(self, user: dict[str, Any]) -> None:
+        user_id = user["id"]
+        self.id_to_user_map[user_id] = user
+
+    def get_user(self, user_id: int) -> dict[str, Any]:
+        user = self.id_to_user_map[user_id]
+        return user
+
+    def get_all_users(self) -> list[dict[str, Any]]:
+        users = list(self.id_to_user_map.values())
+        return users
+
+    def validate_user_emails(self) -> None:
+        all_users = self.get_all_users()
+        validate_user_emails_for_import([user["delivery_email"] for user in all_users])