snapshot of https://github.com/RocketChat/Rocket.Chat.git (develop, 2026-09-16; blobs > 131072B omitted)

apps/meteor/server/lib/import/pending-files/PendingFileImporter.ts

@@ -0,0 +1,237 @@
+import http from 'node:http';
+import https from 'node:https';
+
+import { api } from '@rocket.chat/core-services';
+import type { IImport, MessageAttachment, IUpload, IImporterShortSelection } from '@rocket.chat/core-typings';
+import { Messages, Users } from '@rocket.chat/models';
+import { Random } from '@rocket.chat/random';
+
+import { Importer, ProgressStep } from '..';
+import { parseFileIntoMessageAttachments } from '../../../meteor-methods/messages/sendFileMessage';
+import { FileUpload } from '../../media/file-upload';
+import type { ConverterOptions } from '../classes/ImportDataConverter';
+import type { ImporterProgress } from '../classes/ImporterProgress';
+import type { ImporterInfo } from '../definitions/ImporterInfo';
+
+export class PendingFileImporter extends Importer {
+	constructor(info: ImporterInfo, importRecord: IImport, converterOptions: ConverterOptions = {}) {
+		super(info, importRecord, converterOptions);
+	}
+
+	async prepareFileCount() {
+		this.logger.debug('start preparing import operation');
+		await super.updateProgress(ProgressStep.PREPARING_STARTED);
+
+		const fileCount = await Messages.countAllImportedMessagesWithFilesToDownload();
+		if (fileCount === 0) {
+			await super.updateProgress(ProgressStep.DONE);
+			return 0;
+		}
+
+		this.progress.count.total += fileCount;
+		await this.updateRecord({
+			'count.messages': fileCount,
+			'count.total': fileCount,
+			'messagesstatus': null,
+			'status': ProgressStep.IMPORTING_FILES,
+		});
+		this.reportProgress();
+
+		setImmediate(() => {
+			void this.startImport({});
+		});
+
+		return fileCount;
+	}
+
+	override async startImport(importSelection: IImporterShortSelection): Promise<ImporterProgress> {
+		const downloadedFileIds: string[] = [];
+		const maxFileCount = 10;
+		const maxFileSize = 1024 * 1024 * 500;
+
+		let count = 0;
+		let currentSize = 0;
+		let nextSize = 0;
+
+		const waitForFiles = async () => {
+			if (count + 1 < maxFileCount && currentSize + nextSize < maxFileSize) {
+				return;
+			}
+
+			return new Promise<void>((resolve) => {
+				const handler = setInterval(() => {
+					if (count + 1 >= maxFileCount) {
+						return;
+					}
+
+					if (currentSize + nextSize >= maxFileSize && count > 0) {
+						return;
+					}
+
+					clearInterval(handler);
+					resolve();
+				}, 1000);
+			});
+		};
+
+		const completeFile = async (details: { size: number }) => {
+			await this.addCountCompleted(1);
+			count--;
+			currentSize -= details.size;
+		};
+
+		const failFile = async (details: { size: number }) => {
+			await this.addCountError(1);
+			count--;
+			currentSize -= details.size;
+		};
+
+		const logError = this.logger.error.bind(this.logger);
+
+		try {
+			const pendingFileMessageList = Messages.findAllImportedMessagesWithFilesToDownload();
+			const importedRoomIds = new Set<string>();
+			for await (const message of pendingFileMessageList) {
+				try {
+					const { _importFile } = message;
+
+					if (!_importFile || _importFile.downloaded || downloadedFileIds.includes(_importFile.id)) {
+						await this.addCountCompleted(1);
+						continue;
+					}
+
+					const url = _importFile.downloadUrl;
+					if (!url?.startsWith('http')) {
+						await this.addCountCompleted(1);
+						continue;
+					}
+
+					const details: { message_id: string; name: string; size: number; userId: string; rid: string; type?: string } = {
+						message_id: `${message._id}-file-${_importFile.id}`,
+						name: _importFile.name || Random.id(),
+						size: _importFile.size || 0,
+						userId: message.u._id,
+						rid: message.rid,
+					};
+
+					const fileStore = FileUpload.getStore('Uploads');
+
+					nextSize = details.size;
+					await waitForFiles();
+					count++;
+					currentSize += nextSize;
+					downloadedFileIds.push(_importFile.id);
+
+					void this.downloadFile(url, details)
+						.then(async (rawData) => {
+							// Bypass the fileStore filters
+							const file = await fileStore._doInsert(details, rawData);
+
+							const rocketChatUrl = FileUpload.getPath(`${file._id}/${encodeURI(file.name || '')}`);
+							const user = await Users.findOneById(message.u._id);
+							const attachment = user
+								? (await parseFileIntoMessageAttachments(file, message.rid, user)).attachments[0]
+								: this.getMessageAttachment(file, rocketChatUrl);
+
+							await Messages.setImportFileRocketChatAttachment(_importFile.id, rocketChatUrl, attachment);
+							await completeFile(details);
+							importedRoomIds.add(message.rid);
+						})
+						.catch(async (err) => {
+							logError({ msg: 'Failed to download pending file', url, err });
+							await failFile(details);
+						});
+				} catch (err) {
+					this.logger.error({ err });
+				}
+			}
+
+			void api.broadcast('notify.importedMessages', { roomIds: Array.from(importedRoomIds) });
+		} catch (error) {
+			// If the cursor expired, restart the method
+			if (this.isCursorNotFoundError(error)) {
+				this.logger.info('CursorNotFound');
+				return this.startImport(importSelection);
+			}
+
+			await super.updateProgress(ProgressStep.ERROR);
+			throw error;
+		}
+
+		await super.updateProgress(ProgressStep.DONE);
+		return this.getProgress();
+	}
+
+	private downloadFile(url: string, details: { type?: string }): Promise<Buffer> {
+		return new Promise((resolve, reject) => {
+			const requestModule = /https/i.test(url) ? https : http;
+
+			requestModule
+				.get(url, (res) => {
+					res.on('error', reject);
+
+					if (!res.statusCode || res.statusCode < 200 || res.statusCode >= 300) {
+						// Error responses (e.g. a redirect to a login page) would otherwise be saved as the file's content.
+						res.resume(); // discard the body and free the socket
+						reject(new Error(`Unexpected response status ${res.statusCode}`));
+						return;
+					}
+
+					const contentType = res.headers['content-type'];
+					if (!details.type && contentType) {
+						details.type = contentType;
+					}
+
+					const rawData: Uint8Array[] = [];
+					res.on('data', (chunk) => {
+						rawData.push(chunk);
+
+						// Update progress more often on large files
+						this.reportProgress();
+					});
+					res.on('end', () => resolve(Buffer.concat(rawData)));
+				})
+				.on('error', reject);
+		});
+	}
+
+	getMessageAttachment(file: IUpload, url: string): MessageAttachment {
+		if (file.type) {
+			if (/^image\/.+/.test(file.type)) {
+				return {
+					title: file.name,
+					title_link: url,
+					image_url: url,
+					image_type: file.type,
+					image_size: file.size,
+					image_dimensions: file.identify ? file.identify.size : undefined,
+				};
+			}
+
+			if (/^audio\/.+/.test(file.type)) {
+				return {
+					title: file.name,
+					title_link: url,
+					audio_url: url,
+					audio_type: file.type,
+					audio_size: file.size,
+				};
+			}
+
+			if (/^video\/.+/.test(file.type)) {
+				return {
+					title: file.name,
+					title_link: url,
+					video_url: url,
+					video_type: file.type,
+					video_size: file.size,
+				};
+			}
+		}
+
+		return {
+			title: file.name,
+			title_link: url,
+		};
+	}
+}

apps/meteor/server/lib/import/pending-files/index.ts

@@ -0,0 +1,9 @@
+import { Importers } from '..';
+import { PendingFileImporter } from './PendingFileImporter';
+
+Importers.add({
+	key: 'pending-files',
+	name: 'Pending Files',
+	visible: false,
+	importer: PendingFileImporter,
+});