upstream-rocketchat/apps/meteor/server/lib/dataExport/processDataDownloads.ts

import { randomUUID } from 'node:crypto';
import { createWriteStream } from 'node:fs';
import { access, mkdir, rm, writeFile } from 'node:fs/promises';

import type { IExportOperation, IUser, RoomType } from '@rocket.chat/core-typings';
import { Avatars, ExportOperations, UserDataFiles, Subscriptions } from '@rocket.chat/models';
import { escapeHTML } from '@rocket.chat/tools';
import moment from 'moment';

import { settings } from '../../settings';
import { joinPath } from '../fileUtils';
import { i18n } from '../i18n';
import { copyFileUpload } from './copyFileUpload';
import { exportRoomMessagesToFile } from './exportRoomMessagesToFile';
import { getPath } from './getPath';
import { getRoomData } from './getRoomData';
import { makeZipFile } from './makeZipFile';
import { sendEmail } from './sendEmail';
import { uploadZipFile } from './uploadZipFile';
import { FileUpload } from '../media/file-upload';
import { getURL } from '../utils/getURL';

const loadUserSubscriptions = async (_exportOperation: IExportOperation, fileType: 'json' | 'html', userId: IUser['_id']) => {
	const roomList: (
		| {
				roomId: string;
				roomName: string;
				userId: string | undefined;
				exportedCount: number;
				status: string;
				type: RoomType;
				targetFile: string;
		  }
		| Record<string, never>
	)[] = [];

	const cursor = Subscriptions.findByUserId(userId);
	for await (const subscription of cursor) {
		const roomData = await getRoomData(subscription.rid, userId);
		roomData.targetFile = `${(fileType === 'json' && roomData.roomName) || subscription.rid}.${fileType}`;

		roomList.push(roomData);
	}

	return roomList;
};

const generateUserFile = async (exportOperation: IExportOperation, userData?: IUser) => {
	if (!userData) {
		return;
	}

	await mkdir(exportOperation.exportPath, { recursive: true });

	const { username, name, statusText, emails, roles, services } = userData;

	const dataToSave = {
		username,
		name,
		statusText,
		emails: emails?.map(({ address }) => address),
		roles,
		services: services ? Object.keys(services) : undefined,
	};

	const fileName = joinPath(exportOperation.exportPath, exportOperation.fullExport ? 'user.json' : 'user.html');

	if (exportOperation.fullExport) {
		await writeFile(fileName, JSON.stringify(dataToSave), { encoding: 'utf8' });

		exportOperation.generatedUserFile = true;
		return;
	}

	return new Promise<void>((resolve, reject) => {
		const stream = createWriteStream(fileName, { encoding: 'utf8' });

		stream.on('finish', resolve);
		stream.on('error', reject);

		stream.write('<!DOCTYPE html>\n');
		stream.write('<meta http-equiv="content-type" content="text/html; charset=utf-8">\n');
		stream.write(`<meta http-equiv="Content-Security-Policy" content="default-src 'none'; style-src 'unsafe-inline'; img-src data:;">\n`);
		for (const [key, value] of Object.entries(dataToSave)) {
			stream.write(`<p><strong>${escapeHTML(key)}</strong>:`);
			if (typeof value === 'string') {
				stream.write(escapeHTML(value));
			} else if (Array.isArray(value)) {
				stream.write('<br/>');

				for (const item of value) {
					stream.write(`${escapeHTML(String(item))}<br/>`);
				}
			}

			stream.write('</p>\n');
		}

		stream.end();
	});
};

const generateUserAvatarFile = async (exportOperation: IExportOperation, userData?: IUser) => {
	if (!userData?.username) {
		return;
	}

	await mkdir(exportOperation.exportPath, { recursive: true });

	const file = await Avatars.findOneByName(userData.username);
	if (!file) {
		return;
	}

	const filePath = joinPath(exportOperation.exportPath, 'avatar');
	if (await FileUpload.copy?.(file, filePath)) {
		exportOperation.generatedAvatar = true;
	}
};

const generateChannelsFile = async (type: 'json' | 'html', exportPath: string, exportOperation: IExportOperation) => {
	if (type !== 'json') {
		return;
	}

	await mkdir(exportOperation.exportPath, { recursive: true });

	const fileName = joinPath(exportPath, 'channels.json');
	await writeFile(
		fileName,
		exportOperation.roomList
			?.map((roomData) =>
				JSON.stringify({
					roomId: roomData.roomId,
					roomName: roomData.roomName,
					type: roomData.type,
				}),
			)
			?.join('\n') ?? '',
	);
};

const isExportComplete = (exportOperation: IExportOperation) => {
	const incomplete = exportOperation.roomList?.some((exportOpRoomData) => exportOpRoomData.status !== 'completed') ?? false;

	return !incomplete;
};

const continueExportOperation = async function (exportOperation: IExportOperation): Promise<void> {
	if (exportOperation.status === 'completed') {
		return;
	}

	const exportType = exportOperation.fullExport ? 'json' : 'html';

	if (!exportOperation.roomList) {
		exportOperation.roomList = await loadUserSubscriptions(exportOperation, exportType, exportOperation.userId);

		if (exportOperation.fullExport) {
			exportOperation.status = 'exporting-rooms';
		} else {
			exportOperation.status = 'exporting';
		}
	}

	try {
		if (!exportOperation.generatedUserFile) {
			await generateUserFile(exportOperation, exportOperation.userData);
		}

		if (!exportOperation.generatedAvatar) {
			await generateUserAvatarFile(exportOperation, exportOperation.userData);
		}

		if (exportOperation.status === 'exporting-rooms') {
			await generateChannelsFile(exportType, exportOperation.exportPath, exportOperation);

			exportOperation.status = 'exporting';
		}

		// Run every room on every request, to avoid missing new messages on the rooms that finished first.
		if (exportOperation.status === 'exporting') {
			if (!exportOperation.userNameTable) {
				exportOperation.userNameTable = {};
			}
			const { fileList } = await exportRoomMessagesToFile(
				exportOperation.exportPath,
				exportOperation.assetsPath,
				exportType,
				exportOperation.roomList,
				exportOperation.userData,
				{},
				exportOperation.userNameTable,
			);
			if (!exportOperation.fileList) {
				exportOperation.fileList = [];
			}
			exportOperation.fileList.push(...fileList);

			if (isExportComplete(exportOperation)) {
				exportOperation.status = 'downloading';
			}
		}

		const generatedFileName = randomUUID();
		const zipFolder = settings.get<string>('UserData_FileSystemZipPath')?.trim() || '/tmp/zipFiles';

		if (exportOperation.status === 'downloading') {
			for await (const attachmentData of exportOperation.fileList) {
				await copyFileUpload(attachmentData, exportOperation.assetsPath);
			}

			const targetFile = joinPath(zipFolder, `${generatedFileName}.zip`);
			await rm(targetFile, { force: true });

			exportOperation.status = 'compressing';
		}

		if (exportOperation.status === 'compressing') {
			await mkdir(zipFolder, { recursive: true });

			exportOperation.generatedFile = joinPath(zipFolder, `${generatedFileName}.zip`);
			try {
				await access(exportOperation.generatedFile);
			} catch (error) {
				await makeZipFile(exportOperation.exportPath, exportOperation.generatedFile);
			}

			exportOperation.status = 'uploading';
		}

		if (exportOperation.status === 'uploading') {
			if (!exportOperation.generatedFile) {
				throw new Error('No generated file');
			}

			const { _id: fileId } = await uploadZipFile(exportOperation.generatedFile, exportOperation.userId, exportType);
			exportOperation.fileId = fileId;

			exportOperation.status = 'completed';
			await ExportOperations.updateOperation(exportOperation);
		}

		await ExportOperations.updateOperation(exportOperation);
	} catch (e) {
		console.error(e);
	}
};

export async function processDataDownloads(): Promise<void> {
	const operation = await ExportOperations.findOnePending();
	if (!operation) {
		return;
	}

	if (operation.status === 'completed') {
		return;
	}

	if (operation.status !== 'pending') {
		// If the operation has started but was not updated in over a day, then skip it
		if (operation._updatedAt && moment().diff(moment(operation._updatedAt), 'days') > 1) {
			operation.status = 'skipped';
			await ExportOperations.updateOperation(operation);
			return processDataDownloads();
		}
	}

	await continueExportOperation(operation);
	await ExportOperations.updateOperation(operation);

	if (operation.status === 'completed') {
		const file = operation.fileId
			? await UserDataFiles.findOneById(operation.fileId)
			: await UserDataFiles.findLastFileByUser(operation.userId);
		if (!file) {
			return;
		}

		const subject = i18n.t('UserDataDownload_EmailSubject');
		const body = i18n.t('UserDataDownload_EmailBody', {
			download_link: getURL(getPath(file._id), { cdn: false, full: true }),
		});

		await sendEmail(operation.userData, subject, body);
	}
}