snapshot of https://github.com/RocketChat/Rocket.Chat.git (develop, 2026-09-16; blobs > 131072B omitted)

apps/meteor/server/lib/import/classes/ImportDataConverter.ts

@@ -0,0 +1,180 @@
+import type { IImportRecord, IImportUser, IImportMessage, IImportChannel, IImportContact } from '@rocket.chat/core-typings';
+import type { Logger } from '@rocket.chat/logger';
+import { ImportData } from '@rocket.chat/models';
+import { pick } from '@rocket.chat/tools';
+
+import type { IConversionCallbacks } from '../definitions/IConversionCallbacks';
+import { ContactConverter } from './converters/ContactConverter';
+import { ConverterCache } from './converters/ConverterCache';
+import { type MessageConversionCallbacks, MessageConverter } from './converters/MessageConverter';
+import type { RecordConverter, RecordConverterOptions } from './converters/RecordConverter';
+import { RoomConverter } from './converters/RoomConverter';
+import { UserConverter, type UserConverterOptions } from './converters/UserConverter';
+
+export type ConverterOptions = UserConverterOptions & Omit<RecordConverterOptions, 'deleteDbData'>;
+
+export class ImportDataConverter {
+	protected _options: ConverterOptions;
+
+	protected _userConverter: UserConverter;
+
+	protected _roomConverter: RoomConverter;
+
+	protected _messageConverter: MessageConverter;
+
+	protected _contactConverter: ContactConverter;
+
+	protected _cache = new ConverterCache();
+
+	public get options(): ConverterOptions {
+		return this._options;
+	}
+
+	constructor(logger: Logger, options?: ConverterOptions) {
+		this._options = {
+			workInMemory: false,
+			...(options || {}),
+		};
+
+		this.initializeUserConverter(logger);
+		this.initializeContactConverter(logger);
+		this.initializeRoomConverter(logger);
+		this.initializeMessageConverter(logger);
+	}
+
+	protected getRecordConverterOptions(): RecordConverterOptions {
+		return {
+			...pick(this._options, 'workInMemory'),
+			// DbData is deleted by this class directly, so the converters don't need to do it individually
+			deleteDbData: false,
+		};
+	}
+
+	protected getUserConverterOptions(): UserConverterOptions {
+		return {
+			flagEmailsAsVerified: false,
+			skipExistingUsers: false,
+			skipNewUsers: false,
+
+			...pick(
+				this._options,
+				'flagEmailsAsVerified',
+				'skipExistingUsers',
+				'skipNewUsers',
+				'skipUserCallbacks',
+				'skipDefaultChannels',
+				'quickUserInsertion',
+				'enableEmail2fa',
+			),
+		};
+	}
+
+	protected initializeUserConverter(logger: Logger): void {
+		const userOptions = {
+			...this.getRecordConverterOptions(),
+			...this.getUserConverterOptions(),
+		};
+
+		this._userConverter = new UserConverter(userOptions, logger, this._cache);
+	}
+
+	protected initializeContactConverter(logger: Logger): void {
+		const contactOptions = {
+			...this.getRecordConverterOptions(),
+		};
+
+		this._contactConverter = new ContactConverter(contactOptions, logger, this._cache);
+	}
+
+	protected initializeRoomConverter(logger: Logger): void {
+		const roomOptions = {
+			...this.getRecordConverterOptions(),
+		};
+
+		this._roomConverter = new RoomConverter(roomOptions, logger, this._cache);
+	}
+
+	protected initializeMessageConverter(logger: Logger): void {
+		const messageOptions = {
+			...this.getRecordConverterOptions(),
+		};
+
+		this._messageConverter = new MessageConverter(messageOptions, logger, this._cache);
+	}
+
+	async addContact(data: IImportContact): Promise<void> {
+		return this._contactConverter.addObject(data);
+	}
+
+	async addUser(data: IImportUser): Promise<void> {
+		return this._userConverter.addObject(data);
+	}
+
+	async addChannel(data: IImportChannel): Promise<void> {
+		return this._roomConverter.addObject(data);
+	}
+
+	async addMessage(data: IImportMessage, useQuickInsert = false): Promise<void> {
+		return this._messageConverter.addObject(data, {
+			useQuickInsert: useQuickInsert || undefined,
+		});
+	}
+
+	async convertContacts(callbacks: IConversionCallbacks): Promise<void> {
+		return this._contactConverter.convertData(callbacks);
+	}
+
+	async convertUsers(callbacks: IConversionCallbacks): Promise<void> {
+		return this._userConverter.convertData(callbacks);
+	}
+
+	async convertChannels(startedByUserId: string, callbacks: IConversionCallbacks): Promise<void> {
+		return this._roomConverter.convertChannels(startedByUserId, callbacks);
+	}
+
+	async convertMessages(callbacks: MessageConversionCallbacks): Promise<void> {
+		return this._messageConverter.convertData(callbacks);
+	}
+
+	async convertData(startedByUserId: string, callbacks: IConversionCallbacks = {}): Promise<void> {
+		await this.convertUsers(callbacks);
+		await this.convertContacts(callbacks);
+		await this.convertChannels(startedByUserId, callbacks);
+		await this.convertMessages(callbacks);
+
+		process.nextTick(async () => {
+			await this.clearSuccessfullyImportedData();
+		});
+	}
+
+	protected getAllConverters(): RecordConverter<IImportRecord>[] {
+		return [this._userConverter, this._roomConverter, this._messageConverter];
+	}
+
+	public async clearImportData(): Promise<void> {
+		if (!this._options.workInMemory) {
+			// Using raw collection since its faster
+			await ImportData.col.deleteMany({});
+		}
+
+		await Promise.all(this.getAllConverters().map((converter) => converter.clearImportData()));
+	}
+
+	async clearSuccessfullyImportedData(): Promise<void> {
+		if (!this._options.workInMemory) {
+			await ImportData.col.deleteMany({
+				errors: {
+					$exists: false,
+				},
+			});
+		}
+
+		await Promise.all(this.getAllConverters().map((converter) => converter.clearSuccessfullyImportedData()));
+	}
+
+	public abort(): void {
+		this.getAllConverters().forEach((converter) => {
+			converter.aborted = true;
+		});
+	}
+}

apps/meteor/server/lib/import/classes/Importer.ts

@@ -0,0 +1,420 @@
+import { api } from '@rocket.chat/core-services';
+import type {
+	IImport,
+	IImportRecord,
+	IImportChannel,
+	IImportUser,
+	IImportProgress,
+	IImporterShortSelection,
+	IImportContact,
+} from '@rocket.chat/core-typings';
+import { Logger } from '@rocket.chat/logger';
+import { Settings, ImportData, Imports } from '@rocket.chat/models';
+import AdmZip from 'adm-zip';
+import type { MatchKeysAndValues, MongoServerError } from 'mongodb';
+
+import { ImportDataConverter } from './ImportDataConverter';
+import type { ConverterOptions } from './ImportDataConverter';
+import { ImporterProgress } from './ImporterProgress';
+import { ImporterWebsocket } from './ImporterWebsocket';
+import { ProgressStep, ImportPreparingStartedStates } from '../../../../app/importer/lib/ImporterProgressStep';
+import { t } from '../../../../app/utils/lib/i18n';
+import { notifyOnSettingChanged, notifyOnSettingChangedById } from '../../notifyListener';
+import type { ImporterInfo } from '../definitions/ImporterInfo';
+import { Selection, SelectionChannel, SelectionUser } from '../index';
+
+type OldSettings = {
+	allowedDomainList?: string | null;
+	allowUsernameChange?: boolean | null;
+	maxFileSize?: number | null;
+	mediaTypeWhiteList?: string | null;
+	mediaTypeBlackList?: string | null;
+};
+
+/**
+ * Base class for all of the importers.
+ */
+export class Importer {
+	private _reportProgressHandler: ReturnType<typeof setTimeout> | undefined;
+
+	protected AdmZip = AdmZip;
+
+	protected converter: ImportDataConverter;
+
+	protected info: ImporterInfo;
+
+	protected logger: Logger;
+
+	protected oldSettings: OldSettings;
+
+	protected _lastProgressReportTotal = 0;
+
+	public importRecord: IImport;
+
+	public progress: ImporterProgress;
+
+	constructor(info: ImporterInfo, importRecord: IImport, converterOptions: ConverterOptions = {}) {
+		if (!info.key || !info.importer) {
+			throw new Error('Information passed in must be a valid ImporterInfo instance.');
+		}
+
+		this.info = info;
+		this.logger = new Logger(`${this.info.name} Importer`);
+
+		this.converter = new ImportDataConverter(this.logger, converterOptions);
+
+		this.importRecord = importRecord;
+		this.progress = new ImporterProgress(this.info.key, this.info.name);
+		this.oldSettings = {};
+
+		this.progress.step = this.importRecord.status;
+		this._lastProgressReportTotal = 0;
+		this.reloadCount();
+
+		this.logger.debug({ msg: 'Constructed a new Importer.', importerName: this.info.name });
+	}
+
+	/**
+	 * Registers the file name and content type on the import operation
+	 */
+	async startFileUpload(fileName: string, contentType?: string): Promise<IImport> {
+		await this.updateProgress(ProgressStep.UPLOADING);
+		return this.updateRecord({ file: fileName, ...(contentType ? { contentType } : {}) });
+	}
+
+	/**
+	 * Takes the uploaded file and extracts the users, channels, and messages from it.
+	 *
+	 * @param {string} _fullFilePath the full path of the uploaded file
+	 * @returns {ImporterProgress} The progress record of the import.
+	 */
+	async prepareUsingLocalFile(_fullFilePath: string): Promise<ImporterProgress> {
+		return this.updateProgress(ProgressStep.PREPARING_STARTED);
+	}
+
+	/**
+	 * Starts the import process. The implementing method should defer
+	 * as soon as the selection is set, so the user who started the process
+	 * doesn't end up with a "locked" UI while Meteor waits for a response.
+	 * The returned object should be the progress.
+	 *
+	 * @param {IImporterShortSelection} importSelection The selection data.
+	 * @returns {ImporterProgress} The progress record of the import.
+	 */
+	async startImport(importSelection: IImporterShortSelection, startedByUserId: string): Promise<ImporterProgress> {
+		await this.updateProgress(ProgressStep.IMPORTING_STARTED);
+		this.reloadCount();
+		const started = Date.now();
+
+		const beforeImportFn = async ({ data, dataType: type }: IImportRecord) => {
+			if (this.importRecord.valid === false) {
+				this.converter.abort();
+				throw new Error('The import operation is no longer valid.');
+			}
+
+			switch (type) {
+				case 'channel': {
+					if (importSelection.channels?.all) {
+						return true;
+					}
+					if (!importSelection.channels?.list?.length) {
+						return false;
+					}
+
+					const channelData = data as IImportChannel;
+
+					const id = channelData.t === 'd' ? '__directMessages__' : channelData.importIds[0];
+					return importSelection.channels.list?.includes(id);
+				}
+				case 'user': {
+					if (importSelection.users?.all) {
+						return true;
+					}
+					if (!importSelection.users?.list?.length) {
+						return false;
+					}
+
+					const userData = data as IImportUser;
+					const id = userData.importIds[0];
+					return importSelection.users.list.includes(id);
+				}
+
+				case 'contact': {
+					if (importSelection.contacts?.all) {
+						return true;
+					}
+					if (!importSelection.contacts?.list?.length) {
+						return false;
+					}
+
+					const contactData = data as IImportContact;
+
+					const id = contactData.importIds[0];
+					return importSelection.contacts.list.includes(id);
+				}
+			}
+
+			return false;
+		};
+
+		const afterImportFn = async () => {
+			await this.addCountCompleted(1);
+
+			if (this.importRecord.valid === false) {
+				this.converter.abort();
+				throw new Error('The import operation is no longer valid.');
+			}
+		};
+
+		const afterImportAllMessagesFn = async (importedRoomIds: string[]): Promise<void> =>
+			api.broadcast('notify.importedMessages', { roomIds: importedRoomIds });
+
+		const afterBatchFn = async (successCount: number, errorCount: number) => {
+			if (successCount) {
+				await this.addCountCompleted(successCount);
+			}
+			if (errorCount) {
+				await this.addCountError(errorCount);
+			}
+
+			if (this.importRecord.valid === false) {
+				this.converter.abort();
+				throw new Error('The import operation is no longer valid.');
+			}
+		};
+
+		const afterContactsBatchFn = async (successCount: number) => {
+			const value = await Settings.incrementValueById('Contacts_Importer_Count', successCount, { returnDocument: 'after' });
+			if (value) {
+				void notifyOnSettingChanged(value);
+			}
+		};
+
+		const onErrorFn = async () => {
+			await this.addCountCompleted(1);
+		};
+
+		process.nextTick(async () => {
+			await this.backupSettingValues();
+
+			try {
+				await this.applySettingValues({});
+
+				await this.updateProgress(ProgressStep.IMPORTING_USERS);
+				await this.converter.convertUsers({ beforeImportFn, afterImportFn, onErrorFn, afterBatchFn });
+
+				await this.updateProgress(ProgressStep.IMPORTING_CONTACTS);
+				await this.converter.convertContacts({ beforeImportFn, afterImportFn, onErrorFn, afterBatchFn: afterContactsBatchFn });
+
+				await this.updateProgress(ProgressStep.IMPORTING_CHANNELS);
+				await this.converter.convertChannels(startedByUserId, { beforeImportFn, afterImportFn, onErrorFn });
+
+				await this.updateProgress(ProgressStep.IMPORTING_MESSAGES);
+				await this.converter.convertMessages({ afterImportFn, onErrorFn, afterImportAllMessagesFn });
+
+				await this.updateProgress(ProgressStep.FINISHING);
+
+				process.nextTick(async () => {
+					await this.converter.clearSuccessfullyImportedData();
+				});
+
+				await this.updateProgress(ProgressStep.DONE);
+			} catch (e) {
+				this.logger.error({ msg: 'Importer encountered an error during import', err: e });
+				await this.updateProgress(ProgressStep.ERROR);
+			} finally {
+				await this.applySettingValues(this.oldSettings);
+			}
+
+			const timeTook = Date.now() - started;
+			this.logger.log({ msg: 'Import completed', durationMs: timeTook });
+		});
+
+		return this.getProgress();
+	}
+
+	async backupSettingValues() {
+		const allowUsernameChange = (await Settings.findOneById('Accounts_AllowUsernameChange'))?.value as boolean | null;
+		const maxFileSize = (await Settings.findOneById('FileUpload_MaxFileSize'))?.value as number | null;
+		const mediaTypeWhiteList = (await Settings.findOneById('FileUpload_MediaTypeWhiteList'))?.value as string | null;
+		const mediaTypeBlackList = (await Settings.findOneById('FileUpload_MediaTypeBlackList'))?.value as string | null;
+
+		this.oldSettings = {
+			allowUsernameChange,
+			maxFileSize,
+			mediaTypeWhiteList,
+			mediaTypeBlackList,
+		};
+	}
+
+	async applySettingValues(settingValues: OldSettings) {
+		const settingsIds = [
+			{ _id: 'Accounts_AllowUsernameChange', value: settingValues.allowUsernameChange ?? true },
+			{ _id: 'FileUpload_MaxFileSize', value: settingValues.maxFileSize ?? -1 },
+			{ _id: 'FileUpload_MediaTypeWhiteList', value: settingValues.mediaTypeWhiteList ?? '*' },
+			{ _id: 'FileUpload_MediaTypeBlackList', value: settingValues.mediaTypeBlackList ?? '' },
+		];
+
+		const promises = settingsIds.map((setting) => Settings.updateValueById(setting._id, setting.value));
+
+		(await Promise.all(promises)).forEach((value, index) => {
+			if (value?.modifiedCount) {
+				void notifyOnSettingChangedById(settingsIds[index]._id);
+			}
+		});
+	}
+
+	getProgress(): ImporterProgress {
+		return this.progress;
+	}
+
+	/**
+	 * Updates the progress step of this importer.
+	 * It also changes some internal settings at various stages of the import.
+	 * This way the importer can adjust user/room information at will.
+	 *
+	 * @param {ProgressStep} step The progress step which this import is currently at.
+	 * @returns {ImporterProgress} The progress record of the import.
+	 */
+	async updateProgress(step: IImportProgress['step']): Promise<ImporterProgress> {
+		this.progress.step = step;
+
+		this.logger.debug({ msg: 'Importer progress step updated', importerName: this.info.name, step });
+		await this.updateRecord({ status: this.progress.step });
+
+		// Do not send the default progress report during the preparing stage - the classes are sending their own report in a different format.
+		if (!ImportPreparingStartedStates.includes(this.progress.step)) {
+			this.reportProgress();
+		}
+
+		return this.progress;
+	}
+
+	reloadCount() {
+		this.progress.count.total = this.importRecord.count?.total || 0;
+		this.progress.count.completed = this.importRecord.count?.completed || 0;
+		this.progress.count.error = this.importRecord.count?.error || 0;
+	}
+
+	/**
+	 * Adds the passed in value to the total amount of items needed to complete.
+	 *
+	 * @param {number} count The amount to add to the total count of items.
+	 * @returns {ImporterProgress} The progress record of the import.
+	 */
+	async addCountToTotal(count: number): Promise<ImporterProgress> {
+		this.progress.count.total += count;
+		await this.updateRecord({ 'count.total': this.progress.count.total });
+
+		return this.progress;
+	}
+
+	/**
+	 * Adds the passed in value to the total amount of items completed.
+	 *
+	 * @param {number} count The amount to add to the total count of finished items.
+	 * @returns {ImporterProgress} The progress record of the import.
+	 */
+	async addCountCompleted(count: number): Promise<ImporterProgress> {
+		this.progress.count.completed += count;
+
+		return this.maybeUpdateRecord();
+	}
+
+	async addCountError(count: number): Promise<ImporterProgress> {
+		this.progress.count.error += count;
+
+		return this.maybeUpdateRecord();
+	}
+
+	async maybeUpdateRecord() {
+		// Only update the database every 500 messages (or 50 for other records)
+		// Or the completed is greater than or equal to the total amount
+		const count = this.progress.count.completed + this.progress.count.error;
+		const range = this.progress.step === ProgressStep.IMPORTING_MESSAGES ? 500 : 50;
+
+		if (count % range === 0 || count >= this.progress.count.total || count - this._lastProgressReportTotal > range) {
+			this._lastProgressReportTotal = this.progress.count.completed + this.progress.count.error;
+			await this.updateRecord({ 'count.completed': this.progress.count.completed, 'count.error': this.progress.count.error });
+			this.reportProgress();
+		} else if (!this._reportProgressHandler) {
+			this._reportProgressHandler = setTimeout(() => {
+				this.reportProgress();
+			}, 250);
+		}
+
+		this.logger.log({
+			msg: 'Import progress update',
+			completed: this.progress.count.completed,
+			failed: this.progress.count.error,
+		});
+
+		return this.progress;
+	}
+
+	/**
+	 * Sends an updated progress to the websocket
+	 */
+	reportProgress() {
+		if (this._reportProgressHandler) {
+			clearTimeout(this._reportProgressHandler);
+			this._reportProgressHandler = undefined;
+		}
+		ImporterWebsocket.progressUpdated(this.progress);
+	}
+
+	/**
+	 * Updates the import record with the given fields being `set`.
+	 */
+	async updateRecord(fields: MatchKeysAndValues<IImport>): Promise<IImport> {
+		if (!this.importRecord) {
+			return this.importRecord;
+		}
+
+		await Imports.update({ _id: this.importRecord._id }, { $set: fields });
+		// #TODO: Remove need for the typecast
+		this.importRecord = (await Imports.findOne(this.importRecord._id)) as IImport;
+
+		return this.importRecord;
+	}
+
+	async buildSelection(): Promise<Selection> {
+		await this.updateProgress(ProgressStep.USER_SELECTION);
+
+		const users = await ImportData.getAllUsersForSelection();
+		const channels = await ImportData.getAllChannelsForSelection();
+		const contacts = await ImportData.getAllContactsForSelection();
+		const hasDM = await ImportData.checkIfDirectMessagesExists();
+
+		const selectionUsers = users.map(
+			(u) =>
+				new SelectionUser(u.data.importIds[0], u.data.username, u.data.emails[0], Boolean(u.data.deleted), u.data.type === 'bot', true),
+		);
+		const selectionChannels = channels.map(
+			(c) => new SelectionChannel(c.data.importIds[0], c.data.name, Boolean(c.data.archived), true, c.data.t === 'p', c.data.t === 'd'),
+		);
+		const selectionContacts = contacts.map((c) => ({
+			id: c.data.importIds[0],
+			name: c.data.name || '',
+			emails: c.data.emails || [],
+			phones: c.data.phones || [],
+			do_import: true,
+		}));
+		const selectionMessages = await ImportData.countMessages();
+
+		if (hasDM) {
+			selectionChannels.push(new SelectionChannel('__directMessages__', t('Direct_Messages'), false, true, true, true));
+		}
+
+		const results = new Selection(this.info.name, selectionUsers, selectionChannels, selectionMessages, selectionContacts);
+
+		return results;
+	}
+
+	/**
+	 * Utility method to check if the passed in error is a `MongoServerError` with the `codeName` of `'CursorNotFound'`.
+	 */
+	protected isCursorNotFoundError(error: unknown): error is MongoServerError & { codeName: 'CursorNotFound' } {
+		return typeof error === 'object' && error !== null && 'codeName' in error && error.codeName === 'CursorNotFound';
+	}
+}

apps/meteor/server/lib/import/classes/ImporterProgress.ts

@@ -0,0 +1,30 @@
+import type { IImportProgress } from '@rocket.chat/core-typings';
+
+import { ProgressStep } from '../../../../app/importer/lib/ImporterProgressStep';
+
+export class ImporterProgress implements IImportProgress {
+	public key: string;
+
+	public name: string;
+
+	public step: IImportProgress['step'];
+
+	public count: {
+		completed: number;
+		total: number;
+		error: number;
+	};
+
+	/**
+	 * Creates a new progress container for the importer.
+	 *
+	 * @param {string} key The unique key of the importer.
+	 * @param {string} name The name of the importer.
+	 */
+	constructor(key: string, name: string) {
+		this.key = key;
+		this.name = name;
+		this.step = ProgressStep.NEW;
+		this.count = { completed: 0, total: 0, error: 0 };
+	}
+}

apps/meteor/server/lib/import/classes/ImporterSelection.ts

@@ -0,0 +1,40 @@
+import type {
+	IImporterSelection,
+	IImporterSelectionChannel,
+	IImporterSelectionUser,
+	IImporterSelectionContact,
+} from '@rocket.chat/core-typings';
+
+export class ImporterSelection implements IImporterSelection {
+	public name: string;
+
+	public users: IImporterSelectionUser[];
+
+	public channels: IImporterSelectionChannel[];
+
+	public contacts: IImporterSelectionContact[];
+
+	public message_count: number;
+
+	/**
+	 * Constructs a new importer selection object.
+	 *
+	 * @param name the name of the importer
+	 * @param users the users which can be selected
+	 * @param channels the channels which can be selected
+	 * @param messageCount the number of messages
+	 */
+	constructor(
+		name: string,
+		users: IImporterSelectionUser[],
+		channels: IImporterSelectionChannel[],
+		messageCount: number,
+		contacts: IImporterSelectionContact[],
+	) {
+		this.name = name;
+		this.users = users;
+		this.channels = channels;
+		this.message_count = messageCount;
+		this.contacts = contacts;
+	}
+}

apps/meteor/server/lib/import/classes/ImporterSelectionChannel.ts

@@ -0,0 +1,34 @@
+import type { IImporterSelectionChannel } from '@rocket.chat/core-typings';
+
+export class SelectionChannel implements IImporterSelectionChannel {
+	public channel_id: string;
+
+	public name: string | undefined;
+
+	public is_archived: boolean;
+
+	public do_import: boolean;
+
+	public is_private: boolean;
+
+	public is_direct: boolean;
+
+	public creator: undefined;
+
+	/**
+	 * Constructs a new selection channel.
+	 *
+	 * @param channelId the unique identifier of the channel
+	 * @param name the name of the channel
+	 * @param isArchived whether the channel was archived or not
+	 * @param doImport whether we will be importing the channel or not
+	 */
+	constructor(channelId: string, name: string | undefined, isArchived: boolean, doImport: boolean, isPrivate: boolean, isDirect: boolean) {
+		this.channel_id = channelId;
+		this.name = name;
+		this.is_archived = isArchived;
+		this.do_import = doImport;
+		this.is_private = isPrivate;
+		this.is_direct = isDirect;
+	}
+}

apps/meteor/server/lib/import/classes/ImporterSelectionUser.ts

@@ -0,0 +1,46 @@
+import type { IImporterSelectionUser } from '@rocket.chat/core-typings';
+
+export class SelectionUser implements IImporterSelectionUser {
+	public user_id: string;
+
+	public username: string | undefined;
+
+	public email: string;
+
+	public is_deleted: boolean;
+
+	public is_bot: boolean;
+
+	public do_import: boolean;
+
+	public is_email_taken: boolean;
+
+	/**
+	 * Constructs a new selection user.
+	 *
+	 * @param userId the unique user identifier
+	 * @param username the user's username
+	 * @param email the user's email
+	 * @param isDeleted whether the user was deleted or not
+	 * @param isBot whether the user is a bot or not
+	 * @param doImport whether we are going to import this user or not
+	 * @param isEmailTaken whether there's an existing user with the same email
+	 */
+	constructor(
+		userId: string,
+		username: string | undefined,
+		email: string,
+		isDeleted: boolean,
+		isBot: boolean,
+		doImport: boolean,
+		isEmailTaken = false,
+	) {
+		this.user_id = userId;
+		this.username = username;
+		this.email = email;
+		this.is_deleted = isDeleted;
+		this.is_bot = isBot;
+		this.do_import = doImport;
+		this.is_email_taken = isEmailTaken;
+	}
+}

apps/meteor/server/lib/import/classes/ImporterWebsocket.ts

@@ -0,0 +1,23 @@
+import type { IImportProgress } from '@rocket.chat/core-typings';
+
+import type { IStreamer } from '../../../modules/streamer/types';
+import notifications from '../../notifications/core/lib/Notifications';
+
+class ImporterWebsocketDef {
+	private streamer: IStreamer<'importers'>;
+
+	constructor() {
+		this.streamer = notifications.streamImporters;
+	}
+
+	/**
+	 * Called when the progress is updated.
+	 *
+	 * @param {Progress} progress The progress of the import.
+	 */
+	progressUpdated(progress: { rate: number } | IImportProgress) {
+		this.streamer.emit('progress', progress);
+	}
+}
+
+export const ImporterWebsocket = new ImporterWebsocketDef();

apps/meteor/server/lib/import/classes/ImportersContainer.ts

@@ -0,0 +1,27 @@
+import type { ImporterInfo } from '../definitions/ImporterInfo';
+
+/** Container class which holds all of the importer details. */
+export class ImportersContainer {
+	private importers: Map<ImporterInfo['key'], ImporterInfo>;
+
+	constructor() {
+		this.importers = new Map();
+	}
+
+	add({ key, name, importer, visible }: Omit<ImporterInfo, 'visible'> & { visible?: boolean }) {
+		this.importers.set(key, {
+			key,
+			name,
+			visible: visible !== false,
+			importer,
+		});
+	}
+
+	get(key: string): ImporterInfo | undefined {
+		return this.importers.get(key);
+	}
+
+	getAllVisible(): ImporterInfo[] {
+		return Array.from(this.importers.values()).filter(({ visible }) => visible);
+	}
+}

apps/meteor/server/lib/import/classes/converters/ContactConverter.ts

@@ -0,0 +1,42 @@
+import type { IImportContact, IImportContactRecord } from '@rocket.chat/core-typings';
+import { LivechatVisitors } from '@rocket.chat/models';
+
+import { RecordConverter } from './RecordConverter';
+import { createContact } from '../../../omnichannel/contacts/createContact';
+import { getAllowedCustomFields } from '../../../omnichannel/contacts/getAllowedCustomFields';
+import { validateCustomFields } from '../../../omnichannel/contacts/validateCustomFields';
+
+export class ContactConverter extends RecordConverter<IImportContactRecord> {
+	protected async convertCustomFields(customFields: IImportContact['customFields']): Promise<IImportContact['customFields']> {
+		if (!customFields) {
+			return;
+		}
+
+		const allowedCustomFields = await getAllowedCustomFields();
+		return validateCustomFields(allowedCustomFields, customFields, { ignoreAdditionalFields: true });
+	}
+
+	protected override async convertRecord(record: IImportContactRecord): Promise<boolean> {
+		const { data } = record;
+
+		await createContact({
+			name: data.name || (await this.generateNewContactName()),
+			emails: data.emails,
+			phones: data.phones,
+			customFields: await this.convertCustomFields(data.customFields),
+			contactManager: await this._cache.getIdOfUsername(data.contactManager),
+			unknown: false,
+			importIds: data.importIds,
+		});
+
+		return true;
+	}
+
+	protected async generateNewContactName(): Promise<string> {
+		return LivechatVisitors.getNextVisitorUsername();
+	}
+
+	protected override getDataType(): 'contact' {
+		return 'contact';
+	}
+}

apps/meteor/server/lib/import/classes/converters/ConverterCache.ts

@@ -0,0 +1,224 @@
+import type { IImportUser, IUser } from '@rocket.chat/core-typings';
+import { Rooms, Users } from '@rocket.chat/models';
+
+export type UserIdentification = {
+	_id: string;
+	username: string | undefined;
+};
+
+export type MentionedChannel = {
+	_id: string;
+	name: string;
+};
+
+export class ConverterCache {
+	private _userCache = new Map<string, UserIdentification>();
+
+	// display name uses a different cache because it's only used on mentions so we don't need to load it every time we load an user
+	private _userDisplayNameCache = new Map<string, string>();
+
+	private _userNameToIdCache = new Map<string, string | undefined>();
+
+	private _roomCache = new Map<string, string>();
+
+	private _roomNameCache = new Map<string, string>();
+
+	addUser(importId: string, _id: string, username: string | undefined): UserIdentification {
+		const cache = {
+			_id,
+			username,
+		};
+
+		this._userCache.set(importId, cache);
+		if (username) {
+			this._userNameToIdCache.set(username, _id);
+		}
+		return cache;
+	}
+
+	addUserDisplayName(importId: string, name: string): string {
+		this._userDisplayNameCache.set(importId, name);
+		return name;
+	}
+
+	addRoom(importId: string, rid: string): string {
+		this._roomCache.set(importId, rid);
+		return rid;
+	}
+
+	addRoomName(importId: string, name: string): string {
+		this._roomNameCache.set(importId, name);
+		return name;
+	}
+
+	addUserData(userData: IImportUser): void {
+		if (!userData._id) {
+			return;
+		}
+		if (!userData.importIds.length) {
+			return;
+		}
+
+		this.addUser(userData.importIds[0], userData._id, userData.username);
+	}
+
+	addUsernameToId(username: string, id: string): void {
+		this._userNameToIdCache.set(username, id);
+	}
+
+	async findImportedRoomId(importId: string): Promise<string | null> {
+		if (this._roomCache.has(importId)) {
+			return this._roomCache.get(importId) as string;
+		}
+
+		const options = {
+			projection: {
+				_id: 1,
+			},
+		};
+
+		const room = await Rooms.findOneByImportId(importId, options);
+		if (room) {
+			return this.addRoom(importId, room._id);
+		}
+
+		return null;
+	}
+
+	async findImportedRoomName(importId: string): Promise<string | undefined> {
+		if (this._roomNameCache.has(importId)) {
+			return this._roomNameCache.get(importId) as string;
+		}
+
+		const options = {
+			projection: {
+				_id: 1,
+				name: 1,
+			},
+		};
+
+		const room = await Rooms.findOneByImportId(importId, options);
+		if (room) {
+			if (!this._roomCache.has(importId)) {
+				this.addRoom(importId, room._id);
+			}
+			if (room?.name) {
+				return this.addRoomName(importId, room.name);
+			}
+		}
+	}
+
+	async findImportedUser(importId: string): Promise<UserIdentification | null> {
+		if (importId === 'rocket.cat') {
+			return {
+				_id: 'rocket.cat',
+				username: 'rocket.cat',
+			};
+		}
+
+		const options = {
+			projection: {
+				_id: 1,
+				username: 1,
+			},
+		};
+
+		if (this._userCache.has(importId)) {
+			return this._userCache.get(importId) as UserIdentification;
+		}
+
+		const user = await Users.findOneByImportId(importId, options);
+		if (user) {
+			return this.addUser(importId, user._id, user.username);
+		}
+
+		return null;
+	}
+
+	async findImportedUserId(_id: string): Promise<string | undefined> {
+		const data = await this.findImportedUser(_id);
+		return data?._id;
+	}
+
+	async findImportedUsername(_id: string): Promise<string | undefined> {
+		const data = await this.findImportedUser(_id);
+		return data?.username;
+	}
+
+	async findImportedUserDisplayName(importId: string): Promise<string | undefined> {
+		const options = {
+			projection: {
+				_id: 1,
+				name: 1,
+				username: 1,
+			},
+		};
+
+		if (this._userDisplayNameCache.has(importId)) {
+			return this._userDisplayNameCache.get(importId);
+		}
+
+		const user =
+			importId === 'rocket.cat' ? await Users.findOneById('rocket.cat', options) : await Users.findOneByImportId(importId, options);
+		if (user) {
+			if (!this._userCache.has(importId)) {
+				this.addUser(importId, user._id, user.username);
+			}
+
+			if (!user.name) {
+				return;
+			}
+
+			return this.addUserDisplayName(importId, user.name);
+		}
+	}
+
+	async convertImportedIdsToUsernames(importedIds: Array<string>, idToRemove: string | undefined = undefined): Promise<Array<string>> {
+		return (
+			await Promise.all(
+				importedIds.map(async (user) => {
+					if (user === 'rocket.cat') {
+						return user;
+					}
+
+					if (this._userCache.has(user)) {
+						const cache = this._userCache.get(user);
+						if (cache) {
+							return cache.username;
+						}
+					}
+
+					const obj = await Users.findOneByImportId(user, { projection: { _id: 1, username: 1 } });
+					if (obj) {
+						this.addUser(user, obj._id, obj.username);
+
+						if (idToRemove && obj._id === idToRemove) {
+							return false;
+						}
+
+						return obj.username;
+					}
+
+					return false;
+				}),
+			)
+		).filter((user) => user) as string[];
+	}
+
+	async getIdOfUsername(username: string | undefined): Promise<string | undefined> {
+		if (!username) {
+			return;
+		}
+
+		if (this._userNameToIdCache.has(username)) {
+			return this._userNameToIdCache.get(username);
+		}
+
+		const user = await Users.findOneByUsername<Pick<IUser, '_id'>>(username, { projection: { _id: 1 } });
+		if (user) {
+			this.addUsernameToId(username, user._id);
+		}
+
+		return user?._id;
+	}
+}

apps/meteor/server/lib/import/classes/converters/MessageConverter.ts

@@ -0,0 +1,263 @@
+import type { IImportMessageRecord, IMessage as IDBMessage, IImportMessage, IImportMessageReaction } from '@rocket.chat/core-typings';
+import { Rooms } from '@rocket.chat/models';
+import { removeEmpty } from '@rocket.chat/tools';
+import limax from 'limax';
+
+import type { UserIdentification, MentionedChannel } from './ConverterCache';
+import { RecordConverter } from './RecordConverter';
+import { insertMessage } from '../../../messages/insertMessage';
+import type { IConversionCallbacks } from '../../definitions/IConversionCallbacks';
+
+export type MessageConversionCallbacks = IConversionCallbacks & { afterImportAllMessagesFn?: (roomIds: string[]) => Promise<void> };
+
+type MessageObject = Record<string, any>;
+
+type MentionedUser = {
+	_id: string;
+	username: string;
+	name?: string;
+};
+
+type IMessageReaction = {
+	name: string;
+	usernames: string[];
+};
+
+type IMessageReactions = Record<string, IMessageReaction>;
+
+export class MessageConverter extends RecordConverter<IImportMessageRecord> {
+	private rids: string[] = [];
+
+	override async convertData({ afterImportAllMessagesFn, ...callbacks }: MessageConversionCallbacks = {}): Promise<void> {
+		this.rids = [];
+		await super.convertData(callbacks);
+
+		await this.resetLastMessages();
+		if (afterImportAllMessagesFn) {
+			await afterImportAllMessagesFn(this.rids);
+		}
+	}
+
+	protected async resetLastMessages(): Promise<void> {
+		for (const rid of this.rids) {
+			try {
+				await Rooms.resetLastMessageById(rid, null);
+			} catch (err) {
+				this._logger.error({ msg: 'Failed to update last message of room', roomId: rid, err });
+			}
+		}
+	}
+
+	protected async insertMessage(data: IImportMessage): Promise<void> {
+		if (!data.ts || isNaN(data.ts as unknown as number)) {
+			throw new Error('importer-message-invalid-timestamp');
+		}
+
+		const creator = await this._cache.findImportedUser(data.u._id);
+		if (!creator) {
+			this._logger.warn({ msg: 'Imported user not found', userId: data.u._id });
+			throw new Error('importer-message-unknown-user');
+		}
+		const rid = await this._cache.findImportedRoomId(data.rid);
+		if (!rid) {
+			throw new Error('importer-message-unknown-room');
+		}
+		if (!this.rids.includes(rid)) {
+			this.rids.push(rid);
+		}
+
+		const msgObj = await this.buildMessageObject(data, rid, creator);
+
+		try {
+			await insertMessage(creator, msgObj as unknown as IDBMessage, rid, true);
+		} catch (err) {
+			this._logger.error({ msg: 'Failed to import message', timestamp: msgObj.ts, roomId: rid, err });
+		}
+	}
+
+	protected override async convertRecord(record: IImportMessageRecord): Promise<boolean> {
+		await this.insertMessage(record.data);
+		return true;
+	}
+
+	protected async buildMessageObject(data: IImportMessage, rid: string, creator: UserIdentification): Promise<MessageObject> {
+		// Convert the mentions and channels first because these conversions can also modify the msg in the message object
+		const mentions = data.mentions && (await this.convertMessageMentions(data));
+		const channels = data.channels && (await this.convertMessageChannels(data));
+
+		return removeEmpty({
+			rid,
+			u: {
+				_id: creator._id,
+				username: creator.username,
+			},
+			msg: data.msg,
+			ts: data.ts,
+			t: data.t || undefined,
+			groupable: data.groupable,
+			tmid: data.tmid,
+			tlm: data.tlm,
+			tcount: data.tcount,
+			replies: data.replies && (await this.convertMessageReplies(data.replies)),
+			editedAt: data.editedAt,
+			editedBy: data.editedBy && ((await this._cache.findImportedUser(data.editedBy)) || undefined),
+			mentions,
+			channels,
+			_importFile: data._importFile,
+			_hidden: data._hidden,
+			url: data.url,
+			attachments: data.attachments,
+			bot: data.bot,
+			emoji: data.emoji,
+			alias: data.alias,
+			...(data._id ? { _id: data._id } : {}),
+			...(data.reactions ? { reactions: await this.convertMessageReactions(data.reactions) } : {}),
+		});
+	}
+
+	protected async convertMessageChannels(message: IImportMessage): Promise<MentionedChannel[] | undefined> {
+		const { channels } = message;
+		if (!channels) {
+			return;
+		}
+
+		const result: MentionedChannel[] = [];
+		for (const importId of channels) {
+			const { name, _id } = (await this.getMentionedChannelData(importId)) || {};
+
+			if (!_id || !name) {
+				this._logger.warn({ msg: 'Mentioned room not found', importId });
+				continue;
+			}
+
+			message.msg = message.msg.replace(new RegExp(`\#${importId}`, 'gi'), `#${name}`);
+
+			result.push({
+				_id,
+				name,
+			});
+		}
+
+		return result;
+	}
+
+	protected async convertMessageMentions(message: IImportMessage): Promise<MentionedUser[] | undefined> {
+		const { mentions } = message;
+		if (!mentions) {
+			return undefined;
+		}
+
+		const result: MentionedUser[] = [];
+		for (const importId of mentions) {
+			if (importId === ('all' as 'string') || importId === 'here') {
+				result.push({
+					_id: importId,
+					username: importId,
+				});
+				continue;
+			}
+
+			// Loading the name will also store the remaining data on the cache if it's missing, so this won't run two queries
+			const name = await this._cache.findImportedUserDisplayName(importId);
+			const data = await this._cache.findImportedUser(importId);
+
+			if (!data) {
+				this._logger.warn({ msg: 'Mentioned user not found', importId });
+				continue;
+			}
+
+			if (!data.username) {
+				this._logger.debug({ msg: 'Mentioned user has no username', importId });
+				throw new Error('importer-message-mentioned-username-not-found');
+			}
+
+			message.msg = message.msg.replace(new RegExp(`\@${importId}`, 'gi'), `@${data.username}`);
+
+			result.push({
+				_id: data._id,
+				username: data.username as 'string',
+				name,
+			});
+		}
+		return result;
+	}
+
+	protected async convertMessageReactions(
+		importedReactions: Record<string, IImportMessageReaction>,
+	): Promise<undefined | IMessageReactions> {
+		const reactions: IMessageReactions = {};
+
+		for (const name of Object.keys(importedReactions)) {
+			if (!importedReactions.hasOwnProperty(name)) {
+				continue;
+			}
+			const { users } = importedReactions[name];
+
+			if (!users.length) {
+				continue;
+			}
+
+			const reaction: IMessageReaction = {
+				name,
+				usernames: [],
+			};
+
+			for (const importId of users) {
+				const username = await this._cache.findImportedUsername(importId);
+				if (username && !reaction.usernames.includes(username)) {
+					reaction.usernames.push(username);
+				}
+			}
+
+			if (reaction.usernames.length) {
+				reactions[name] = reaction;
+			}
+		}
+
+		if (Object.keys(reactions).length > 0) {
+			return reactions;
+		}
+	}
+
+	protected async convertMessageReplies(replies: string[]): Promise<string[]> {
+		const result: string[] = [];
+		for (const importId of replies) {
+			const userId = await this._cache.findImportedUserId(importId);
+			if (userId && !result.includes(userId)) {
+				result.push(userId);
+			}
+		}
+		return result;
+	}
+
+	protected async getMentionedChannelData(importId: string): Promise<MentionedChannel | undefined> {
+		// loading the name will also store the id on the cache if it's missing, so this won't run two queries
+		const name = await this._cache.findImportedRoomName(importId);
+		const _id = await this._cache.findImportedRoomId(importId);
+
+		if (name && _id) {
+			return {
+				name,
+				_id,
+			};
+		}
+
+		// If the importId was not found, check if we have a room with that name
+		const roomName = limax(importId.trim(), { maintainCase: true });
+
+		const room = await Rooms.findOneByNonValidatedName(roomName, { projection: { name: 1 } });
+		if (room?.name) {
+			this._cache.addRoom(importId, room._id);
+			this._cache.addRoomName(importId, room.name);
+
+			return {
+				name: room.name,
+				_id: room._id,
+			};
+		}
+	}
+
+	protected override getDataType(): 'message' {
+		return 'message';
+	}
+}

apps/meteor/server/lib/import/classes/converters/RecordConverter.ts

@@ -0,0 +1,252 @@
+import type { IImportRecord } from '@rocket.chat/core-typings';
+import { Logger } from '@rocket.chat/logger';
+import { ImportData } from '@rocket.chat/models';
+import { Random } from '@rocket.chat/random';
+import { type FindCursor, ObjectId } from 'mongodb';
+
+import { ConverterCache } from './ConverterCache';
+import type { IConversionCallbacks } from '../../definitions/IConversionCallbacks';
+
+export type RecordConverterOptions = {
+	workInMemory?: boolean;
+	deleteDbData?: boolean;
+};
+
+export class RecordConverter<R extends IImportRecord, T extends RecordConverterOptions = RecordConverterOptions> {
+	protected _logger: Logger;
+
+	protected _cache: ConverterCache;
+
+	protected _converterOptions: RecordConverterOptions;
+
+	protected _options: Omit<T, keyof RecordConverterOptions>;
+
+	protected _records: R[];
+
+	protected skippedCount = 0;
+
+	protected failedCount = 0;
+
+	protected newCount = 0;
+
+	public aborted = false;
+
+	constructor(options?: T, logger?: Logger, cache?: ConverterCache) {
+		const { workInMemory = false, deleteDbData = false, ...customOptions } = options || ({} as T);
+		this._converterOptions = {
+			workInMemory,
+			deleteDbData,
+		};
+		this._options = customOptions;
+
+		this._logger = logger || new Logger(`Data Importer - ${this.constructor.name}`);
+		this._cache = cache || new ConverterCache();
+		this._records = [];
+	}
+
+	private skipMemoryRecord(_id: string): void {
+		const record = this.getMemoryRecordById(_id);
+		if (!record) {
+			return;
+		}
+
+		record.skipped = true;
+	}
+
+	private async skipDatabaseRecord(_id: string): Promise<void> {
+		await ImportData.updateOne(
+			{
+				_id,
+			},
+			{
+				$set: {
+					skipped: true,
+				},
+			},
+		);
+	}
+
+	protected async skipRecord(_id: string): Promise<void> {
+		this.skippedCount++;
+		this.skipMemoryRecord(_id);
+		if (!this._converterOptions.workInMemory) {
+			return this.skipDatabaseRecord(_id);
+		}
+	}
+
+	private saveErrorToMemory(importId: string, error: Error): void {
+		const record = this.getMemoryRecordById(importId);
+
+		if (!record) {
+			return;
+		}
+
+		if (!record.errors) {
+			record.errors = [];
+		}
+
+		record.errors.push({
+			message: error.message,
+			stack: error.stack,
+		});
+	}
+
+	private async saveErrorToDatabase(importId: string, error: Error): Promise<void> {
+		await ImportData.updateOne(
+			{
+				_id: importId,
+			},
+			{
+				$push: {
+					errors: {
+						message: error.message,
+						stack: error.stack,
+					},
+				},
+			},
+		);
+	}
+
+	protected async saveError(importId: string, error: Error): Promise<void> {
+		this._logger.error({
+			msg: 'Import record conversion failed',
+			importId,
+			err: error,
+		});
+		this.saveErrorToMemory(importId, error);
+
+		if (!this._converterOptions.workInMemory) {
+			return this.saveErrorToDatabase(importId, error);
+		}
+	}
+
+	public async clearImportData(): Promise<void> {
+		this._records = [];
+
+		// On regular import operations this data will be deleted by the importer class with one single operation for all dataTypes (aka with no filter)
+		if (!this._converterOptions.workInMemory && this._converterOptions.deleteDbData) {
+			await ImportData.col.deleteMany({ dataType: this.getDataType() });
+		}
+	}
+
+	public async clearSuccessfullyImportedData(): Promise<void> {
+		this._records = this._records.filter((record) => !record.errors?.length);
+
+		// On regular import operations this data will be deleted by the importer class with one single operation for all dataTypes (aka with no filter)
+		if (!this._converterOptions.workInMemory && this._converterOptions.deleteDbData) {
+			await ImportData.col.deleteMany({ dataType: this.getDataType(), error: { $exists: false } });
+		}
+	}
+
+	private getMemoryRecordById(id: string): R | undefined {
+		for (const record of this._records) {
+			if (record._id === id) {
+				return record;
+			}
+		}
+
+		return undefined;
+	}
+
+	protected getDataType(): R['dataType'] {
+		throw new Error('Unspecified type');
+	}
+
+	protected async addObjectToDatabase(data: R['data'], options: R['options'] = {}): Promise<void> {
+		await ImportData.col.insertOne({
+			_id: new ObjectId().toHexString(),
+			data,
+			dataType: this.getDataType(),
+			options,
+			_updatedAt: new Date(),
+		});
+	}
+
+	public addObjectToMemory(data: R['data'], options: R['options'] = {}): void {
+		this._records.push({
+			_id: Random.id(),
+			data,
+			dataType: this.getDataType(),
+			options,
+		} as R);
+	}
+
+	public async addObject(data: R['data'], options: R['options'] = {}): Promise<void> {
+		if (this._converterOptions.workInMemory) {
+			return this.addObjectToMemory(data, options);
+		}
+
+		return this.addObjectToDatabase(data, options);
+	}
+
+	protected getDatabaseDataToImport(): Promise<R[]> {
+		return (ImportData.find({ dataType: this.getDataType() }) as FindCursor<R>).toArray();
+	}
+
+	protected async getDataToImport(): Promise<R[]> {
+		if (this._converterOptions.workInMemory) {
+			return this._records;
+		}
+
+		const dbRecords = await this.getDatabaseDataToImport();
+		if (this._records.length) {
+			return [...this._records, ...dbRecords];
+		}
+
+		return dbRecords;
+	}
+
+	protected async iterateRecords({
+		beforeImportFn,
+		afterImportFn,
+		onErrorFn,
+		processRecord,
+		afterBatchFn,
+	}: IConversionCallbacks & { processRecord?: (record: R) => Promise<boolean | undefined> } = {}): Promise<void> {
+		const records = await this.getDataToImport();
+
+		this.skippedCount = 0;
+		this.failedCount = 0;
+		this.newCount = 0;
+
+		for (const record of records) {
+			const { _id } = record;
+			if (this.aborted) {
+				return;
+			}
+
+			try {
+				if (beforeImportFn && !(await beforeImportFn(record))) {
+					await this.skipRecord(_id);
+					continue;
+				}
+
+				const isNew = await (processRecord || this.convertRecord).call(this, record);
+
+				if (typeof isNew === 'boolean') {
+					this.newCount++;
+					if (afterImportFn) {
+						await afterImportFn(record, isNew);
+					}
+				}
+			} catch (e) {
+				this.failedCount++;
+				await this.saveError(_id, e instanceof Error ? e : new Error(String(e)));
+				if (onErrorFn) {
+					await onErrorFn();
+				}
+			}
+		}
+		if (afterBatchFn) {
+			await afterBatchFn(this.newCount, this.failedCount);
+		}
+	}
+
+	async convertData(callbacks: IConversionCallbacks = {}): Promise<void> {
+		return this.iterateRecords(callbacks);
+	}
+
+	protected async convertRecord(_record: R): Promise<boolean | undefined> {
+		return undefined;
+	}
+}

apps/meteor/server/lib/import/classes/converters/RoomConverter.ts

@@ -0,0 +1,201 @@
+import type { IImportChannel, IImportChannelRecord, IRoom } from '@rocket.chat/core-typings';
+import { Subscriptions, Rooms, Users } from '@rocket.chat/models';
+import { removeEmpty } from '@rocket.chat/tools';
+import limax from 'limax';
+
+import { RecordConverter } from './RecordConverter';
+import { createDirectMessage } from '../../../../meteor-methods/messages/createDirectMessage';
+import { createChannelMethod } from '../../../../meteor-methods/rooms/createChannel';
+import { createPrivateGroupMethod } from '../../../../meteor-methods/rooms/createPrivateGroup';
+import { saveRoomSettings } from '../../../../meteor-methods/rooms/saveRoomSettings';
+import { notifyOnSubscriptionChangedByRoomId } from '../../../notifyListener';
+import type { IConversionCallbacks } from '../../definitions/IConversionCallbacks';
+
+export class RoomConverter extends RecordConverter<IImportChannelRecord> {
+	public startedByUserId: string;
+
+	async convertChannels(startedByUserId: string, callbacks: IConversionCallbacks = {}): Promise<void> {
+		this.startedByUserId = startedByUserId;
+
+		return this.convertData(callbacks);
+	}
+
+	protected override async convertRecord(record: IImportChannelRecord): Promise<boolean> {
+		const { data } = record;
+
+		if (!data.name && data.t !== 'd') {
+			throw new Error('importer-channel-missing-name');
+		}
+
+		data.importIds = data.importIds.filter((item) => item);
+		data.users = [...new Set(data.users)];
+
+		if (!data.importIds.length) {
+			throw new Error('importer-channel-missing-import-id');
+		}
+
+		const existingRoom = await this.findExistingRoom(data);
+		await this.insertOrUpdateRoom(existingRoom, data, this.startedByUserId);
+
+		return !existingRoom;
+	}
+
+	async insertOrUpdateRoom(existingRoom: IRoom | null, data: IImportChannel, startedByUserId: string): Promise<void> {
+		if (existingRoom) {
+			await this.updateRoom(existingRoom, data, startedByUserId);
+		} else {
+			await this.insertRoom(data, startedByUserId);
+		}
+
+		if (data.archived && data._id) {
+			await this.archiveRoomById(data._id);
+		}
+	}
+
+	async findExistingRoom(data: IImportChannel): Promise<IRoom | null> {
+		if (data._id && data._id.toUpperCase() === 'GENERAL') {
+			const room = await Rooms.findOneById('GENERAL', {});
+			// Prevent the importer from trying to create a new general
+			if (!room) {
+				throw new Error('importer-channel-general-not-found');
+			}
+
+			return room;
+		}
+
+		if (data.t === 'd') {
+			const users = await this._cache.convertImportedIdsToUsernames(data.users);
+			if (users.length !== data.users.length) {
+				throw new Error('importer-channel-missing-users');
+			}
+
+			return Rooms.findDirectRoomContainingAllUsernames(users, {});
+		}
+
+		if (!data.name) {
+			return null;
+		}
+
+		// Imported room names always allow special chars
+		const roomName = limax(data.name.trim(), { maintainCase: true });
+		return Rooms.findOneByNonValidatedName(roomName, {});
+	}
+
+	async updateRoom(room: IRoom, roomData: IImportChannel, startedByUserId: string): Promise<void> {
+		roomData._id = room._id;
+
+		if ((roomData._id as string).toUpperCase() === 'GENERAL' && roomData.name !== room.name) {
+			await saveRoomSettings(startedByUserId, 'GENERAL', 'roomName', roomData.name);
+		}
+
+		await this.updateRoomId(room._id, roomData);
+	}
+
+	async insertRoom(roomData: IImportChannel, startedByUserId: string): Promise<void> {
+		// Find the rocketchatId of the user who created this channel
+		const creatorId = await this.getRoomCreatorId(roomData, startedByUserId);
+		const members = await this._cache.convertImportedIdsToUsernames(roomData.users, roomData.t !== 'd' ? creatorId : undefined);
+
+		if (roomData.t === 'd') {
+			if (members.length < roomData.users.length) {
+				this._logger.warn({
+					msg: 'One or more imported users not found',
+					users: roomData.users,
+				});
+				throw new Error('importer-channel-missing-users');
+			}
+		}
+
+		// Create the channel
+		try {
+			let roomInfo;
+			if (roomData.t === 'd') {
+				roomInfo = await createDirectMessage(members, startedByUserId, true);
+			} else {
+				if (!roomData.name) {
+					return;
+				}
+				if (roomData.t === 'p') {
+					const user = await Users.findOneById(creatorId);
+					if (!user) {
+						throw new Error('importer-channel-invalid-creator');
+					}
+					roomInfo = await createPrivateGroupMethod(user, roomData.name, members, false, {}, {});
+				} else {
+					roomInfo = await createChannelMethod(creatorId, roomData.name, members, false, {}, {});
+				}
+			}
+
+			roomData._id = roomInfo.rid;
+		} catch (e) {
+			this._logger.warn({ msg: 'Failed to create new room', name: roomData.name, members, err: e });
+			throw e;
+		}
+
+		await this.updateRoomId(roomData._id as 'string', roomData);
+	}
+
+	async archiveRoomById(rid: string) {
+		const responses = await Promise.all([Rooms.archiveById(rid), Subscriptions.archiveByRoomId(rid)]);
+
+		if (responses[1]?.modifiedCount) {
+			void notifyOnSubscriptionChangedByRoomId(rid);
+		}
+	}
+
+	async updateRoomId(_id: string, roomData: IImportChannel): Promise<void> {
+		const set = {
+			ts: roomData.ts,
+			topic: roomData.topic,
+			description: roomData.description,
+		};
+
+		const roomUpdate: { $set?: Record<string, any>; $addToSet?: Record<string, any> } = {};
+
+		if (Object.keys(set).length > 0) {
+			roomUpdate.$set = removeEmpty(set);
+		}
+
+		if (roomData.importIds.length) {
+			roomUpdate.$addToSet = {
+				importIds: {
+					$each: roomData.importIds,
+				},
+			};
+		}
+
+		if (roomUpdate.$set || roomUpdate.$addToSet) {
+			await Rooms.updateOne({ _id: roomData._id }, roomUpdate);
+		}
+	}
+
+	async getRoomCreatorId(roomData: IImportChannel, startedByUserId: string): Promise<string> {
+		if (roomData.u) {
+			const creatorId = await this._cache.findImportedUserId(roomData.u._id);
+			if (creatorId) {
+				return creatorId;
+			}
+
+			if (roomData.t !== 'd') {
+				return startedByUserId;
+			}
+
+			throw new Error('importer-channel-invalid-creator');
+		}
+
+		if (roomData.t === 'd') {
+			for await (const member of roomData.users) {
+				const userId = await this._cache.findImportedUserId(member);
+				if (userId) {
+					return userId;
+				}
+			}
+		}
+
+		throw new Error('importer-channel-invalid-creator');
+	}
+
+	protected override getDataType(): 'channel' {
+		return 'channel';
+	}
+}

apps/meteor/server/lib/import/classes/converters/UserConverter.ts

@@ -0,0 +1,430 @@
+import type { IImportUser, IImportUserRecord, IUser, IUserEmail } from '@rocket.chat/core-typings';
+import { Users } from '@rocket.chat/models';
+import { Random } from '@rocket.chat/random';
+import { SHA256 } from '@rocket.chat/sha256';
+import { hash as bcryptHash } from 'bcrypt';
+import { Accounts } from 'meteor/accounts-base';
+
+import { RecordConverter, type RecordConverterOptions } from './RecordConverter';
+import { generateTempPassword } from './generateTempPassword';
+import { callbacks as systemCallbacks } from '../../../callbacks';
+import { notifyOnUserChange } from '../../../notifyListener';
+import { addUserToDefaultChannels } from '../../../rooms/addUserToDefaultChannels';
+import { generateUsernameSuggestion } from '../../../users/getUsernameSuggestion';
+import { saveUserIdentity } from '../../../users/saveUserIdentity';
+import { setUserActiveStatus } from '../../../users/setUserActiveStatus';
+import type { IConversionCallbacks } from '../../definitions/IConversionCallbacks';
+
+export type UserConverterOptions = {
+	flagEmailsAsVerified?: boolean;
+	skipExistingUsers?: boolean;
+	skipNewUsers?: boolean;
+	skipUserCallbacks?: boolean;
+	skipDefaultChannels?: boolean;
+
+	quickUserInsertion?: boolean;
+	enableEmail2fa?: boolean;
+	syncVoipExtension?: boolean;
+};
+
+export class UserConverter extends RecordConverter<IImportUserRecord, UserConverterOptions & RecordConverterOptions> {
+	private insertedIds = new Set<IUser['_id']>();
+
+	private updatedIds = new Set<IUser['_id']>();
+
+	protected override async convertRecord(record: IImportUserRecord): Promise<boolean | undefined> {
+		const { data, _id } = record;
+
+		data.importIds = data.importIds.filter((item) => item);
+
+		if (!data.emails.length && !data.username) {
+			throw new Error('importer-user-missing-email-and-username');
+		}
+
+		const existingUser = await this.findExistingUser(data);
+		if (existingUser && this._options.skipExistingUsers) {
+			await this.skipRecord(_id);
+			return;
+		}
+		if (!existingUser && this._options.skipNewUsers) {
+			await this.skipRecord(_id);
+			return;
+		}
+
+		await this.insertOrUpdateUser(existingUser, data);
+		return !existingUser;
+	}
+
+	override async convertData(userCallbacks: IConversionCallbacks = {}): Promise<void> {
+		this.insertedIds.clear();
+		this.updatedIds.clear();
+
+		if (this._options.quickUserInsertion) {
+			await this.batchConversion(userCallbacks);
+		} else {
+			await super.convertData(userCallbacks);
+		}
+
+		await systemCallbacks.run('afterUserImport', {
+			inserted: [...this.insertedIds],
+			updated: [...this.updatedIds],
+			skipped: this.skippedCount,
+			failed: this.failedCount,
+		});
+	}
+
+	public async batchConversion({ afterBatchFn, ...callbacks }: IConversionCallbacks = {}): Promise<void> {
+		const batchToInsert = new Set<IImportUser>();
+
+		await this.iterateRecords({
+			...callbacks,
+			processRecord: async (record: IImportUserRecord) => {
+				const { data } = record;
+
+				data.importIds = data.importIds.filter((item) => item);
+
+				if (!data.emails.length && !data.username) {
+					throw new Error('importer-user-missing-email-and-username');
+				}
+
+				batchToInsert.add(data);
+
+				if (batchToInsert.size >= 50) {
+					const usersToInsert = await this.buildUserBatch([...batchToInsert]);
+					batchToInsert.clear();
+
+					const newIds = await this.insertUserBatch(usersToInsert, { afterBatchFn });
+					newIds.forEach((id) => this.insertedIds.add(id));
+				}
+
+				return undefined;
+			},
+		});
+
+		if (batchToInsert.size > 0) {
+			const usersToInsert = await this.buildUserBatch([...batchToInsert]);
+			const newIds = await this.insertUserBatch(usersToInsert, { afterBatchFn });
+			newIds.forEach((id) => this.insertedIds.add(id));
+		}
+	}
+
+	private async insertUserBatch(users: IUser[], { afterBatchFn }: IConversionCallbacks): Promise<string[]> {
+		let newIds: string[] | null = null;
+
+		try {
+			newIds = Object.values((await Users.insertMany(users, { ordered: false })).insertedIds);
+			if (afterBatchFn) {
+				await afterBatchFn(newIds.length, 0);
+			}
+		} catch (e: any) {
+			newIds = (e.result?.result?.insertedIds || []) as string[];
+			const errorCount = users.length - (e.result?.result?.nInserted || 0);
+
+			if (afterBatchFn) {
+				await afterBatchFn(Math.min(newIds.length, users.length - errorCount), errorCount);
+			}
+		}
+
+		return newIds;
+	}
+
+	async findExistingUser(data: IImportUser): Promise<IUser | null | undefined> {
+		if (data.emails.length) {
+			const emailUser = await Users.findOneByEmailAddress(data.emails[0], {});
+
+			if (emailUser) {
+				return emailUser;
+			}
+		}
+
+		// If we couldn't find one by their email address, try to find an existing user by their username
+		if (data.username) {
+			return Users.findOneByUsernameIgnoringCase<IUser>(data.username, {});
+		}
+	}
+
+	addUserImportId(updateData: Record<string, any>, userData: IImportUser): void {
+		if (userData.importIds?.length) {
+			updateData.$addToSet = {
+				importIds: {
+					$each: userData.importIds,
+				},
+			};
+		}
+	}
+
+	addUserEmails(updateData: Record<string, any>, userData: IImportUser, existingEmails: Array<IUserEmail>): void {
+		if (!userData.emails?.length) {
+			return;
+		}
+
+		const verifyEmails = Boolean(this._options.flagEmailsAsVerified);
+		const newEmailList: Array<IUserEmail> = [];
+
+		for (const email of userData.emails) {
+			const verified = verifyEmails || existingEmails.find((ee) => ee.address === email)?.verified || false;
+
+			newEmailList.push({
+				address: email,
+				verified,
+			});
+		}
+
+		updateData.$set.emails = newEmailList;
+	}
+
+	addUserServices(updateData: Record<string, any>, userData: IImportUser): void {
+		if (!userData.services) {
+			return;
+		}
+
+		for (const serviceKey in userData.services) {
+			if (!userData.services[serviceKey]) {
+				continue;
+			}
+
+			const service = userData.services[serviceKey];
+
+			for (const key in service) {
+				if (!service[key]) {
+					continue;
+				}
+
+				updateData.$set[`services.${serviceKey}.${key}`] = service[key];
+			}
+		}
+	}
+
+	addCustomFields(updateData: Record<string, any>, userData: IImportUser): void {
+		if (!userData.customFields) {
+			return;
+		}
+
+		const subset = (source: Record<string, any>, currentPath: string): void => {
+			for (const key in source) {
+				if (!source.hasOwnProperty(key) || source[key] === undefined) {
+					continue;
+				}
+
+				const keyPath = `${currentPath}.${key}`;
+				if (typeof source[key] === 'object' && !Array.isArray(source[key])) {
+					subset(source[key], keyPath);
+					continue;
+				}
+
+				updateData.$set = {
+					...updateData.$set,
+					...{ [keyPath]: source[key] },
+				};
+			}
+		};
+
+		subset(userData.customFields, 'customFields');
+	}
+
+	async insertOrUpdateUser(existingUser: IUser | null | undefined, data: IImportUser): Promise<void> {
+		if (!data.username && !existingUser?.username) {
+			const emails = data.emails.filter(Boolean).map((email) => ({ address: email }));
+			data.username = await generateUsernameSuggestion({
+				name: data.name,
+				emails,
+			});
+		}
+
+		if (existingUser) {
+			await this.updateUser(existingUser, data);
+			this.updatedIds.add(existingUser._id);
+		} else {
+			if (!data.name && data.username) {
+				data.name = this.guessNameFromUsername(data.username);
+			}
+
+			const userId = await this.insertUser(data);
+			data._id = userId;
+			this.insertedIds.add(userId);
+
+			if (!this._options.skipDefaultChannels) {
+				const insertedUser = await Users.findOneById(userId, {});
+				if (!insertedUser) {
+					throw new Error(`User not found: ${userId}`);
+				}
+
+				await addUserToDefaultChannels(insertedUser, true);
+			}
+		}
+	}
+
+	async updateUser(existingUser: IUser, userData: IImportUser): Promise<void> {
+		const { _id } = existingUser;
+		if (!_id) {
+			return;
+		}
+
+		if (Boolean(userData.federated) !== Boolean(existingUser.federated)) {
+			throw new Error("Local and Federated users can't be converted to each other.");
+		}
+
+		userData._id = _id;
+
+		if (!userData.roles && !existingUser.roles) {
+			userData.roles = ['user'];
+		}
+		if (!userData.type && !existingUser.type) {
+			userData.type = 'user';
+		}
+
+		const updateData: Record<string, any> = Object.assign(Object.create(null), {
+			$set: Object.assign(Object.create(null), {
+				...(userData.roles && { roles: userData.roles }),
+				...(userData.type && { type: userData.type }),
+				...(userData.statusText && { statusText: userData.statusText }),
+				...(userData.bio && { bio: userData.bio }),
+				...(userData.services?.ldap && { ldap: true }),
+				...(userData.avatarUrl && { _pendingAvatarUrl: userData.avatarUrl }),
+				...(this._options.syncVoipExtension && userData.voipExtension && { freeSwitchExtension: userData.voipExtension }),
+			}),
+			...(this._options.syncVoipExtension &&
+				!userData.voipExtension && {
+					$unset: {
+						freeSwitchExtension: 1,
+					},
+				}),
+		});
+
+		this.addCustomFields(updateData, userData);
+		this.addUserServices(updateData, userData);
+		this.addUserImportId(updateData, userData);
+		if (!userData.federated) {
+			this.addUserEmails(updateData, userData, existingUser.emails || []);
+		}
+
+		if (Object.keys(updateData.$set).length === 0) {
+			delete updateData.$set;
+		}
+		if (Object.keys(updateData).length > 0) {
+			await Users.updateOne({ _id }, updateData);
+		}
+
+		if (userData.utcOffset) {
+			await Users.setUtcOffset(_id, userData.utcOffset);
+		}
+
+		const localUsername = userData.federated ? undefined : userData.username;
+		if (userData.name || localUsername) {
+			await saveUserIdentity({ _id, name: userData.name, username: localUsername } as Parameters<typeof saveUserIdentity>[0]);
+		}
+
+		if (userData.importIds.length) {
+			this._cache.addUser(userData.importIds[0], existingUser._id, existingUser.username || userData.username);
+		}
+
+		// Deleted users are 'inactive' users in Rocket.Chat
+		if (userData.deleted && existingUser?.active) {
+			await setUserActiveStatus(_id, false, true);
+		} else if (userData.deleted === false && existingUser?.active === false) {
+			await setUserActiveStatus(_id, true);
+		}
+
+		void notifyOnUserChange({ clientAction: 'updated', id: _id, diff: updateData.$set });
+	}
+
+	async hashPassword(password: string): Promise<string> {
+		return bcryptHash(SHA256(password), Accounts._bcryptRounds());
+	}
+
+	generateTempPassword(userData: IImportUser): string {
+		return generateTempPassword(userData);
+	}
+
+	async buildNewUserObject(userData: IImportUser): Promise<Partial<IUser>> {
+		return {
+			type: userData.type || 'user',
+			...(userData.username && { username: userData.username }),
+			...(userData.emails.length && {
+				emails: userData.emails.map((email) => ({ address: email, verified: !!this._options.flagEmailsAsVerified })),
+			}),
+			...(userData.statusText && { statusText: userData.statusText }),
+			...(userData.name && { name: userData.name }),
+			...(userData.bio && { bio: userData.bio }),
+			...(userData.avatarUrl && { _pendingAvatarUrl: userData.avatarUrl }),
+			...(userData.utcOffset !== undefined && { utcOffset: userData.utcOffset }),
+			...{
+				services: {
+					// Add a password service if there's a password string, or if there's no service at all
+					...((!!userData.password || !userData.services || !Object.keys(userData.services).length) && {
+						password: { bcrypt: await this.hashPassword(userData.password || this.generateTempPassword(userData)) },
+					}),
+					...(userData.services || {}),
+				},
+			},
+			...(userData.services?.ldap && { ldap: true }),
+			...(userData.importIds?.length && { importIds: userData.importIds }),
+			...(!!userData.customFields && { customFields: userData.customFields }),
+			...(userData.deleted !== undefined && { active: !userData.deleted }),
+			...(this._options.syncVoipExtension && userData.voipExtension && { freeSwitchExtension: userData.voipExtension }),
+			...(userData.federated !== undefined && { federated: userData.federated }),
+		};
+	}
+
+	private async buildUserBatch(usersData: IImportUser[]): Promise<IUser[]> {
+		return Promise.all(
+			usersData.map(async (userData) => {
+				const user = await this.buildNewUserObject(userData);
+				return {
+					createdAt: new Date(),
+					_id: Random.id(),
+
+					status: 'offline',
+					...user,
+					roles: userData.roles?.length ? userData.roles : ['user'],
+					active: !userData.deleted,
+					services: {
+						...user.services,
+						...(this._options.enableEmail2fa
+							? {
+									email2fa: {
+										enabled: true,
+										changedAt: new Date(),
+									},
+								}
+							: {}),
+					},
+				} as IUser;
+			}),
+		);
+	}
+
+	async insertUser(userData: IImportUser): Promise<IUser['_id']> {
+		const user = await this.buildNewUserObject(userData);
+
+		return Accounts.insertUserDoc(
+			{
+				joinDefaultChannels: false,
+				skipEmailValidation: true,
+				skipAdminCheck: true,
+				skipAdminEmail: true,
+				skipOnCreateUserCallback: this._options.skipUserCallbacks,
+				skipBeforeCreateUserCallback: this._options.skipUserCallbacks,
+				skipAfterCreateUserCallback: this._options.skipUserCallbacks,
+				skipDefaultAvatar: true,
+				skipAppsEngineEvent: !!process.env.IMPORTER_SKIP_APPS_EVENT,
+			},
+			{
+				...user,
+				...(userData.roles?.length ? { globalRoles: userData.roles } : {}),
+			},
+		);
+	}
+
+	protected guessNameFromUsername(username: string): string {
+		return username
+			.replace(/\W/g, ' ')
+			.replace(/\s(.)/g, (u) => u.toUpperCase())
+			.replace(/^(.)/, (u) => u.toLowerCase())
+			.replace(/^\w/, (u) => u.toUpperCase());
+	}
+
+	protected override getDataType(): 'user' {
+		return 'user';
+	}
+}

apps/meteor/server/lib/import/classes/converters/generateTempPassword.ts

@@ -0,0 +1,5 @@
+import type { IImportUser } from '@rocket.chat/core-typings';
+
+export function generateTempPassword(userData: IImportUser): string {
+	return `${Date.now()}${userData.name || ''}${userData.emails.length ? userData.emails[0].toUpperCase() : ''}`;
+}