import type { IMessage, IRoom, MessageAttachment, IProviderMetadata, ISupportedLanguage, ITranslationResult, } from '@rocket.chat/core-typings'; import { Logger } from '@rocket.chat/logger'; import { Messages, Subscriptions } from '@rocket.chat/models'; import { escapeHTML, isTruthy } from '@rocket.chat/tools'; import { Meteor } from 'meteor/meteor'; import _ from 'underscore'; import { settings } from '../../settings'; import { callbacks } from '../callbacks'; import { Markdown } from '../messaging/markdown'; import { notifyOnMessageChange } from '../notifyListener'; const translationLogger = new Logger('AutoTranslate'); // Constant tokenizer patterns, compiled once instead of on every message. // Safe to share with /g flags because String.prototype.replace resets // lastIndex on each call. const urlSchemes = 'http,https'; const markdownLinkRegex = new RegExp(`(!?\\[)([^\\]]+)(\\]\\((?:${urlSchemes}):\\/\\/[^\\)]+\\))`, 'gm'); const pipedLinkRegex = new RegExp(`((?:<|<)(?:${urlSchemes}):\\/\\/[^\\|]+\\|)(.+?)(?=>|>)((?:>|>))`, 'gm'); const wrappedParagraphRegex = new RegExp('^\\s*

|

\\s*$', 'gm'); const Providers = Symbol('Providers'); const Provider = Symbol('Provider'); /** * This class allows translation providers to * register,load and also returns the active provider. */ export class TranslationProviderRegistry { static [Providers]: { [k: string]: AutoTranslate } = {}; static enabled = false; static [Provider]: string | null = null; /** * Registers the translation provider into the registry. * @param {*} provider */ static registerProvider(provider: AutoTranslate): void { // get provider information const metadata = provider._getProviderMetadata(); if (!metadata) { translationLogger.error('Provider metadata is not defined'); return; } TranslationProviderRegistry[Providers][metadata.name] = provider; } /** * Return the active Translation provider */ static getActiveProvider(): AutoTranslate | null { if (!TranslationProviderRegistry.enabled) { return null; } const provider = TranslationProviderRegistry[Provider]; if (!provider) { return null; } return TranslationProviderRegistry[Providers][provider]; } static async getSupportedLanguages(target: string): Promise { return TranslationProviderRegistry.enabled ? TranslationProviderRegistry.getActiveProvider()?.getSupportedLanguages(target) : undefined; } static async translateMessage(message: IMessage, room: IRoom, targetLanguage?: string): Promise { if (!TranslationProviderRegistry.enabled) { return null; } const provider = TranslationProviderRegistry.getActiveProvider(); if (!provider) { return null; } return provider.translateMessage(message, { room, targetLanguage }); } static getProviders(): AutoTranslate[] { return Object.values(TranslationProviderRegistry[Providers]); } static setCurrentProvider(provider: string): void { TranslationProviderRegistry[Provider] = provider; } static setEnable(enabled: boolean): void { TranslationProviderRegistry.enabled = enabled; TranslationProviderRegistry.registerCallbacks(); } static registerCallbacks(): void { if (!TranslationProviderRegistry.enabled) { callbacks.remove('afterSaveMessage', 'autotranslate'); return; } callbacks.add( 'afterSaveMessage', (message, { room }) => TranslationProviderRegistry.translateMessage(message, room), callbacks.priority.MEDIUM, 'autotranslate', ); } } /** * Generic auto translate base implementation. * This class provides generic parts of implementation for * tokenization, detokenization, call back register and unregister. * @abstract * @class */ export abstract class AutoTranslate { name: string; languages: string[]; supportedLanguages: { [language: string]: ISupportedLanguage[]; }; /** * Encapsulate the api key and provider settings. * @constructor */ constructor() { this.name = ''; this.languages = []; this.supportedLanguages = {}; } /** * Extracts non-translatable parts of a message * @param {object} message * @return {object} message */ tokenize(message: IMessage): IMessage { if (!message.tokens || !Array.isArray(message.tokens)) { message.tokens = []; } message = this.tokenizeEmojis(message); message = this.tokenizeCode(message); message = this.tokenizeURLs(message); message = this.tokenizeMentions(message); return message; } tokenizeEmojis(message: IMessage): IMessage { let count = message.tokens?.length || 0; message.msg = message.msg.replace(/:[+\w\d]+:/g, (match) => { const token = `{${count++}}`; message.tokens?.push({ token, text: match, }); return token; }); return message; } tokenizeURLs(message: IMessage): IMessage { let count = message.tokens?.length || 0; // Support ![alt text](http://image url) and [text](http://link) message.msg = message.msg.replace(markdownLinkRegex, (_match, pre, text, post) => { const pretoken = `{${count++}}`; message.tokens?.push({ token: pretoken, text: pre, }); const posttoken = `{${count++}}`; message.tokens?.push({ token: posttoken, text: post, }); return pretoken + text + posttoken; }); // Support message.msg = message.msg.replace(pipedLinkRegex, (_match, pre, text, post) => { const pretoken = `{${count++}}`; message.tokens?.push({ token: pretoken, text: pre, }); const posttoken = `{${count++}}`; message.tokens?.push({ token: posttoken, text: post, }); return pretoken + text + posttoken; }); return message; } tokenizeCode(message: IMessage): IMessage { let count = message.tokens?.length || 0; message.html = message.msg; message = Markdown.parseMessageNotEscaped(message); // Some parsers (e. g. Marked) wrap the complete message in a

- this is unnecessary and should be ignored with respect to translations message.msg = message.msg.replace(wrappedParagraphRegex, ''); for (const [tokenIndex, value] of message.tokens?.entries() ?? []) { const { token } = value; if (token.indexOf('notranslate') === -1) { const newToken = `{${count++}}`; message.msg = message.msg.replace(token, newToken); message.tokens ? (message.tokens[tokenIndex].token = newToken) : undefined; } } return message; } tokenizeMentions(message: IMessage): IMessage { let count = message.tokens?.length || 0; if (message.mentions && message.mentions.length > 0) { message.mentions.forEach((mention) => { message.msg = message.msg.replace(new RegExp(`(@${mention.username})`, 'gm'), (match) => { const token = `{${count++}}`; message.tokens?.push({ token, text: match, }); return token; }); }); } if (message.channels && message.channels.length > 0) { message.channels.forEach((channel) => { message.msg = message.msg.replace(new RegExp(`(#${channel.name})`, 'gm'), (match) => { const token = `{${count++}}`; message.tokens?.push({ token, text: match, }); return token; }); }); } return message; } deTokenize(message: IMessage): string { if (message.tokens && message.tokens?.length > 0) { for (const { token, text, noHtml } of message.tokens) { message.msg = message.msg.replace(token, () => noHtml || text); } } return message.msg; } /** * Triggers the translation of the prepared (tokenized) message * and persists the result * @public * @param {object} message * @param {object} room * @param {object} targetLanguage * @returns {object} unmodified message object. */ async translateMessage(message: IMessage, { room, targetLanguage }: { room: IRoom; targetLanguage?: string }): Promise { let targetLanguages: string[]; if (targetLanguage) { targetLanguages = [targetLanguage]; } else { targetLanguages = (await Subscriptions.getAutoTranslateLanguagesByRoomAndNotUser(room._id, message.u?._id)).filter(isTruthy); } if (message.msg) { setImmediate(async () => { let targetMessage = Object.assign({}, message); targetMessage.html = escapeHTML(String(targetMessage.msg)); targetMessage = this.tokenize(targetMessage); const translations = await this._translateMessage(targetMessage, targetLanguages); if (!_.isEmpty(translations)) { await Messages.addTranslations(message._id, translations, TranslationProviderRegistry[Provider] || ''); this.notifyTranslatedMessage(message._id); } }); } if (message.attachments && message.attachments.length > 0) { setImmediate(async () => { for (const [index, attachment] of message.attachments?.entries() ?? []) { if (attachment.description || attachment.text) { // Removes the initial link `[ ](quoterl)` from quote message before translation const translatedText = attachment?.text?.replace(/\[(.*?)\]\(.*?\)/g, '$1') || attachment?.text; const attachmentMessage = { ...attachment, text: translatedText }; const translations = await this._translateAttachmentDescriptions(attachmentMessage, targetLanguages); if (!_.isEmpty(translations)) { await Messages.addAttachmentTranslations(message._id, String(index), translations); this.notifyTranslatedMessage(message._id); } } } }); } return Messages.findOneById(message._id); } private notifyTranslatedMessage(messageId: string): void { void notifyOnMessageChange({ id: messageId, }); } /** * Returns metadata information about the service provider which is used by * the generic implementation * @abstract * @protected * @returns { name, displayName, settings } }; */ abstract _getProviderMetadata(): IProviderMetadata; /** * Provides the possible languages _from_ which a message can be translated into a target language * @abstract * @protected * @param {string} target - the language into which shall be translated * @returns [{ language, name }] */ abstract getSupportedLanguages(target: string): Promise; /** * Performs the actual translation of a message, * usually by sending a REST API call to the service provider. * @abstract * @protected * @param {object} message * @param {object} targetLanguages * @return {object} */ abstract _translateMessage(message: IMessage, targetLanguages: string[]): Promise; /** * Performs the actual translation of an attachment (precisely its description), * usually by sending a REST API call to the service provider. * @abstract * @param {object} attachment * @param {object} targetLanguages * @returns {object} translated messages for each target language */ abstract _translateAttachmentDescriptions(attachment: MessageAttachment, targetLanguages: string[]): Promise; } Meteor.startup(() => { /** Register the active service provider on the 'AfterSaveMessage' callback. * So the registered provider will be invoked when a message is saved. * All the other inactive service provider must be deactivated. */ settings.watch('AutoTranslate_ServiceProvider', (providerName) => { TranslationProviderRegistry.setCurrentProvider(providerName); }); // Get Auto Translate Active flag settings.watch('AutoTranslate_Enabled', (value) => { TranslationProviderRegistry.setEnable(value); }); });