snapshot of https://github.com/RocketChat/Rocket.Chat.git (develop, 2026-09-16; blobs > 131072B omitted)
Some checks failed
Create issues from TODOs / todos (push) Failing after 3s
Close inactive issues / close-issues (push) Has been cancelled
Deploy GitHub Pages / deploy-preview (push) Has been cancelled
CI / ⚙️ Variables Setup (push) Has been cancelled
CI / ⚙️ Test Guard (push) Has been cancelled
CI / 🚀 Notify external services - draft (push) Has been cancelled
CI / 📦 Build Packages (push) Has been cancelled
CI / 📦 Meteor Build (coverage) (push) Has been cancelled
CI / 📦 Meteor Build (production) (push) Has been cancelled
CI / 🚢 Build Docker (amd64, [account-service presence-service omnichannel-transcript-service], ${{ (github.event_name != 'release' && github.ref != 'refs/heads/develop') && 'coverage' || 'production' }}) (push) Has been cancelled
CI / 🚢 Build Docker (amd64, [account-service presence-service omnichannel-transcript-service], fips) (push) Has been cancelled
CI / 🚢 Build Docker (amd64, [authorization-service queue-worker-service ddp-streamer-service], ${{ (github.event_name != 'release' && github.ref != 'refs/heads/develop') && 'coverage' || 'production' }}) (push) Has been cancelled
CI / 🚢 Build Docker (amd64, [authorization-service queue-worker-service ddp-streamer-service], fips) (push) Has been cancelled
CI / 🚢 Build Docker (amd64, [rocketchat], ${{ (github.event_name != 'release' && github.ref != 'refs/heads/develop') && 'coverage' || 'production' }}) (push) Has been cancelled
CI / 🚢 Build Docker (amd64, [rocketchat], coverage) (push) Has been cancelled
CI / 🚢 Build Docker (amd64, [rocketchat], fips) (push) Has been cancelled
CI / 🚢 Build Docker (arm64, [account-service presence-service omnichannel-transcript-service], ${{ (github.event_name != 'release' && github.ref != 'refs/heads/develop') && 'coverage' || 'production' }}) (push) Has been cancelled
CI / 🚢 Build Docker (arm64, [authorization-service queue-worker-service ddp-streamer-service], ${{ (github.event_name != 'release' && github.ref != 'refs/heads/develop') && 'coverage' || 'production' }}) (push) Has been cancelled
CI / 🚢 Build Docker (arm64, [rocketchat], ${{ (github.event_name != 'release' && github.ref != 'refs/heads/develop') && 'coverage' || 'production' }}) (push) Has been cancelled
CI / 🚢 Build Docker (arm64, [rocketchat], coverage) (push) Has been cancelled
CI / 🚢 Publish Docker Images (ghcr.io) (push) Has been cancelled
CI / 📦 Track Image Sizes (push) Has been cancelled
CI / 🔎 Code Check - Action Lint (push) Has been cancelled
CI / 🔎 Code Check (push) Has been cancelled
CI / 🔨 Test Storybook (push) Has been cancelled
CI / 🔨 Test Unit (push) Has been cancelled
CI / 🔨 Test API (CE) (push) Has been cancelled
CI / 🔨 Test API Livechat (CE) (push) Has been cancelled
CI / 🔨 Test UI (CE) (push) Has been cancelled
CI / 🔨 Test API (EE) (push) Has been cancelled
CI / 🔨 Test API Livechat (EE) (push) Has been cancelled
CI / 🔨 Test API Apps (deno-runtime - EE) (push) Has been cancelled
CI / 🔨 Test UI (EE) (push) Has been cancelled
CI / 🔨 Test API (FIPS) (push) Has been cancelled
CI / 🔨 Test API Livechat (FIPS) (push) Has been cancelled
CI / 🔨 Test UI (FIPS) (push) Has been cancelled
CI / 🔨 Test Federation Matrix (push) Has been cancelled
CI / 📊 Report Coverage (push) Has been cancelled
CI / ✅ Tests Done (push) Has been cancelled
CI / 🚀 Publish build assets (push) Has been cancelled
CI / 🚀 Publish Docker Images (DockerHub) (push) Has been cancelled
CI / 🚀 Notify external services (push) Has been cancelled
CI / Update Version Durability (push) Has been cancelled
Release / Release (push) Has been cancelled
Auto-close duplicate issues / auto-close-duplicates (push) Failing after 1s
Code scanning - action / CodeQL-Build (push) Failing after 1s
Some checks failed
Create issues from TODOs / todos (push) Failing after 3s
Close inactive issues / close-issues (push) Has been cancelled
Deploy GitHub Pages / deploy-preview (push) Has been cancelled
CI / ⚙️ Variables Setup (push) Has been cancelled
CI / ⚙️ Test Guard (push) Has been cancelled
CI / 🚀 Notify external services - draft (push) Has been cancelled
CI / 📦 Build Packages (push) Has been cancelled
CI / 📦 Meteor Build (coverage) (push) Has been cancelled
CI / 📦 Meteor Build (production) (push) Has been cancelled
CI / 🚢 Build Docker (amd64, [account-service presence-service omnichannel-transcript-service], ${{ (github.event_name != 'release' && github.ref != 'refs/heads/develop') && 'coverage' || 'production' }}) (push) Has been cancelled
CI / 🚢 Build Docker (amd64, [account-service presence-service omnichannel-transcript-service], fips) (push) Has been cancelled
CI / 🚢 Build Docker (amd64, [authorization-service queue-worker-service ddp-streamer-service], ${{ (github.event_name != 'release' && github.ref != 'refs/heads/develop') && 'coverage' || 'production' }}) (push) Has been cancelled
CI / 🚢 Build Docker (amd64, [authorization-service queue-worker-service ddp-streamer-service], fips) (push) Has been cancelled
CI / 🚢 Build Docker (amd64, [rocketchat], ${{ (github.event_name != 'release' && github.ref != 'refs/heads/develop') && 'coverage' || 'production' }}) (push) Has been cancelled
CI / 🚢 Build Docker (amd64, [rocketchat], coverage) (push) Has been cancelled
CI / 🚢 Build Docker (amd64, [rocketchat], fips) (push) Has been cancelled
CI / 🚢 Build Docker (arm64, [account-service presence-service omnichannel-transcript-service], ${{ (github.event_name != 'release' && github.ref != 'refs/heads/develop') && 'coverage' || 'production' }}) (push) Has been cancelled
CI / 🚢 Build Docker (arm64, [authorization-service queue-worker-service ddp-streamer-service], ${{ (github.event_name != 'release' && github.ref != 'refs/heads/develop') && 'coverage' || 'production' }}) (push) Has been cancelled
CI / 🚢 Build Docker (arm64, [rocketchat], ${{ (github.event_name != 'release' && github.ref != 'refs/heads/develop') && 'coverage' || 'production' }}) (push) Has been cancelled
CI / 🚢 Build Docker (arm64, [rocketchat], coverage) (push) Has been cancelled
CI / 🚢 Publish Docker Images (ghcr.io) (push) Has been cancelled
CI / 📦 Track Image Sizes (push) Has been cancelled
CI / 🔎 Code Check - Action Lint (push) Has been cancelled
CI / 🔎 Code Check (push) Has been cancelled
CI / 🔨 Test Storybook (push) Has been cancelled
CI / 🔨 Test Unit (push) Has been cancelled
CI / 🔨 Test API (CE) (push) Has been cancelled
CI / 🔨 Test API Livechat (CE) (push) Has been cancelled
CI / 🔨 Test UI (CE) (push) Has been cancelled
CI / 🔨 Test API (EE) (push) Has been cancelled
CI / 🔨 Test API Livechat (EE) (push) Has been cancelled
CI / 🔨 Test API Apps (deno-runtime - EE) (push) Has been cancelled
CI / 🔨 Test UI (EE) (push) Has been cancelled
CI / 🔨 Test API (FIPS) (push) Has been cancelled
CI / 🔨 Test API Livechat (FIPS) (push) Has been cancelled
CI / 🔨 Test UI (FIPS) (push) Has been cancelled
CI / 🔨 Test Federation Matrix (push) Has been cancelled
CI / 📊 Report Coverage (push) Has been cancelled
CI / ✅ Tests Done (push) Has been cancelled
CI / 🚀 Publish build assets (push) Has been cancelled
CI / 🚀 Publish Docker Images (DockerHub) (push) Has been cancelled
CI / 🚀 Notify external services (push) Has been cancelled
CI / Update Version Durability (push) Has been cancelled
Release / Release (push) Has been cancelled
Auto-close duplicate issues / auto-close-duplicates (push) Failing after 1s
Code scanning - action / CodeQL-Build (push) Failing after 1s
This commit is contained in:
5
apps/meteor/server/lib/autotranslate/README.md
Normal file
5
apps/meteor/server/lib/autotranslate/README.md
Normal file
@@ -0,0 +1,5 @@
|
||||
# Rocket.Chat Auto Translate
|
||||
|
||||
Rocket.Chat supports auto translate through Google Cloud Translation API
|
||||
Through this you will be able to read messages in your native languages
|
||||
regardless of language the message was written on.
|
||||
387
apps/meteor/server/lib/autotranslate/autotranslate.ts
Normal file
387
apps/meteor/server/lib/autotranslate/autotranslate.ts
Normal file
@@ -0,0 +1,387 @@
|
||||
import type {
|
||||
IMessage,
|
||||
IRoom,
|
||||
MessageAttachment,
|
||||
IProviderMetadata,
|
||||
ISupportedLanguage,
|
||||
ITranslationResult,
|
||||
} from '@rocket.chat/core-typings';
|
||||
import { Logger } from '@rocket.chat/logger';
|
||||
import { Messages, Subscriptions } from '@rocket.chat/models';
|
||||
import { escapeHTML, isTruthy } from '@rocket.chat/tools';
|
||||
import { Meteor } from 'meteor/meteor';
|
||||
import _ from 'underscore';
|
||||
|
||||
import { settings } from '../../settings';
|
||||
import { callbacks } from '../callbacks';
|
||||
import { Markdown } from '../messaging/markdown';
|
||||
import { notifyOnMessageChange } from '../notifyListener';
|
||||
|
||||
const translationLogger = new Logger('AutoTranslate');
|
||||
|
||||
// Constant tokenizer patterns, compiled once instead of on every message.
|
||||
// Safe to share with /g flags because String.prototype.replace resets
|
||||
// lastIndex on each call.
|
||||
const urlSchemes = 'http,https';
|
||||
const markdownLinkRegex = new RegExp(`(!?\\[)([^\\]]+)(\\]\\((?:${urlSchemes}):\\/\\/[^\\)]+\\))`, 'gm');
|
||||
const pipedLinkRegex = new RegExp(`((?:<|<)(?:${urlSchemes}):\\/\\/[^\\|]+\\|)(.+?)(?=>|>)((?:>|>))`, 'gm');
|
||||
const wrappedParagraphRegex = new RegExp('^\\s*<p>|</p>\\s*$', 'gm');
|
||||
|
||||
const Providers = Symbol('Providers');
|
||||
const Provider = Symbol('Provider');
|
||||
|
||||
/**
|
||||
* This class allows translation providers to
|
||||
* register,load and also returns the active provider.
|
||||
*/
|
||||
export class TranslationProviderRegistry {
|
||||
static [Providers]: { [k: string]: AutoTranslate } = {};
|
||||
|
||||
static enabled = false;
|
||||
|
||||
static [Provider]: string | null = null;
|
||||
|
||||
/**
|
||||
* Registers the translation provider into the registry.
|
||||
* @param {*} provider
|
||||
*/
|
||||
static registerProvider(provider: AutoTranslate): void {
|
||||
// get provider information
|
||||
const metadata = provider._getProviderMetadata();
|
||||
if (!metadata) {
|
||||
translationLogger.error('Provider metadata is not defined');
|
||||
return;
|
||||
}
|
||||
|
||||
TranslationProviderRegistry[Providers][metadata.name] = provider;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the active Translation provider
|
||||
*/
|
||||
static getActiveProvider(): AutoTranslate | null {
|
||||
if (!TranslationProviderRegistry.enabled) {
|
||||
return null;
|
||||
}
|
||||
const provider = TranslationProviderRegistry[Provider];
|
||||
if (!provider) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return TranslationProviderRegistry[Providers][provider];
|
||||
}
|
||||
|
||||
static async getSupportedLanguages(target: string): Promise<ISupportedLanguage[] | undefined> {
|
||||
return TranslationProviderRegistry.enabled ? TranslationProviderRegistry.getActiveProvider()?.getSupportedLanguages(target) : undefined;
|
||||
}
|
||||
|
||||
static async translateMessage(message: IMessage, room: IRoom, targetLanguage?: string): Promise<IMessage | null> {
|
||||
if (!TranslationProviderRegistry.enabled) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const provider = TranslationProviderRegistry.getActiveProvider();
|
||||
if (!provider) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return provider.translateMessage(message, { room, targetLanguage });
|
||||
}
|
||||
|
||||
static getProviders(): AutoTranslate[] {
|
||||
return Object.values(TranslationProviderRegistry[Providers]);
|
||||
}
|
||||
|
||||
static setCurrentProvider(provider: string): void {
|
||||
TranslationProviderRegistry[Provider] = provider;
|
||||
}
|
||||
|
||||
static setEnable(enabled: boolean): void {
|
||||
TranslationProviderRegistry.enabled = enabled;
|
||||
|
||||
TranslationProviderRegistry.registerCallbacks();
|
||||
}
|
||||
|
||||
static registerCallbacks(): void {
|
||||
if (!TranslationProviderRegistry.enabled) {
|
||||
callbacks.remove('afterSaveMessage', 'autotranslate');
|
||||
return;
|
||||
}
|
||||
|
||||
callbacks.add(
|
||||
'afterSaveMessage',
|
||||
(message, { room }) => TranslationProviderRegistry.translateMessage(message, room),
|
||||
callbacks.priority.MEDIUM,
|
||||
'autotranslate',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generic auto translate base implementation.
|
||||
* This class provides generic parts of implementation for
|
||||
* tokenization, detokenization, call back register and unregister.
|
||||
* @abstract
|
||||
* @class
|
||||
*/
|
||||
export abstract class AutoTranslate {
|
||||
name: string;
|
||||
|
||||
languages: string[];
|
||||
|
||||
supportedLanguages: {
|
||||
[language: string]: ISupportedLanguage[];
|
||||
};
|
||||
|
||||
/**
|
||||
* Encapsulate the api key and provider settings.
|
||||
* @constructor
|
||||
*/
|
||||
constructor() {
|
||||
this.name = '';
|
||||
this.languages = [];
|
||||
this.supportedLanguages = {};
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts non-translatable parts of a message
|
||||
* @param {object} message
|
||||
* @return {object} message
|
||||
*/
|
||||
tokenize(message: IMessage): IMessage {
|
||||
if (!message.tokens || !Array.isArray(message.tokens)) {
|
||||
message.tokens = [];
|
||||
}
|
||||
message = this.tokenizeEmojis(message);
|
||||
message = this.tokenizeCode(message);
|
||||
message = this.tokenizeURLs(message);
|
||||
message = this.tokenizeMentions(message);
|
||||
return message;
|
||||
}
|
||||
|
||||
tokenizeEmojis(message: IMessage): IMessage {
|
||||
let count = message.tokens?.length || 0;
|
||||
message.msg = message.msg.replace(/:[+\w\d]+:/g, (match) => {
|
||||
const token = `<i class=notranslate>{${count++}}</i>`;
|
||||
message.tokens?.push({
|
||||
token,
|
||||
text: match,
|
||||
});
|
||||
return token;
|
||||
});
|
||||
|
||||
return message;
|
||||
}
|
||||
|
||||
tokenizeURLs(message: IMessage): IMessage {
|
||||
let count = message.tokens?.length || 0;
|
||||
|
||||
// Support  and [text](http://link)
|
||||
message.msg = message.msg.replace(markdownLinkRegex, (_match, pre, text, post) => {
|
||||
const pretoken = `<i class=notranslate>{${count++}}</i>`;
|
||||
message.tokens?.push({
|
||||
token: pretoken,
|
||||
text: pre,
|
||||
});
|
||||
|
||||
const posttoken = `<i class=notranslate>{${count++}}</i>`;
|
||||
message.tokens?.push({
|
||||
token: posttoken,
|
||||
text: post,
|
||||
});
|
||||
|
||||
return pretoken + text + posttoken;
|
||||
});
|
||||
|
||||
// Support <http://link|Text>
|
||||
message.msg = message.msg.replace(pipedLinkRegex, (_match, pre, text, post) => {
|
||||
const pretoken = `<i class=notranslate>{${count++}}</i>`;
|
||||
message.tokens?.push({
|
||||
token: pretoken,
|
||||
text: pre,
|
||||
});
|
||||
|
||||
const posttoken = `<i class=notranslate>{${count++}}</i>`;
|
||||
message.tokens?.push({
|
||||
token: posttoken,
|
||||
text: post,
|
||||
});
|
||||
|
||||
return pretoken + text + posttoken;
|
||||
});
|
||||
|
||||
return message;
|
||||
}
|
||||
|
||||
tokenizeCode(message: IMessage): IMessage {
|
||||
let count = message.tokens?.length || 0;
|
||||
message.html = message.msg;
|
||||
message = Markdown.parseMessageNotEscaped(message);
|
||||
|
||||
// Some parsers (e. g. Marked) wrap the complete message in a <p> - this is unnecessary and should be ignored with respect to translations
|
||||
message.msg = message.msg.replace(wrappedParagraphRegex, '');
|
||||
|
||||
for (const [tokenIndex, value] of message.tokens?.entries() ?? []) {
|
||||
const { token } = value;
|
||||
if (token.indexOf('notranslate') === -1) {
|
||||
const newToken = `<i class=notranslate>{${count++}}</i>`;
|
||||
message.msg = message.msg.replace(token, newToken);
|
||||
message.tokens ? (message.tokens[tokenIndex].token = newToken) : undefined;
|
||||
}
|
||||
}
|
||||
|
||||
return message;
|
||||
}
|
||||
|
||||
tokenizeMentions(message: IMessage): IMessage {
|
||||
let count = message.tokens?.length || 0;
|
||||
|
||||
if (message.mentions && message.mentions.length > 0) {
|
||||
message.mentions.forEach((mention) => {
|
||||
message.msg = message.msg.replace(new RegExp(`(@${mention.username})`, 'gm'), (match) => {
|
||||
const token = `<i class=notranslate>{${count++}}</i>`;
|
||||
message.tokens?.push({
|
||||
token,
|
||||
text: match,
|
||||
});
|
||||
return token;
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
if (message.channels && message.channels.length > 0) {
|
||||
message.channels.forEach((channel) => {
|
||||
message.msg = message.msg.replace(new RegExp(`(#${channel.name})`, 'gm'), (match) => {
|
||||
const token = `<i class=notranslate>{${count++}}</i>`;
|
||||
message.tokens?.push({
|
||||
token,
|
||||
text: match,
|
||||
});
|
||||
return token;
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
return message;
|
||||
}
|
||||
|
||||
deTokenize(message: IMessage): string {
|
||||
if (message.tokens && message.tokens?.length > 0) {
|
||||
for (const { token, text, noHtml } of message.tokens) {
|
||||
message.msg = message.msg.replace(token, () => noHtml || text);
|
||||
}
|
||||
}
|
||||
return message.msg;
|
||||
}
|
||||
|
||||
/**
|
||||
* Triggers the translation of the prepared (tokenized) message
|
||||
* and persists the result
|
||||
* @public
|
||||
* @param {object} message
|
||||
* @param {object} room
|
||||
* @param {object} targetLanguage
|
||||
* @returns {object} unmodified message object.
|
||||
*/
|
||||
async translateMessage(message: IMessage, { room, targetLanguage }: { room: IRoom; targetLanguage?: string }): Promise<IMessage | null> {
|
||||
let targetLanguages: string[];
|
||||
if (targetLanguage) {
|
||||
targetLanguages = [targetLanguage];
|
||||
} else {
|
||||
targetLanguages = (await Subscriptions.getAutoTranslateLanguagesByRoomAndNotUser(room._id, message.u?._id)).filter(isTruthy);
|
||||
}
|
||||
if (message.msg) {
|
||||
setImmediate(async () => {
|
||||
let targetMessage = Object.assign({}, message);
|
||||
targetMessage.html = escapeHTML(String(targetMessage.msg));
|
||||
targetMessage = this.tokenize(targetMessage);
|
||||
|
||||
const translations = await this._translateMessage(targetMessage, targetLanguages);
|
||||
if (!_.isEmpty(translations)) {
|
||||
await Messages.addTranslations(message._id, translations, TranslationProviderRegistry[Provider] || '');
|
||||
this.notifyTranslatedMessage(message._id);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (message.attachments && message.attachments.length > 0) {
|
||||
setImmediate(async () => {
|
||||
for (const [index, attachment] of message.attachments?.entries() ?? []) {
|
||||
if (attachment.description || attachment.text) {
|
||||
// Removes the initial link `[ ](quoterl)` from quote message before translation
|
||||
const translatedText = attachment?.text?.replace(/\[(.*?)\]\(.*?\)/g, '$1') || attachment?.text;
|
||||
const attachmentMessage = { ...attachment, text: translatedText };
|
||||
const translations = await this._translateAttachmentDescriptions(attachmentMessage, targetLanguages);
|
||||
|
||||
if (!_.isEmpty(translations)) {
|
||||
await Messages.addAttachmentTranslations(message._id, String(index), translations);
|
||||
this.notifyTranslatedMessage(message._id);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
return Messages.findOneById(message._id);
|
||||
}
|
||||
|
||||
private notifyTranslatedMessage(messageId: string): void {
|
||||
void notifyOnMessageChange({
|
||||
id: messageId,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns metadata information about the service provider which is used by
|
||||
* the generic implementation
|
||||
* @abstract
|
||||
* @protected
|
||||
* @returns { name, displayName, settings }
|
||||
};
|
||||
*/
|
||||
abstract _getProviderMetadata(): IProviderMetadata;
|
||||
|
||||
/**
|
||||
* Provides the possible languages _from_ which a message can be translated into a target language
|
||||
* @abstract
|
||||
* @protected
|
||||
* @param {string} target - the language into which shall be translated
|
||||
* @returns [{ language, name }]
|
||||
*/
|
||||
abstract getSupportedLanguages(target: string): Promise<ISupportedLanguage[]>;
|
||||
|
||||
/**
|
||||
* Performs the actual translation of a message,
|
||||
* usually by sending a REST API call to the service provider.
|
||||
* @abstract
|
||||
* @protected
|
||||
* @param {object} message
|
||||
* @param {object} targetLanguages
|
||||
* @return {object}
|
||||
*/
|
||||
abstract _translateMessage(message: IMessage, targetLanguages: string[]): Promise<ITranslationResult>;
|
||||
|
||||
/**
|
||||
* Performs the actual translation of an attachment (precisely its description),
|
||||
* usually by sending a REST API call to the service provider.
|
||||
* @abstract
|
||||
* @param {object} attachment
|
||||
* @param {object} targetLanguages
|
||||
* @returns {object} translated messages for each target language
|
||||
*/
|
||||
abstract _translateAttachmentDescriptions(attachment: MessageAttachment, targetLanguages: string[]): Promise<ITranslationResult>;
|
||||
}
|
||||
|
||||
Meteor.startup(() => {
|
||||
/** Register the active service provider on the 'AfterSaveMessage' callback.
|
||||
* So the registered provider will be invoked when a message is saved.
|
||||
* All the other inactive service provider must be deactivated.
|
||||
*/
|
||||
settings.watch<string>('AutoTranslate_ServiceProvider', (providerName) => {
|
||||
TranslationProviderRegistry.setCurrentProvider(providerName);
|
||||
});
|
||||
|
||||
// Get Auto Translate Active flag
|
||||
settings.watch<boolean>('AutoTranslate_Enabled', (value) => {
|
||||
TranslationProviderRegistry.setEnable(value);
|
||||
});
|
||||
});
|
||||
220
apps/meteor/server/lib/autotranslate/deeplTranslate.ts
Normal file
220
apps/meteor/server/lib/autotranslate/deeplTranslate.ts
Normal file
@@ -0,0 +1,220 @@
|
||||
/**
|
||||
* @author Vigneshwaran Odayappan <vickyokrm@gmail.com>
|
||||
*/
|
||||
|
||||
import type { IMessage, MessageAttachment, IProviderMetadata, ITranslationResult, ISupportedLanguage } from '@rocket.chat/core-typings';
|
||||
import { serverFetch as fetch } from '@rocket.chat/server-fetch';
|
||||
import _ from 'underscore';
|
||||
|
||||
import { TranslationProviderRegistry, AutoTranslate } from './autotranslate';
|
||||
import { settings } from '../../settings';
|
||||
import { i18n } from '../i18n';
|
||||
import { SystemLogger } from '../logger/system';
|
||||
|
||||
const proApiEndpoint = 'https://api.deepl.com/v2/translate';
|
||||
const freeApiEndpoint = 'https://api-free.deepl.com/v2/translate';
|
||||
|
||||
interface IDeepLTranslation {
|
||||
detected_source_language: string;
|
||||
text: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* DeepL translation service provider class representation.
|
||||
* Encapsulates the service provider settings and information.
|
||||
* Provides languages supported by the service provider.
|
||||
* Resolves API call to service provider to resolve the translation request.
|
||||
* @class
|
||||
* @augments AutoTranslate
|
||||
*/
|
||||
class DeeplAutoTranslate extends AutoTranslate {
|
||||
apiKey: string;
|
||||
|
||||
apiEndPointUrl: string;
|
||||
|
||||
private supportedLanguageEndpointUrl: string;
|
||||
|
||||
/**
|
||||
* setup api reference to deepl translate to be used as message translation provider.
|
||||
* @constructor
|
||||
*/
|
||||
constructor() {
|
||||
super();
|
||||
this.name = 'deepl-translate';
|
||||
this.apiEndPointUrl = proApiEndpoint;
|
||||
|
||||
// Get the service provide API key.
|
||||
settings.watch<string>('AutoTranslate_DeepLAPIKey', (value) => {
|
||||
this.apiKey = value;
|
||||
|
||||
// if the api key ends with `:fx` it is a free api key
|
||||
if (/:fx$/.test(value)) {
|
||||
this.apiEndPointUrl = freeApiEndpoint;
|
||||
this.supportedLanguageEndpointUrl = `https://api-free.deepl.com/v2/languages`;
|
||||
return;
|
||||
}
|
||||
this.apiEndPointUrl = proApiEndpoint;
|
||||
this.supportedLanguageEndpointUrl = `https://api.deepl.com/v2/languages`;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns metadata information about the service provide
|
||||
* @private implements super abstract method.
|
||||
* @return {object}
|
||||
*/
|
||||
_getProviderMetadata(): IProviderMetadata {
|
||||
return {
|
||||
name: this.name,
|
||||
displayName: i18n.t('AutoTranslate_DeepL'),
|
||||
settings: this._getSettings(),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns necessary settings information about the translation service provider.
|
||||
* @private implements super abstract method.
|
||||
* @return {object}
|
||||
*/
|
||||
_getSettings(): IProviderMetadata['settings'] {
|
||||
return {
|
||||
apiKey: this.apiKey,
|
||||
apiEndPointUrl: this.apiEndPointUrl,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns supported languages for translation by the active service provider.
|
||||
* Deepl does not provide an endpoint yet to retrieve the supported languages.
|
||||
* So each supported languages are explicitly maintained.
|
||||
* @private implements super abstract method.
|
||||
* @param {string} target
|
||||
* @returns {object} code : value pair
|
||||
*/
|
||||
async getSupportedLanguages(target: string): Promise<ISupportedLanguage[]> {
|
||||
if (!this.apiKey) {
|
||||
return [];
|
||||
}
|
||||
|
||||
if (this.supportedLanguages[target]) {
|
||||
return this.supportedLanguages[target];
|
||||
}
|
||||
let result: (ISupportedLanguage & { supports_formality?: boolean })[] = [];
|
||||
|
||||
// SECURITY: the URL is a default hardcoded value or an envvar/setting set by an admin. It's safe to disable this check.
|
||||
const request = await fetch(this.supportedLanguageEndpointUrl, {
|
||||
ignoreSsrfValidation: true,
|
||||
params: { type: 'target' },
|
||||
headers: {
|
||||
Authorization: `DeepL-Auth-Key ${this.apiKey}`,
|
||||
},
|
||||
});
|
||||
if (!request.ok) {
|
||||
throw new Error('Failed to fetch supported languages');
|
||||
}
|
||||
|
||||
result = (await request.json()) as typeof result;
|
||||
|
||||
if (this.supportedLanguages[target]) {
|
||||
return this.supportedLanguages[target];
|
||||
}
|
||||
this.supportedLanguages[target || 'en'] = result.map(({ language, ...other }) => ({
|
||||
...other,
|
||||
language: new Intl.Locale(language).toString(),
|
||||
}));
|
||||
return this.supportedLanguages[target || 'en'];
|
||||
}
|
||||
|
||||
/**
|
||||
* Send Request REST API call to the service provider.
|
||||
* Returns translated message for each target language in target languages.
|
||||
* @private
|
||||
* @param {object} message
|
||||
* @param {object} targetLanguages
|
||||
* @returns {object} translations: Translated messages for each language
|
||||
*/
|
||||
async _translateMessage(message: IMessage, targetLanguages: string[]): Promise<ITranslationResult> {
|
||||
const translations: { [k: string]: string } = {};
|
||||
const msgs = message.msg.split('\n');
|
||||
const supportedLanguages = await this.getSupportedLanguages('en');
|
||||
for (let language of targetLanguages) {
|
||||
if (language.indexOf('-') !== -1 && !_.findWhere(supportedLanguages, { language })) {
|
||||
language = language.substr(0, 2);
|
||||
}
|
||||
try {
|
||||
// SECURITY: the URL is a default hardcoded value or an envvar/setting set by an admin. It's safe to disable this check.
|
||||
const result = await fetch(this.apiEndPointUrl, {
|
||||
ignoreSsrfValidation: true,
|
||||
params: { target_lang: language, text: msgs },
|
||||
headers: {
|
||||
Authorization: `DeepL-Auth-Key ${this.apiKey}`,
|
||||
},
|
||||
method: 'POST',
|
||||
});
|
||||
|
||||
if (!result.ok) {
|
||||
throw new Error(result.statusText);
|
||||
}
|
||||
|
||||
const body = await result.json();
|
||||
|
||||
if (result.status === 200 && body.translations && Array.isArray(body.translations) && body.translations.length > 0) {
|
||||
// store translation only when the source and target language are different.
|
||||
// multiple lines might contain different languages => Mix the text between source and detected target if neccessary
|
||||
const translatedText = body.translations
|
||||
.map((translation: IDeepLTranslation, index: number) =>
|
||||
translation.detected_source_language !== language ? translation.text : msgs[index],
|
||||
)
|
||||
.join('\n');
|
||||
translations[language] = this.deTokenize(Object.assign({}, message, { msg: translatedText }));
|
||||
}
|
||||
} catch (err) {
|
||||
SystemLogger.error({ msg: 'Error translating message', err });
|
||||
}
|
||||
}
|
||||
return translations;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns translated message attachment description in target languages.
|
||||
* @private
|
||||
* @param {object} attachment
|
||||
* @param {object} targetLanguages
|
||||
* @returns {object} translated messages for each target language
|
||||
*/
|
||||
async _translateAttachmentDescriptions(attachment: MessageAttachment, targetLanguages: string[]): Promise<ITranslationResult> {
|
||||
const translations: { [k: string]: string } = {};
|
||||
const supportedLanguages = await this.getSupportedLanguages('en');
|
||||
for (let language of targetLanguages) {
|
||||
if (language.indexOf('-') !== -1 && !_.findWhere(supportedLanguages, { language })) {
|
||||
language = language.substr(0, 2);
|
||||
}
|
||||
try {
|
||||
// SECURITY: the URL is a default hardcoded value or an envvar/setting set by an admin. It's safe to disable this check.
|
||||
const result = await fetch(this.apiEndPointUrl, {
|
||||
ignoreSsrfValidation: true,
|
||||
params: {
|
||||
auth_key: this.apiKey,
|
||||
target_lang: language,
|
||||
text: attachment.description || attachment.text || '',
|
||||
},
|
||||
});
|
||||
if (!result.ok) {
|
||||
throw new Error(result.statusText);
|
||||
}
|
||||
const body = await result.json();
|
||||
if (result.status === 200 && body.translations && Array.isArray(body.translations) && body.translations.length > 0) {
|
||||
if (body.translations.map((translation: IDeepLTranslation) => translation.detected_source_language).join() !== language) {
|
||||
translations[language] = body.translations.map((translation: IDeepLTranslation) => translation.text);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
SystemLogger.error({ msg: 'Error translating message attachment', err });
|
||||
}
|
||||
}
|
||||
return translations;
|
||||
}
|
||||
}
|
||||
|
||||
// Register DeepL translation provider to the registry.
|
||||
TranslationProviderRegistry.registerProvider(new DeeplAutoTranslate());
|
||||
@@ -0,0 +1,19 @@
|
||||
import { Meteor } from 'meteor/meteor';
|
||||
|
||||
import { settings } from '../../../settings';
|
||||
import { hasPermissionAsync } from '../../authorization/hasPermission';
|
||||
import { TranslationProviderRegistry } from '../index';
|
||||
|
||||
export const getSupportedLanguages = async (userId: string, targetLanguage: string) => {
|
||||
if (!settings.get('AutoTranslate_Enabled')) {
|
||||
throw new Meteor.Error('error-autotranslate-disabled', 'Auto-Translate is disabled');
|
||||
}
|
||||
|
||||
if (!(await hasPermissionAsync(userId, 'auto-translate'))) {
|
||||
throw new Meteor.Error('error-action-not-allowed', 'Auto-Translate is not allowed', {
|
||||
method: 'autoTranslate.getSupportedLanguages',
|
||||
});
|
||||
}
|
||||
|
||||
return TranslationProviderRegistry.getSupportedLanguages(targetLanguage);
|
||||
};
|
||||
@@ -0,0 +1,78 @@
|
||||
import { Subscriptions, Rooms } from '@rocket.chat/models';
|
||||
import { check } from 'meteor/check';
|
||||
import { Meteor } from 'meteor/meteor';
|
||||
|
||||
import { hasPermissionAsync } from '../../authorization/hasPermission';
|
||||
import { notifyOnSubscriptionChangedById } from '../../notifyListener';
|
||||
|
||||
export const saveAutoTranslateSettings = async (
|
||||
userId: string,
|
||||
rid: string,
|
||||
field: string,
|
||||
value: string,
|
||||
options: { defaultLanguage: string },
|
||||
) => {
|
||||
if (!(await hasPermissionAsync(userId, 'auto-translate'))) {
|
||||
throw new Meteor.Error('error-action-not-allowed', 'Auto-Translate is not allowed', {
|
||||
method: 'autoTranslate.saveSettings',
|
||||
});
|
||||
}
|
||||
|
||||
check(rid, String);
|
||||
check(field, String);
|
||||
check(value, String);
|
||||
|
||||
if (['autoTranslate', 'autoTranslateLanguage'].indexOf(field) === -1) {
|
||||
throw new Meteor.Error('error-invalid-settings', 'Invalid settings field', {
|
||||
method: 'saveAutoTranslateSettings',
|
||||
});
|
||||
}
|
||||
|
||||
const subscription = await Subscriptions.findOneByRoomIdAndUserId(rid, userId);
|
||||
if (!subscription) {
|
||||
throw new Meteor.Error('error-invalid-subscription', 'Invalid subscription', {
|
||||
method: 'saveAutoTranslateSettings',
|
||||
});
|
||||
}
|
||||
|
||||
let shouldNotifySubscriptionChanged = false;
|
||||
|
||||
switch (field) {
|
||||
case 'autoTranslate':
|
||||
const room = await Rooms.findE2ERoomById(rid, { projection: { _id: 1 } });
|
||||
if (room && value === '1') {
|
||||
throw new Meteor.Error('error-e2e-enabled', 'Enabling auto-translation in E2E encrypted rooms is not allowed', {
|
||||
method: 'saveAutoTranslateSettings',
|
||||
});
|
||||
}
|
||||
|
||||
const updateAutoTranslateResponse = await Subscriptions.updateAutoTranslateById(subscription._id, value === '1');
|
||||
if (updateAutoTranslateResponse.modifiedCount) {
|
||||
shouldNotifySubscriptionChanged = true;
|
||||
}
|
||||
|
||||
if (!subscription.autoTranslateLanguage && options.defaultLanguage) {
|
||||
const updateAutoTranslateLanguageResponse = await Subscriptions.updateAutoTranslateLanguageById(
|
||||
subscription._id,
|
||||
options.defaultLanguage,
|
||||
);
|
||||
if (updateAutoTranslateLanguageResponse.modifiedCount) {
|
||||
shouldNotifySubscriptionChanged = true;
|
||||
}
|
||||
}
|
||||
|
||||
break;
|
||||
case 'autoTranslateLanguage':
|
||||
const updateAutoTranslateLanguage = await Subscriptions.updateAutoTranslateLanguageById(subscription._id, value);
|
||||
if (updateAutoTranslateLanguage.modifiedCount) {
|
||||
shouldNotifySubscriptionChanged = true;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
if (shouldNotifySubscriptionChanged) {
|
||||
void notifyOnSubscriptionChangedById(subscription._id);
|
||||
}
|
||||
|
||||
return true;
|
||||
};
|
||||
@@ -0,0 +1,26 @@
|
||||
import type { IMessage } from '@rocket.chat/core-typings';
|
||||
import { Rooms } from '@rocket.chat/models';
|
||||
|
||||
import { TranslationProviderRegistry } from '../index';
|
||||
|
||||
export const translateMessage = async (targetLanguage?: string, message?: IMessage) => {
|
||||
if (!TranslationProviderRegistry.enabled) {
|
||||
return;
|
||||
}
|
||||
if (!message?.rid) {
|
||||
return;
|
||||
}
|
||||
|
||||
const room = await Rooms.findOneById(message?.rid);
|
||||
let translatedMessage;
|
||||
|
||||
if (message && room) {
|
||||
translatedMessage = await TranslationProviderRegistry.translateMessage(message, room, targetLanguage);
|
||||
}
|
||||
|
||||
if (!translatedMessage) {
|
||||
return;
|
||||
}
|
||||
|
||||
return translatedMessage;
|
||||
};
|
||||
223
apps/meteor/server/lib/autotranslate/googleTranslate.ts
Normal file
223
apps/meteor/server/lib/autotranslate/googleTranslate.ts
Normal file
@@ -0,0 +1,223 @@
|
||||
/**
|
||||
* @author Vigneshwaran Odayappan <vickyokrm@gmail.com>
|
||||
*/
|
||||
|
||||
import type { IMessage, IProviderMetadata, ISupportedLanguage, ITranslationResult, MessageAttachment } from '@rocket.chat/core-typings';
|
||||
import { serverFetch as fetch } from '@rocket.chat/server-fetch';
|
||||
import _ from 'underscore';
|
||||
|
||||
import { AutoTranslate, TranslationProviderRegistry } from './autotranslate';
|
||||
import { settings } from '../../settings';
|
||||
import { i18n } from '../i18n';
|
||||
import { SystemLogger } from '../logger/system';
|
||||
|
||||
interface IGoogleTranslation {
|
||||
translatedText: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents google translate class
|
||||
* @class
|
||||
* @augments AutoTranslate
|
||||
*/
|
||||
|
||||
class GoogleAutoTranslate extends AutoTranslate {
|
||||
apiKey: string;
|
||||
|
||||
apiEndPointUrl: string;
|
||||
|
||||
/**
|
||||
* setup api reference to Google translate to be used as message translation provider.
|
||||
* @constructor
|
||||
*/
|
||||
constructor() {
|
||||
super();
|
||||
this.name = 'google-translate';
|
||||
this.apiEndPointUrl = 'https://translation.googleapis.com/language/translate/v2';
|
||||
// Get the service provide API key.
|
||||
settings.watch<string>('AutoTranslate_GoogleAPIKey', (value) => {
|
||||
this.apiKey = value;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns metadata information about the service provider
|
||||
* @private implements super abstract method.
|
||||
* @returns {object}
|
||||
*/
|
||||
_getProviderMetadata(): IProviderMetadata {
|
||||
return {
|
||||
name: this.name,
|
||||
displayName: i18n.t('AutoTranslate_Google'),
|
||||
settings: this._getSettings(),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns necessary settings information about the translation service provider.
|
||||
* @private implements super abstract method.
|
||||
* @returns {object}
|
||||
*/
|
||||
_getSettings(): IProviderMetadata['settings'] {
|
||||
return {
|
||||
apiKey: this.apiKey,
|
||||
apiEndPointUrl: this.apiEndPointUrl,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns supported languages for translation by the active service provider.
|
||||
* Google Translate api provides the list of supported languages.
|
||||
* @private implements super abstract method.
|
||||
* @param {string} target : user language setting or 'en'
|
||||
* @returns {object} code : value pair
|
||||
*/
|
||||
async getSupportedLanguages(target: string): Promise<ISupportedLanguage[]> {
|
||||
if (!this.apiKey) {
|
||||
return [];
|
||||
}
|
||||
|
||||
if (this.supportedLanguages[target]) {
|
||||
return this.supportedLanguages[target];
|
||||
}
|
||||
|
||||
let result: { data?: { languages: ISupportedLanguage[] } } = {};
|
||||
const params = {
|
||||
key: this.apiKey,
|
||||
...(target && { target }),
|
||||
};
|
||||
|
||||
try {
|
||||
// SECURITY: the URL is a default hardcoded value or an envvar/setting set by an admin. It's safe to disable this check.
|
||||
const request = await fetch(`https://translation.googleapis.com/language/translate/v2/languages`, {
|
||||
ignoreSsrfValidation: true,
|
||||
params,
|
||||
});
|
||||
if (!request.ok && request.status === 400 && request.statusText === 'INVALID_ARGUMENT') {
|
||||
throw new Error('Failed to fetch supported languages');
|
||||
}
|
||||
|
||||
result = (await request.json()) as typeof result;
|
||||
} catch (e: any) {
|
||||
// Fallback: Get the English names of the target languages
|
||||
if (e.message === 'Failed to fetch supported languages') {
|
||||
params.target = 'en';
|
||||
target = 'en';
|
||||
if (!this.supportedLanguages[target]) {
|
||||
// SECURITY: the URL is a default hardcoded value or an envvar/setting set by an admin. It's safe to disable this check.
|
||||
const request = await fetch(`https://translation.googleapis.com/language/translate/v2/languages`, {
|
||||
ignoreSsrfValidation: true,
|
||||
params,
|
||||
});
|
||||
result = (await request.json()) as typeof result;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (this.supportedLanguages[target]) {
|
||||
return this.supportedLanguages[target];
|
||||
}
|
||||
this.supportedLanguages[target || 'en'] = result?.data?.languages || [];
|
||||
return this.supportedLanguages[target || 'en'];
|
||||
}
|
||||
|
||||
/**
|
||||
* Send Request REST API call to the service provider.
|
||||
* Returns translated message for each target language in target languages.
|
||||
* @private
|
||||
* @param {object} message
|
||||
* @param {object} targetLanguages
|
||||
* @returns {object} translations: Translated messages for each language
|
||||
*/
|
||||
async _translateMessage(message: IMessage, targetLanguages: string[]): Promise<ITranslationResult> {
|
||||
const translations: { [k: string]: string } = {};
|
||||
|
||||
const supportedLanguages = await this.getSupportedLanguages('en');
|
||||
|
||||
for (let language of targetLanguages) {
|
||||
if (language.indexOf('-') !== -1 && !_.findWhere(supportedLanguages, { language })) {
|
||||
language = language.slice(0, 2);
|
||||
}
|
||||
|
||||
try {
|
||||
// SECURITY: the URL is a default hardcoded value or an envvar/setting set by an admin. It's safe to disable this check.
|
||||
const result = await fetch(this.apiEndPointUrl, {
|
||||
ignoreSsrfValidation: true,
|
||||
params: {
|
||||
key: this.apiKey,
|
||||
target: language,
|
||||
format: 'text',
|
||||
q: message.msg.split('\n'),
|
||||
},
|
||||
});
|
||||
if (!result.ok) {
|
||||
throw new Error(result.statusText);
|
||||
}
|
||||
const body = await result.json();
|
||||
|
||||
if (
|
||||
result.status === 200 &&
|
||||
body.data?.translations &&
|
||||
Array.isArray(body.data.translations) &&
|
||||
body.data.translations.length > 0
|
||||
) {
|
||||
const txt = body.data.translations.map((translation: IGoogleTranslation) => translation.translatedText).join('\n');
|
||||
translations[language] = this.deTokenize(Object.assign({}, message, { msg: txt }));
|
||||
}
|
||||
} catch (err) {
|
||||
SystemLogger.error({ msg: 'Error translating message', err });
|
||||
}
|
||||
}
|
||||
return translations;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns translated message attachment description in target languages.
|
||||
* @private
|
||||
* @param {object} attachment
|
||||
* @param {object} targetLanguages
|
||||
* @returns {object} translated attachment descriptions for each target language
|
||||
*/
|
||||
async _translateAttachmentDescriptions(attachment: MessageAttachment, targetLanguages: string[]): Promise<ITranslationResult> {
|
||||
const translations: { [k: string]: string } = {};
|
||||
const supportedLanguages = await this.getSupportedLanguages('en');
|
||||
|
||||
for (let language of targetLanguages) {
|
||||
if (language.indexOf('-') !== -1 && !_.findWhere(supportedLanguages, { language })) {
|
||||
language = language.slice(0, 2);
|
||||
}
|
||||
|
||||
try {
|
||||
// SECURITY: the URL is a default hardcoded value or an envvar/setting set by an admin. It's safe to disable this check.
|
||||
const result = await fetch(this.apiEndPointUrl, {
|
||||
ignoreSsrfValidation: true,
|
||||
params: {
|
||||
key: this.apiKey,
|
||||
target: language,
|
||||
format: 'text',
|
||||
q: attachment.description || attachment.text || '',
|
||||
},
|
||||
});
|
||||
if (!result.ok) {
|
||||
throw new Error(result.statusText);
|
||||
}
|
||||
const body = await result.json();
|
||||
|
||||
if (
|
||||
result.status === 200 &&
|
||||
body.data?.translations &&
|
||||
Array.isArray(body.data.translations) &&
|
||||
body.data.translations.length > 0
|
||||
) {
|
||||
translations[language] = body.data.translations.map((translation: IGoogleTranslation) => translation.translatedText).join('\n');
|
||||
}
|
||||
} catch (err) {
|
||||
SystemLogger.error({ msg: 'Error translating message', err });
|
||||
}
|
||||
}
|
||||
return translations;
|
||||
}
|
||||
}
|
||||
|
||||
// Register Google translation provider.
|
||||
TranslationProviderRegistry.registerProvider(new GoogleAutoTranslate());
|
||||
13
apps/meteor/server/lib/autotranslate/index.ts
Normal file
13
apps/meteor/server/lib/autotranslate/index.ts
Normal file
@@ -0,0 +1,13 @@
|
||||
/**
|
||||
* This file contains the exported members of the package shall be re-used.
|
||||
* @module AutoTranslate, TranslationProviderRegistry
|
||||
*/
|
||||
|
||||
import { TranslationProviderRegistry } from './autotranslate';
|
||||
import './permissions';
|
||||
import './googleTranslate';
|
||||
import './deeplTranslate';
|
||||
import './libreTranslate';
|
||||
import './msTranslate';
|
||||
|
||||
export { TranslationProviderRegistry };
|
||||
179
apps/meteor/server/lib/autotranslate/libreTranslate.ts
Normal file
179
apps/meteor/server/lib/autotranslate/libreTranslate.ts
Normal file
@@ -0,0 +1,179 @@
|
||||
import type { IMessage, MessageAttachment, IProviderMetadata, ITranslationResult, ISupportedLanguage } from '@rocket.chat/core-typings';
|
||||
import { serverFetch as fetch } from '@rocket.chat/server-fetch';
|
||||
|
||||
import { TranslationProviderRegistry, AutoTranslate } from './autotranslate';
|
||||
import { libreLogger } from './logger';
|
||||
import { settings } from '../../settings';
|
||||
import { i18n } from '../i18n';
|
||||
|
||||
interface ILibreTranslateLanguage {
|
||||
code: string;
|
||||
name: string;
|
||||
targets?: string[];
|
||||
}
|
||||
|
||||
const REQUEST_TIMEOUT = 10000;
|
||||
|
||||
const toLanguageTag = (code: string): string => {
|
||||
try {
|
||||
return new Intl.Locale(code).toString();
|
||||
} catch (err) {
|
||||
libreLogger.error({ msg: 'Invalid language code returned by LibreTranslate', code, err });
|
||||
return code;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* LibreTranslate translation service provider class representation.
|
||||
* @class
|
||||
* @augments AutoTranslate
|
||||
*/
|
||||
class LibreTranslateAutoTranslate extends AutoTranslate {
|
||||
apiKey: string;
|
||||
|
||||
apiEndPointUrl: string;
|
||||
|
||||
constructor() {
|
||||
super();
|
||||
this.name = 'libre-translate';
|
||||
this.apiEndPointUrl = '';
|
||||
this.apiKey = '';
|
||||
|
||||
settings.watch<string>('AutoTranslate_LibreTranslateAPIURL', (value) => {
|
||||
this.apiEndPointUrl = (value || '').replace(/\/$/, '');
|
||||
});
|
||||
|
||||
settings.watch<string>('AutoTranslate_LibreTranslateAPIKey', (value) => {
|
||||
this.apiKey = value;
|
||||
});
|
||||
}
|
||||
|
||||
_getProviderMetadata(): IProviderMetadata {
|
||||
return {
|
||||
name: this.name,
|
||||
displayName: i18n.t('AutoTranslate_LibreTranslate'),
|
||||
settings: this._getSettings(),
|
||||
};
|
||||
}
|
||||
|
||||
_getSettings(): IProviderMetadata['settings'] {
|
||||
return {
|
||||
apiKey: this.apiKey,
|
||||
apiEndPointUrl: this.apiEndPointUrl,
|
||||
};
|
||||
}
|
||||
|
||||
async getSupportedLanguages(target: string): Promise<ISupportedLanguage[]> {
|
||||
if (!this.apiEndPointUrl) {
|
||||
return [];
|
||||
}
|
||||
|
||||
if (this.supportedLanguages[target]) {
|
||||
return this.supportedLanguages[target];
|
||||
}
|
||||
|
||||
try {
|
||||
// SECURITY: the URL comes from a setting only an admin can change, so disabling SSRF validation is safe here.
|
||||
const request = await fetch(`${this.apiEndPointUrl}/languages`, {
|
||||
ignoreSsrfValidation: true,
|
||||
timeout: REQUEST_TIMEOUT,
|
||||
});
|
||||
if (!request.ok) {
|
||||
throw new Error('Failed to fetch supported languages');
|
||||
}
|
||||
|
||||
const result = (await request.json()) as ILibreTranslateLanguage[];
|
||||
|
||||
this.supportedLanguages[target || 'en'] = result.map(({ code, name }) => ({
|
||||
language: toLanguageTag(code),
|
||||
name,
|
||||
}));
|
||||
return this.supportedLanguages[target || 'en'];
|
||||
} catch (err) {
|
||||
libreLogger.error({ msg: 'Error fetching supported languages', err });
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
private resolveTargetLanguage(language: string, supportedLanguages: ISupportedLanguage[]): string {
|
||||
if (language.includes('-') && !supportedLanguages.find((supported) => supported.language === language)) {
|
||||
return language.slice(0, 2);
|
||||
}
|
||||
return language;
|
||||
}
|
||||
|
||||
// `q` is sent as an array of lines; LibreTranslate returns `translatedText` as an array (one entry per line).
|
||||
private async _query(lines: string[], targetLanguage: string): Promise<string | null> {
|
||||
const body: Record<string, unknown> = {
|
||||
q: lines,
|
||||
source: 'auto',
|
||||
target: targetLanguage,
|
||||
format: 'text',
|
||||
...(this.apiKey && { api_key: this.apiKey }),
|
||||
};
|
||||
|
||||
// SECURITY: the URL comes from a setting only an admin can change, so disabling SSRF validation is safe here.
|
||||
const result = await fetch(`${this.apiEndPointUrl}/translate`, {
|
||||
method: 'POST',
|
||||
ignoreSsrfValidation: true,
|
||||
timeout: REQUEST_TIMEOUT,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body,
|
||||
});
|
||||
|
||||
if (!result.ok) {
|
||||
throw new Error(result.statusText);
|
||||
}
|
||||
|
||||
const json = (await result.json()) as { translatedText?: string | string[] };
|
||||
if (json.translatedText === undefined) {
|
||||
return null;
|
||||
}
|
||||
return Array.isArray(json.translatedText) ? json.translatedText.join('\n') : json.translatedText;
|
||||
}
|
||||
|
||||
async _translateMessage(message: IMessage, targetLanguages: string[]): Promise<ITranslationResult> {
|
||||
const translations: { [k: string]: string } = {};
|
||||
if (!this.apiEndPointUrl) {
|
||||
return translations;
|
||||
}
|
||||
|
||||
const msgs = message.msg.split('\n');
|
||||
const supportedLanguages = await this.getSupportedLanguages('en');
|
||||
for (const targetLanguage of targetLanguages) {
|
||||
const language = this.resolveTargetLanguage(targetLanguage, supportedLanguages);
|
||||
try {
|
||||
const translatedText = await this._query(msgs, language);
|
||||
if (translatedText !== null) {
|
||||
translations[language] = this.deTokenize(Object.assign({}, message, { msg: translatedText }));
|
||||
}
|
||||
} catch (err) {
|
||||
libreLogger.error({ msg: 'Error translating message', err });
|
||||
}
|
||||
}
|
||||
return translations;
|
||||
}
|
||||
|
||||
async _translateAttachmentDescriptions(attachment: MessageAttachment, targetLanguages: string[]): Promise<ITranslationResult> {
|
||||
const translations: { [k: string]: string } = {};
|
||||
if (!this.apiEndPointUrl) {
|
||||
return translations;
|
||||
}
|
||||
|
||||
const supportedLanguages = await this.getSupportedLanguages('en');
|
||||
for (const targetLanguage of targetLanguages) {
|
||||
const language = this.resolveTargetLanguage(targetLanguage, supportedLanguages);
|
||||
try {
|
||||
const translatedText = await this._query([attachment.description || attachment.text || ''], language);
|
||||
if (translatedText !== null) {
|
||||
translations[language] = translatedText;
|
||||
}
|
||||
} catch (err) {
|
||||
libreLogger.error({ msg: 'Error translating message attachment', err });
|
||||
}
|
||||
}
|
||||
return translations;
|
||||
}
|
||||
}
|
||||
|
||||
TranslationProviderRegistry.registerProvider(new LibreTranslateAutoTranslate());
|
||||
7
apps/meteor/server/lib/autotranslate/logger.ts
Normal file
7
apps/meteor/server/lib/autotranslate/logger.ts
Normal file
@@ -0,0 +1,7 @@
|
||||
import { Logger } from '@rocket.chat/logger';
|
||||
|
||||
const logger = new Logger('AutoTranslate');
|
||||
|
||||
export const msLogger = logger.section('Microsoft');
|
||||
|
||||
export const libreLogger = logger.section('LibreTranslate');
|
||||
208
apps/meteor/server/lib/autotranslate/msTranslate.ts
Normal file
208
apps/meteor/server/lib/autotranslate/msTranslate.ts
Normal file
@@ -0,0 +1,208 @@
|
||||
/**
|
||||
* @author Vigneshwaran Odayappan <vickyokrm@gmail.com>
|
||||
*/
|
||||
|
||||
import type { IMessage, IProviderMetadata, ISupportedLanguage, ITranslationResult, MessageAttachment } from '@rocket.chat/core-typings';
|
||||
import { serverFetch as fetch } from '@rocket.chat/server-fetch';
|
||||
import _ from 'underscore';
|
||||
|
||||
import { TranslationProviderRegistry, AutoTranslate } from './autotranslate';
|
||||
import { msLogger } from './logger';
|
||||
import { settings } from '../../settings';
|
||||
import { i18n } from '../i18n';
|
||||
|
||||
/**
|
||||
* Microsoft translation service provider class representation.
|
||||
* Encapsulates the service provider settings and information.
|
||||
* Provides languages supported by the service provider.
|
||||
* Resolves API call to service provider to resolve the translation request.
|
||||
* @class
|
||||
* @augments AutoTranslate
|
||||
*/
|
||||
class MsAutoTranslate extends AutoTranslate {
|
||||
apiKey: string;
|
||||
|
||||
apiEndPointUrl: string;
|
||||
|
||||
apiDetectText: string;
|
||||
|
||||
apiGetLanguages: string;
|
||||
|
||||
breakSentence: string;
|
||||
|
||||
/**
|
||||
* setup api reference to Microsoft translate to be used as message translation provider.
|
||||
* @constructor
|
||||
*/
|
||||
constructor() {
|
||||
super();
|
||||
this.name = 'microsoft-translate';
|
||||
this.apiEndPointUrl = 'https://api.cognitive.microsofttranslator.com/translate?api-version=3.0';
|
||||
this.apiDetectText = 'https://api.cognitive.microsofttranslator.com/detect?api-version=3.0';
|
||||
this.apiGetLanguages = 'https://api.cognitive.microsofttranslator.com/languages?api-version=3.0';
|
||||
this.breakSentence = 'https://api.cognitive.microsofttranslator.com/breaksentence?api-version=3.0';
|
||||
// Get the service provide API key.
|
||||
settings.watch<string>('AutoTranslate_MicrosoftAPIKey', (value) => {
|
||||
this.apiKey = value;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns metadata information about the service provide
|
||||
* @private implements super abstract method.
|
||||
* @return {object}
|
||||
*/
|
||||
_getProviderMetadata(): IProviderMetadata {
|
||||
return {
|
||||
name: this.name,
|
||||
displayName: i18n.t('AutoTranslate_Microsoft'),
|
||||
settings: this._getSettings(),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns necessary settings information about the translation service provider.
|
||||
* @private implements super abstract method.
|
||||
* @return {object}
|
||||
*/
|
||||
_getSettings(): IProviderMetadata['settings'] {
|
||||
return {
|
||||
apiKey: this.apiKey,
|
||||
apiEndPointUrl: this.apiEndPointUrl,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns supported languages for translation by the active service provider.
|
||||
* Microsoft does not provide an endpoint yet to retrieve the supported languages.
|
||||
* So each supported languages are explicitly maintained.
|
||||
* @private implements super abstract method.
|
||||
* @param {string} target
|
||||
* @returns {object} code : value pair
|
||||
*/
|
||||
async getSupportedLanguages(target: string): Promise<ISupportedLanguage[]> {
|
||||
if (!this.apiKey) {
|
||||
return [];
|
||||
}
|
||||
if (this.supportedLanguages[target]) {
|
||||
return this.supportedLanguages[target];
|
||||
}
|
||||
// SECURITY: the URL is a default hardcoded value or an envvar/setting set by an admin. It's safe to disable this check.
|
||||
const request = await fetch(this.apiGetLanguages, {
|
||||
ignoreSsrfValidation: true,
|
||||
});
|
||||
if (!request.ok) {
|
||||
throw new Error(request.statusText);
|
||||
}
|
||||
const languages = await request.json();
|
||||
this.supportedLanguages[target] = Object.keys(languages.translation).map((language) => ({
|
||||
language,
|
||||
name: languages.translation[language].name,
|
||||
}));
|
||||
return this.supportedLanguages[target || 'en'];
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-use method for REST API consumption of MS translate.
|
||||
* @private
|
||||
* @param {object} message
|
||||
* @param {object} targetLanguages
|
||||
* @throws Communication Errors
|
||||
* @returns {object} translations: Translated messages for each language
|
||||
*/
|
||||
async _translate(
|
||||
data: {
|
||||
Text: string;
|
||||
}[],
|
||||
targetLanguages: string[],
|
||||
): Promise<ITranslationResult> {
|
||||
let translations: { [k: string]: string } = {};
|
||||
const supportedLanguages = await this.getSupportedLanguages('en');
|
||||
targetLanguages = targetLanguages.map((language) => {
|
||||
if (language.indexOf('-') !== -1 && !_.findWhere(supportedLanguages, { language })) {
|
||||
language = language.substr(0, 2);
|
||||
}
|
||||
return language;
|
||||
});
|
||||
// SECURITY: the URL is a default hardcoded value or an envvar/setting set by an admin. It's safe to disable this check.
|
||||
const request = await fetch(this.apiEndPointUrl, {
|
||||
ignoreSsrfValidation: true,
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Ocp-Apim-Subscription-Key': this.apiKey,
|
||||
'Content-Type': 'application/json; charset=UTF-8',
|
||||
},
|
||||
body: data,
|
||||
params: {
|
||||
to: targetLanguages,
|
||||
},
|
||||
});
|
||||
if (!request.ok) {
|
||||
throw new Error(request.statusText);
|
||||
}
|
||||
const result = await request.json();
|
||||
|
||||
if (request.status === 200 && result.length > 0) {
|
||||
// store translation only when the source and target language are different.
|
||||
translations = Object.assign(
|
||||
{},
|
||||
...targetLanguages.map((language) => ({
|
||||
[language]: result
|
||||
.map(
|
||||
(line: { translations: { to: string; text: string }[] }) =>
|
||||
line.translations.find((translation) => translation.to === language)?.text,
|
||||
)
|
||||
.join('\n'),
|
||||
})),
|
||||
);
|
||||
}
|
||||
|
||||
return translations;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns translated message for each target language.
|
||||
* @private
|
||||
* @param {object} message
|
||||
* @param {object} targetLanguages
|
||||
* @returns {object} translations: Translated messages for each language
|
||||
*/
|
||||
async _translateMessage(message: IMessage, targetLanguages: string[]): Promise<ITranslationResult> {
|
||||
// There are multi-sentence-messages where multiple sentences come from different languages
|
||||
// This is a problem for translation services since the language detection fails.
|
||||
// Thus, we'll split the message in sentences, get them translated, and join them again after translation
|
||||
const msgs = message.msg.split('\n').map((msg) => ({ Text: msg }));
|
||||
try {
|
||||
return this._translate(msgs, targetLanguages);
|
||||
} catch (e) {
|
||||
msLogger.error({ err: e, msg: 'Error translating message' });
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns translated message attachment description in target languages.
|
||||
* @private
|
||||
* @param {object} attachment
|
||||
* @param {object} targetLanguages
|
||||
* @returns {object} translated messages for each target language
|
||||
*/
|
||||
async _translateAttachmentDescriptions(attachment: MessageAttachment, targetLanguages: string[]): Promise<ITranslationResult> {
|
||||
try {
|
||||
return this._translate(
|
||||
[
|
||||
{
|
||||
Text: attachment.description || attachment.text || '',
|
||||
},
|
||||
],
|
||||
targetLanguages,
|
||||
);
|
||||
} catch (e) {
|
||||
msLogger.error({ err: e, msg: 'Error translating message attachment' });
|
||||
}
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
// Register Microsoft translation provider to the registry.
|
||||
TranslationProviderRegistry.registerProvider(new MsAutoTranslate());
|
||||
8
apps/meteor/server/lib/autotranslate/permissions.ts
Normal file
8
apps/meteor/server/lib/autotranslate/permissions.ts
Normal file
@@ -0,0 +1,8 @@
|
||||
import { Permissions } from '@rocket.chat/models';
|
||||
import { Meteor } from 'meteor/meteor';
|
||||
|
||||
Meteor.startup(async () => {
|
||||
if (!(await Permissions.findOne({ _id: 'auto-translate' }))) {
|
||||
await Permissions.create('auto-translate', ['admin']);
|
||||
}
|
||||
});
|
||||
Reference in New Issue
Block a user