snapshot of https://github.com/twentyhq/twenty.git (main, 2026-09-17; blobs > 131072B omitted)

packages/twenty-utils/crowdin-normalizer/utils/__tests__/evaluate-rules.util.spec.ts

@@ -0,0 +1,49 @@
+import { NORMALIZATION_RULES } from '../../constants/normalization-rules.constant';
+import { evaluateRules } from '../evaluate-rules.util';
+
+describe('evaluateRules', () => {
+  it('reports every rule that fired and applies them in sequence', () => {
+    const result = evaluateRules({
+      rules: NORMALIZATION_RULES,
+      sourceText: 'Replace `<path>` with your path',
+      translationText: '\\u62db `&lt;path&gt;`',
+    });
+
+    expect(result.fixedText).toBe('招 `<path>`');
+    expect(result.ruleNames).toEqual([
+      'escaped-unicode',
+      'escaped-inline-code-tags',
+    ]);
+  });
+
+  it('skips source-filtered rules when the source string is unknown', () => {
+    const result = evaluateRules({
+      rules: NORMALIZATION_RULES,
+      sourceText: undefined,
+      translationText: 'Remplacez `&lt;path&gt;` par votre chemin',
+    });
+
+    expect(result.ruleNames).toEqual([]);
+  });
+
+  it('skips source-filtered rules when the source does not match', () => {
+    const result = evaluateRules({
+      rules: NORMALIZATION_RULES,
+      sourceText: 'Escape a literal &lt; in prose',
+      translationText: 'Échappez un &lt; littéral dans le texte',
+    });
+
+    expect(result.ruleNames).toEqual([]);
+  });
+
+  it('returns the translation untouched when no rule fires', () => {
+    const result = evaluateRules({
+      rules: NORMALIZATION_RULES,
+      sourceText: 'Replace `<path>` with your path',
+      translationText: 'Remplacez `<path>` par votre chemin',
+    });
+
+    expect(result.fixedText).toBe('Remplacez `<path>` par votre chemin');
+    expect(result.ruleNames).toEqual([]);
+  });
+});

packages/twenty-utils/crowdin-normalizer/utils/__tests__/is-translation-already-present-error.util.spec.ts

@@ -0,0 +1,67 @@
+import { CrowdinApiError } from '../../errors/crowdin-api.error';
+import { isTranslationAlreadyPresentError } from '../is-translation-already-present-error.util';
+
+const buildError = (body: string) => new CrowdinApiError(400, body);
+
+const buildDetailError = (detail: { code: string; message: string }) =>
+  buildError(
+    JSON.stringify({
+      errors: [{ error: { key: 'translation', errors: [detail] } }],
+    }),
+  );
+
+describe('isTranslationAlreadyPresentError', () => {
+  it('recognises the identical-translation error code', () => {
+    expect(
+      isTranslationAlreadyPresentError(
+        buildDetailError({
+          code: 'identicalTranslation',
+          message: 'Identical translation already saved',
+        }),
+      ),
+    ).toBe(true);
+  });
+
+  it('recognises the duplicate-translation validation error', () => {
+    expect(
+      isTranslationAlreadyPresentError(
+        buildDetailError({
+          code: 'validationError',
+          message:
+            'Duplicate translation. Please vote or approve the original.',
+        }),
+      ),
+    ).toBe(true);
+  });
+
+  it('does not swallow an unrelated validation error', () => {
+    expect(
+      isTranslationAlreadyPresentError(
+        buildDetailError({
+          code: 'validationError',
+          message: 'Translation is too long',
+        }),
+      ),
+    ).toBe(false);
+  });
+
+  it('does not treat another failure as present because of its wording', () => {
+    expect(
+      isTranslationAlreadyPresentError(
+        buildDetailError({
+          code: 'notFound',
+          message: 'String is identical to a deleted one',
+        }),
+      ),
+    ).toBe(false);
+  });
+
+  it('rejects a non-JSON body and any non-Crowdin error', () => {
+    expect(
+      isTranslationAlreadyPresentError(buildError('<html>502</html>')),
+    ).toBe(false);
+    expect(isTranslationAlreadyPresentError(new Error('identical'))).toBe(
+      false,
+    );
+  });
+});

packages/twenty-utils/crowdin-normalizer/utils/__tests__/map-with-concurrency.util.spec.ts

@@ -0,0 +1,51 @@
+import { mapWithConcurrency } from '../map-with-concurrency.util';
+
+describe('mapWithConcurrency', () => {
+  it('keeps results in input order regardless of completion order', async () => {
+    const results = await mapWithConcurrency({
+      items: [30, 10, 20],
+      limit: 3,
+      handler: (delay) =>
+        new Promise<number>((resolve) =>
+          setTimeout(() => resolve(delay), delay),
+        ),
+    });
+
+    expect(results).toEqual([30, 10, 20]);
+  });
+
+  it('never runs more than `limit` handlers at once', async () => {
+    let running = 0;
+    let peak = 0;
+
+    await mapWithConcurrency({
+      items: Array.from({ length: 20 }, (_, index) => index),
+      limit: 3,
+      handler: async (item) => {
+        running++;
+        peak = Math.max(peak, running);
+        await new Promise((resolve) => setTimeout(resolve, 1));
+        running--;
+
+        return item;
+      },
+    });
+
+    expect(peak).toBe(3);
+  });
+
+  it('rejects a limit that would leave every item unhandled', async () => {
+    await expect(
+      mapWithConcurrency({ items: [1], limit: 0, handler: jest.fn() }),
+    ).rejects.toThrow('at least 1');
+  });
+
+  it('returns an empty array without spawning workers', async () => {
+    const handler = jest.fn();
+
+    expect(await mapWithConcurrency({ items: [], limit: 5, handler })).toEqual(
+      [],
+    );
+    expect(handler).not.toHaveBeenCalled();
+  });
+});

packages/twenty-utils/crowdin-normalizer/utils/add-translation.util.ts

@@ -0,0 +1,25 @@
+import { type CrowdinContext } from '../types/crowdin-context.type';
+import { crowdinRequest } from './crowdin-request.util';
+import { isTranslationAlreadyPresentError } from './is-translation-already-present-error.util';
+
+export async function addTranslation(
+  context: CrowdinContext,
+  {
+    stringId,
+    languageId,
+    text,
+  }: { stringId: number; languageId: string; text: string },
+): Promise<void> {
+  try {
+    await crowdinRequest(
+      context,
+      `/projects/${context.projectId}/translations`,
+      {
+        method: 'POST',
+        body: JSON.stringify({ stringId, languageId, text }),
+      },
+    );
+  } catch (error) {
+    if (!isTranslationAlreadyPresentError(error)) throw error;
+  }
+}

packages/twenty-utils/crowdin-normalizer/utils/crowdin-request.util.ts

@@ -0,0 +1,31 @@
+import { CrowdinApiError } from '../errors/crowdin-api.error';
+import { type CrowdinContext } from '../types/crowdin-context.type';
+
+const CROWDIN_BASE_URL = 'https://twenty.api.crowdin.com/api/v2';
+
+export async function crowdinRequest<T>(
+  { token }: CrowdinContext,
+  endpoint: string,
+  options: RequestInit = {},
+): Promise<T | undefined> {
+  const response = await fetch(`${CROWDIN_BASE_URL}${endpoint}`, {
+    ...options,
+    headers: {
+      Authorization: `Bearer ${token}`,
+      'Content-Type': 'application/json',
+      ...options.headers,
+    },
+  });
+
+  const body = await response.text();
+
+  if (!response.ok) {
+    if (response.status === 404) return undefined;
+
+    throw new CrowdinApiError(response.status, body);
+  }
+
+  if (!body) return undefined;
+
+  return JSON.parse(body) as T;
+}

packages/twenty-utils/crowdin-normalizer/utils/delete-translation.util.ts

@@ -0,0 +1,13 @@
+import { type CrowdinContext } from '../types/crowdin-context.type';
+import { crowdinRequest } from './crowdin-request.util';
+
+export async function deleteTranslation(
+  context: CrowdinContext,
+  { translationId }: { translationId: number },
+): Promise<void> {
+  await crowdinRequest(
+    context,
+    `/projects/${context.projectId}/translations/${translationId}`,
+    { method: 'DELETE' },
+  );
+}

packages/twenty-utils/crowdin-normalizer/utils/evaluate-rules.util.ts

@@ -0,0 +1,29 @@
+import { type NormalizationRule } from '../types/normalization-rule.type';
+
+export function evaluateRules({
+  rules,
+  sourceText,
+  translationText,
+}: {
+  rules: NormalizationRule[];
+  sourceText: string | undefined;
+  translationText: string;
+}): { fixedText: string; ruleNames: string[] } {
+  return rules.reduce<{ fixedText: string; ruleNames: string[] }>(
+    (accumulator, rule) => {
+      const isFilteredOut =
+        rule.sourceFilter !== undefined &&
+        (sourceText === undefined || !rule.sourceFilter(sourceText));
+
+      if (isFilteredOut || !rule.detect(accumulator.fixedText, sourceText)) {
+        return accumulator;
+      }
+
+      return {
+        fixedText: rule.fix(accumulator.fixedText, sourceText),
+        ruleNames: [...accumulator.ruleNames, rule.name],
+      };
+    },
+    { fixedText: translationText, ruleNames: [] },
+  );
+}

packages/twenty-utils/crowdin-normalizer/utils/fetch-all-pages.util.ts

@@ -0,0 +1,31 @@
+import { type CrowdinContext } from '../types/crowdin-context.type';
+import { crowdinRequest } from './crowdin-request.util';
+
+const PAGE_SIZE = 500;
+
+export async function fetchAllPages<TItem>(
+  context: CrowdinContext,
+  buildEndpoint: (pagination: string) => string,
+): Promise<TItem[]> {
+  const items: TItem[] = [];
+  let offset = 0;
+
+  while (true) {
+    type PagedResponse = { data: Array<{ data: TItem }> };
+
+    const response = await crowdinRequest<PagedResponse>(
+      context,
+      buildEndpoint(`limit=${PAGE_SIZE}&offset=${offset}`),
+    );
+
+    if (!response || response.data.length === 0) break;
+
+    items.push(...response.data.map((item) => item.data));
+
+    if (response.data.length < PAGE_SIZE) break;
+
+    offset += PAGE_SIZE;
+  }
+
+  return items;
+}

packages/twenty-utils/crowdin-normalizer/utils/fetch-language-translations.util.ts

@@ -0,0 +1,47 @@
+import { type CrowdinContext } from '../types/crowdin-context.type';
+import { type CrowdinTranslation } from '../types/crowdin-translation.type';
+import { fetchAllPages } from './fetch-all-pages.util';
+
+const MAX_STRING_IDS_PER_REQUEST = 50;
+
+function chunkStringIds(stringIds: number[]): number[][] {
+  const batches: number[][] = [];
+
+  for (
+    let index = 0;
+    index < stringIds.length;
+    index += MAX_STRING_IDS_PER_REQUEST
+  ) {
+    batches.push(stringIds.slice(index, index + MAX_STRING_IDS_PER_REQUEST));
+  }
+
+  return batches;
+}
+
+export async function fetchLanguageTranslations(
+  context: CrowdinContext,
+  { languageId, stringIds }: { languageId: string; stringIds?: number[] },
+): Promise<CrowdinTranslation[]> {
+  const stringIdBatches = stringIds ? chunkStringIds(stringIds) : [undefined];
+
+  const translations: Array<Partial<CrowdinTranslation>> = [];
+
+  for (const batch of stringIdBatches) {
+    const stringIdsFilter = batch ? `&stringIds=${batch.join(',')}` : '';
+
+    translations.push(
+      ...(await fetchAllPages<Partial<CrowdinTranslation>>(
+        context,
+        (pagination) =>
+          `/projects/${context.projectId}/languages/${languageId}/translations?${pagination}${stringIdsFilter}`,
+      )),
+    );
+  }
+
+  return translations.filter(
+    (translation): translation is CrowdinTranslation =>
+      typeof translation.stringId === 'number' &&
+      typeof translation.translationId === 'number' &&
+      typeof translation.text === 'string',
+  );
+}

packages/twenty-utils/crowdin-normalizer/utils/fetch-source-strings-by-id.util.ts

@@ -0,0 +1,26 @@
+import { type CrowdinContext } from '../types/crowdin-context.type';
+import { fetchAllPages } from './fetch-all-pages.util';
+
+export async function fetchSourceStringsById(
+  context: CrowdinContext,
+): Promise<Map<number, string>> {
+  type SourceString = {
+    id?: number;
+    text?: string | Record<string, string>;
+  };
+
+  const sourceStrings = await fetchAllPages<SourceString>(
+    context,
+    (pagination) => `/projects/${context.projectId}/strings?${pagination}`,
+  );
+
+  return new Map(
+    sourceStrings
+      .filter(
+        (sourceString): sourceString is { id: number; text: string } =>
+          typeof sourceString.id === 'number' &&
+          typeof sourceString.text === 'string',
+      )
+      .map((sourceString) => [sourceString.id, sourceString.text]),
+  );
+}

packages/twenty-utils/crowdin-normalizer/utils/fetch-target-language-ids.util.ts

@@ -0,0 +1,15 @@
+import { type CrowdinContext } from '../types/crowdin-context.type';
+import { crowdinRequest } from './crowdin-request.util';
+
+export async function fetchTargetLanguageIds(
+  context: CrowdinContext,
+): Promise<string[]> {
+  type ProjectResponse = { data: { targetLanguageIds: string[] } };
+
+  const response = await crowdinRequest<ProjectResponse>(
+    context,
+    `/projects/${context.projectId}`,
+  );
+
+  return response?.data.targetLanguageIds ?? [];
+}

packages/twenty-utils/crowdin-normalizer/utils/get-crowdin-token-or-throw.util.ts

@@ -0,0 +1,11 @@
+export function getCrowdinTokenOrThrow(): string {
+  const token = process.env.CROWDIN_PERSONAL_TOKEN;
+
+  if (!token) {
+    throw new Error(
+      'CROWDIN_PERSONAL_TOKEN environment variable not set. Get your token from: https://twenty.crowdin.com/u/settings#api-key',
+    );
+  }
+
+  return token;
+}

packages/twenty-utils/crowdin-normalizer/utils/is-translation-already-present-error.util.ts

@@ -0,0 +1,33 @@
+import { CrowdinApiError } from '../errors/crowdin-api.error';
+
+const IDENTICAL_TRANSLATION_CODE = 'identicalTranslation';
+const VALIDATION_ERROR_CODE = 'validationError';
+
+// validationError also covers unrelated refusals, so the wording has to narrow
+// it down to the one that means the text we wanted is already there.
+const DUPLICATE_TRANSLATION_REGEX = /duplicate translation/i;
+
+type CrowdinErrorBody = {
+  errors?: Array<{
+    error?: { errors?: Array<{ code?: string; message?: string }> };
+  }>;
+};
+
+export function isTranslationAlreadyPresentError(error: unknown): boolean {
+  if (!(error instanceof CrowdinApiError)) return false;
+
+  try {
+    const body = JSON.parse(error.body) as CrowdinErrorBody;
+
+    return (body.errors ?? []).some((entry) =>
+      (entry.error?.errors ?? []).some(
+        (detail) =>
+          detail.code === IDENTICAL_TRANSLATION_CODE ||
+          (detail.code === VALIDATION_ERROR_CODE &&
+            DUPLICATE_TRANSLATION_REGEX.test(detail.message ?? '')),
+      ),
+    );
+  } catch {
+    return false;
+  }
+}

packages/twenty-utils/crowdin-normalizer/utils/map-with-concurrency.util.ts

@@ -0,0 +1,32 @@
+export async function mapWithConcurrency<TItem, TResult>({
+  items,
+  limit,
+  handler,
+}: {
+  items: TItem[];
+  limit: number;
+  handler: (item: TItem) => Promise<TResult>;
+}): Promise<TResult[]> {
+  if (limit < 1) {
+    throw new Error(
+      `mapWithConcurrency needs a limit of at least 1, got ${limit}`,
+    );
+  }
+
+  const results: TResult[] = Array.from({ length: items.length });
+  let cursor = 0;
+
+  const worker = async () => {
+    while (cursor < items.length) {
+      const index = cursor++;
+
+      results[index] = await handler(items[index]);
+    }
+  };
+
+  await Promise.all(
+    Array.from({ length: Math.min(limit, items.length) }, worker),
+  );
+
+  return results;
+}