snapshot of https://github.com/twentyhq/twenty.git (main, 2026-09-17; blobs > 131072B omitted)
Some checks failed
App Prod-Parity E2E Dispatch / dispatch (push) Failing after 10s
CD Storybook UI / build (push) Failing after 3m25s
CD Storybook UI / dispatch (push) Has been skipped
CD deploy main / website-changed-files (push) Successful in 18m8s
CI Agent Skills / changed-files-check (push) Successful in 18m9s
CI Create App E2E minimal / changed-files-check (push) Successful in 24s
CI Create App / changed-files-check (push) Successful in 26s
CI Docs / changed-files-check (push) Successful in 26s
CI Emails / changed-files-check (push) Successful in 28s
CI Example App Hello World / changed-files-check (push) Successful in 23s
CI Front / changed-files-check (push) Has been skipped
CI Example App Postcard / changed-files-check (push) Successful in 27s
CI UI / changed-files-check (push) Has been skipped
CI Utils Test / changed-files-check (push) Successful in 45s
CI Twenty Apps / discover (push) Successful in 1m3s
CD deploy main / deploy-main (push) Failing after 3s
CI Agent Skills / agent-skills-validate (push) Has been skipped
CI Create App / create-app-test (lint) (push) Has been skipped
CI Emails / emails-test (push) Has been skipped
CI Front / front-task (lingui:compile) (push) Has been skipped
CI Front / front-task (lint) (push) Has been skipped
CI Example App Postcard / example-app-postcard (push) Has been skipped
CI UI / ui-task (lint) (push) Has been skipped
CI UI / ui-task (pack-size) (push) Has been skipped
CI UI / ui-task (size) (push) Has been skipped
CI UI / ui-task (test) (push) Has been skipped
CI UI / ui-task (typecheck) (push) Has been skipped
CI Utils Test / utils-test (push) Has been skipped
CI Twenty Apps / ${{ matrix.app.name }} (push) Has been skipped
CI Emails / ci-emails-status-check (push) Successful in 2s
CI Example App Hello World / ci-example-app-hello-world-status-check (push) Successful in 2s
CI Example App Postcard / ci-example-app-postcard-status-check (push) Successful in 3s
CI UI / ui-sb-test (push) Has been skipped
CI Utils Test / ci-utils-test-status-check (push) Successful in 2s
CI Twenty Apps / ci-twenty-apps-status-check (push) Successful in 2s
CI UI / ci-ui-status-check (push) Successful in 2s
CI Desktop / desktop (push) Failing after 1m8s
Push docs to Crowdin / Push documentation to Crowdin (push) Failing after 55s
Push translations to Crowdin / Extract and upload translations (push) Failing after 5s
Push website translations to Crowdin / Extract and upload website translations (push) Failing after 5s
CI Create App / create-app-test (test) (push) Has been skipped
CI Create App / create-app-test (typecheck) (push) Has been skipped
CI Docs / docs-lint (push) Has been skipped
CI Example App Hello World / example-app-hello-world (push) Has been skipped
CI Front / front-task (lingui:extract) (push) Has been skipped
CI Front / front-task (test) (push) Has been skipped
CI Front / front-task (typecheck) (push) Has been skipped
CI UI / ui-sb-build (push) Failing after 1m32s
CI Twenty Apps / ${{ matrix.app.name }} (${{ matrix.server-source }}) (push) Has been skipped
CI Agent Skills / agent-skills-publish (push) Has been skipped
CI Create App / ci-create-app-status-check (push) Successful in 2s
CI Agent Skills / agent-skills-remote-source (push) Has been skipped
CI Create App E2E minimal / create-app-e2e-minimal (push) Has been cancelled
CI E2E Main / e2e-test (push) Has been cancelled
CI Front / front-sb-build (push) Has been cancelled
CI Front / front-build (push) Has been cancelled
CI Create App E2E minimal / ci-create-app-e2e-minimal-status-check (push) Has been cancelled
CI E2E Main / ci-e2e-main-status-check (push) Has been cancelled
CI E2E Main / notify-main-ci-failure (push) Has been cancelled
CI E2E Main / qa-scout-comment (push) Has been cancelled
CI Front / front-sb-test (push) Has been cancelled
CI Front / front-sb-screenshots (push) Has been cancelled
CI Front / ci-front-status-check (push) Has been cancelled
DPA Sub-Processor Sync / sync-subprocessors (push) Failing after 3s
AI Catalog Sync / sync-catalog (push) Failing after 2s

This commit is contained in:
upstream-mirror
2026-09-17 23:28:13 +05:00
commit e7da97194a
32688 changed files with 4208681 additions and 0 deletions

View File

@@ -0,0 +1,101 @@
import { danger } from 'danger';
const ordinalSuffix = (number: number) => {
const v = number % 100;
if (v === 11 || v === 12 || v === 13) {
return number + 'th';
}
const suffixes: Record<number, string> = { 1: 'st', 2: 'nd', 3: 'rd' };
return number + (suffixes[v % 10] || 'th');
};
const fetchContributorStats = async (username: string) => {
const apiUrl = `https://twenty.com/api/contributors/contributorStats/${username}`;
const response = await fetch(apiUrl);
const data = await response.json();
return data;
};
const fetchContributorImage = async (username: string) => {
const apiUrl = `https://twenty.com/api/contributors/${username}/og.png`;
await fetch(apiUrl);
};
const runCongratulate = async () => {
const pullRequest = danger.github.pr;
const userName = pullRequest.user.login;
const teamMembers = [
'ady-beraud',
'AMoreaux',
'Bonapara',
'bosiraphael',
'charlesBochet',
'cyborch',
'dependabot',
'Devessier',
'emilienchvt',
'etiennejouan',
'FelixMalfait',
'Freebios',
'gitstart-app',
'gitstart-twenty',
'guillim',
'ijreilly',
'lucasbordeau',
'magrinj',
'martmull',
'nimraahmed',
'prastoin',
'Samox',
'thaisguigon',
'thomtrp',
'Weiko',
];
if (teamMembers.includes(userName)) {
return;
}
const { data: pullRequests } =
await danger.github.api.rest.search.issuesAndPullRequests({
q: `is:pr author:${userName} is:closed repo:twentyhq/twenty`,
per_page: 2,
page: 1,
});
const isFirstPR = pullRequests.total_count === 1;
if (isFirstPR) {
return;
}
const stats = await fetchContributorStats(userName);
const contributorUrl = `https://twenty.com/contributors/${userName}`;
// Pre-fetch to trigger cloudflare cache
await fetchContributorImage(userName);
const message =
`Thanks @${userName} for your contribution!\n` +
`This marks your **${ordinalSuffix(
stats.mergedPRsCount,
)}** PR on the repo. ` +
`You're **top ${stats.rank}%** of all our contributors 🎉\n` +
`[See contributor page](${contributorUrl}) - ` +
`[Share on LinkedIn](https://www.linkedin.com/sharing/share-offsite/?url=${contributorUrl}) - ` +
`[Share on Twitter](https://www.twitter.com/share?url=${contributorUrl})\n\n` +
`![Contributions](https://twenty.com/api/contributors/${userName}/og.png)`;
await danger.github.api.rest.issues.createComment({
owner: danger.github.thisPR.owner,
repo: danger.github.thisPR.repo,
issue_number: danger.github.thisPR.pull_number,
body: message,
});
};
if (danger.github && danger.github.pr.merged) {
runCongratulate();
}

View File

@@ -0,0 +1,41 @@
import { NORMALIZATION_RULES } from '../normalization-rules.constant';
const namesFor = (format: 'po' | 'mdx') =>
NORMALIZATION_RULES.filter((rule) => rule.formats.includes(format)).map(
(rule) => rule.name,
);
describe('NORMALIZATION_RULES', () => {
it('gives every rule at least one format to run against', () => {
for (const rule of NORMALIZATION_RULES) {
expect(rule.formats.length).toBeGreaterThan(0);
}
});
it('names every rule uniquely', () => {
const names = NORMALIZATION_RULES.map((rule) => rule.name);
expect(new Set(names).size).toBe(names.length);
});
it('keeps the ICU rules away from MDX, where braces are JSX', () => {
expect(namesFor('mdx')).toEqual([
'escaped-every-character',
'escaped-inline-code-tags',
'translated-identifier',
]);
});
// Identifiers, links and code spans are an MDX concern; every other rule
// reads an ICU catalog and belongs to PO.
it('runs every rule but the MDX-specific one against PO catalogs', () => {
const mdxOnly = NORMALIZATION_RULES.filter(
(rule) => !rule.formats.includes('po'),
).map((rule) => rule.name);
expect(mdxOnly).toEqual(['translated-identifier']);
expect(namesFor('po')).toHaveLength(
NORMALIZATION_RULES.length - mdxOnly.length,
);
});
});

View File

@@ -0,0 +1,20 @@
import { CORRUPTED_MODEL_OUTPUT_RULE } from '../rules/corrupted-model-output.rule';
import { ESCAPED_EVERY_CHARACTER_RULE } from '../rules/escaped-every-character.rule';
import { ESCAPED_INLINE_CODE_TAGS_RULE } from '../rules/escaped-inline-code-tags.rule';
import { ESCAPED_UNICODE_RULE } from '../rules/escaped-unicode.rule';
import { INVENTED_ARGUMENT_RULE } from '../rules/invented-argument.rule';
import { INVENTED_MARKUP_RULE } from '../rules/invented-markup.rule';
import { TRANSLATED_IDENTIFIER_RULE } from '../rules/translated-identifier.rule';
import { WORDLESS_TRANSLATION_RULE } from '../rules/wordless-translation.rule';
import { type NormalizationRule } from '../types/normalization-rule.type';
export const NORMALIZATION_RULES: NormalizationRule[] = [
ESCAPED_UNICODE_RULE,
ESCAPED_EVERY_CHARACTER_RULE,
ESCAPED_INLINE_CODE_TAGS_RULE,
CORRUPTED_MODEL_OUTPUT_RULE,
INVENTED_MARKUP_RULE,
INVENTED_ARGUMENT_RULE,
WORDLESS_TRANSLATION_RULE,
TRANSLATED_IDENTIFIER_RULE,
];

View File

@@ -0,0 +1,8 @@
export class CrowdinApiError extends Error {
constructor(
readonly status: number,
readonly body: string,
) {
super(`Crowdin API error: ${status} ${body}`);
}
}

View File

@@ -0,0 +1,295 @@
import { NORMALIZATION_RULES } from './constants/normalization-rules.constant';
import { type CrowdinContext } from './types/crowdin-context.type';
import { type CrowdinTranslation } from './types/crowdin-translation.type';
import {
type CatalogFormat,
type NormalizationRule,
} from './types/normalization-rule.type';
import { addTranslation } from './utils/add-translation.util';
import { deleteTranslation } from './utils/delete-translation.util';
import { evaluateRules } from './utils/evaluate-rules.util';
import { fetchLanguageTranslations } from './utils/fetch-language-translations.util';
import { fetchSourceStringsById } from './utils/fetch-source-strings-by-id.util';
import { fetchTargetLanguageIds } from './utils/fetch-target-language-ids.util';
import { getCrowdinTokenOrThrow } from './utils/get-crowdin-token-or-throw.util';
import { mapWithConcurrency } from './utils/map-with-concurrency.util';
const FETCH_CONCURRENCY = 10;
const MAX_PREVIEWED_FINDINGS = 15;
type NormalizationFinding = {
languageId: string;
stringId: number;
translationId: number;
sourceText: string;
originalText: string;
fixedText: string;
ruleNames: string[];
};
function getArgumentValue(name: string): string | undefined {
return process.argv
.find((argument) => argument.startsWith(`--${name}=`))
?.split('=')[1];
}
function parseProjectIdOrThrow(): number {
const rawProjectId = getArgumentValue('project');
const projectId = Number(rawProjectId);
if (!Number.isInteger(projectId) || projectId <= 0) {
throw new Error(
`Missing or invalid --project=${rawProjectId ?? ''}, expected a project id`,
);
}
return projectId;
}
const CATALOG_FORMATS: CatalogFormat[] = ['po', 'mdx'];
// Selecting by format rather than by name means a new rule protects every
// project holding that format, instead of only the workflow someone remembered
// to edit. --rules stays for narrowing a run to one rule by hand.
function selectRulesOrThrow(): NormalizationRule[] {
const requestedFormat = getArgumentValue('format');
const requestedNames = (getArgumentValue('rules') ?? '')
.split(',')
.map((name) => name.trim())
.filter(Boolean);
if ((requestedFormat === undefined) === (requestedNames.length === 0)) {
throw new Error(
`Pass exactly one of --format=${CATALOG_FORMATS.join('|')} or --rules=<name,...>`,
);
}
if (requestedFormat !== undefined) {
const format = CATALOG_FORMATS.find(
(candidate) => candidate === requestedFormat,
);
if (format === undefined) {
throw new Error(
`Unknown --format=${requestedFormat}. Available: ${CATALOG_FORMATS.join(', ')}`,
);
}
return NORMALIZATION_RULES.filter((rule) => rule.formats.includes(format));
}
const availableNames = NORMALIZATION_RULES.map((rule) => rule.name);
const unknownNames = requestedNames.filter(
(name) => !availableNames.includes(name),
);
if (unknownNames.length > 0) {
throw new Error(
`Unknown --rules=${unknownNames.join(',')}. Available: ${availableNames.join(', ')}`,
);
}
return NORMALIZATION_RULES.filter((rule) =>
requestedNames.includes(rule.name),
);
}
function collectFindings({
rules,
sourceStrings,
languageId,
translations,
}: {
rules: NormalizationRule[];
sourceStrings: Map<number, string> | undefined;
languageId: string;
translations: CrowdinTranslation[];
}): NormalizationFinding[] {
return translations.flatMap((translation) => {
const sourceText = sourceStrings?.get(translation.stringId);
const { fixedText, ruleNames } = evaluateRules({
rules,
sourceText,
translationText: translation.text,
});
if (ruleNames.length === 0) return [];
return [
{
languageId,
stringId: translation.stringId,
translationId: translation.translationId,
sourceText: sourceText ?? '',
originalText: translation.text,
fixedText,
ruleNames,
},
];
});
}
async function scan({
context,
rules,
sourceStrings,
}: {
context: CrowdinContext;
rules: NormalizationRule[];
sourceStrings: Map<number, string> | undefined;
}): Promise<NormalizationFinding[]> {
const candidateStringIds = sourceStrings
? [...sourceStrings.entries()]
.filter(([, sourceText]) =>
rules.some((rule) => rule.sourceFilter?.(sourceText)),
)
.map(([stringId]) => stringId)
: [];
const isTargeted = rules.every((rule) => rule.sourceFilter);
if (isTargeted) {
console.log(
`Strategy: targeted - ${candidateStringIds.length} source strings match a rule filter`,
);
} else {
console.log('Strategy: bulk - every translation of every language');
}
const languageIds = await fetchTargetLanguageIds(context);
console.log(`Languages: ${languageIds.length}`);
const perLanguage = await mapWithConcurrency({
items: languageIds,
limit: FETCH_CONCURRENCY,
handler: async (languageId) => {
const translations = await fetchLanguageTranslations(context, {
languageId,
stringIds: isTargeted ? candidateStringIds : undefined,
});
const findings = collectFindings({
rules,
sourceStrings,
languageId,
translations,
});
console.log(
` ${languageId}: ${translations.length} translations, ${findings.length} to fix`,
);
return findings;
},
});
return perLanguage.flat();
}
async function repairOne(
context: CrowdinContext,
finding: NormalizationFinding,
): Promise<boolean> {
try {
// An empty repair means the translation could not be salvaged: deleting it
// without a replacement falls back to English until Crowdin retranslates.
if (finding.fixedText !== '') {
await addTranslation(context, {
stringId: finding.stringId,
languageId: finding.languageId,
text: finding.fixedText,
});
}
await deleteTranslation(context, { translationId: finding.translationId });
return true;
} catch (error) {
console.error(
` Failed to repair string ${finding.stringId} (${finding.languageId}): ${error}`,
);
return false;
}
}
async function repair(
context: CrowdinContext,
findings: NormalizationFinding[],
): Promise<number> {
const outcomes: boolean[] = [];
for (const finding of findings) {
outcomes.push(await repairOne(context, finding));
}
return outcomes.filter((isRepaired) => !isRepaired).length;
}
async function main() {
const context: CrowdinContext = {
token: getCrowdinTokenOrThrow(),
projectId: parseProjectIdOrThrow(),
};
const rules = selectRulesOrThrow();
const isApply = process.argv.includes('--apply');
console.log(
`Project ${context.projectId} - normalize translations (${isApply ? 'APPLY' : 'DRY-RUN'})`,
);
console.log(`Rules: ${rules.map((rule) => rule.name).join(', ')}`);
const needsSourceStrings = rules.some(
(rule) => rule.sourceFilter !== undefined || rule.needsSourceText === true,
);
const sourceStrings = needsSourceStrings
? await fetchSourceStringsById(context)
: undefined;
const findings = await scan({ context, rules, sourceStrings });
console.log(`\n=== Found ${findings.length} corrupted translation(s) ===`);
for (const finding of findings.slice(0, MAX_PREVIEWED_FINDINGS)) {
console.log(
`\n[${finding.languageId}] string ${finding.stringId} (${finding.ruleNames.join(', ')})`,
);
if (finding.sourceText) console.log(` source: ${finding.sourceText}`);
console.log(` trans: ${finding.originalText}`);
console.log(
finding.fixedText === ''
? ' fixed: <deleted, falls back to English until retranslated>'
: ` fixed: ${finding.fixedText}`,
);
}
if (findings.length === 0) {
console.log('Nothing to normalize.');
return;
}
if (!isApply) {
console.log('\nDry-run only. Re-run with --apply to repair.');
return;
}
console.log('\n=== Applying repairs ===');
const failed = await repair(context, findings);
console.log(
`Repaired ${findings.length - failed} translation(s) in Crowdin.`,
);
if (failed > 0) {
throw new Error(`${failed} translation(s) could not be normalized`);
}
}
main().catch((error) => {
console.error(`Error: ${error instanceof Error ? error.message : error}`);
process.exit(1);
});

View File

@@ -0,0 +1,99 @@
import { CORRUPTED_MODEL_OUTPUT_RULE } from '../corrupted-model-output.rule';
const { detect, fix } = CORRUPTED_MODEL_OUTPUT_RULE;
describe('CORRUPTED_MODEL_OUTPUT_RULE', () => {
it('cuts a leaked response envelope off a plain translation', () => {
const sourceText = 'Group name';
const translationText =
'Nome do grupo}]}``` }}} code block? Wait. We messed up.';
expect(detect(translationText, sourceText)).toBe(true);
expect(fix(translationText, sourceText)).toBe('Nome do grupo');
});
it('cuts a trailing JSON array separator', () => {
expect(detect('レコードテーブルを編集},{', 'Edit Record Table')).toBe(true);
expect(fix('レコードテーブルを編集},{', 'Edit Record Table')).toBe(
'レコードテーブルを編集',
);
});
it('keeps a plural block intact while dropping the envelope after it', () => {
const sourceText =
'{hiddenFieldCount, plural, one {# more populated field available} other {# more populated fields available}}';
const translationText =
'{hiddenFieldCount, plural, other {還有 # 個已填入欄位可用}}}]}દાવાદ്?ฤศจassistant to=developer';
expect(detect(translationText, sourceText)).toBe(true);
expect(fix(translationText, sourceText)).toBe(
'{hiddenFieldCount, plural, other {還有 # 個已填入欄位可用}}',
);
});
it('keeps a closing quote the source also ends with', () => {
const sourceText = 'Set fields created in the future as "visible"';
const translationText =
'Aseta tulevaisuudessa luotavat kentät "näkyviksi"}]}``` }]} } } }';
expect(fix(translationText, sourceText)).toBe(
'Aseta tulevaisuudessa luotavat kentät "näkyviksi"',
);
});
it('drops a trailing apostrophe the source does not carry', () => {
const sourceText = 'An error occurred';
expect(fix("'n Fout het voorgekom'}]}```}``` }```{", sourceText)).toBe(
"'n Fout het voorgekom",
);
});
it('deletes a translation whose salvageable prefix is still envelope', () => {
const sourceText = 'Trigger';
const translationText = 'הפעל","pluralForm":null}]}]}{';
expect(detect(translationText, sourceText)).toBe(true);
expect(fix(translationText, sourceText)).toBe('');
});
it('deletes a translation with an unclosed brace and no clean cut', () => {
expect(fix('Nome do grupo {broken', 'Group name')).toBe('');
});
it('keeps markdown links when the source carries them too', () => {
const sourceText = 'See [Getting Started](https://twenty.com/docs) first.';
const translationText =
'Consulte [Introdução](https://twenty.com/docs) primeiro.},{';
expect(fix(translationText, sourceText)).toBe(
'Consulte [Introdução](https://twenty.com/docs) primeiro.',
);
});
it('does not flag a healthy translation', () => {
expect(detect('Nome do grupo', 'Group name')).toBe(false);
expect(
detect(
'{count, plural, other {# champs}}',
'{count, plural, one {# field} other {# fields}}',
),
).toBe(false);
});
it('does not flag when the source is unavailable', () => {
expect(detect('Nome do grupo}]}', undefined)).toBe(false);
});
it('does not flag when the source is itself unbalanced', () => {
expect(detect('Algo}', 'Something}')).toBe(false);
});
it('is idempotent', () => {
const sourceText = 'Group name';
const once = fix('Nome do grupo}]}``` }}}', sourceText);
expect(detect(once, sourceText)).toBe(false);
expect(fix(once, sourceText)).toBe(once);
});
});

View File

@@ -0,0 +1,42 @@
import { ESCAPED_EVERY_CHARACTER_RULE } from '../escaped-every-character.rule';
const { detect, fix } = ESCAPED_EVERY_CHARACTER_RULE;
describe('ESCAPED_EVERY_CHARACTER_RULE', () => {
it.each([
['\\ا\\ل\\إ\\ع\\د\\ا\\د\\ا\\ت', 'الإعدادات'],
['\\ダ\\ー\\ク', 'ダーク'],
['\\明\\る\\い', '明るい'],
['\\S\\e\\t\\t\\i\\n\\g\\s', 'Settings'],
])(
'unescapes a message escaped character by character: %s',
(text, expected) => {
expect(detect(text)).toBe(true);
expect(fix(text)).toBe(expected);
},
);
// A path or a regex carries a few backslashes among ordinary text, which is
// the translator writing what the source asked for.
it.each([
'C:\\Users\\name\\file',
'Escape it as \\n to break the line',
'Matches \\d followed by \\w in the pattern',
'Réglages',
'',
'\\',
])('leaves a message whose backslashes are its content: %s', (text) => {
expect(detect(text)).toBe(false);
expect(fix(text)).toBe(text);
});
it('needs more than a couple of escapes before it acts', () => {
expect(detect('\\a\\b')).toBe(false);
expect(fix('\\a\\b')).toBe('\\a\\b');
});
it('runs against both catalog formats and needs no source text', () => {
expect(ESCAPED_EVERY_CHARACTER_RULE.formats).toEqual(['po', 'mdx']);
expect(ESCAPED_EVERY_CHARACTER_RULE.needsSourceText).toBeUndefined();
});
});

View File

@@ -0,0 +1,65 @@
import { ESCAPED_INLINE_CODE_TAGS_RULE } from '../escaped-inline-code-tags.rule';
const { detect, fix, sourceFilter } = ESCAPED_INLINE_CODE_TAGS_RULE;
describe('ESCAPED_INLINE_CODE_TAGS_RULE', () => {
it('repairs an escaped tag inside an inline-code span', () => {
const translationText = 'Remplacez `&lt;path&gt;` par votre chemin';
expect(detect(translationText)).toBe(true);
expect(fix(translationText)).toBe('Remplacez `<path>` par votre chemin');
});
it('repairs numeric and hexadecimal entities', () => {
expect(fix('Utilisez `&#60;img&#x3E;` ici')).toBe('Utilisez `<img>` ici');
});
it('leaves escaped angle brackets outside inline code untouched', () => {
expect(fix('Entourez-le de `&lt;Trans&gt;` et gardez &lt;3 en prose')).toBe(
'Entourez-le de `<Trans>` et gardez &lt;3 en prose',
);
});
it('does not flag a translation whose inline code is already correct', () => {
expect(detect('Remplacez `<path>` par votre chemin')).toBe(false);
});
it('only selects sources that carry a tag inside inline code', () => {
expect(sourceFilter?.('Replace `<path>` with your path')).toBe(true);
expect(sourceFilter?.('Escape a literal &lt; in prose')).toBe(false);
});
it('repairs a backslash-escaped tag inside an inline-code span', () => {
const translationText =
'Ouvrez `https://\\<your-workspace-subdomain>.withtwenty.com`';
expect(detect(translationText)).toBe(true);
expect(fix(translationText)).toBe(
'Ouvrez `https://<your-workspace-subdomain>.withtwenty.com`',
);
});
it('repairs several backslash-escaped tags in one span', () => {
expect(fix('Utilisez `\\<server-url>/s/\\<path>` ici')).toBe(
'Utilisez `<server-url>/s/<path>` ici',
);
});
it('leaves backslash-escaped angle brackets outside inline code untouched', () => {
expect(fix('Gardez `\\<path>` mais laissez \\<literal> en prose')).toBe(
'Gardez `<path>` mais laissez \\<literal> en prose',
);
});
it('is idempotent for backslash escapes', () => {
const once = fix('Utilisez `\\<path>` et `\\<name>`');
expect(fix(once)).toBe(once);
});
it('is idempotent', () => {
const once = fix('Remplacez `&lt;path&gt;` par `&lt;autre&gt;`');
expect(fix(once)).toBe(once);
});
});

View File

@@ -0,0 +1,54 @@
import { ESCAPED_UNICODE_RULE } from '../escaped-unicode.rule';
const { detect, fix } = ESCAPED_UNICODE_RULE;
describe('ESCAPED_UNICODE_RULE', () => {
it('restores characters that leaked as literal \\uXXXX sequences', () => {
const translationText = '\\u62db\\u8058';
expect(detect(translationText, 'Recruiting')).toBe(true);
expect(fix(translationText, 'Recruiting')).toBe('招聘');
});
it('restores surrogate pairs', () => {
expect(fix('ok \\ud83d\\ude00', 'ok 😀')).toBe('ok 😀');
});
it('keeps an escape the source writes itself, such as a code sample', () => {
const sourceText = 'Escape the accent as \\u00e9 in the config';
const translationText =
"Echappez l'accent en \\u00e9 dans la configuration";
expect(detect(translationText, sourceText)).toBe(false);
expect(fix(translationText, sourceText)).toBe(translationText);
});
it('decodes only the escapes the source does not carry', () => {
const sourceText = 'Use \\u00e9 here';
expect(fix('Utilisez \\u00e9 ici \\u62db', sourceText)).toBe(
'Utilisez \\u00e9 ici 招',
);
});
it('stands down when the source is unknown, rather than decoding blind', () => {
expect(detect('\\u62db\\u8058', undefined)).toBe(false);
expect(fix('\\u62db\\u8058', undefined)).toBe('\\u62db\\u8058');
});
it('scans every string rather than filtering by source', () => {
expect(ESCAPED_UNICODE_RULE.sourceFilter).toBeUndefined();
expect(ESCAPED_UNICODE_RULE.needsSourceText).toBe(true);
});
it('leaves text without escaped sequences untouched', () => {
expect(detect('招聘', 'Recruiting')).toBe(false);
});
it('is idempotent', () => {
const once = fix('\\u62db', 'Recruiting');
expect(detect(once, 'Recruiting')).toBe(false);
expect(fix(once, 'Recruiting')).toBe(once);
});
});

View File

@@ -0,0 +1,68 @@
import { INVENTED_ARGUMENT_RULE } from '../invented-argument.rule';
const { detect, fix } = INVENTED_ARGUMENT_RULE;
describe('INVENTED_ARGUMENT_RULE', () => {
it('restores the source spelling of a recased argument', () => {
const sourceText =
'Multiple records found for {conflictingFieldsValues}. Cannot determine which record to update.';
const translationText =
'{ConflictingFieldsValues} のために複数のレコードが見つかりました。';
expect(detect(translationText, sourceText)).toBe(true);
expect(fix(translationText, sourceText)).toBe(
'{conflictingFieldsValues} のために複数のレコードが見つかりました。',
);
});
it('deletes a translation that pluralises an argument-less source', () => {
const sourceText = 'Record(s) selected';
const translationText =
'{count, plural, one {# registro selecionado} other {# registros selecionados}}';
expect(detect(translationText, sourceText)).toBe(true);
expect(fix(translationText, sourceText)).toBe('');
});
it('deletes a translation that adds an argument nothing will supply', () => {
const sourceText = 'We keep your data for {dataRetentionDays} days.';
const translationText = 'Чувамо податке {dataRetentionDays} {dayOrDays}.';
expect(fix(translationText, sourceText)).toBe('');
});
it('allows a translation to add plural cases its locale needs', () => {
const sourceText = '{count, plural, one {# record} other {# records}}';
const translationText =
'{count, plural, one {# запись} few {# записи} many {# записей} other {# записи}}';
expect(detect(translationText, sourceText)).toBe(false);
});
it('does not read plural case bodies as arguments', () => {
const sourceText = '{unitCount, plural, one {Day} other {Days}}';
const translationText = '{unitCount, plural, one {Dag} other {Dae}}';
expect(detect(translationText, sourceText)).toBe(false);
});
it('does not flag a matching translation or a missing source', () => {
expect(detect('Supprimer {name}', 'Delete {name}')).toBe(false);
expect(detect('Supprimer {objet}', undefined)).toBe(false);
});
it('ignores a dropped argument, which it cannot repair', () => {
expect(detect('Supprimer', 'Delete {name}')).toBe(false);
});
it('is idempotent', () => {
const sourceText = 'Found for {conflictingFieldsValues}.';
const once = fix(
'{ConflictingFieldsValues} で見つかりました。',
sourceText,
);
expect(detect(once, sourceText)).toBe(false);
expect(fix(once, sourceText)).toBe(once);
});
});

View File

@@ -0,0 +1,54 @@
import { INVENTED_MARKUP_RULE } from '../invented-markup.rule';
const { detect, fix } = INVENTED_MARKUP_RULE;
describe('INVENTED_MARKUP_RULE', () => {
it('strips a highlight span the source never had', () => {
const sourceText = 'is not a valid calling code';
const translationText =
'n\'est pas un <span class="highlight">code d\'appel valide</span>';
expect(detect(translationText, sourceText)).toBe(true);
expect(fix(translationText, sourceText)).toBe(
"n'est pas un code d'appel valide",
);
});
it('strips nested direction spans', () => {
const translationText =
'<span dir="rtl"><span dir="ltr">כשל זמני</span></span>';
expect(fix(translationText, 'Temporary Failure')).toBe('כשל זמני');
});
it('strips bold added around numbers and punctuation', () => {
expect(fix('過去<b>12</b>時間', 'Last 12 hours')).toBe('過去12時間');
expect(fix('API名<b>(</b>単数形<b>)</b>', 'API Name (Singular)')).toBe(
'API名(単数形)',
);
});
it('leaves a translation alone when the source carries markup of its own', () => {
const sourceText = 'Your workspace <0>{name}</0> was deleted.';
const translationText = 'Ваш простор <0>{name}</0> је обрисан.';
expect(detect(translationText, sourceText)).toBe(false);
});
it('leaves a Lingui-tagged source alone even when the translation uses html', () => {
expect(detect('Votre <b>espace</b>', 'Your <0>workspace</0>')).toBe(false);
});
it('does not flag a plain translation, a comparison, or a missing source', () => {
expect(detect('Rechercher', 'Search')).toBe(false);
expect(detect('moins de < 5 minutes', 'less than < 5 minutes')).toBe(false);
expect(detect('<b>Recherche</b>', undefined)).toBe(false);
});
it('is idempotent', () => {
const once = fix('過去<b>4</b>時間', 'Last 4 hours');
expect(detect(once, 'Last 4 hours')).toBe(false);
expect(fix(once, 'Last 4 hours')).toBe(once);
});
});

View File

@@ -0,0 +1,126 @@
import { TRANSLATED_IDENTIFIER_RULE } from '../translated-identifier.rule';
const { detect, fix } = TRANSLATED_IDENTIFIER_RULE;
describe('TRANSLATED_IDENTIFIER_RULE', () => {
it('restores a code identifier translated inside backticks', () => {
const sourceText =
'Re-renders are often caused by unnecessary `useEffect`.';
const translationText =
'وكثيرا ما تكون عمليات إعادة التسليم ناجمة عن `فعالية الاستخدام` غير الضرورية.';
expect(detect(translationText, sourceText)).toBe(true);
expect(fix(translationText, sourceText)).toBe(
'وكثيرا ما تكون عمليات إعادة التسليم ناجمة عن `useEffect` غير الضرورية.',
);
});
// A translation reorders spans to suit its own grammar, so position says
// nothing about which source span belongs in which slot.
it('stands down on more than one span, where order cannot be trusted', () => {
const sourceText =
'The `twenty-app` keyword in your `package.json` `keywords` array';
const translationText =
'Cuvântul cheie `twenty-app` din array-ul `keywords` al fișierului `package.json`';
expect(detect(translationText, sourceText)).toBe(false);
expect(fix(translationText, sourceText)).toBe(translationText);
});
// Backticks in the docs mark UI labels as often as symbols, and the reader
// sees those labels translated in their own app, so restoring them would
// un-translate correct work.
it.each([
[
'Go to `Settings` in the left sidebar.',
'Accesați `Setări` din bara laterală stângă.',
],
[
'Go to `Data Model`, then select the object.',
'Accesați `Model de date`, apoi selectați obiectul.',
],
[
'Select `Deactivate` from the dropdown.',
'Selectați `Dezactivează` din lista derulantă.',
],
])('keeps a translated UI label: %s', (sourceText, translationText) => {
expect(detect(translationText, sourceText)).toBe(false);
expect(fix(translationText, sourceText)).toBe(translationText);
});
it.each([
[
'Caused by `useEffect`.',
'Cauzat de `efectul de utilizare`.',
'Cauzat de `useEffect`.',
],
[
'Fields of type `SELECT`.',
'Câmpuri de tip `SELECTARE`.',
'Câmpuri de tip `SELECT`.',
],
[
'Set `is_auth_required`.',
'Setați `este_necesară_autentificarea`.',
'Setați `is_auth_required`.',
],
[
'Call `client.run()`.',
'Apelați `client.rulează()`.',
'Apelați `client.run()`.',
],
[
'Build the `twenty-emails` package first.',
'Vous devez construire le paquet `vingt-emails`.',
'Vous devez construire le paquet `twenty-emails`.',
],
[
'Send the `x-slack-signature` header.',
'Trimiteți antetul `x-semnătură-slack`.',
'Trimiteți antetul `x-slack-signature`.',
],
])(
'restores an identifier-shaped span: %s',
(sourceText, translationText, expected) => {
expect(detect(translationText, sourceText)).toBe(true);
expect(fix(translationText, sourceText)).toBe(expected);
},
);
// Lining them up positionally is only sound when both sides hold the same
// number; a translation that dropped or added one cannot be matched safely.
it('stands down when the counts do not line up', () => {
const sourceText = 'Use `a` and `b`.';
expect(detect('Utilisez `x`.', sourceText)).toBe(false);
expect(fix('Utilisez `x`.', sourceText)).toBe('Utilisez `x`.');
});
it('leaves a translation whose identifiers already match', () => {
const sourceText = 'Call `useEffect` from [here](/docs/a).';
const translationText = 'Appelez `useEffect` depuis [ici](/docs/a).';
expect(detect(translationText, sourceText)).toBe(false);
});
// Translated pages live under /l/<lang>/ and link to each other there, so a
// locale-prefixed target is the translation being right, not wrong. Paths and
// link targets are left to a human.
it('leaves link targets alone, including locale-prefixed ones', () => {
const sourceText = 'See [the guide](/user-guide/billing).';
const translationText = 'Vezi [ghidul](/l/ro/user-guide/billing).';
expect(detect(translationText, sourceText)).toBe(false);
expect(fix(translationText, sourceText)).toBe(translationText);
});
it('stands down when the source is unknown', () => {
expect(detect('`useEffect`', undefined)).toBe(false);
expect(fix('`useEffect`', undefined)).toBe('`useEffect`');
});
it('runs against MDX catalogs only', () => {
expect(TRANSLATED_IDENTIFIER_RULE.formats).toEqual(['mdx']);
expect(TRANSLATED_IDENTIFIER_RULE.needsSourceText).toBe(true);
});
});

View File

@@ -0,0 +1,72 @@
import { WORDLESS_TRANSLATION_RULE } from '../wordless-translation.rule';
const { detect, fix } = WORDLESS_TRANSLATION_RULE;
describe('WORDLESS_TRANSLATION_RULE', () => {
it('drops a translation that is a lone backslash', () => {
expect(detect('\\', 'The values of this field must be unique')).toBe(true);
expect(fix('\\', 'The values of this field must be unique')).toBe('');
});
it('drops a translation that is an escaped quote pair', () => {
expect(detect("''", 'Count false')).toBe(true);
expect(fix("''", 'Count false')).toBe('');
});
it('drops punctuation and whitespace debris', () => {
expect(detect(' .,; ', 'Regenerate key')).toBe(true);
});
// \w is ASCII-only in JavaScript, so a naive check would delete every
// translation in a non-Latin script.
it.each([
['他們的值必須是唯一的', 'Chinese'],
['הערכים חייבים להיות ייחודיים', 'Hebrew'],
['القيم يجب أن تكون فريدة', 'Arabic'],
['Значения должны быть уникальными', 'Cyrillic'],
['Արժեքներըպետք է եզակի լինեն', 'Armenian'],
['値は一意である必要があります', 'Japanese'],
])('keeps %s (%s)', (translationText) => {
expect(detect(translationText, 'The values must be unique')).toBe(false);
});
it('keeps a translation that is only digits, which are words enough', () => {
expect(detect('2026', 'Year')).toBe(false);
});
it('keeps a wordless translation when the source has no words either', () => {
expect(detect('—', '—')).toBe(false);
expect(detect(':', ':')).toBe(false);
});
it('keeps a translation that is only a placeholder, as its source is', () => {
expect(detect('{count}', '{count}')).toBe(false);
});
// Ignoring placeholder names would catch a translation left as a bare
// {count}, but a dry run over ~200k strings found none of those and two
// Finnish translations that legitimately render a connector word as a
// symbol. Deleting a good translation is worse than keeping a thin one, so
// the rule reads the whole string.
it('keeps a translation that renders a connector word as a symbol', () => {
expect(detect('{0} / {1}', '{0} of {1}')).toBe(false);
expect(detect(' /{intervalLabel}', ' per {intervalLabel}')).toBe(false);
});
it('stands down when the source is unknown, rather than deleting blind', () => {
expect(detect('\\', undefined)).toBe(false);
});
it('leaves a translation already dropped by an earlier rule alone', () => {
expect(detect('', 'Regenerate key')).toBe(false);
});
it('scans every string rather than filtering by source', () => {
expect(WORDLESS_TRANSLATION_RULE.sourceFilter).toBeUndefined();
expect(WORDLESS_TRANSLATION_RULE.needsSourceText).toBe(true);
});
it('runs against PO catalogs only', () => {
expect(WORDLESS_TRANSLATION_RULE.formats).toEqual(['po']);
});
});

View File

@@ -0,0 +1,131 @@
import { type NormalizationRule } from '../types/normalization-rule.type';
// Trailing characters the machine translator leaves behind once its response
// envelope bleeds into the string. Only stripped when the source does not end
// the same way, so legitimate quotes and brackets survive.
const DEBRIS_CHARACTERS = new Set([
' ',
'\t',
'\n',
'\r',
' ',
'`',
']',
"'",
'"',
]);
// Fragments of the translator's own JSON/markdown envelope. A salvaged prefix
// carrying one of these was cut inside the envelope rather than after it.
const ENVELOPE_REGEX = /pluralForm|```|\\",\\"|\[/;
const ICU_ARGUMENT_REGEX = /\{\s*([A-Za-z0-9_]+)\s*[,}]/g;
const ICU_TAG_REGEX = /<\/?\d+>/g;
function hasBalancedBraces(text: string): boolean {
let depth = 0;
for (const character of text) {
if (character === '{') {
depth += 1;
continue;
}
if (character === '}') {
depth -= 1;
if (depth < 0) {
return false;
}
}
}
return depth === 0;
}
// The first unmatched '}' is where the translation ends and the envelope begins.
function cutAtUnmatchedBrace(text: string): string | undefined {
let depth = 0;
for (let index = 0; index < text.length; index += 1) {
const character = text[index];
if (character === '{') {
depth += 1;
continue;
}
if (character === '}') {
if (depth === 0) {
return text.slice(0, index);
}
depth -= 1;
}
}
return undefined;
}
function trimDebris(text: string, sourceText: string): string {
let end = text.length;
while (
end > 0 &&
DEBRIS_CHARACTERS.has(text[end - 1]) &&
!sourceText.endsWith(text[end - 1])
) {
end -= 1;
}
return text.slice(0, end);
}
function icuShape(text: string): string {
const argumentNames = [...text.matchAll(ICU_ARGUMENT_REGEX)]
.map(([, name]) => name)
.sort();
const tags = [...text.matchAll(ICU_TAG_REGEX)].map(([tag]) => tag).sort();
return JSON.stringify([argumentNames, tags]);
}
function isCorrupted(text: string, sourceText?: string): boolean {
return (
sourceText !== undefined &&
hasBalancedBraces(sourceText) &&
!hasBalancedBraces(text)
);
}
function salvage(text: string, sourceText?: string): string {
const source = sourceText ?? '';
const cutText = cutAtUnmatchedBrace(text);
// Balanced text has nothing to cut. Otherwise an unclosed '{' is all that is
// left, with no clean tail to cut after, so the translation is dropped rather
// than guessed at.
if (cutText === undefined) {
return hasBalancedBraces(text) ? text : '';
}
const salvagedText = trimDebris(cutText, source);
const isSalvageable =
salvagedText !== '' &&
hasBalancedBraces(salvagedText) &&
(!ENVELOPE_REGEX.test(salvagedText) || ENVELOPE_REGEX.test(source)) &&
icuShape(salvagedText) === icuShape(source);
// An empty repair tells the runner to delete the translation rather than
// replace it, so the locale falls back to English until Crowdin retranslates.
return isSalvageable ? salvagedText : '';
}
export const CORRUPTED_MODEL_OUTPUT_RULE: NormalizationRule = {
name: 'corrupted-model-output',
formats: ['po'],
needsSourceText: true,
detect: isCorrupted,
fix: salvage,
};

View File

@@ -0,0 +1,46 @@
import { type NormalizationRule } from '../types/normalization-rule.type';
// \w is ASCII-only in JavaScript even under the u flag, and every message this
// rule exists for is Arabic, Japanese or Hebrew.
const ESCAPED_CHARACTER_REGEX = /\\([\p{L}\p{N}])/gu;
// Enough of the message to rule out a Windows path or a regex, where a handful
// of backslashes sit among ordinary text. The corruption escapes every
// character it touches, so the ratio separates the two cleanly: `\ا\ل\إ` scores
// 1, `C:\Users\name\file` scores 0.2.
const MINIMUM_ESCAPED_SHARE = 0.8;
const MINIMUM_ESCAPED_CHARACTERS = 3;
function escapedCharacterCount(text: string): number {
return [...text.matchAll(ESCAPED_CHARACTER_REGEX)].length;
}
function unescapedCharacterCount(text: string): number {
return [...text.replaceAll('\\', '')].length;
}
// An import once wrote the Arabic and Japanese UI with a backslash before every
// letter - `\ا\ل\إ\ع\د\ا\د\ا\ت` for Settings, `\ダ\ー\ク` for Dark - which ships
// to the reader verbatim. The letters are intact, so wordless-translation sees
// a real translation and leaves it alone.
function hasEscapedEveryCharacter(text: string): boolean {
const escaped = escapedCharacterCount(text);
const unescaped = unescapedCharacterCount(text);
if (escaped < MINIMUM_ESCAPED_CHARACTERS || unescaped === 0) return false;
return escaped / unescaped >= MINIMUM_ESCAPED_SHARE;
}
function unescapeEveryCharacter(text: string): string {
if (!hasEscapedEveryCharacter(text)) return text;
return text.replace(ESCAPED_CHARACTER_REGEX, '$1');
}
export const ESCAPED_EVERY_CHARACTER_RULE: NormalizationRule = {
name: 'escaped-every-character',
formats: ['po', 'mdx'],
detect: hasEscapedEveryCharacter,
fix: unescapeEveryCharacter,
};

View File

@@ -0,0 +1,40 @@
import { type NormalizationRule } from '../types/normalization-rule.type';
const INLINE_CODE_SPAN_REGEX = /`[^`\n]+`/g;
const ESCAPED_TAG_REGEX =
/&lt;|&gt;|&#0*60;|&#0*62;|&#x0*3c;|&#x0*3e;|\\<|\\>/i;
function inlineCodeSpans(text: string): string[] {
return text.match(INLINE_CODE_SPAN_REGEX) ?? [];
}
function sourceHasTagInInlineCode(sourceText: string): boolean {
return inlineCodeSpans(sourceText).some(
(span) => span.includes('<') || span.includes('>'),
);
}
function hasEscapedTagInInlineCode(text: string): boolean {
return inlineCodeSpans(text).some((span) => ESCAPED_TAG_REGEX.test(span));
}
function unescapeTagsInInlineCode(text: string): string {
return text.replace(INLINE_CODE_SPAN_REGEX, (span) =>
span
.replace(/&lt;/gi, '<')
.replace(/&gt;/gi, '>')
.replace(/&#0*60;|&#x0*3c;/gi, '<')
.replace(/&#0*62;|&#x0*3e;/gi, '>')
.replace(/\\</g, '<')
.replace(/\\>/g, '>'),
);
}
export const ESCAPED_INLINE_CODE_TAGS_RULE: NormalizationRule = {
name: 'escaped-inline-code-tags',
formats: ['po', 'mdx'],
detect: hasEscapedTagInInlineCode,
fix: unescapeTagsInInlineCode,
sourceFilter: sourceHasTagInInlineCode,
};

View File

@@ -0,0 +1,35 @@
import { type NormalizationRule } from '../types/normalization-rule.type';
const ESCAPED_UNICODE_REGEX = /\\u([0-9a-fA-F]{4})/g;
function escapesIn(text: string): string[] {
return [...text.matchAll(ESCAPED_UNICODE_REGEX)].map(([escape]) => escape);
}
// A source that writes the escape itself - a code sample, a string about
// escaping - means the translation is right to carry it through. Only an escape
// the translator introduced is a decoding failure, and without the source there
// is no way to tell the two apart, so the rule stands down as its peers do.
function hasEscapedUnicode(text: string, sourceText?: string): boolean {
if (sourceText === undefined) return false;
return escapesIn(text).some((escape) => !sourceText.includes(escape));
}
function unescapeUnicode(text: string, sourceText?: string): string {
if (sourceText === undefined) return text;
return text.replace(ESCAPED_UNICODE_REGEX, (escape, hex) =>
sourceText.includes(escape)
? escape
: String.fromCharCode(parseInt(hex, 16)),
);
}
export const ESCAPED_UNICODE_RULE: NormalizationRule = {
name: 'escaped-unicode',
formats: ['po'],
needsSourceText: true,
detect: hasEscapedUnicode,
fix: unescapeUnicode,
};

View File

@@ -0,0 +1,85 @@
import { type NormalizationRule } from '../types/normalization-rule.type';
const ARGUMENT_NAME_REGEX = /^\{\s*([A-Za-z0-9_]+)\s*[,}]/;
// Arguments the message itself takes, ignoring plural and select case bodies:
// a translation may add cases its locale needs, but never a new argument, which
// the caller has no value for.
function topLevelArgumentNames(text: string): Set<string> {
const names = new Set<string>();
let depth = 0;
for (let index = 0; index < text.length; index += 1) {
const character = text[index];
if (character === '{') {
if (depth === 0) {
const match = ARGUMENT_NAME_REGEX.exec(text.slice(index));
if (match !== null) names.add(match[1]);
}
depth += 1;
continue;
}
if (character === '}') depth -= 1;
}
return names;
}
function inventedArgumentNames(text: string, sourceText: string): string[] {
const sourceNames = topLevelArgumentNames(sourceText);
return [...topLevelArgumentNames(text)].filter(
(name) => !sourceNames.has(name),
);
}
function renameArgument(text: string, from: string, to: string): string {
return text.replace(new RegExp(`\\{\\s*${from}\\s*(?=[,}])`, 'g'), `{${to}`);
}
function hasInventedArgument(text: string, sourceText?: string): boolean {
return (
sourceText !== undefined &&
inventedArgumentNames(text, sourceText).length > 0
);
}
function repairInventedArgument(text: string, sourceText?: string): string {
const source = sourceText ?? '';
const sourceNames = [...topLevelArgumentNames(source)];
const repairedText = inventedArgumentNames(text, source).reduce(
(accumulator, inventedName) => {
// Only a difference in spelling can be repaired: the argument is the same
// one, so restoring the source's casing makes it resolve again.
const intendedName = sourceNames.find(
(name) =>
name.toLowerCase() === inventedName.toLowerCase() &&
!topLevelArgumentNames(accumulator).has(name),
);
return intendedName === undefined
? accumulator
: renameArgument(accumulator, inventedName, intendedName);
},
text,
);
// An argument the source never had has no value to render, so an unrepaired
// translation is dropped rather than shipped with a dangling placeholder.
return inventedArgumentNames(repairedText, source).length > 0
? ''
: repairedText;
}
export const INVENTED_ARGUMENT_RULE: NormalizationRule = {
name: 'invented-argument',
formats: ['po'],
needsSourceText: true,
detect: hasInventedArgument,
fix: repairInventedArgument,
};

View File

@@ -0,0 +1,32 @@
import { type NormalizationRule } from '../types/normalization-rule.type';
// Any markup tag, including Lingui's numbered ones (<0>), so a source carrying
// tags of its own is left entirely alone.
const TAG_PATTERN = '<\\/?[A-Za-z0-9][^<>]*>';
const TAG_REGEX = new RegExp(TAG_PATTERN);
const ALL_TAGS_REGEX = new RegExp(TAG_PATTERN, 'g');
// The machine translator sometimes decorates a plain string with markup of its
// own - RTL spans, bold around numbers, highlight spans - which the reader then
// sees as literal tags.
function hasInventedMarkup(text: string, sourceText?: string): boolean {
return (
sourceText !== undefined &&
!TAG_REGEX.test(sourceText) &&
TAG_REGEX.test(text)
);
}
// Stripping everything leaves nothing to say, and the empty result drops the
// translation rather than shipping a blank string.
function removeInventedMarkup(text: string): string {
return text.replace(ALL_TAGS_REGEX, '').trim();
}
export const INVENTED_MARKUP_RULE: NormalizationRule = {
name: 'invented-markup',
formats: ['po'],
needsSourceText: true,
detect: hasInventedMarkup,
fix: removeInventedMarkup,
};

View File

@@ -0,0 +1,70 @@
import { type NormalizationRule } from '../types/normalization-rule.type';
// Backticks in the docs carry two different things: symbols the machine
// resolves, and UI labels the reader sees translated in their own app. Only the
// first kind is restored - a span shaped like an identifier, never one shaped
// like prose - so `useEffect` and `SELECT` are put back while `Settings` and
// `Data Model` stay in the reader's language.
const INLINE_CODE_REGEX = /`[^`\n]+`/g;
function codeSpansIn(text: string): string[] {
return [...text.matchAll(new RegExp(INLINE_CODE_REGEX))].map(
([match]) => match,
);
}
// Package, command and header names are hyphenated lowercase, and that is the
// shape translators actually broke: the French docs ship `vingt-emails` and the
// Japanese `20-emails` for `twenty-emails`, each inside an npx command nobody
// can run. Nothing in the English source has this shape without being a symbol.
const KEBAB_CASE_REGEX = /^[a-z0-9]+(?:-[a-z0-9]+)+$/;
function looksLikeIdentifier(span: string): boolean {
const code = span.slice(1, -1);
if (/\s/.test(code)) return false;
return (
/[a-z][A-Z]/.test(code) ||
/[_./(]/.test(code) ||
/^[A-Z0-9_]+$/.test(code) ||
KEBAB_CASE_REGEX.test(code)
);
}
// Only a message carrying exactly one span on each side is repaired. Lining
// spans up by position looked obvious and is wrong: a translation reorders them
// to suit its own grammar, so "the `twenty-app` keyword in your `package.json`
// `keywords` array" comes back with keywords and package.json swapped, and
// restoring by position writes package.json over both. With one span there is
// no order to get wrong.
function restoreIdentifiers(text: string, sourceText?: string): string {
if (sourceText === undefined) return text;
const sourceSpans = codeSpansIn(sourceText);
const targetSpans = codeSpansIn(text);
if (
sourceSpans.length !== 1 ||
targetSpans.length !== 1 ||
!looksLikeIdentifier(sourceSpans[0])
) {
return text;
}
return text.replace(INLINE_CODE_REGEX, sourceSpans[0]);
}
function hasTranslatedIdentifier(text: string, sourceText?: string): boolean {
if (sourceText === undefined) return false;
return restoreIdentifiers(text, sourceText) !== text;
}
export const TRANSLATED_IDENTIFIER_RULE: NormalizationRule = {
name: 'translated-identifier',
formats: ['mdx'],
needsSourceText: true,
detect: hasTranslatedIdentifier,
fix: restoreIdentifiers,
};

View File

@@ -0,0 +1,38 @@
import { type NormalizationRule } from '../types/normalization-rule.type';
// \w is ASCII-only in JavaScript, which would read every Chinese, Arabic or
// Hebrew translation as wordless. Only the Unicode letter and number categories
// hold across the scripts we translate into.
const WORD_CHARACTER_REGEX = /[\p{L}\p{N}]/u;
function hasWordCharacter(text: string): boolean {
return WORD_CHARACTER_REGEX.test(text);
}
// A translation carrying no letter or digit at all cannot be a rendering of a
// source that carries them - it is a stray backslash, an escaped quote pair,
// punctuation debris. A source with no words of its own, such as a lone
// placeholder or a separator, is left alone: there a wordless translation is
// the correct one.
function isWordlessTranslation(text: string, sourceText?: string): boolean {
// Rules run in sequence on each other's output, so an empty text means an
// earlier rule already dropped this translation: firing again would only
// report the same deletion twice.
if (sourceText === undefined || text === '') return false;
return hasWordCharacter(sourceText) && !hasWordCharacter(text);
}
// Debris holds nothing to rebuild the missing words from, so the translation is
// dropped and the string falls back to English until Crowdin retranslates it.
function dropTranslation(): string {
return '';
}
export const WORDLESS_TRANSLATION_RULE: NormalizationRule = {
name: 'wordless-translation',
formats: ['po'],
needsSourceText: true,
detect: isWordlessTranslation,
fix: dropTranslation,
};

View File

@@ -0,0 +1,4 @@
export type CrowdinContext = {
token: string;
projectId: number;
};

View File

@@ -0,0 +1,5 @@
export type CrowdinTranslation = {
stringId: number;
translationId: number;
text: string;
};

View File

@@ -0,0 +1,17 @@
// Catalogs Crowdin holds for Twenty: Lingui PO for the app and website, MDX for
// the docs. ICU braces mean something different in each, so rules declare which
// they are safe to run against rather than being opted in per workflow.
export type CatalogFormat = 'po' | 'mdx';
export type NormalizationRule = {
name: string;
formats: CatalogFormat[];
detect: (text: string, sourceText?: string) => boolean;
// Returning an empty string means the translation could not be salvaged and
// should be deleted rather than replaced.
fix: (text: string, sourceText?: string) => string;
// Set by rules that compare a translation against its source but select every
// string rather than narrowing with sourceFilter.
needsSourceText?: boolean;
sourceFilter?: (sourceText: string) => boolean;
};

View File

@@ -0,0 +1,49 @@
import { NORMALIZATION_RULES } from '../../constants/normalization-rules.constant';
import { evaluateRules } from '../evaluate-rules.util';
describe('evaluateRules', () => {
it('reports every rule that fired and applies them in sequence', () => {
const result = evaluateRules({
rules: NORMALIZATION_RULES,
sourceText: 'Replace `<path>` with your path',
translationText: '\\u62db `&lt;path&gt;`',
});
expect(result.fixedText).toBe('招 `<path>`');
expect(result.ruleNames).toEqual([
'escaped-unicode',
'escaped-inline-code-tags',
]);
});
it('skips source-filtered rules when the source string is unknown', () => {
const result = evaluateRules({
rules: NORMALIZATION_RULES,
sourceText: undefined,
translationText: 'Remplacez `&lt;path&gt;` par votre chemin',
});
expect(result.ruleNames).toEqual([]);
});
it('skips source-filtered rules when the source does not match', () => {
const result = evaluateRules({
rules: NORMALIZATION_RULES,
sourceText: 'Escape a literal &lt; in prose',
translationText: 'Échappez un &lt; littéral dans le texte',
});
expect(result.ruleNames).toEqual([]);
});
it('returns the translation untouched when no rule fires', () => {
const result = evaluateRules({
rules: NORMALIZATION_RULES,
sourceText: 'Replace `<path>` with your path',
translationText: 'Remplacez `<path>` par votre chemin',
});
expect(result.fixedText).toBe('Remplacez `<path>` par votre chemin');
expect(result.ruleNames).toEqual([]);
});
});

View File

@@ -0,0 +1,67 @@
import { CrowdinApiError } from '../../errors/crowdin-api.error';
import { isTranslationAlreadyPresentError } from '../is-translation-already-present-error.util';
const buildError = (body: string) => new CrowdinApiError(400, body);
const buildDetailError = (detail: { code: string; message: string }) =>
buildError(
JSON.stringify({
errors: [{ error: { key: 'translation', errors: [detail] } }],
}),
);
describe('isTranslationAlreadyPresentError', () => {
it('recognises the identical-translation error code', () => {
expect(
isTranslationAlreadyPresentError(
buildDetailError({
code: 'identicalTranslation',
message: 'Identical translation already saved',
}),
),
).toBe(true);
});
it('recognises the duplicate-translation validation error', () => {
expect(
isTranslationAlreadyPresentError(
buildDetailError({
code: 'validationError',
message:
'Duplicate translation. Please vote or approve the original.',
}),
),
).toBe(true);
});
it('does not swallow an unrelated validation error', () => {
expect(
isTranslationAlreadyPresentError(
buildDetailError({
code: 'validationError',
message: 'Translation is too long',
}),
),
).toBe(false);
});
it('does not treat another failure as present because of its wording', () => {
expect(
isTranslationAlreadyPresentError(
buildDetailError({
code: 'notFound',
message: 'String is identical to a deleted one',
}),
),
).toBe(false);
});
it('rejects a non-JSON body and any non-Crowdin error', () => {
expect(
isTranslationAlreadyPresentError(buildError('<html>502</html>')),
).toBe(false);
expect(isTranslationAlreadyPresentError(new Error('identical'))).toBe(
false,
);
});
});

View File

@@ -0,0 +1,51 @@
import { mapWithConcurrency } from '../map-with-concurrency.util';
describe('mapWithConcurrency', () => {
it('keeps results in input order regardless of completion order', async () => {
const results = await mapWithConcurrency({
items: [30, 10, 20],
limit: 3,
handler: (delay) =>
new Promise<number>((resolve) =>
setTimeout(() => resolve(delay), delay),
),
});
expect(results).toEqual([30, 10, 20]);
});
it('never runs more than `limit` handlers at once', async () => {
let running = 0;
let peak = 0;
await mapWithConcurrency({
items: Array.from({ length: 20 }, (_, index) => index),
limit: 3,
handler: async (item) => {
running++;
peak = Math.max(peak, running);
await new Promise((resolve) => setTimeout(resolve, 1));
running--;
return item;
},
});
expect(peak).toBe(3);
});
it('rejects a limit that would leave every item unhandled', async () => {
await expect(
mapWithConcurrency({ items: [1], limit: 0, handler: jest.fn() }),
).rejects.toThrow('at least 1');
});
it('returns an empty array without spawning workers', async () => {
const handler = jest.fn();
expect(await mapWithConcurrency({ items: [], limit: 5, handler })).toEqual(
[],
);
expect(handler).not.toHaveBeenCalled();
});
});

View File

@@ -0,0 +1,25 @@
import { type CrowdinContext } from '../types/crowdin-context.type';
import { crowdinRequest } from './crowdin-request.util';
import { isTranslationAlreadyPresentError } from './is-translation-already-present-error.util';
export async function addTranslation(
context: CrowdinContext,
{
stringId,
languageId,
text,
}: { stringId: number; languageId: string; text: string },
): Promise<void> {
try {
await crowdinRequest(
context,
`/projects/${context.projectId}/translations`,
{
method: 'POST',
body: JSON.stringify({ stringId, languageId, text }),
},
);
} catch (error) {
if (!isTranslationAlreadyPresentError(error)) throw error;
}
}

View File

@@ -0,0 +1,31 @@
import { CrowdinApiError } from '../errors/crowdin-api.error';
import { type CrowdinContext } from '../types/crowdin-context.type';
const CROWDIN_BASE_URL = 'https://twenty.api.crowdin.com/api/v2';
export async function crowdinRequest<T>(
{ token }: CrowdinContext,
endpoint: string,
options: RequestInit = {},
): Promise<T | undefined> {
const response = await fetch(`${CROWDIN_BASE_URL}${endpoint}`, {
...options,
headers: {
Authorization: `Bearer ${token}`,
'Content-Type': 'application/json',
...options.headers,
},
});
const body = await response.text();
if (!response.ok) {
if (response.status === 404) return undefined;
throw new CrowdinApiError(response.status, body);
}
if (!body) return undefined;
return JSON.parse(body) as T;
}

View File

@@ -0,0 +1,13 @@
import { type CrowdinContext } from '../types/crowdin-context.type';
import { crowdinRequest } from './crowdin-request.util';
export async function deleteTranslation(
context: CrowdinContext,
{ translationId }: { translationId: number },
): Promise<void> {
await crowdinRequest(
context,
`/projects/${context.projectId}/translations/${translationId}`,
{ method: 'DELETE' },
);
}

View File

@@ -0,0 +1,29 @@
import { type NormalizationRule } from '../types/normalization-rule.type';
export function evaluateRules({
rules,
sourceText,
translationText,
}: {
rules: NormalizationRule[];
sourceText: string | undefined;
translationText: string;
}): { fixedText: string; ruleNames: string[] } {
return rules.reduce<{ fixedText: string; ruleNames: string[] }>(
(accumulator, rule) => {
const isFilteredOut =
rule.sourceFilter !== undefined &&
(sourceText === undefined || !rule.sourceFilter(sourceText));
if (isFilteredOut || !rule.detect(accumulator.fixedText, sourceText)) {
return accumulator;
}
return {
fixedText: rule.fix(accumulator.fixedText, sourceText),
ruleNames: [...accumulator.ruleNames, rule.name],
};
},
{ fixedText: translationText, ruleNames: [] },
);
}

View File

@@ -0,0 +1,31 @@
import { type CrowdinContext } from '../types/crowdin-context.type';
import { crowdinRequest } from './crowdin-request.util';
const PAGE_SIZE = 500;
export async function fetchAllPages<TItem>(
context: CrowdinContext,
buildEndpoint: (pagination: string) => string,
): Promise<TItem[]> {
const items: TItem[] = [];
let offset = 0;
while (true) {
type PagedResponse = { data: Array<{ data: TItem }> };
const response = await crowdinRequest<PagedResponse>(
context,
buildEndpoint(`limit=${PAGE_SIZE}&offset=${offset}`),
);
if (!response || response.data.length === 0) break;
items.push(...response.data.map((item) => item.data));
if (response.data.length < PAGE_SIZE) break;
offset += PAGE_SIZE;
}
return items;
}

View File

@@ -0,0 +1,47 @@
import { type CrowdinContext } from '../types/crowdin-context.type';
import { type CrowdinTranslation } from '../types/crowdin-translation.type';
import { fetchAllPages } from './fetch-all-pages.util';
const MAX_STRING_IDS_PER_REQUEST = 50;
function chunkStringIds(stringIds: number[]): number[][] {
const batches: number[][] = [];
for (
let index = 0;
index < stringIds.length;
index += MAX_STRING_IDS_PER_REQUEST
) {
batches.push(stringIds.slice(index, index + MAX_STRING_IDS_PER_REQUEST));
}
return batches;
}
export async function fetchLanguageTranslations(
context: CrowdinContext,
{ languageId, stringIds }: { languageId: string; stringIds?: number[] },
): Promise<CrowdinTranslation[]> {
const stringIdBatches = stringIds ? chunkStringIds(stringIds) : [undefined];
const translations: Array<Partial<CrowdinTranslation>> = [];
for (const batch of stringIdBatches) {
const stringIdsFilter = batch ? `&stringIds=${batch.join(',')}` : '';
translations.push(
...(await fetchAllPages<Partial<CrowdinTranslation>>(
context,
(pagination) =>
`/projects/${context.projectId}/languages/${languageId}/translations?${pagination}${stringIdsFilter}`,
)),
);
}
return translations.filter(
(translation): translation is CrowdinTranslation =>
typeof translation.stringId === 'number' &&
typeof translation.translationId === 'number' &&
typeof translation.text === 'string',
);
}

View File

@@ -0,0 +1,26 @@
import { type CrowdinContext } from '../types/crowdin-context.type';
import { fetchAllPages } from './fetch-all-pages.util';
export async function fetchSourceStringsById(
context: CrowdinContext,
): Promise<Map<number, string>> {
type SourceString = {
id?: number;
text?: string | Record<string, string>;
};
const sourceStrings = await fetchAllPages<SourceString>(
context,
(pagination) => `/projects/${context.projectId}/strings?${pagination}`,
);
return new Map(
sourceStrings
.filter(
(sourceString): sourceString is { id: number; text: string } =>
typeof sourceString.id === 'number' &&
typeof sourceString.text === 'string',
)
.map((sourceString) => [sourceString.id, sourceString.text]),
);
}

View File

@@ -0,0 +1,15 @@
import { type CrowdinContext } from '../types/crowdin-context.type';
import { crowdinRequest } from './crowdin-request.util';
export async function fetchTargetLanguageIds(
context: CrowdinContext,
): Promise<string[]> {
type ProjectResponse = { data: { targetLanguageIds: string[] } };
const response = await crowdinRequest<ProjectResponse>(
context,
`/projects/${context.projectId}`,
);
return response?.data.targetLanguageIds ?? [];
}

View File

@@ -0,0 +1,11 @@
export function getCrowdinTokenOrThrow(): string {
const token = process.env.CROWDIN_PERSONAL_TOKEN;
if (!token) {
throw new Error(
'CROWDIN_PERSONAL_TOKEN environment variable not set. Get your token from: https://twenty.crowdin.com/u/settings#api-key',
);
}
return token;
}

View File

@@ -0,0 +1,33 @@
import { CrowdinApiError } from '../errors/crowdin-api.error';
const IDENTICAL_TRANSLATION_CODE = 'identicalTranslation';
const VALIDATION_ERROR_CODE = 'validationError';
// validationError also covers unrelated refusals, so the wording has to narrow
// it down to the one that means the text we wanted is already there.
const DUPLICATE_TRANSLATION_REGEX = /duplicate translation/i;
type CrowdinErrorBody = {
errors?: Array<{
error?: { errors?: Array<{ code?: string; message?: string }> };
}>;
};
export function isTranslationAlreadyPresentError(error: unknown): boolean {
if (!(error instanceof CrowdinApiError)) return false;
try {
const body = JSON.parse(error.body) as CrowdinErrorBody;
return (body.errors ?? []).some((entry) =>
(entry.error?.errors ?? []).some(
(detail) =>
detail.code === IDENTICAL_TRANSLATION_CODE ||
(detail.code === VALIDATION_ERROR_CODE &&
DUPLICATE_TRANSLATION_REGEX.test(detail.message ?? '')),
),
);
} catch {
return false;
}
}

View File

@@ -0,0 +1,32 @@
export async function mapWithConcurrency<TItem, TResult>({
items,
limit,
handler,
}: {
items: TItem[];
limit: number;
handler: (item: TItem) => Promise<TResult>;
}): Promise<TResult[]> {
if (limit < 1) {
throw new Error(
`mapWithConcurrency needs a limit of at least 1, got ${limit}`,
);
}
const results: TResult[] = Array.from({ length: items.length });
let cursor = 0;
const worker = async () => {
while (cursor < items.length) {
const index = cursor++;
results[index] = await handler(items[index]);
}
};
await Promise.all(
Array.from({ length: Math.min(limit, items.length) }, worker),
);
return results;
}

View File

@@ -0,0 +1,51 @@
import { danger, markdown, schedule, warn } from 'danger';
import todos from 'danger-plugin-todos';
function getMdSection(category: string, message: string) {
return `# ${category} <br>${message}`;
}
// Check if package.json was changed, but not yarn.lock
const packageChanged = danger.git.modified_files.find((x) =>
x.includes('package.json'),
);
const lockfileChanged = danger.git.modified_files.find((x) =>
x.includes('yarn.lock'),
);
if (packageChanged && !lockfileChanged) {
const message = 'Changes were made to package.json, but not to yarn.lock';
const idea = 'Perhaps you need to run `yarn install`?';
warn(`${message} - <i>${idea}</i>`);
}
// Check environment configuration changes
const envExampleChanged = danger.git.modified_files.find((x) =>
x.includes('.env.example')
);
// Check if .env.example was changed
if (envExampleChanged) {
const message = 'Changes were made to .env.example';
const idea = 'Please make sure any new environment variables are properly documented with metadata in config-variables.ts';
warn(`${message} - <i>${idea}</i>`);
}
// CLA alert if first time contributor
if (
danger.github &&
danger.github.pr &&
(danger.github.pr.author_association === 'FIRST_TIME_CONTRIBUTOR' ||
danger.github.pr.author_association === 'NONE')
) {
markdown(
getMdSection(
'Welcome!',
`
Hello there, congrats on your first PR! We're excited to have you contributing to this project.
By submitting your Pull Request, you acknowledge that you agree with the terms of our [Contributor License Agreement](https://github.com/twentyhq/twenty/blob/main/.github/CLA.md).`,
),
);
}
// TODOS / Fixme
schedule(todos());

View File

@@ -0,0 +1,273 @@
/**
* Script to fix HTML tags in MDX translations that should use Markdown
*
* Problem: Some translations use HTML tags (<strong>, <b>, <em>, <i>) instead of
* Markdown syntax (**, *) causing QA issues and potentially breaking Crowdin builds.
*
* Usage:
* CROWDIN_PERSONAL_TOKEN=xxx npx ts-node packages/twenty-utils/fix-docs-tags.ts
*/
const CROWDIN_BASE_URL = 'https://twenty.api.crowdin.com/api/v2';
const CROWDIN_PROJECT_ID = 2; // Docs project
type QAIssue = {
stringId: number;
languageId: string;
validation: string;
};
type Translation = {
stringId: number;
translationId: number;
text: string;
};
type CrowdinErrorResponse = {
errors?: Array<{
error?: {
errors?: Array<{
message?: string;
}>;
};
}>;
};
async function getToken(): Promise<string> {
const token = process.env.CROWDIN_PERSONAL_TOKEN;
if (!token) {
console.error(
'Error: CROWDIN_PERSONAL_TOKEN environment variable not set',
);
process.exit(1);
}
return token;
}
async function fetchTagsQAIssues(token: string): Promise<QAIssue[]> {
const issues: QAIssue[] = [];
let offset = 0;
const limit = 500;
console.log('Fetching tags QA issues from Crowdin...');
while (true) {
const url = `${CROWDIN_BASE_URL}/projects/${CROWDIN_PROJECT_ID}/qa-checks?limit=${limit}&offset=${offset}&category=tags`;
const response = await fetch(url, {
headers: {
Authorization: `Bearer ${token}`,
'Content-Type': 'application/json',
},
});
if (!response.ok) {
throw new Error(`Crowdin API error: ${response.status}`);
}
type QAResponse = {
data: Array<{ data: QAIssue }>;
};
const data = (await response.json()) as QAResponse;
if (data.data.length === 0) break;
for (const item of data.data) {
issues.push(item.data);
}
if (data.data.length < limit) break;
offset += limit;
}
return issues;
}
async function getTranslation(
token: string,
stringId: number,
languageId: string,
): Promise<Translation | null> {
const url = `${CROWDIN_BASE_URL}/projects/${CROWDIN_PROJECT_ID}/languages/${languageId}/translations?stringIds=${stringId}`;
const response = await fetch(url, {
headers: {
Authorization: `Bearer ${token}`,
'Content-Type': 'application/json',
},
});
if (!response.ok) {
console.error(`Failed to get translation for string ${stringId}`);
return null;
}
type TransResponse = {
data: Array<{
data: Translation;
}>;
};
const data = (await response.json()) as TransResponse;
return data.data[0]?.data || null;
}
async function deleteTranslation(
token: string,
translationId: number,
): Promise<boolean> {
const url = `${CROWDIN_BASE_URL}/projects/${CROWDIN_PROJECT_ID}/translations/${translationId}`;
const response = await fetch(url, {
method: 'DELETE',
headers: {
Authorization: `Bearer ${token}`,
},
});
return response.ok || response.status === 404;
}
async function addTranslation(
token: string,
stringId: number,
languageId: string,
text: string,
): Promise<boolean> {
const url = `${CROWDIN_BASE_URL}/projects/${CROWDIN_PROJECT_ID}/translations`;
const response = await fetch(url, {
method: 'POST',
headers: {
Authorization: `Bearer ${token}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({ stringId, languageId, text }),
});
if (response.ok) return true;
const data = (await response.json()) as CrowdinErrorResponse;
const errorMsg = data?.errors?.[0]?.error?.errors?.[0]?.message ?? '';
// Identical translation already exists - that's fine
if (errorMsg.includes('identical')) return true;
console.error(`Failed to add translation: ${JSON.stringify(data)}`);
return false;
}
function fixHtmlTags(text: string): string {
let fixed = text;
// Fix bold: <strong>...</strong> or <b>...</b> → **...**
fixed = fixed.replace(/<strong>(.*?)<\/strong>/gi, '**$1**');
fixed = fixed.replace(/<b>(.*?)<\/b>/gi, '**$1**');
// Fix italic: <em>...</em> or <i>...</i> → *...*
fixed = fixed.replace(/<em>(.*?)<\/em>/gi, '*$1*');
fixed = fixed.replace(/<i>(.*?)<\/i>/gi, '*$1*');
// Fix code: <code>...</code> → `...`
fixed = fixed.replace(/<code>(.*?)<\/code>/gi, '`$1`');
return fixed;
}
function hasHtmlFormattingTags(text: string): boolean {
return /<(strong|b|em|i|code)[^>]*>/i.test(text);
}
async function main() {
const token = await getToken();
const issues = await fetchTagsQAIssues(token);
console.log(`Found ${issues.length} tags QA issues`);
if (issues.length === 0) {
console.log('No tags issues to fix!');
return;
}
// Group by stringId + languageId to avoid duplicates
const uniqueIssues = new Map<string, QAIssue>();
for (const issue of issues) {
const key = `${issue.stringId}-${issue.languageId}`;
uniqueIssues.set(key, issue);
}
console.log(`Processing ${uniqueIssues.size} unique string-language pairs...`);
let fixed = 0;
let skipped = 0;
let errors = 0;
for (const [key, issue] of uniqueIssues) {
const translation = await getTranslation(
token,
issue.stringId,
issue.languageId,
);
if (!translation) {
console.log(` [${key}] No translation found, skipping`);
skipped++;
continue;
}
if (!hasHtmlFormattingTags(translation.text)) {
console.log(` [${key}] No HTML formatting tags, skipping`);
skipped++;
continue;
}
const fixedText = fixHtmlTags(translation.text);
if (fixedText === translation.text) {
console.log(` [${key}] No changes needed, skipping`);
skipped++;
continue;
}
console.log(` [${key}] Fixing HTML tags...`);
// Delete old translation
const deleted = await deleteTranslation(token, translation.translationId);
if (!deleted) {
console.error(` [${key}] Failed to delete old translation`);
errors++;
continue;
}
// Add corrected translation
const added = await addTranslation(
token,
issue.stringId,
issue.languageId,
fixedText,
);
if (added) {
console.log(` [${key}] ✓ Fixed`);
fixed++;
} else {
errors++;
}
// Small delay to avoid rate limiting
await new Promise((resolve) => setTimeout(resolve, 100));
}
console.log(`
Done!
Fixed: ${fixed}
Skipped: ${skipped}
Errors: ${errors}
`);
}
main().catch((error) => {
console.error('Error:', error);
process.exit(1);
});

View File

@@ -0,0 +1,428 @@
/**
* Script to fix QA issues detected by Crowdin
*
* Fixes:
* - Variables mismatch (translated placeholder names)
* - Empty translations
* - Tags mismatch
*
* Usage:
* CROWDIN_PERSONAL_TOKEN=xxx npx ts-node packages/twenty-utils/fix-qa-issues.ts
*/
const CROWDIN_BASE_URL = 'https://twenty.api.crowdin.com/api/v2';
const CROWDIN_PROJECT_ID = 1;
type QACheck = {
stringId: number;
languageId: string;
category: string;
validation: string;
text: string;
};
async function getToken(): Promise<string> {
const token = process.env.CROWDIN_PERSONAL_TOKEN;
if (!token) {
console.error('Error: CROWDIN_PERSONAL_TOKEN not set');
process.exit(1);
}
return token;
}
async function crowdinGet<T>(endpoint: string, token: string): Promise<T> {
const response = await fetch(`${CROWDIN_BASE_URL}${endpoint}`, {
headers: { Authorization: `Bearer ${token}` },
});
if (!response.ok) {
throw new Error(`API error: ${response.status}`);
}
return response.json() as Promise<T>;
}
async function getSourceString(
token: string,
stringId: number,
): Promise<string> {
type Response = { data: { text: string } };
const data = await crowdinGet<Response>(
`/projects/${CROWDIN_PROJECT_ID}/strings/${stringId}`,
token,
);
return data.data.text;
}
async function getTranslation(
token: string,
stringId: number,
languageId: string,
): Promise<{ translationId: number; text: string } | null> {
type Response = {
data: Array<{ data: { translationId: number; text: string } }>;
};
const data = await crowdinGet<Response>(
`/projects/${CROWDIN_PROJECT_ID}/languages/${languageId}/translations?stringIds=${stringId}`,
token,
);
if (data.data.length === 0) return null;
return {
translationId: data.data[0].data.translationId,
text: data.data[0].data.text,
};
}
async function deleteTranslation(
token: string,
translationId: number,
): Promise<void> {
const response = await fetch(
`${CROWDIN_BASE_URL}/projects/${CROWDIN_PROJECT_ID}/translations/${translationId}`,
{
method: 'DELETE',
headers: { Authorization: `Bearer ${token}` },
},
);
if (!response.ok && response.status !== 404) {
throw new Error(`Delete failed: ${response.status}`);
}
}
async function addTranslation(
token: string,
stringId: number,
languageId: string,
text: string,
): Promise<void> {
const response = await fetch(
`${CROWDIN_BASE_URL}/projects/${CROWDIN_PROJECT_ID}/translations`,
{
method: 'POST',
headers: {
Authorization: `Bearer ${token}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({ stringId, languageId, text }),
},
);
// OK if added or if identical exists
if (!response.ok) {
const data = await response.json();
const msg = JSON.stringify(data);
if (!msg.includes('identical')) {
throw new Error(`Add failed: ${msg}`);
}
}
}
// Extract placeholder names from text (e.g., {days}, {count}, ${price})
function extractPlaceholders(text: string): string[] {
const matches = text.match(/\$?\{[a-zA-Z_][a-zA-Z0-9_]*\}/g) || [];
return [...new Set(matches)];
}
// Fix translated placeholder names back to source names
function fixPlaceholderNames(
sourceText: string,
translationText: string,
): string | null {
const sourcePlaceholders = extractPlaceholders(sourceText);
const translationPlaceholders = extractPlaceholders(translationText);
if (sourcePlaceholders.length === 0) return null;
// Check if any placeholders were translated
const missingInTranslation = sourcePlaceholders.filter(
(p) => !translationPlaceholders.includes(p),
);
if (missingInTranslation.length === 0) return null;
// Try to find translated versions and replace them
let fixedText = translationText;
for (const sourcePlaceholder of missingInTranslation) {
// Extract the name from source placeholder
const sourceNameMatch = sourcePlaceholder.match(/\$?\{([^}]+)\}/);
if (!sourceNameMatch) continue;
const sourceName = sourceNameMatch[1];
// Find potential translations of this placeholder
// Common patterns: {days} -> {jours}, {dae}, {dni}, {días}, etc.
for (const transPlaceholder of translationPlaceholders) {
const transNameMatch = transPlaceholder.match(/\$?\{([^}]+)\}/);
if (!transNameMatch) continue;
const transName = transNameMatch[1];
// If trans placeholder is not in source, it might be a translated version
if (
!sourcePlaceholders.includes(transPlaceholder) &&
transName !== sourceName
) {
// Check if this looks like a translation of the source name
// (same position in ICU structure, similar pattern)
// For ICU plural messages, check if the placeholder appears in same position
const sourcePattern = new RegExp(
`\\{${sourceName}\\}`,
'g',
);
const transPattern = new RegExp(`\\{${transName}\\}`, 'g');
const sourceMatches = sourceText.match(sourcePattern)?.length || 0;
const transMatches = translationText.match(transPattern)?.length || 0;
if (sourceMatches > 0 && transMatches > 0 && sourceMatches === transMatches) {
// Replace translated placeholder with source placeholder
fixedText = fixedText.replace(
new RegExp(`\\{${transName}\\}`, 'g'),
`{${sourceName}}`,
);
console.log(` Replacing {${transName}} -> {${sourceName}}`);
}
}
}
// Handle $ prefix for currency
if (sourcePlaceholder.startsWith('$')) {
const withoutDollar = sourcePlaceholder.slice(1);
if (translationText.includes(withoutDollar)) {
fixedText = fixedText.replace(withoutDollar, sourcePlaceholder);
console.log(` Adding $ prefix: ${withoutDollar} -> ${sourcePlaceholder}`);
}
}
}
// Return null if nothing changed
if (fixedText === translationText) return null;
return fixedText;
}
async function fetchQAChecks(
token: string,
category: string,
): Promise<QACheck[]> {
const checks: QACheck[] = [];
let offset = 0;
while (true) {
type Response = { data: Array<{ data: QACheck }> };
const data = await crowdinGet<Response>(
`/projects/${CROWDIN_PROJECT_ID}/qa-checks?limit=500&offset=${offset}&category=${category}`,
token,
);
if (data.data.length === 0) break;
for (const item of data.data) {
checks.push(item.data);
}
if (data.data.length < 500) break;
offset += 500;
}
return checks;
}
async function fixVariablesIssues(token: string): Promise<number> {
console.log('\n=== Fixing Variables Mismatch Issues ===\n');
const checks = await fetchQAChecks(token, 'variables');
console.log(`Found ${checks.length} variables issues\n`);
let fixed = 0;
for (const check of checks) {
console.log(
`String ${check.stringId} (${check.languageId}): ${check.text.slice(0, 80)}...`,
);
try {
const sourceText = await getSourceString(token, check.stringId);
const translation = await getTranslation(
token,
check.stringId,
check.languageId,
);
if (!translation) {
console.log(' -> No translation found, skipping\n');
continue;
}
const fixedText = fixPlaceholderNames(sourceText, translation.text);
if (!fixedText) {
console.log(' -> Could not auto-fix, manual review needed\n');
continue;
}
console.log(` Source: ${sourceText.slice(0, 60)}...`);
console.log(` Before: ${translation.text.slice(0, 60)}...`);
console.log(` After: ${fixedText.slice(0, 60)}...`);
// Delete old translation and add fixed one
await deleteTranslation(token, translation.translationId);
await addTranslation(token, check.stringId, check.languageId, fixedText);
console.log(' -> Fixed!\n');
fixed++;
} catch (error) {
console.log(` -> Error: ${error}\n`);
}
}
return fixed;
}
async function fixEmptyTranslations(token: string): Promise<number> {
console.log('\n=== Fixing Empty Translation Issues ===\n');
const checks = await fetchQAChecks(token, 'empty');
console.log(`Found ${checks.length} empty translation issues\n`);
// Empty translations should be deleted so they fall back to source
let fixed = 0;
for (const check of checks) {
console.log(`String ${check.stringId} (${check.languageId})`);
try {
const translation = await getTranslation(
token,
check.stringId,
check.languageId,
);
if (!translation) {
console.log(' -> No translation found\n');
continue;
}
if (translation.text.trim() === '') {
await deleteTranslation(token, translation.translationId);
console.log(' -> Deleted empty translation\n');
fixed++;
}
} catch (error) {
console.log(` -> Error: ${error}\n`);
}
}
return fixed;
}
async function fixTagsIssues(token: string): Promise<number> {
console.log('\n=== Fixing Tags Mismatch Issues ===\n');
const checks = await fetchQAChecks(token, 'tags');
console.log(`Found ${checks.length} tags issues\n`);
let fixed = 0;
for (const check of checks) {
console.log(`String ${check.stringId} (${check.languageId}): ${check.text.slice(0, 60)}...`);
try {
const sourceText = await getSourceString(token, check.stringId);
const translation = await getTranslation(
token,
check.stringId,
check.languageId,
);
if (!translation) {
console.log(' -> No translation found\n');
continue;
}
// Extract tags from source and translation
const sourceTags = sourceText.match(/<\/?[a-zA-Z][^>]*>/g) || [];
const translationTags = translation.text.match(/<\/?[a-zA-Z][^>]*>/g) || [];
// If translation has extra tags not in source, remove them
if (check.text.includes('extra formatting tags')) {
let fixedText = translation.text;
for (const tag of translationTags) {
if (!sourceTags.includes(tag)) {
// Remove extra tag
fixedText = fixedText.replace(new RegExp(tag.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g'), '');
}
}
// Clean up any double spaces
fixedText = fixedText.replace(/\s+/g, ' ').trim();
if (fixedText !== translation.text) {
console.log(` Source: ${sourceText.slice(0, 60)}`);
console.log(` Before: ${translation.text.slice(0, 60)}`);
console.log(` After: ${fixedText.slice(0, 60)}`);
await deleteTranslation(token, translation.translationId);
await addTranslation(token, check.stringId, check.languageId, fixedText);
console.log(' -> Fixed!\n');
fixed++;
} else {
console.log(' -> Could not auto-fix\n');
}
} else {
console.log(' -> Manual review needed\n');
}
} catch (error) {
console.log(` -> Error: ${error}\n`);
}
}
return fixed;
}
async function main() {
const token = await getToken();
console.log('Crowdin QA Issues Fixer\n');
console.log('This script will fix:');
console.log('- Variables mismatch (translated placeholder names)');
console.log('- Empty translations');
console.log('- Tags mismatch (extra formatting tags)\n');
let totalFixed = 0;
// Fix variables issues
totalFixed += await fixVariablesIssues(token);
// Fix empty translations
totalFixed += await fixEmptyTranslations(token);
// Fix tags issues
totalFixed += await fixTagsIssues(token);
console.log(`\n=== Done! Fixed ${totalFixed} issues ===`);
}
main().catch((error) => {
console.error('Error:', error);
process.exit(1);
});

View File

@@ -0,0 +1,234 @@
query IntrospectionQuery {
__schema {
queryType { name }
mutationType { name }
subscriptionType { name }
types {
kind
name
description
fields(includeDeprecated: true) {
name
description
args {
name
description
type {
kind
name
ofType {
kind
name
ofType {
kind
name
ofType {
kind
name
ofType {
kind
name
ofType {
kind
name
ofType {
kind
name
}
}
}
}
}
}
}
defaultValue
}
type {
kind
name
ofType {
kind
name
ofType {
kind
name
ofType {
kind
name
ofType {
kind
name
ofType {
kind
name
ofType {
kind
name
ofType {
kind
name
}
}
}
}
}
}
}
}
isDeprecated
deprecationReason
}
inputFields(includeDeprecated: true) {
name
description
type {
kind
name
ofType {
kind
name
ofType {
kind
name
ofType {
kind
name
ofType {
kind
name
ofType {
kind
name
ofType {
kind
name
ofType {
kind
name
}
}
}
}
}
}
}
}
defaultValue
isDeprecated
deprecationReason
}
interfaces {
kind
name
ofType {
kind
name
ofType {
kind
name
ofType {
kind
name
ofType {
kind
name
ofType {
kind
name
ofType {
kind
name
ofType {
kind
name
}
}
}
}
}
}
}
}
enumValues(includeDeprecated: true) {
name
description
isDeprecated
deprecationReason
}
possibleTypes {
kind
name
ofType {
kind
name
ofType {
kind
name
ofType {
kind
name
ofType {
kind
name
ofType {
kind
name
ofType {
kind
name
ofType {
kind
name
}
}
}
}
}
}
}
}
}
directives {
name
description
locations
args {
name
description
type {
kind
name
ofType {
kind
name
ofType {
kind
name
ofType {
kind
name
ofType {
kind
name
ofType {
kind
name
ofType {
kind
name
ofType {
kind
name
}
}
}
}
}
}
}
}
defaultValue
}
}
}
}

View File

@@ -0,0 +1,16 @@
const jestConfig = {
displayName: 'twenty-utils',
preset: '../../jest.preset.js',
testEnvironment: 'node',
testEnvironmentOptions: {},
transform: {
'^.+\\.[tj]s$': [
'@swc/jest',
{ jsc: { parser: { syntax: 'typescript' } } },
],
},
moduleFileExtensions: ['ts', 'js'],
coverageDirectory: './coverage',
};
export default jestConfig;

View File

@@ -0,0 +1,14 @@
{
"name": "twenty-utils",
"private": true,
"scripts": {
"nx": "NX_DEFAULT_PROJECT=twenty-front node ../../node_modules/nx/bin/nx.js",
"danger:ci": "danger ci --use-github-checks --failOnErrors",
"danger:congratulate": "danger ci --dangerfile ./congratulate-dangerfile.ts --use-github-checks --failOnErrors",
"release": "node release.js"
},
"devDependencies": {
"danger": "patch:danger@npm%3A13.0.8#~/.yarn/patches/danger-npm-13.0.8-48aba2788c.patch",
"danger-plugin-todos": "^1.3.1"
}
}

View File

@@ -0,0 +1,9 @@
{
"name": "twenty-utils",
"$schema": "../../node_modules/nx/schemas/project-schema.json",
"sourceRoot": "packages/twenty-utils",
"projectType": "application",
"targets": {
"test": {}
}
}

View File

@@ -0,0 +1,281 @@
#!/bin/bash
# =============================================================================
# Twenty CRM — Development Environment Setup
# =============================================================================
# Single entry point for setting up a dev environment. Idempotent.
#
# What it does:
# 1. Starts Postgres + Redis (local services or Docker, auto-detected)
# 2. Creates 'default' and 'test' databases
# 3. Copies .env.example -> .env for front and server
# 4. Initializes database schema (runs migrations) when not already present
#
# Usage (from repo root):
# bash packages/twenty-utils/setup-dev-env.sh # start + configure
# bash packages/twenty-utils/setup-dev-env.sh --down # stop services
# bash packages/twenty-utils/setup-dev-env.sh --reset # wipe data + restart
# bash packages/twenty-utils/setup-dev-env.sh --docker # force Docker mode
# =============================================================================
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
COMPOSE_FILE="$REPO_ROOT/packages/twenty-docker/docker-compose.dev.yml"
info() { echo "=> $*"; }
ok() { echo " done: $*"; }
fail() { echo " FAIL: $*" >&2; }
# --------------- detection helpers ---------------
has_local_pg() {
command -v pg_ctlcluster &>/dev/null && pg_lsclusters 2>/dev/null | grep -q "16"
}
has_local_redis() {
command -v redis-server &>/dev/null
}
can_use_docker() {
docker compose version &>/dev/null 2>&1
}
pg_is_up() {
if command -v pg_isready &>/dev/null; then
pg_isready -h localhost -p 5432 -U postgres -q 2>/dev/null
elif command -v psql &>/dev/null; then
PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -c "SELECT 1" &>/dev/null
elif can_use_docker && docker compose -f "$COMPOSE_FILE" ps --quiet db 2>/dev/null | grep -q .; then
docker compose -f "$COMPOSE_FILE" exec -T db pg_isready -U postgres -q 2>/dev/null
else
return 1
fi
}
redis_is_up() {
if command -v redis-cli &>/dev/null; then
redis-cli -h localhost -p 6379 ping 2>/dev/null | grep -q PONG
elif can_use_docker && docker compose -f "$COMPOSE_FILE" ps --quiet redis 2>/dev/null | grep -q .; then
docker compose -f "$COMPOSE_FILE" exec -T redis redis-cli ping 2>/dev/null | grep -q PONG
else
# Portable fallback using bash /dev/tcp (no nc -q dependency)
timeout 2 bash -c 'exec 3<>/dev/tcp/localhost/6379; echo PING >&3; read -r reply <&3; exec 3>&-; echo "$reply"' 2>/dev/null | grep -q PONG
fi
}
wait_for_pg() {
local retries=30
while ! pg_is_up; do
retries=$((retries - 1))
if [ "$retries" -le 0 ]; then fail "PostgreSQL did not start in time"; exit 1; fi
sleep 1
done
}
wait_for_redis() {
local retries=30
while ! redis_is_up; do
retries=$((retries - 1))
if [ "$retries" -le 0 ]; then fail "Redis did not start in time"; exit 1; fi
sleep 1
done
}
schema_exists() {
if command -v psql &>/dev/null; then
PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d default -t -c "SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname = 'core'" 2>/dev/null | grep -q 1
elif can_use_docker && docker compose -f "$COMPOSE_FILE" ps --quiet db 2>/dev/null | grep -q .; then
docker compose -f "$COMPOSE_FILE" exec -T db psql -U postgres -d default -t -c "SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname = 'core'" 2>/dev/null | grep -q 1
else
return 1
fi
}
# --------------- parse flags ---------------
USE_DOCKER=false
ACTION="up"
while [ $# -gt 0 ]; do
case "$1" in
--docker) USE_DOCKER=true ;;
--down) ACTION="down" ;;
--reset) ACTION="reset" ;;
*) echo "Unknown flag: $1"; exit 1 ;;
esac
shift
done
# --------------- stop ---------------
stop_docker() {
if can_use_docker && docker compose -f "$COMPOSE_FILE" ps -a --quiet 2>/dev/null | grep -q .; then
docker compose -f "$COMPOSE_FILE" down "$@"
fi
}
stop_local() {
if has_local_pg; then sudo pg_ctlcluster 16 main stop 2>/dev/null || true; fi
if has_local_redis && pgrep -x redis-server &>/dev/null; then
sudo service redis-server stop 2>/dev/null || true
fi
}
stop_services() {
if [ "$USE_DOCKER" = true ]; then
stop_docker "$@"
else
stop_docker "$@"
stop_local
fi
}
if [ "$ACTION" = "down" ]; then
info "Stopping dev services..."
stop_services
ok "Services stopped"
exit 0
fi
if [ "$ACTION" = "reset" ]; then
info "Resetting dev services (wiping data)..."
# Wipe local Redis data while it's still running
if [ "$USE_DOCKER" = false ] && has_local_redis && pgrep -x redis-server &>/dev/null; then
info "Flushing local Redis data..."
redis-cli flushall 2>/dev/null || true
fi
# Wipe local PostgreSQL data while it's still running
if [ "$USE_DOCKER" = false ] && has_local_pg; then
info "Dropping local databases..."
sudo pg_ctlcluster 16 main start 2>/dev/null || true
wait_for_pg
sudo -u postgres psql -c 'DROP DATABASE IF EXISTS "default";' 2>/dev/null || true
sudo -u postgres psql -c 'DROP DATABASE IF EXISTS "test";' 2>/dev/null || true
fi
# Stop Docker with -v to remove volumes
stop_docker -v 2>/dev/null || stop_docker
# Stop local services
if [ "$USE_DOCKER" = false ]; then
stop_local
fi
fi
# =============================================================================
# 1. Start services (auto-detect: local > Docker)
# =============================================================================
start_pg() {
if pg_is_up; then
ok "PostgreSQL already running"
return
fi
if [ "$USE_DOCKER" = false ] && has_local_pg; then
info "Starting local PostgreSQL..."
sudo pg_ctlcluster 16 main start
wait_for_pg
sudo -u postgres psql -c "ALTER USER postgres PASSWORD 'postgres';" 2>/dev/null || true
elif can_use_docker; then
info "Starting PostgreSQL via Docker..."
docker compose -f "$COMPOSE_FILE" up -d db
wait_for_pg
else
fail "No PostgreSQL available. Install PostgreSQL 16 or Docker."
exit 1
fi
}
start_redis() {
if redis_is_up; then
ok "Redis already running"
return
fi
if [ "$USE_DOCKER" = false ] && has_local_redis; then
info "Starting local Redis..."
sudo service redis-server start 2>/dev/null || redis-server --daemonize yes 2>/dev/null || true
wait_for_redis
elif can_use_docker; then
info "Starting Redis via Docker..."
docker compose -f "$COMPOSE_FILE" up -d redis
wait_for_redis
else
fail "No Redis available. Install Redis or Docker."
exit 1
fi
}
if [ "$USE_DOCKER" = true ]; then
info "Starting services via Docker Compose..."
docker compose -f "$COMPOSE_FILE" up -d
wait_for_pg
wait_for_redis
else
start_pg
start_redis
fi
ok "PostgreSQL on localhost:5432"
ok "Redis on localhost:6379"
# =============================================================================
# 2. Create databases
# =============================================================================
info "Creating databases..."
run_psql() {
if command -v psql &>/dev/null; then
PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d postgres -c "$1" 2>/dev/null || true
elif can_use_docker && docker compose -f "$COMPOSE_FILE" ps --quiet db 2>/dev/null | grep -q .; then
docker compose -f "$COMPOSE_FILE" exec -T db psql -U postgres -d postgres -c "$1" 2>/dev/null || true
else
fail "No psql client available and no Docker db container running"
return 1
fi
}
run_psql 'CREATE DATABASE "default";'
run_psql 'CREATE DATABASE "test";'
ok "Databases 'default' and 'test' ready"
# =============================================================================
# 3. Environment files (via Nx when available, fallback to cp)
# =============================================================================
info "Setting up .env files..."
cd "$REPO_ROOT"
if command -v npx &>/dev/null && [ -d node_modules ]; then
npx nx reset:env twenty-front
npx nx reset:env twenty-server
else
for pkg in twenty-front twenty-server; do
src="packages/$pkg/.env.example"
dst="packages/$pkg/.env"
if [ -f "$src" ] && [ ! -f "$dst" ]; then
cp "$src" "$dst"
ok "$pkg/.env created"
fi
done
fi
# =============================================================================
# 4. Initialize database schema
# =============================================================================
if command -v npx &>/dev/null && [ -d node_modules ]; then
if schema_exists; then
ok "Database schema already initialized"
else
info "Initializing database schema (running migrations)..."
if npx nx database:init twenty-server; then
ok "Database schema initialized"
else
fail "Database schema initialization failed"
exit 1
fi
fi
else
info "Run 'npx nx database:init twenty-server' to initialize the schema"
fi
# =============================================================================
echo ""
echo "Dev environment ready."
echo ""
echo " yarn start # start everything"
echo " npx nx start twenty-front # frontend -> http://localhost:3001"
echo " npx nx start twenty-server # backend -> http://localhost:3000"
echo ""

View File

@@ -0,0 +1,245 @@
/**
* Script to fetch and display QA issues from Crowdin
*
* This script uses Crowdin's native QA checks API to:
* - Fetch all QA issues detected by Crowdin
* - Group and display them by category and language
* - Provide actionable information for fixing
*
* Usage:
* CROWDIN_PERSONAL_TOKEN=xxx npx ts-node packages/twenty-utils/translation-qa-report.ts
*
* The token can be obtained from: https://twenty.crowdin.com/u/settings#api-key
*/
import * as fs from 'fs';
import * as path from 'path';
const CROWDIN_BASE_URL = 'https://twenty.api.crowdin.com/api/v2';
const CROWDIN_PROJECT_ID = 1;
type QACheck = {
stringId: number;
languageId: string;
category: string;
categoryDescription: string;
validation: string;
validationDescription: string;
pluralId: number;
text: string;
};
async function getToken(): Promise<string> {
const token = process.env.CROWDIN_PERSONAL_TOKEN;
if (!token) {
console.error(
'Error: CROWDIN_PERSONAL_TOKEN environment variable not set',
);
console.error(
'Get your token from: https://twenty.crowdin.com/u/settings#api-key',
);
process.exit(1);
}
return token;
}
async function fetchAllQAChecks(token: string): Promise<QACheck[]> {
const allChecks: QACheck[] = [];
let offset = 0;
const limit = 500;
console.log('Fetching QA issues from Crowdin...');
while (true) {
const url = `${CROWDIN_BASE_URL}/projects/${CROWDIN_PROJECT_ID}/qa-checks?limit=${limit}&offset=${offset}`;
const response = await fetch(url, {
headers: {
Authorization: `Bearer ${token}`,
'Content-Type': 'application/json',
},
});
if (!response.ok) {
throw new Error(`Crowdin API error: ${response.status}`);
}
type QAResponse = {
data: Array<{ data: QACheck }>;
};
const data = (await response.json()) as QAResponse;
if (data.data.length === 0) break;
for (const item of data.data) {
allChecks.push(item.data);
}
console.log(` Fetched ${allChecks.length} issues...`);
if (data.data.length < limit) break;
offset += limit;
}
return allChecks;
}
function generateReport(checks: QACheck[]): string {
// Group by category
const byCategory = new Map<string, QACheck[]>();
for (const check of checks) {
const key = check.category;
const existing = byCategory.get(key) || [];
existing.push(check);
byCategory.set(key, existing);
}
// Group by language within each category
let report = `# Crowdin QA Issues Report
Generated: ${new Date().toISOString()}
## Summary
- **Total QA Issues**: ${checks.length}
- **Categories**: ${byCategory.size}
| Category | Count | Description |
|----------|-------|-------------|
`;
for (const [category, categoryChecks] of byCategory) {
const desc = categoryChecks[0]?.categoryDescription || category;
report += `| ${category} | ${categoryChecks.length} | ${desc} |\n`;
}
report += `
## Issues by Category
`;
// Sort categories by count (most issues first), but put spellcheck last
const sortedCategories = Array.from(byCategory.entries()).sort((a, b) => {
if (a[0] === 'spellcheck') return 1;
if (b[0] === 'spellcheck') return -1;
return b[1].length - a[1].length;
});
for (const [category, categoryChecks] of sortedCategories) {
const desc = categoryChecks[0]?.categoryDescription || category;
// Group by language
const byLang = new Map<string, QACheck[]>();
for (const check of categoryChecks) {
const existing = byLang.get(check.languageId) || [];
existing.push(check);
byLang.set(check.languageId, existing);
}
report += `### ${desc} (${categoryChecks.length} issues)
`;
// Show top issues per language
const sortedLangs = Array.from(byLang.entries()).sort(
(a, b) => b[1].length - a[1].length,
);
for (const [lang, langChecks] of sortedLangs.slice(0, 10)) {
report += `**${lang}** (${langChecks.length} issues):\n`;
for (const check of langChecks.slice(0, 5)) {
const truncatedText =
check.text.length > 100 ? check.text.slice(0, 100) + '...' : check.text;
report += `- String #${check.stringId}: ${truncatedText}\n`;
}
if (langChecks.length > 5) {
report += `- ... and ${langChecks.length - 5} more\n`;
}
report += '\n';
}
if (sortedLangs.length > 10) {
report += `*... and ${sortedLangs.length - 10} more languages with issues*\n\n`;
}
}
report += `## How to Fix
### Variables Mismatch
These are the most critical - placeholders must match exactly:
- Check that all \`{placeholder}\` variables from source appear in translation
- Don't translate placeholder names (e.g., \`{days}\` should stay \`{days}\`, not \`{jours}\`)
- Include \`$\` for currency placeholders (e.g., \`\${price}\`)
### Punctuation/Special Characters
- Ensure ending punctuation matches (periods, question marks, etc.)
- Check parentheses and brackets are balanced
### Spellcheck
- Often false positives for technical terms - can usually be ignored
- Review for actual typos
---
**Fix in Crowdin**: https://twenty.crowdin.com/u/projects/1/all?filter=qa-issue
*To auto-fix encoding issues, run:*
\`\`\`
CROWDIN_PERSONAL_TOKEN=xxx npx tsx packages/twenty-utils/crowdin-normalizer/normalize-crowdin-translations.ts --project=1 --apply --rules=escaped-unicode
\`\`\`
`;
return report;
}
async function main() {
const token = await getToken();
const checks = await fetchAllQAChecks(token);
console.log(`\nTotal QA issues: ${checks.length}`);
// Group by category for summary
const byCategory = new Map<string, number>();
for (const check of checks) {
byCategory.set(check.category, (byCategory.get(check.category) || 0) + 1);
}
console.log('\nBy category:');
for (const [cat, count] of byCategory) {
console.log(` ${cat}: ${count}`);
}
// Generate report
const report = generateReport(checks);
const reportPath = path.join(process.cwd(), 'TRANSLATION_QA_REPORT.md');
fs.writeFileSync(reportPath, report, 'utf-8');
console.log(`\nReport written to: ${reportPath}`);
// Return non-zero if there are critical issues (not spellcheck)
const criticalIssues = checks.filter((c) => c.category !== 'spellcheck');
if (criticalIssues.length > 0) {
console.log(`\n⚠️ ${criticalIssues.length} critical issues found (excluding spellcheck)`);
process.exit(1);
}
}
main().catch((error) => {
console.error('Error:', error);
process.exit(1);
});