upstream-univer/packages/ui/scripts/generate-emojis.mts

import type { PathLike } from 'node:fs';
import { spawnSync } from 'node:child_process';
import fs from 'node:fs';
import path from 'node:path';
import process from 'node:process';
import { fileURLToPath } from 'node:url';
import { discoverUniverUiLocales } from '@univerjs-infra/shared/locale';

interface IEmojiVariant {
    emoji: string;
    title: string;
}

interface IEmojiItem extends IEmojiVariant {
    skinToneVariants?: IEmojiVariant[];
}

interface ICldrAnnotationsData {
    annotations?: {
        annotations?: Record<string, {
            default?: string[];
            tts?: string[];
        }>;
    };
    annotationsDerived?: {
        annotations?: Record<string, {
            default?: string[];
            tts?: string[];
        }>;
    };
}

interface IGeneratedEmojiLocale {
    emojiSearchIndex: Record<string, string>;
    emojiTitles: Record<string, string>;
}

type EmojiCategory = 'people' | 'nature' | 'foods' | 'activity' | 'places' | 'objects' | 'symbols';
type EmojiGroups = Record<'frequent' | EmojiCategory, IEmojiItem[]>;
type UnicodeEmojiGroup =
    | 'Activities'
    | 'Animals & Nature'
    | 'Flags'
    | 'Food & Drink'
    | 'Objects'
    | 'People & Body'
    | 'Smileys & Emotion'
    | 'Symbols'
    | 'Travel & Places';

const __dirname = path.dirname(fileURLToPath(import.meta.url));
const root = path.resolve(__dirname, '../../..');
const outputPath = path.resolve(__dirname, '../src/views/emoji-picker/emojis.generated.ts');
const emojiLocaleOutputDir = path.resolve(__dirname, '../src/locale/emoji-locale');
const unicodeEmojiTestUrl = 'https://www.unicode.org/Public/17.0.0/emoji/emoji-test.txt';
const cldrJsonVersion = '48.2.0';
const cldrJsonBaseUrl = `https://cdn.jsdelivr.net/gh/unicode-org/cldr-json@${cldrJsonVersion}/cldr-json`;
const cldrJsonDownloadConcurrency = 4;
const downloadMaxAttempts = 3;
const downloadRetryBaseDelayMs = 500;
const downloadTimeoutMs = 30_000;
const envProxyNodeArg = '--use-env-proxy';
const envProxyRelaunchEnv = 'UNIVER_UI_EMOJI_ENV_PROXY_REEXEC';
const emojiCategories: EmojiCategory[] = ['people', 'nature', 'foods', 'activity', 'places', 'objects', 'symbols'];
const emojiSkinToneModifierRegex = /[\u{1F3FB}-\u{1F3FF}]/u;
const emojiSkinToneNames = new Set([
    'Light Skin Tone',
    'Medium-light Skin Tone',
    'Medium Skin Tone',
    'Medium-dark Skin Tone',
    'Dark Skin Tone',
]);
const cldrLocaleMap: Record<string, string> = {
    'ar-SA': 'ar',
    'ca-ES': 'ca',
    'de-DE': 'de',
    'en-US': 'en',
    'es-ES': 'es',
    'fa-IR': 'fa',
    'fr-FR': 'fr',
    'id-ID': 'id',
    'it-IT': 'it',
    'ja-JP': 'ja',
    'ko-KR': 'ko',
    'pl-PL': 'pl',
    'pt-BR': 'pt',
    'ru-RU': 'ru',
    'sk-SK': 'sk',
    'vi-VN': 'vi',
    'zh-CN': 'zh',
    'zh-HK': 'zh-Hant-HK',
    'zh-TW': 'zh-Hant',
};
const localeFiles = discoverUniverUiLocales({ rootDir: root }).map((locale) => {
    const cldrLocale = cldrLocaleMap[locale];
    if (!cldrLocale) {
        throw new Error(`Missing CLDR locale mapping for ${locale}`);
    }

    return [locale, cldrLocale] as const;
});
const frequentEmojis = [
    '👍',
    '😀',
    '😘',
    '😍',
    '😆',
    '😜',
    '😅',
    '😂',
    '😱',
    '😞',
    '😒',
];

const groupCategoryMap: Partial<Record<UnicodeEmojiGroup, EmojiCategory>> = {
    Activities: 'activity',
    'Animals & Nature': 'nature',
    'Food & Drink': 'foods',
    Objects: 'objects',
    'People & Body': 'people',
    'Smileys & Emotion': 'people',
    Symbols: 'symbols',
    'Travel & Places': 'places',
};

export function buildEmojisFromEmojiTest(emojiTest: string, options: { supportedEmojis?: Set<string> } = {}): EmojiGroups {
    const groups = createEmptyEmojiGroups();
    let currentGroup: UnicodeEmojiGroup | null = null;
    const titleByEmoji = new Map<string, string>();

    emojiTest.split(/\r?\n/).forEach((line) => {
        const group = line.match(/^# group: (.+)$/);
        if (group) {
            currentGroup = group[1] as UnicodeEmojiGroup;
            return;
        }

        const category = currentGroup ? groupCategoryMap[currentGroup] : undefined;
        if (!category) {
            return;
        }

        const item = parseEmojiTestLine(line);
        if (!item || (options.supportedEmojis && !options.supportedEmojis.has(item.emoji))) {
            return;
        }

        titleByEmoji.set(item.emoji, item.title);
        groups[category].push(item);
    });

    groups.frequent = frequentEmojis
        .map((emoji) => {
            const title = titleByEmoji.get(emoji);
            return title ? { emoji, title } : null;
        })
        .filter(isEmojiItem);

    emojiCategories.forEach((category) => {
        groups[category] = groupSkinToneVariants(groups[category]);
    });

    return groups;
}

export function groupSkinToneVariants(items: IEmojiItem[]): IEmojiItem[] {
    const baseItemsByTitle = new Map(
        items
            .filter((item) => !hasSkinToneModifier(item.emoji))
            .map((item) => [item.title, item])
    );
    const variantsByBaseEmoji = new Map<string, IEmojiVariant[]>();
    const groupedVariants = new Set<string>();

    items.forEach((item) => {
        if (!hasSkinToneModifier(item.emoji)) {
            return;
        }

        const familyTitle = getSkinToneFamilyTitle(item.title, baseItemsByTitle);
        const baseItem = baseItemsByTitle.get(familyTitle);
        if (!baseItem) {
            return;
        }

        const variants = variantsByBaseEmoji.get(baseItem.emoji) ?? [];
        variants.push({ emoji: item.emoji, title: item.title });
        variantsByBaseEmoji.set(baseItem.emoji, variants);
        groupedVariants.add(item.emoji);
    });

    return items
        .filter((item) => !groupedVariants.has(item.emoji))
        .map((item) => {
            const skinToneVariants = variantsByBaseEmoji.get(item.emoji);
            return skinToneVariants?.length ? { ...item, skinToneVariants } : item;
        });
}

export function buildEmojiSearchIndexFromCldrAnnotations(emojis: Array<string | IEmojiItem>, ...sources: ICldrAnnotationsData[]): IGeneratedEmojiLocale {
    const emojiTitles: Record<string, string> = {};
    const emojiSearchIndex: Record<string, string> = {};

    emojis.forEach((entry) => {
        const emoji = typeof entry === 'string' ? entry : entry.emoji;
        const fallbackTitle = typeof entry === 'string' ? undefined : entry.title;
        const annotations = sources
            .map((source) => getCldrAnnotation(source, emoji))
            .filter((annotation): annotation is NonNullable<ReturnType<typeof getCldrAnnotation>> => Boolean(annotation));

        const title = annotations.find((annotation) => annotation.tts?.[0])?.tts?.[0] ?? fallbackTitle;
        const keywords = [...new Set(annotations.flatMap((annotation) => [
            ...(annotation.tts ?? []),
            ...(annotation.default ?? []),
        ]).filter(Boolean))];
        if (!keywords.length && fallbackTitle) {
            keywords.push(fallbackTitle);
        }

        if (title) {
            emojiTitles[emoji] = title;
        }
        if (keywords.length) {
            emojiSearchIndex[emoji] = keywords.join(' ');
        }
    });

    return {
        emojiSearchIndex,
        emojiTitles,
    };
}

function createEmptyEmojiGroups(): EmojiGroups {
    return Object.fromEntries([
        ['frequent', []],
        ...emojiCategories.map((category) => [category, []]),
    ]) as EmojiGroups;
}

function parseEmojiTestLine(line: string): IEmojiItem | null {
    const match = line.match(/^\S(?:.+?)\s+;\s+fully-qualified\s+#\s+(\S+)\s+E[\d.]+\s+(.+)$/);
    if (!match) {
        return null;
    }

    return {
        emoji: match[1],
        title: toTitleCase(match[2]),
    };
}

function isEmojiItem(item: IEmojiItem | null): item is IEmojiItem {
    return item !== null;
}

function getCldrAnnotation(source: ICldrAnnotationsData, emoji: string) {
    const normalizedEmoji = stripEmojiVariationSelectors(emoji);
    const annotations = source.annotations?.annotations ?? source.annotationsDerived?.annotations;
    return annotations?.[emoji] ?? annotations?.[normalizedEmoji];
}

function stripEmojiVariationSelectors(emoji: string): string {
    return emoji.replace(/\uFE0F/g, '');
}

function hasSkinToneModifier(emoji: string): boolean {
    return emojiSkinToneModifierRegex.test(emoji);
}

function getSkinToneFamilyTitle(title: string, baseItemsByTitle: Map<string, IEmojiItem>): string {
    const separatorIndex = title.indexOf(': ');
    if (separatorIndex < 0) {
        return title;
    }

    const baseTitle = title.slice(0, separatorIndex);
    const descriptors = title
        .slice(separatorIndex + 2)
        .split(', ')
        .filter((descriptor) => !emojiSkinToneNames.has(descriptor));
    const familyTitle = descriptors.length ? `${baseTitle}: ${descriptors.join(', ')}` : baseTitle;

    // Unicode uses neutral person ZWJ sequences for mixed-tone variants while the unmodified
    // kiss/couple emoji is a single code point with the shorter base title.
    if (familyTitle.endsWith(': Person, Person') && baseItemsByTitle.has(baseTitle)) {
        return baseTitle;
    }

    return familyTitle;
}

function toTitleCase(value: string): string {
    return value
        .split(' ')
        .map((word) => word ? `${word[0].toUpperCase()}${word.slice(1)}` : word)
        .join(' ');
}

interface IShouldRelaunchWithEnvProxyOptions {
    allowedFlags: ReadonlySet<string>;
    env: NodeJS.ProcessEnv | Record<string, string | undefined>;
    execArgv: readonly string[];
}

export function shouldRelaunchWithEnvProxy(options: IShouldRelaunchWithEnvProxyOptions): boolean {
    const proxyConfigured = [
        'HTTP_PROXY',
        'HTTPS_PROXY',
        'ALL_PROXY',
        'http_proxy',
        'https_proxy',
        'all_proxy',
    ].some((name) => Boolean(options.env[name]));

    if (!proxyConfigured || options.env[envProxyRelaunchEnv] || !options.allowedFlags.has(envProxyNodeArg)) {
        return false;
    }

    return !options.execArgv.includes(envProxyNodeArg) && !options.env.NODE_OPTIONS?.split(/\s+/).includes(envProxyNodeArg);
}

function relaunchWithEnvProxyIfNeeded(): boolean {
    if (!shouldRelaunchWithEnvProxy({
        allowedFlags: process.allowedNodeEnvironmentFlags,
        env: process.env,
        execArgv: process.execArgv,
    })) {
        return false;
    }

    const result = spawnSync(process.execPath, [
        envProxyNodeArg,
        ...process.execArgv,
        ...process.argv.slice(1),
    ], {
        env: {
            ...process.env,
            [envProxyRelaunchEnv]: '1',
        },
        stdio: 'inherit',
    });

    if (result.error) {
        throw result.error;
    }

    process.exitCode = result.status ?? 1;
    return true;
}

function getCldrAnnotationsUrl(cldrLocale: string, type: 'annotations' | 'annotationsDerived'): string {
    if (type === 'annotations') {
        return `${cldrJsonBaseUrl}/cldr-annotations-full/annotations/${cldrLocale}/annotations.json`;
    }

    return `${cldrJsonBaseUrl}/cldr-annotations-derived-full/annotationsDerived/${cldrLocale}/annotations.json`;
}

export function getEmojiSourceUrls(): string[] {
    return [
        unicodeEmojiTestUrl,
        ...localeFiles.flatMap(([, cldrLocale]) => [
            getCldrAnnotationsUrl(cldrLocale, 'annotations'),
            getCldrAnnotationsUrl(cldrLocale, 'annotationsDerived'),
        ]),
    ];
}

function isRetryableDownloadStatus(status: number): boolean {
    return status === 408 || status === 429 || status >= 500;
}

async function waitForDownloadRetry(url: string, attempt: number, reason: unknown): Promise<void> {
    const delayMs = downloadRetryBaseDelayMs * 2 ** (attempt - 1);
    const message = reason instanceof Error ? reason.message : String(reason);
    console.warn(`Failed to download ${url}: ${message}. Retrying in ${delayMs}ms (${attempt}/${downloadMaxAttempts - 1}).`);
    await new Promise<void>((resolve) => setTimeout(resolve, delayMs));
}

async function downloadText(url: string): Promise<string> {
    for (let attempt = 1; attempt <= downloadMaxAttempts; attempt += 1) {
        let response: Response;
        try {
            response = await fetch(url, {
                signal: AbortSignal.timeout(downloadTimeoutMs),
            });
        } catch (error) {
            if (attempt === downloadMaxAttempts) {
                throw new Error(`Failed to download ${url} after ${downloadMaxAttempts} attempts`, { cause: error });
            }

            await waitForDownloadRetry(url, attempt, error);
            continue;
        }

        if (response.ok) {
            return response.text();
        }

        const error = new Error(`${response.status} ${response.statusText}`);
        if (!isRetryableDownloadStatus(response.status) || attempt === downloadMaxAttempts) {
            throw new Error(`Failed to download ${url}: ${error.message}`);
        }

        await waitForDownloadRetry(url, attempt, error);
    }

    throw new Error(`Failed to download ${url}`);
}

async function downloadUnicodeEmojiTest(): Promise<string> {
    return downloadText(unicodeEmojiTestUrl);
}

async function downloadJson<T>(url: string): Promise<T> {
    return JSON.parse(await downloadText(url)) as T;
}

export async function mapWithConcurrencyLimit<T, R>(
    values: readonly T[],
    limit: number,
    mapper: (value: T, index: number) => Promise<R>
): Promise<R[]> {
    const results = new Array<R>(values.length);
    let nextIndex = 0;
    const workerCount = Math.min(Math.max(1, limit), values.length);

    await Promise.all(Array.from({ length: workerCount }, async () => {
        while (nextIndex < values.length) {
            const currentIndex = nextIndex;
            nextIndex += 1;
            results[currentIndex] = await mapper(values[currentIndex], currentIndex);
        }
    }));

    return results;
}

function serializeEmojis(groups: EmojiGroups): string {
    return [
        '/* eslint-disable */',
        '// Generated by scripts/generate-emojis.mts. Do not edit manually.',
        '',
        `export const emojis = ${JSON.stringify(groups, null, 4)};`,
        '',
    ].join('\n');
}

function writeGeneratedEmojis(filePath: PathLike, groups: EmojiGroups): void {
    fs.mkdirSync(path.dirname(filePath.toString()), { recursive: true });
    fs.writeFileSync(filePath, serializeEmojis(groups));
}

function serializeEmojiLocale(data: IGeneratedEmojiLocale): string {
    return [
        '/* eslint-disable */',
        '// Generated by scripts/generate-emojis.mts. Do not edit manually.',
        '',
        `const emojiLocale: { emojiSearchIndex: Record<string, string>; emojiTitles: Record<string, string> } = ${JSON.stringify(data, null, 4)};`,
        '',
        'export default emojiLocale;',
        '',
    ].join('\n');
}

function writeGeneratedEmojiLocale(locale: string, data: IGeneratedEmojiLocale): void {
    fs.mkdirSync(emojiLocaleOutputDir, { recursive: true });
    fs.writeFileSync(path.resolve(emojiLocaleOutputDir, `${locale}.generated.ts`), serializeEmojiLocale(data));
}

async function generate(): Promise<void> {
    const emojiTest = await downloadUnicodeEmojiTest();
    const emojis = buildEmojisFromEmojiTest(emojiTest);
    const allEmojis = [
        ...new Map(
            Object.values(emojis)
                .flat()
                .flatMap((item) => [item, ...(item.skinToneVariants ?? [])])
                .map((item) => [item.emoji, item])
        ).values(),
    ];

    writeGeneratedEmojis(outputPath, emojis);

    await mapWithConcurrencyLimit(localeFiles, cldrJsonDownloadConcurrency, async ([locale, cldrLocale]) => {
        const annotations = await downloadJson<ICldrAnnotationsData>(getCldrAnnotationsUrl(cldrLocale, 'annotations'));
        const annotationsDerived = await downloadJson<ICldrAnnotationsData>(getCldrAnnotationsUrl(cldrLocale, 'annotationsDerived'));

        writeGeneratedEmojiLocale(locale, buildEmojiSearchIndexFromCldrAnnotations(allEmojis, annotations, annotationsDerived));
    });
}

if (process.argv[1] === fileURLToPath(import.meta.url)) {
    const task = relaunchWithEnvProxyIfNeeded()
        ? Promise.resolve()
        : generate();

    task.catch((error) => {
        console.error(error);
        process.exitCode = 1;
    });
}