upstream-twenty/packages/twenty-server/scripts/ai-catalog-sync/index.ts

import * as fs from 'fs';
import * as path from 'path';
import * as prettier from 'prettier';

import { isDefined, isNonEmptyArray } from 'twenty-shared/utils';

import { MODELS_DEV_API_URL } from 'src/engine/metadata-modules/ai/ai-models/constants/models-dev.const';
import { type ModelsDevData } from 'src/engine/metadata-modules/ai/ai-models/types/models-dev-data.type';

import { assertPayloadIsUsable } from './utils/assert-payload-is-usable.util';
import { buildCatalog } from './utils/build-catalog.util';
import { carryOverCommittedFields } from './utils/carry-over-committed-fields.util';
import { enrichCatalog } from './utils/enrich-catalog.util';
import { fetchArtificialAnalysisBenchmarks } from './utils/fetch-artificial-analysis-benchmarks.util';
import { projectCatalog } from './utils/project-catalog.util';
import { readCommittedBenchmarks } from './utils/read-committed-benchmarks.util';
import { buildCoverageReport } from './utils/build-coverage-report.util';
import { renderCoverageReport } from './utils/render-coverage-report.util';
import { type BenchmarkIndex } from './types/benchmark-index.type';
import { type CatalogSpec } from './types/catalog-spec.type';
import { type GeneratedCatalog } from './types/generated-catalog.type';

const AI_MODELS_DIR = path.resolve(
  __dirname,
  '..',
  '..',
  'src',
  'engine',
  'metadata-modules',
  'ai',
  'ai-models',
);

const MODELS_PATH = path.join(AI_MODELS_DIR, 'ai-models.json');
const SELF_HOST_SPEC_PATH = path.join(AI_MODELS_DIR, 'ai-self-host-spec.json');
const CATALOG_PATH = path.join(AI_MODELS_DIR, 'ai-providers.json');
const BENCHMARKS_PATH = path.join(AI_MODELS_DIR, 'ai-model-benchmarks.json');

// oxlint-disable no-console
const log = console.log;
const warn = console.warn;
// oxlint-enable no-console

const readArgument = (flag: string): string | undefined => {
  const index = process.argv.indexOf(flag);

  return index === -1 ? undefined : process.argv[index + 1];
};

const readCommittedModels = (filePath: string): GeneratedCatalog =>
  fs.existsSync(filePath)
    ? (JSON.parse(fs.readFileSync(filePath, 'utf-8')) as GeneratedCatalog)
    : {};

// The vendors the shipped catalog carries are the ones the self-host spec
// serves; nothing else needs to be fetched, checked or written.
const readVendors = (spec: CatalogSpec): string[] => {
  const vendors = [
    ...new Set(
      spec.providers.flatMap((provider) =>
        Array.isArray(provider.models) ? [] : [provider.models.vendor],
      ),
    ),
  ];

  // An empty list would fetch nothing, assert nothing and write an empty
  // catalog over the real one, and the sync PR automerges.
  if (!isNonEmptyArray(vendors)) {
    throw new Error(
      `${SELF_HOST_SPEC_PATH} names no vendor to carry: every route lists models explicitly`,
    );
  }

  return vendors;
};

const fetchModelsDev = async ({
  vendors,
}: {
  vendors: string[];
}): Promise<ModelsDevData> => {
  const response = await fetch(MODELS_DEV_API_URL, {
    signal: AbortSignal.timeout(120000),
  });

  if (!response.ok) {
    throw new Error(`${response.status} ${response.statusText}`);
  }

  const data: ModelsDevData = await response.json();

  assertPayloadIsUsable({ data, vendors });

  return data;
};

// A leaderboard outage must never break the model catalog, so a failed fetch
// degrades to an empty index and the catalog is written without benchmarks.
const fetchBenchmarks = async (): Promise<BenchmarkIndex> => {
  const apiKey = process.env.ARTIFICIAL_ANALYSIS_API_KEY;

  if (!isDefined(apiKey) || apiKey.length === 0) {
    warn('Skipping benchmarks: ARTIFICIAL_ANALYSIS_API_KEY is not set');

    return new Map();
  }

  try {
    const index = await fetchArtificialAnalysisBenchmarks(apiKey);

    log(`Fetched ${index.size} model aliases from Artificial Analysis`);

    return index;
  } catch (error) {
    warn(
      `Skipping benchmarks: ${error instanceof Error ? error.message : String(error)}`,
    );

    return new Map();
  }
};

const writeJson = async (filePath: string, value: unknown): Promise<void> => {
  const prettierConfig = await prettier.resolveConfig(filePath);

  const formatted = await prettier.format(
    JSON.stringify(value, null, 2) + '\n',
    { ...prettierConfig, filepath: filePath },
  );

  fs.writeFileSync(filePath, formatted, 'utf-8');
  log(`Wrote ${filePath}`);
};

const main = async (): Promise<void> => {
  const dryRun = process.argv.includes('--dry-run');
  const reportPath = readArgument('--report');
  const measuredAt = new Date().toISOString().slice(0, 10);

  const selfHostSpec = JSON.parse(
    fs.readFileSync(SELF_HOST_SPEC_PATH, 'utf-8'),
  ) as CatalogSpec;
  const vendors = readVendors(selfHostSpec);

  log('Fetching models.dev API...');

  const modelsDevData = await fetchModelsDev({ vendors });

  log(`Fetched ${Object.keys(modelsDevData).length} providers from models.dev`);

  const fetched = await fetchBenchmarks();

  // A failed fetch must not delete measurements we already published: the
  // catalog PR is automerged, so an empty index would silently strip every
  // benchmark until the next healthy run.
  const benchmarkIndex =
    fetched.size > 0 ? fetched : readCommittedBenchmarks(BENCHMARKS_PATH);

  if (fetched.size === 0 && benchmarkIndex.size > 0) {
    warn(
      `Reusing ${benchmarkIndex.size} committed benchmark aliases from a previous run`,
    );
  }

  const catalog = buildCatalog({ data: modelsDevData, vendors });

  carryOverCommittedFields({
    catalog,
    committedCatalog: readCommittedModels(MODELS_PATH),
  });

  const overlay = enrichCatalog({
    catalog,
    modelsDevData,
    benchmarkIndex,
    measuredAt,
  });

  const renderedReport = renderCoverageReport(buildCoverageReport(catalog));

  log(`\n${renderedReport}\n`);

  if (isDefined(reportPath)) {
    fs.writeFileSync(reportPath, `${renderedReport}\n`, 'utf-8');
  }

  if (dryRun) {
    log('[DRY RUN] Skipping writes');

    return;
  }

  await writeJson(MODELS_PATH, catalog);
  // Self-host runs the same projection cloud does, from a spec that names the
  // five direct routes and lets each serve its whole vendor.
  await writeJson(
    CATALOG_PATH,
    projectCatalog({ canonicalCatalog: catalog, spec: selfHostSpec }),
  );
  await writeJson(BENCHMARKS_PATH, {
    source: 'artificialanalysis.ai',
    measuredAt,
    models: overlay,
  });
};

main().catch((error) => {
  // oxlint-disable-next-line no-console
  console.error('AI catalog sync failed:', error);
  process.exit(1);
});