snapshot of https://github.com/twentyhq/twenty.git (main, 2026-09-17; blobs > 131072B omitted)

packages/twenty-server/scripts/ai-catalog-sync/utils/assert-payload-is-usable.util.ts

@@ -0,0 +1,21 @@
+import { type ModelsDevData } from 'src/engine/metadata-modules/ai/ai-models/types/models-dev-data.type';
+
+// A 200 carrying an empty or reshaped payload would otherwise generate an
+// empty catalog, and the sync PR automerges over the real one.
+export const assertPayloadIsUsable = ({
+  data,
+  vendors,
+}: {
+  data: ModelsDevData;
+  vendors: string[];
+}): void => {
+  const missing = vendors.filter(
+    (vendor) => Object.keys(data[vendor]?.models ?? {}).length === 0,
+  );
+
+  if (missing.length > 0) {
+    throw new Error(
+      `models.dev payload is missing models for ${missing.join(', ')}; refusing to overwrite the catalog`,
+    );
+  }
+};

packages/twenty-server/scripts/ai-catalog-sync/utils/build-catalog.util.ts

@@ -0,0 +1,164 @@
+import { isDefined } from 'twenty-shared/utils';
+
+import { type ModelsDevData } from 'src/engine/metadata-modules/ai/ai-models/types/models-dev-data.type';
+import { type ModelsDevModel } from 'src/engine/metadata-modules/ai/ai-models/types/models-dev-model.type';
+import { inferModelFamily } from 'src/engine/metadata-modules/ai/ai-models/utils/infer-model-family.util';
+
+import { type GeneratedCatalog } from '../types/generated-catalog.type';
+import { type GeneratedModel } from '../types/generated-model.type';
+
+const EXCLUDED_MODEL_PREFIXES = [
+  'text-embedding',
+  'embedding',
+  'dall-e',
+  'tts-',
+  'whisper',
+  'moderation',
+  'davinci',
+  'babbage',
+  'ada',
+  'curie',
+  'text-search',
+  'text-similarity',
+  'code-search',
+  'text-davinci',
+  'text-curie',
+  'text-babbage',
+  'text-ada',
+  'ft:',
+  'canary',
+];
+
+const EXCLUDED_MODEL_SUFFIXES = ['-audio-preview', '-realtime-preview'];
+
+const LONG_CONTEXT_THRESHOLD_TOKENS = 200000;
+
+const isLanguageModel = (modelId: string): boolean => {
+  const lowerId = modelId.toLowerCase();
+
+  if (EXCLUDED_MODEL_PREFIXES.some((prefix) => lowerId.startsWith(prefix))) {
+    return false;
+  }
+
+  return !EXCLUDED_MODEL_SUFFIXES.some((suffix) => lowerId.endsWith(suffix));
+};
+
+const meetsInclusionCriteria = (modelData: ModelsDevModel): boolean =>
+  modelData.status !== 'beta' &&
+  (modelData.tool_call ?? false) &&
+  isDefined(modelData.cost?.input) &&
+  isDefined(modelData.limit?.context);
+
+const extractCost = ({
+  modelData,
+  model,
+}: {
+  modelData: ModelsDevModel;
+  model: GeneratedModel;
+}): void => {
+  const cost = modelData.cost;
+
+  if (!isDefined(cost)) {
+    return;
+  }
+
+  model.inputCostPerMillionTokens = cost.input;
+  model.outputCostPerMillionTokens = cost.output;
+  model.cachedInputCostPerMillionTokens = cost.cache_read;
+  model.cacheCreationCostPerMillionTokens = cost.cache_write;
+
+  const longContextCost = cost.context_over_200k;
+
+  if (!isDefined(longContextCost?.input)) {
+    return;
+  }
+
+  model.longContextCost = {
+    inputCostPerMillionTokens: longContextCost.input,
+    outputCostPerMillionTokens:
+      longContextCost.output ?? model.outputCostPerMillionTokens ?? 0,
+    thresholdTokens: LONG_CONTEXT_THRESHOLD_TOKENS,
+    cachedInputCostPerMillionTokens: longContextCost.cache_read,
+    cacheCreationCostPerMillionTokens: longContextCost.cache_write,
+  };
+};
+
+const buildModel = ({
+  providerName,
+  modelId,
+  modelData,
+}: {
+  providerName: string;
+  modelId: string;
+  modelData: ModelsDevModel;
+}): GeneratedModel => {
+  const model: GeneratedModel = {
+    name: modelId,
+    label: modelData.name ?? modelId,
+    modelFamily: inferModelFamily(providerName, modelId),
+  };
+
+  extractCost({ modelData, model });
+
+  model.contextWindowTokens = modelData.limit?.context;
+  model.maxOutputTokens = modelData.limit?.output;
+
+  const modalities = (modelData.modalities?.input ?? []).filter(
+    (modality) => modality !== 'text',
+  );
+
+  if (modalities.length > 0) {
+    model.modalities = modalities;
+  }
+
+  if (modelData.reasoning) {
+    model.supportsReasoning = true;
+  }
+
+  if (modelData.status === 'deprecated') {
+    model.isDeprecated = true;
+  }
+
+  return model;
+};
+
+// Which vendors we carry is a deployment decision, so it is read from the
+// spec that states it rather than inferred from how the SDK names a package.
+export const buildCatalog = ({
+  data,
+  vendors,
+}: {
+  data: ModelsDevData;
+  vendors: string[];
+}): GeneratedCatalog => {
+  const catalog: GeneratedCatalog = {};
+
+  for (const providerName of vendors) {
+    const providerData = data[providerName];
+
+    if (!isDefined(providerData)) {
+      // oxlint-disable-next-line no-console
+      console.warn(`Provider "${providerName}" not found in models.dev`);
+      continue;
+    }
+
+    const models = Object.entries(providerData.models)
+      .filter(
+        ([modelId, modelData]) =>
+          isLanguageModel(modelId) && meetsInclusionCriteria(modelData),
+      )
+      .map(([modelId, modelData]) =>
+        buildModel({ providerName, modelId, modelData }),
+      );
+
+    if (models.length === 0) {
+      // oxlint-disable-next-line no-console
+      console.warn(`No qualifying models for "${providerName}", skipping`);
+      continue;
+    }
+
+    catalog[providerName] = { models };
+  }
+
+  return catalog;
+};

packages/twenty-server/scripts/ai-catalog-sync/utils/build-coverage-report.util.ts

@@ -0,0 +1,67 @@
+import { isDefined } from 'twenty-shared/utils';
+
+import { type CoverageReport } from '../types/coverage-report.type';
+import { type GeneratedCatalog } from '../types/generated-catalog.type';
+
+// Models whose job is not general reasoning (speech, image, embeddings,
+// computer use, deep research). A capability index for them would be
+// meaningless, so they are excluded from the coverage denominator rather than
+// counted as gaps.
+const SPECIALIZED_MODEL_PATTERN =
+  /realtime|voxtral|tts|audio|-live|computer-use|deep-research|-image|pixtral|codestral|embed/;
+
+export const buildCoverageReport = (
+  catalog: GeneratedCatalog,
+): CoverageReport => {
+  const report: CoverageReport = {
+    generalPurposeModelCount: 0,
+    scoredGeneralPurposeModelCount: 0,
+    costedGeneralPurposeModelCount: 0,
+    unscoredGeneralPurposeModelIds: [],
+    specializedModelCount: 0,
+    declaredEffortVariantCount: 0,
+    scoredEffortVariantCount: 0,
+    unscoredEffortVariantIds: [],
+  };
+
+  for (const [providerName, provider] of Object.entries(catalog)) {
+    for (const model of provider.models) {
+      if (model.isDeprecated) {
+        continue;
+      }
+
+      if (SPECIALIZED_MODEL_PATTERN.test(model.name)) {
+        report.specializedModelCount += 1;
+        continue;
+      }
+
+      report.generalPurposeModelCount += 1;
+
+      if (isDefined(model.benchmark?.costPerTask)) {
+        report.costedGeneralPurposeModelCount += 1;
+      }
+
+      if (isDefined(model.benchmark?.intelligenceIndex)) {
+        report.scoredGeneralPurposeModelCount += 1;
+      } else {
+        report.unscoredGeneralPurposeModelIds.push(
+          `${providerName}/${model.name}`,
+        );
+      }
+
+      for (const effort of model.efforts ?? []) {
+        report.declaredEffortVariantCount += 1;
+
+        if (isDefined(model.benchmarkByEffort?.[effort]?.intelligenceIndex)) {
+          report.scoredEffortVariantCount += 1;
+        } else {
+          report.unscoredEffortVariantIds.push(
+            `${providerName}/${model.name}@${effort}`,
+          );
+        }
+      }
+    }
+  }
+
+  return report;
+};

packages/twenty-server/scripts/ai-catalog-sync/utils/build-effort-lookup-key.util.ts

@@ -0,0 +1,10 @@
+import { type AiModelEffort } from 'twenty-shared/ai';
+
+// A per-effort row is filed beside the bare model key, so the ceiling lookup the
+// catalog has always done keeps working while a variant can ask for the figure
+// taken at its own effort. `@` survives name normalization, so the two key
+// spaces cannot collide.
+export const buildEffortLookupKey = (
+  normalizedModelName: string,
+  effort: AiModelEffort,
+): string => `${normalizedModelName}@${effort}`;

packages/twenty-server/scripts/ai-catalog-sync/utils/build-lookup-candidates.util.ts

@@ -0,0 +1,69 @@
+import { isDefined } from 'twenty-shared/utils';
+
+import { type ModelsDevModel } from 'src/engine/metadata-modules/ai/ai-models/types/models-dev-model.type';
+
+const DATE_SUFFIX = /-\d{4}-?\d{2}-?\d{2}$/;
+const ROLLING_SUFFIX = /-(latest|preview|exp)$/;
+
+const fingerprint = (model: ModelsDevModel): string =>
+  JSON.stringify([model.cost ?? {}, model.limit ?? {}]);
+
+// `mistral-large-latest` is never benchmarked under that name, so it is
+// resolved to the dated release it currently points at by matching price and
+// limits against its non-rolling siblings. Re-resolves on its own when the
+// provider repoints the alias.
+const resolveRollingAlias = (
+  modelName: string,
+  siblingModels: Record<string, ModelsDevModel>,
+): string | undefined => {
+  const model = siblingModels[modelName];
+
+  if (!isDefined(model) || !ROLLING_SUFFIX.test(modelName)) {
+    return undefined;
+  }
+
+  const target = fingerprint(model);
+
+  const twins = Object.entries(siblingModels)
+    .filter(
+      ([siblingName, sibling]) =>
+        siblingName !== modelName &&
+        !ROLLING_SUFFIX.test(siblingName) &&
+        fingerprint(sibling) === target,
+    )
+    // Falling back to the name keeps this deterministic when a provider dates
+    // neither twin, and for the date-stamped ids providers actually use it
+    // still lands on the newer one.
+    .sort(
+      ([nameA, a], [nameB, b]) =>
+        (b.release_date ?? '').localeCompare(a.release_date ?? '') ||
+        nameB.localeCompare(nameA),
+    );
+
+  return twins[0]?.[0];
+};
+
+export const buildLookupCandidates = ({
+  modelName,
+  siblingModels,
+}: {
+  modelName: string;
+  siblingModels: Record<string, ModelsDevModel>;
+}): string[] => {
+  const candidates = [modelName, modelName.replace(DATE_SUFFIX, '')];
+
+  const resolved = resolveRollingAlias(modelName, siblingModels);
+
+  if (isDefined(resolved)) {
+    // Once the release an alias points at is known, only a row naming that
+    // release will do. Every undated spelling is some other release the
+    // publisher happens to have measured: `mistral-large` on the leaderboard is
+    // the Feb '24 model, and standing in for `mistral-large-2512` it published a
+    // two-year-old score as current.
+    return [...new Set([...candidates, resolved])];
+  }
+
+  // No dated release to point at, so the bare name is the only thing left and
+  // whichever release the publisher measured is the best answer available.
+  return [...new Set([...candidates, modelName.replace(ROLLING_SUFFIX, '')])];
+};

packages/twenty-server/scripts/ai-catalog-sync/utils/carry-over-committed-fields.util.ts

@@ -0,0 +1,43 @@
+import { isDefined } from 'twenty-shared/utils';
+
+import { type GeneratedCatalog } from '../types/generated-catalog.type';
+import { type GeneratedModel } from '../types/generated-model.type';
+
+// models.dev knows neither which effort levels a model takes nor its routing
+// facts, so a rebuild would drop them without this.
+const CARRIED_OVER_FIELDS = [
+  'efforts',
+  'dataResidency',
+  'zeroDataRetention',
+] as const satisfies readonly (keyof GeneratedModel)[];
+
+export const carryOverCommittedFields = ({
+  catalog,
+  committedCatalog,
+}: {
+  catalog: GeneratedCatalog;
+  committedCatalog: GeneratedCatalog;
+}): void => {
+  for (const [providerName, provider] of Object.entries(catalog)) {
+    const committedModels = new Map(
+      (committedCatalog[providerName]?.models ?? []).map((model) => [
+        model.name,
+        model,
+      ]),
+    );
+
+    for (const model of provider.models) {
+      const committedModel = committedModels.get(model.name);
+
+      if (!isDefined(committedModel)) {
+        continue;
+      }
+
+      for (const field of CARRIED_OVER_FIELDS) {
+        if (isDefined(committedModel[field])) {
+          Object.assign(model, { [field]: committedModel[field] });
+        }
+      }
+    }
+  }
+};

packages/twenty-server/scripts/ai-catalog-sync/utils/enrich-catalog.util.ts

@@ -0,0 +1,71 @@
+import { type AiModelEffort } from 'twenty-shared/ai';
+import { isDefined } from 'twenty-shared/utils';
+
+import { type AiModelBenchmark } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-benchmark.type';
+
+import {
+  type BenchmarkOverlayEntry,
+  type BenchmarkOverlayReading,
+} from '../types/benchmark-overlay-entry.type';
+import { type EnrichCatalogArgs } from '../types/enrich-catalog-args.type';
+
+import { matchBenchmarks } from './match-benchmarks.util';
+
+// The catalog embeds the figures only; the aliases stay in the overlay, which
+// is where a joining consumer looks for them.
+const toCatalogReadings = (
+  readings: Partial<Record<AiModelEffort, BenchmarkOverlayReading>>,
+): Partial<Record<AiModelEffort, AiModelBenchmark>> =>
+  Object.fromEntries(
+    Object.entries(readings).map(
+      ([effort, { aliases: _aliases, ...reading }]) => [effort, reading],
+    ),
+  );
+
+export const enrichCatalog = ({
+  catalog,
+  modelsDevData,
+  benchmarkIndex,
+  measuredAt,
+}: EnrichCatalogArgs): Record<string, BenchmarkOverlayEntry> => {
+  const overlay: Record<string, BenchmarkOverlayEntry> = {};
+
+  for (const [providerName, provider] of Object.entries(catalog)) {
+    const siblingModels = modelsDevData[providerName]?.models ?? {};
+
+    provider.models = provider.models.map((model) => {
+      const match = matchBenchmarks({
+        modelName: model.name,
+        siblingModels,
+        benchmarkIndex,
+        measuredAt,
+        efforts: model.efforts,
+      });
+
+      if (!isDefined(match)) {
+        return model;
+      }
+
+      const { benchmark, aliases, benchmarkByEffort } = match;
+
+      overlay[model.name] = {
+        ...benchmark,
+        aliases,
+        ...(isDefined(benchmarkByEffort) ? { benchmarkByEffort } : {}),
+      };
+
+      const { isDeprecated, ...rest } = model;
+
+      return {
+        ...rest,
+        benchmark,
+        ...(isDefined(benchmarkByEffort)
+          ? { benchmarkByEffort: toCatalogReadings(benchmarkByEffort) }
+          : {}),
+        ...(isDefined(isDeprecated) ? { isDeprecated } : {}),
+      };
+    });
+  }
+
+  return overlay;
+};

packages/twenty-server/scripts/ai-catalog-sync/utils/fetch-artificial-analysis-benchmarks.util.ts

@@ -0,0 +1,219 @@
+import { isDefined } from 'twenty-shared/utils';
+
+import { buildEffortLookupKey } from './build-effort-lookup-key.util';
+import { normalizeModelName } from './normalize-model-name.util';
+import { parseArtificialAnalysisEffort } from './parse-artificial-analysis-effort.util';
+import { type BenchmarkIndex } from '../types/benchmark-index.type';
+import { type BenchmarkRecord } from '../types/benchmark-record.type';
+
+const DEFAULT_ARTIFICIAL_ANALYSIS_API_URL =
+  'https://artificialanalysis.ai/api/v2/language/models/free';
+
+const PAGE_SIZE = 200;
+
+type ArtificialAnalysisModel = Record<string, unknown> & {
+  id?: string;
+  slug?: string;
+  name?: string;
+};
+
+type ArtificialAnalysisPage = {
+  data?: unknown;
+  pagination?: { has_more?: unknown };
+};
+
+// This endpoint returns 0 for "not measured", and a zero tokens-per-second is a
+// lie rather than a datum.
+const readPositiveNumber = (value: unknown): number | undefined =>
+  typeof value === 'number' && Number.isFinite(value) && value > 0
+    ? value
+    : undefined;
+
+const readNested = ({
+  model,
+  path,
+}: {
+  model: ArtificialAnalysisModel;
+  path: string[];
+}): number | undefined => {
+  let current: unknown = model;
+
+  for (const key of path) {
+    if (typeof current !== 'object' || current === null) {
+      return undefined;
+    }
+
+    current = (current as Record<string, unknown>)[key];
+  }
+
+  return readPositiveNumber(current);
+};
+
+const readModels = (page: ArtificialAnalysisPage): ArtificialAnalysisModel[] =>
+  Array.isArray(page.data) ? (page.data as ArtificialAnalysisModel[]) : [];
+
+// How much a row actually says. The endpoint carries several rows per model and
+// some hosts publish no measurements at all, so an unmeasured row must never
+// displace a measured one under the same key.
+const informationScore = (record: BenchmarkRecord): number =>
+  [
+    record.intelligenceIndex,
+    record.outputTokensPerSecond,
+    record.costPerTask,
+  ].filter(isDefined).length;
+
+// The rows for one model are the same model at different reasoning efforts, and
+// the publisher spells the effort only inside a display name. Taking the
+// best-scoring row gives every model its ceiling, measured the same way, which
+// is the whole reason for standing on one publisher. Picking by how populated a
+// row happened to be scored Sonnet 4.6 at low effort against Sonnet 5 at max,
+// and the gap read as a capability difference.
+const preferOver = (
+  candidate: BenchmarkRecord,
+  existing: BenchmarkRecord,
+): boolean => {
+  const candidateIndex = candidate.intelligenceIndex ?? -1;
+  const existingIndex = existing.intelligenceIndex ?? -1;
+
+  if (candidateIndex !== existingIndex) {
+    return candidateIndex > existingIndex;
+  }
+
+  return informationScore(candidate) > informationScore(existing);
+};
+
+const fetchPage = async ({
+  url,
+  apiKey,
+  page,
+}: {
+  url: string;
+  apiKey: string;
+  page: number;
+}): Promise<ArtificialAnalysisPage> => {
+  const pageUrl = new URL(url);
+
+  pageUrl.searchParams.set('page', String(page));
+  pageUrl.searchParams.set('page_size', String(PAGE_SIZE));
+
+  const response = await fetch(pageUrl, {
+    headers: { 'x-api-key': apiKey },
+    signal: AbortSignal.timeout(60000),
+  });
+
+  if (!response.ok) {
+    throw new Error(`${response.status} ${response.statusText}`);
+  }
+
+  return (await response.json()) as ArtificialAnalysisPage;
+};
+
+// The list is paginated and the free tier allows ten requests a day, so every
+// page is read in one run rather than spread across runs.
+const fetchAllModels = async ({
+  url,
+  apiKey,
+}: {
+  url: string;
+  apiKey: string;
+}): Promise<ArtificialAnalysisModel[]> => {
+  const models: ArtificialAnalysisModel[] = [];
+
+  for (let page = 1; ; page += 1) {
+    const response = await fetchPage({ url, apiKey, page });
+
+    models.push(...readModels(response));
+
+    if (response.pagination?.has_more !== true) {
+      return models;
+    }
+  }
+};
+
+export const fetchArtificialAnalysisBenchmarks = async (
+  apiKey: string,
+): Promise<BenchmarkIndex> => {
+  const url =
+    process.env.ARTIFICIAL_ANALYSIS_API_URL ??
+    DEFAULT_ARTIFICIAL_ANALYSIS_API_URL;
+
+  const models = await fetchAllModels({ url, apiKey });
+
+  if (models.length === 0) {
+    throw new Error(`No models in response from ${url}`);
+  }
+
+  const index: BenchmarkIndex = new Map();
+
+  const file = (key: string, record: BenchmarkRecord): void => {
+    const existing = index.get(key);
+
+    if (!isDefined(existing) || preferOver(record, existing)) {
+      index.set(key, record);
+    }
+  };
+
+  for (const model of models) {
+    // One malformed row must not cost us every measurement in the response.
+    if (typeof model !== 'object' || model === null) {
+      continue;
+    }
+
+    const aliases = [model.slug, model.id, model.name].filter(isDefined);
+
+    // Cost per task is the price of one task of the intelligence index run at
+    // this row's effort, so it reflects reasoning volume where per-token
+    // pricing cannot.
+    const record: BenchmarkRecord = {
+      intelligenceIndex: readNested({
+        model,
+        path: ['evaluations', 'artificial_analysis_intelligence_index'],
+      }),
+      outputTokensPerSecond: readNested({
+        model,
+        path: ['performance', 'median_output_tokens_per_second'],
+      }),
+      costPerTask: readNested({
+        model,
+        path: [
+          'artificial_analysis_intelligence_index_cost',
+          'cost_per_task',
+          'total_cost',
+        ],
+      }),
+      effort:
+        typeof model.name === 'string'
+          ? parseArtificialAnalysisEffort(model.name)
+          : undefined,
+      aliases,
+    };
+
+    if (informationScore(record) === 0) {
+      continue;
+    }
+
+    for (const alias of aliases) {
+      const key = normalizeModelName(alias);
+
+      if (key.length === 0) {
+        continue;
+      }
+
+      // The bare key keeps the ceiling; the effort key lets a pinned variant
+      // read the figure taken at its own effort.
+      file(key, record);
+
+      if (isDefined(record.effort)) {
+        file(buildEffortLookupKey(key, record.effort), record);
+      }
+    }
+  }
+
+  if (index.size === 0) {
+    throw new Error(
+      `${models.length} models returned from ${url} but none carried a measurement`,
+    );
+  }
+
+  return index;
+};

packages/twenty-server/scripts/ai-catalog-sync/utils/match-benchmarks.util.ts

@@ -0,0 +1,94 @@
+import { type AiModelEffort } from 'twenty-shared/ai';
+import { isDefined } from 'twenty-shared/utils';
+
+import { type AiModelBenchmark } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-benchmark.type';
+
+import { type BenchmarkIndex } from '../types/benchmark-index.type';
+import { type BenchmarkMatch } from '../types/benchmark-match.type';
+import { type BenchmarkOverlayReading } from '../types/benchmark-overlay-entry.type';
+import { type BenchmarkRecord } from '../types/benchmark-record.type';
+import { type MatchBenchmarksArgs } from '../types/match-benchmarks-args.type';
+
+import { buildEffortLookupKey } from './build-effort-lookup-key.util';
+import { buildLookupCandidates } from './build-lookup-candidates.util';
+import { normalizeModelName } from './normalize-model-name.util';
+
+const lookup = (
+  index: BenchmarkIndex,
+  keys: string[],
+): BenchmarkRecord | undefined => {
+  for (const key of keys) {
+    const record = index.get(key);
+
+    if (isDefined(record)) {
+      return record;
+    }
+  }
+
+  return undefined;
+};
+
+// The publisher lists plenty of models it has measured nothing about, and a row
+// of pure aliases says nothing worth carrying into either artifact.
+const hasMeasurement = (record: BenchmarkRecord): boolean =>
+  [
+    record.intelligenceIndex,
+    record.outputTokensPerSecond,
+    record.costPerTask,
+  ].some(isDefined);
+
+const toBenchmark = (
+  record: BenchmarkRecord,
+  measuredAt: string,
+): AiModelBenchmark => ({
+  intelligenceIndex: record.intelligenceIndex,
+  outputTokensPerSecond: record.outputTokensPerSecond,
+  costPerTask: record.costPerTask,
+  effort: record.effort,
+  measuredAt: record.measuredAt ?? measuredAt,
+});
+
+export const matchBenchmarks = ({
+  modelName,
+  siblingModels,
+  benchmarkIndex,
+  measuredAt,
+  efforts,
+}: MatchBenchmarksArgs): BenchmarkMatch | undefined => {
+  const keys = buildLookupCandidates({ modelName, siblingModels }).map(
+    normalizeModelName,
+  );
+
+  const record = lookup(benchmarkIndex, keys);
+
+  if (!isDefined(record) || !hasMeasurement(record)) {
+    return undefined;
+  }
+
+  const benchmarkByEffort: Partial<
+    Record<AiModelEffort, BenchmarkOverlayReading>
+  > = {};
+
+  // A declared effort without a row of its own stays blank: the ceiling was
+  // measured at another effort and would overstate it.
+  for (const effort of efforts ?? []) {
+    const effortRecord = lookup(
+      benchmarkIndex,
+      keys.map((key) => buildEffortLookupKey(key, effort)),
+    );
+
+    if (isDefined(effortRecord) && hasMeasurement(effortRecord)) {
+      benchmarkByEffort[effort] = {
+        ...toBenchmark(effortRecord, measuredAt),
+        effort,
+        aliases: [...new Set(effortRecord.aliases)],
+      };
+    }
+  }
+
+  return {
+    benchmark: toBenchmark(record, measuredAt),
+    aliases: [...new Set([modelName, ...record.aliases])],
+    ...(Object.keys(benchmarkByEffort).length > 0 ? { benchmarkByEffort } : {}),
+  };
+};

packages/twenty-server/scripts/ai-catalog-sync/utils/normalize-model-name.util.ts

@@ -0,0 +1,12 @@
+// Benchmark publishers, providers and the AI SDK all spell the same model
+// differently (`claude-sonnet-4-6`, `claude-sonnet-4.6`, `Claude Sonnet 4.6`),
+// so every lookup key goes through this before it is compared. The trailing
+// `-v1:0` strip lets a Bedrock deployment id join the same row as the model it
+// serves.
+export const normalizeModelName = (modelName: string): string =>
+  modelName
+    .toLowerCase()
+    .replace(/:batch$/, '')
+    .replace(/\(.*\)/g, '')
+    .replace(/-v\d+(:\d+)?$/, '')
+    .replace(/[\s.\-_/]/g, '');

packages/twenty-server/scripts/ai-catalog-sync/utils/parse-artificial-analysis-effort.util.ts

@@ -0,0 +1,35 @@
+import { type AiModelEffort, isAiModelEffort } from 'twenty-shared/ai';
+import { isDefined } from 'twenty-shared/utils';
+
+// A row run with reasoning switched off carries no effort word at all.
+const NO_REASONING_MARKERS = new Set(['non-reasoning', 'nonreasoning']);
+
+// The publisher spells the configuration only inside the trailing parentheses
+// of a display name: `GPT-5.6 Sol (max)`, `Gemini 3.8 Flash (high)`,
+// `Claude Opus 5 (Adaptive Reasoning, Max Effort)`. Dates, fallbacks and a
+// bare "Reasoning" in the same place name no effort.
+export const parseArtificialAnalysisEffort = (
+  displayName: string,
+): AiModelEffort | undefined => {
+  const configuration = displayName.match(/\(([^()]*)\)\s*$/)?.[1];
+
+  if (!isDefined(configuration)) {
+    return undefined;
+  }
+
+  const segments = configuration
+    .split(',')
+    .map((segment) => segment.trim().toLowerCase());
+
+  const effort = segments
+    .map((segment) => segment.replace(/\s+effort$/, ''))
+    .find(isAiModelEffort);
+
+  if (isDefined(effort)) {
+    return effort;
+  }
+
+  return segments.some((segment) => NO_REASONING_MARKERS.has(segment))
+    ? 'none'
+    : undefined;
+};

packages/twenty-server/scripts/ai-catalog-sync/utils/project-catalog.util.ts

@@ -0,0 +1,242 @@
+// twenty-shared by source path rather than by package name: the consuming
+// repository runs this from a sparse checkout with no workspace install, so
+// 'twenty-shared/ai' would not resolve there.
+import { isAiSdkPackage } from '../../../../twenty-shared/src/ai/utils/is-ai-sdk-package.util';
+import { isDataResidency } from '../../../../twenty-shared/src/ai/utils/is-data-residency.util';
+
+import {
+  type CatalogSpec,
+  type CatalogSpecModel,
+  type CatalogSpecModelOverrides,
+  type CatalogSpecProvider,
+} from '../types/catalog-spec.type';
+
+// Structural, so the catalog needs no schema here and this file can run from a
+// checkout that has not installed the monorepo.
+type CanonicalModel = { name: string; label?: string } & Record<
+  string,
+  unknown
+>;
+type CanonicalProvider = { models?: CanonicalModel[] } & Record<
+  string,
+  unknown
+>;
+export type CanonicalCatalog = Record<string, CanonicalProvider>;
+
+const OVERRIDABLE_FIELDS = [
+  'label',
+  'isDeprecated',
+  'contextWindowTokens',
+  'maxOutputTokens',
+  'inputCostPerMillionTokens',
+  'outputCostPerMillionTokens',
+  'cachedInputCostPerMillionTokens',
+  'cacheCreationCostPerMillionTokens',
+] as const satisfies readonly (keyof CatalogSpecModelOverrides)[];
+
+const NUMERIC_FIELDS = [
+  'contextWindowTokens',
+  'maxOutputTokens',
+  'inputCostPerMillionTokens',
+  'outputCostPerMillionTokens',
+  'cachedInputCostPerMillionTokens',
+  'cacheCreationCostPerMillionTokens',
+] as const satisfies readonly (keyof CatalogSpecModelOverrides)[];
+
+const PROVIDER_CREDENTIAL_FIELDS = [
+  'label',
+  'apiKey',
+  'baseUrl',
+  'region',
+  'authType',
+  'accessKeyId',
+  'secretAccessKey',
+  'sessionToken',
+] as const;
+
+const asSpecModel = (entry: string | CatalogSpecModel): CatalogSpecModel =>
+  typeof entry === 'string' ? { model: entry } : entry;
+
+const indexCanonicalModels = (
+  catalog: CanonicalCatalog,
+): Map<string, CanonicalModel> => {
+  const models = new Map<string, CanonicalModel>();
+
+  for (const provider of Object.values(catalog)) {
+    for (const model of provider.models ?? []) {
+      models.set(model.name, model);
+    }
+  }
+
+  return models;
+};
+
+const projectModel = ({
+  canonicalModel,
+  specModel,
+  provider,
+}: {
+  canonicalModel: CanonicalModel;
+  specModel: CatalogSpecModel;
+  provider: CatalogSpecProvider;
+}): CanonicalModel => {
+  const overrides = Object.fromEntries(
+    OVERRIDABLE_FIELDS.filter((field) => specModel[field] !== undefined).map(
+      (field) => [field, specModel[field]],
+    ),
+  );
+
+  const routeFields = {
+    ...(provider.dataResidency === undefined
+      ? {}
+      : { dataResidency: provider.dataResidency }),
+    ...(provider.zeroDataRetention === undefined
+      ? {}
+      : { zeroDataRetention: provider.zeroDataRetention }),
+  };
+
+  return {
+    ...canonicalModel,
+    ...routeFields,
+    ...overrides,
+    name: specModel.as ?? canonicalModel.name,
+    // The suffix marks the route, so it is appended to whichever name the model
+    // ends up with rather than to the catalog's.
+    label: `${specModel.label ?? canonicalModel.label ?? canonicalModel.name}${provider.labelSuffix ?? ''}`,
+  };
+};
+
+const assertProviderIsUsable = ({
+  provider,
+  seenNames,
+}: {
+  provider: CatalogSpecProvider;
+  seenNames: Set<string>;
+}): void => {
+  // Providers key an object, so a repeat would drop the first one's credentials
+  // and models on the floor, and `__proto__` would drop its own.
+  if (seenNames.has(provider.name) || provider.name === '__proto__') {
+    throw new Error(`Provider "${provider.name}" is repeated or reserved`);
+  }
+
+  if (!isAiSdkPackage(provider.npm)) {
+    throw new Error(
+      `Provider "${provider.name}" names an unsupported SDK package: ${provider.npm}`,
+    );
+  }
+
+  if (
+    provider.dataResidency !== undefined &&
+    !isDataResidency(provider.dataResidency)
+  ) {
+    throw new Error(
+      `Provider "${provider.name}" names an unsupported data residency: ${provider.dataResidency}`,
+    );
+  }
+};
+
+const assertNumbersAreUsable = ({
+  provider,
+  specModel,
+}: {
+  provider: CatalogSpecProvider;
+  specModel: CatalogSpecModel;
+}): void => {
+  for (const field of NUMERIC_FIELDS) {
+    const value = specModel[field];
+
+    if (value !== undefined && (!Number.isFinite(value) || value < 0)) {
+      throw new Error(
+        `${provider.name}/${specModel.model} sets ${field} to ${value}, which is not a usable number`,
+      );
+    }
+  }
+};
+
+// A spec naming a model the catalog does not carry is the failure this whole
+// pipeline exists to prevent: it publishes a route to nothing, and the tier
+// chains then fall through to a neighbouring rung in silence.
+const resolveModels = ({
+  provider,
+  canonicalCatalog,
+  canonicalModels,
+}: {
+  provider: CatalogSpecProvider;
+  canonicalCatalog: CanonicalCatalog;
+  canonicalModels: Map<string, CanonicalModel>;
+}): { canonicalModel: CanonicalModel; specModel: CatalogSpecModel }[] => {
+  if (!Array.isArray(provider.models)) {
+    const { vendor } = provider.models;
+    const vendorModels = canonicalCatalog[vendor]?.models ?? [];
+
+    if (vendorModels.length === 0) {
+      throw new Error(
+        `Provider "${provider.name}" serves every model from "${vendor}", which the catalog does not carry`,
+      );
+    }
+
+    return vendorModels.map((canonicalModel) => ({
+      canonicalModel,
+      specModel: { model: canonicalModel.name },
+    }));
+  }
+
+  const specModels = provider.models.map(asSpecModel);
+  const unknown = specModels
+    .filter(({ model }) => !canonicalModels.has(model))
+    .map(({ model }) => `${provider.name}/${model}`);
+
+  if (unknown.length > 0) {
+    throw new Error(
+      `The catalog does not carry: ${unknown.join(', ')}. Check the model names, or wait for the catalog sync to pick them up.`,
+    );
+  }
+
+  return specModels.map((specModel) => {
+    assertNumbersAreUsable({ provider, specModel });
+
+    const canonicalModel = canonicalModels.get(specModel.model);
+
+    if (canonicalModel === undefined) {
+      throw new Error(`The catalog does not carry ${specModel.model}`);
+    }
+
+    return { canonicalModel, specModel };
+  });
+};
+
+export const projectCatalog = ({
+  canonicalCatalog,
+  spec,
+}: {
+  canonicalCatalog: CanonicalCatalog;
+  spec: CatalogSpec;
+}): CanonicalCatalog => {
+  const canonicalModels = indexCanonicalModels(canonicalCatalog);
+  const projected: CanonicalCatalog = {};
+  const seenNames = new Set<string>();
+
+  for (const provider of spec.providers) {
+    assertProviderIsUsable({ provider, seenNames });
+    seenNames.add(provider.name);
+
+    const credentials = Object.fromEntries(
+      PROVIDER_CREDENTIAL_FIELDS.filter(
+        (field) => provider[field] !== undefined,
+      ).map((field) => [field, provider[field]]),
+    );
+
+    projected[provider.name] = {
+      npm: provider.npm,
+      label: provider.label ?? provider.name,
+      ...credentials,
+      models: resolveModels({
+        provider,
+        canonicalCatalog,
+        canonicalModels,
+      }).map((resolved) => projectModel({ ...resolved, provider })),
+    };
+  }
+
+  return projected;
+};

packages/twenty-server/scripts/ai-catalog-sync/utils/read-committed-benchmarks.util.ts

@@ -0,0 +1,103 @@
+import * as fs from 'fs';
+
+import { type AiModelEffort, isAiModelEffort } from 'twenty-shared/ai';
+
+import { buildEffortLookupKey } from './build-effort-lookup-key.util';
+import { normalizeModelName } from './normalize-model-name.util';
+import { type BenchmarkIndex } from '../types/benchmark-index.type';
+import { type BenchmarkRecord } from '../types/benchmark-record.type';
+
+type CommittedReading = Partial<BenchmarkRecord> & {
+  measuredAt?: string;
+  aliases?: string[];
+};
+
+type CommittedEntry = CommittedReading & {
+  benchmarkByEffort?: Partial<Record<AiModelEffort, CommittedReading>>;
+};
+
+const toRecord = ({
+  reading,
+  modelName,
+  effort,
+}: {
+  reading: CommittedReading;
+  modelName: string;
+  effort?: AiModelEffort;
+}): BenchmarkRecord => ({
+  intelligenceIndex: reading.intelligenceIndex,
+  outputTokensPerSecond: reading.outputTokensPerSecond,
+  costPerTask: reading.costPerTask,
+  // The key a reading is filed under is the authority on its effort.
+  effort: effort ?? reading.effort,
+  measuredAt: reading.measuredAt,
+  aliases: reading.aliases ?? [modelName],
+});
+
+const file = ({
+  index,
+  record,
+  modelName,
+  toKey,
+}: {
+  index: BenchmarkIndex;
+  record: BenchmarkRecord;
+  modelName: string;
+  toKey: (normalizedName: string) => string;
+}): void => {
+  for (const alias of record.aliases) {
+    const key = normalizeModelName(alias);
+
+    if (key.length > 0 && !index.has(toKey(key))) {
+      index.set(toKey(key), record);
+    }
+  }
+
+  // An entry always claims its own name, whatever a sibling's alias list says.
+  const ownKey = normalizeModelName(modelName);
+
+  if (ownKey.length > 0) {
+    index.set(toKey(ownKey), record);
+  }
+};
+
+// The last published measurements, read back so a failed fetch degrades to
+// stale data rather than deleting the catalog's benchmarks. Each record keeps
+// its own measuredAt, so a stale entry stays visibly stale.
+export const readCommittedBenchmarks = (filePath: string): BenchmarkIndex => {
+  if (!fs.existsSync(filePath)) {
+    return new Map();
+  }
+
+  const parsed = JSON.parse(fs.readFileSync(filePath, 'utf-8')) as {
+    models?: Record<string, CommittedEntry>;
+  };
+
+  const index: BenchmarkIndex = new Map();
+
+  for (const [modelName, entry] of Object.entries(parsed.models ?? {})) {
+    file({
+      index,
+      record: toRecord({ reading: entry, modelName }),
+      modelName,
+      toKey: (normalizedName) => normalizedName,
+    });
+
+    for (const [effort, reading] of Object.entries(
+      entry.benchmarkByEffort ?? {},
+    )) {
+      if (!isAiModelEffort(effort)) {
+        continue;
+      }
+
+      file({
+        index,
+        record: toRecord({ reading, modelName, effort }),
+        modelName,
+        toKey: (normalizedName) => buildEffortLookupKey(normalizedName, effort),
+      });
+    }
+  }
+
+  return index;
+};

packages/twenty-server/scripts/ai-catalog-sync/utils/render-coverage-report.util.ts

@@ -0,0 +1,55 @@
+import { type CoverageReport } from '../types/coverage-report.type';
+
+const asPercentage = ({
+  numerator,
+  denominator,
+}: {
+  numerator: number;
+  denominator: number;
+}): string =>
+  denominator === 0 ? 'n/a' : `${Math.round((numerator / denominator) * 100)}%`;
+
+const renderGapList = ({
+  summary,
+  modelIds,
+}: {
+  summary: string;
+  modelIds: string[];
+}): string[] =>
+  modelIds.length === 0
+    ? []
+    : [
+        '',
+        `<details><summary>${summary}</summary>`,
+        '',
+        ...modelIds.map((modelId) => `- ${modelId}`),
+        '',
+        '</details>',
+      ];
+
+export const renderCoverageReport = (report: CoverageReport): string => {
+  const {
+    generalPurposeModelCount: total,
+    scoredGeneralPurposeModelCount: scored,
+    costedGeneralPurposeModelCount: costed,
+    declaredEffortVariantCount: declaredVariants,
+    scoredEffortVariantCount: scoredVariants,
+  } = report;
+
+  return [
+    '### Benchmark coverage',
+    '',
+    `- General-purpose models with an intelligence index: **${scored}/${total}** (${asPercentage({ numerator: scored, denominator: total })})`,
+    `- Declared effort variants with an index measured at their own effort: **${scoredVariants}/${declaredVariants}** (${asPercentage({ numerator: scoredVariants, denominator: declaredVariants })})`,
+    `- General-purpose models with a cost per task: **${costed}/${total}** (${asPercentage({ numerator: costed, denominator: total })})`,
+    `- Specialized models excluded from the denominator: ${report.specializedModelCount}`,
+    ...renderGapList({
+      summary: 'General-purpose models with no matched intelligence index',
+      modelIds: report.unscoredGeneralPurposeModelIds,
+    }),
+    ...renderGapList({
+      summary: 'Effort variants with no reading at their own effort',
+      modelIds: report.unscoredEffortVariantIds,
+    }),
+  ].join('\n');
+};