snapshot of https://github.com/twentyhq/twenty.git (main, 2026-09-17; blobs > 131072B omitted)

packages/twenty-server/scripts/ai-catalog-sync/README.md

@@ -0,0 +1,93 @@
+# AI catalog
+
+Three files in `src/engine/metadata-modules/ai/ai-models`, and two entry points
+over them.
+
+| File | What it is |
+| --- | --- |
+| `ai-models.json` | What every model is: identity, pricing, limits, modalities, efforts, benchmarks. No routes, no credentials. Synced daily. |
+| `ai-self-host-spec.json` | What a self-hosted deployment serves: the five direct routes and their key templates. Hand-maintained. |
+| `ai-providers.json` | The catalog the server bundles. Generated from the two above; a test fails if it is hand-edited. |
+
+Cloud works the same way, from a private spec in twenty-infra, so a deployment
+catalog is always a projection rather than a second copy of the truth.
+
+## `index.ts` — sync the model catalog
+
+Run daily by `.github/workflows/ci-ai-catalog-sync.yaml`. Reads models.dev for
+model identity, pricing, context windows and availability, overlays Artificial
+Analysis for intelligence, speed and cost per task, and writes `ai-models.json`
+and `ai-model-benchmarks.json`, then projects the self-host spec over the first
+to write `ai-providers.json`. Hand-maintained fields (`efforts`,
+`dataResidency`, `zeroDataRetention`) survive the rebuild.
+
+```bash
+npx nx run twenty-server:ts-node-no-deps-transpile-only -- \
+  ./scripts/ai-catalog-sync/index.ts --dry-run
+```
+
+## `project.ts` — derive a deployment catalog
+
+A deployment serves a subset of the catalog through its own routes: Azure, a
+Bedrock region, a gateway, or a first-party key. It declares only what is its
+own — which routes exist, their credentials, and which catalog models each one
+serves — and this projects the catalog through that spec.
+
+```bash
+npx tsx ./scripts/ai-catalog-sync/project.ts \
+  --spec ./my-deployment.json \
+  --out ./ai-catalog.json
+```
+
+`--catalog` points at a different `ai-models.json`; it defaults to the
+committed one. The output has the same shape, so a server loads it through
+`AI_CATALOG_STORAGE_PATH` with no further processing.
+
+A spec looks like this:
+
+```json
+{
+  "providers": [
+    {
+      "name": "azure-foundry",
+      "npm": "@ai-sdk/azure",
+      "label": "Azure AI Foundry",
+      "apiKey": "{{AZURE_FOUNDRY_API_KEY}}",
+      "baseUrl": "{{AZURE_FOUNDRY_BASE_URL}}",
+      "dataResidency": "eu",
+      "labelSuffix": " (Azure)",
+      "models": [
+        "gpt-5.6-luna",
+        { "model": "gpt-5.6-sol", "cachedInputCostPerMillionTokens": 0.5 }
+      ]
+    },
+    {
+      "name": "amazon-bedrock",
+      "npm": "@ai-sdk/amazon-bedrock",
+      "region": "eu-central-1",
+      "models": [
+        { "model": "claude-opus-4-7", "as": "eu.anthropic.claude-opus-4-7" }
+      ]
+    }
+  ]
+}
+```
+
+A route that serves a whole vendor says so instead of listing its models, which
+is how self-host picks up new models with the daily sync:
+
+```json
+{ "name": "openai", "npm": "@ai-sdk/openai", "models": { "vendor": "openai" } }
+```
+
+A spec can rename a model for the route that deploys it (`as`) and override the
+prices it negotiated, the limits its route caps, its label and its deprecation.
+It cannot restate what a model is: modalities, reasoning support, efforts and
+benchmarks always come from the catalog, which is what keeps a deployment from drifting away from the
+measured truth. Naming a model the catalog does not carry fails the run rather
+than publishing a route to nothing.
+
+`project.ts` resolves everything it needs by source path, so a repository
+holding a private spec can run it from a sparse checkout of these files and
+`packages/twenty-shared/src/ai` without installing the monorepo and without any
+API key.

packages/twenty-server/scripts/ai-catalog-sync/__tests__/build-catalog.spec.ts

@@ -0,0 +1,63 @@
+import { type ModelsDevData } from 'src/engine/metadata-modules/ai/ai-models/types/models-dev-data.type';
+
+import { assertPayloadIsUsable } from '../utils/assert-payload-is-usable.util';
+import { buildCatalog } from '../utils/build-catalog.util';
+
+const languageModel = {
+  id: 'model',
+  name: 'Model',
+  tool_call: true,
+  cost: { input: 2, output: 6 },
+  limit: { context: 128000, output: 8192 },
+};
+
+const VENDORS = ['openai', 'anthropic', 'google', 'xai', 'mistral'];
+
+const payload = (): ModelsDevData => ({
+  openai: { id: 'openai', models: { 'gpt-x': languageModel } },
+  anthropic: { id: 'anthropic', models: { 'claude-x': languageModel } },
+  google: { id: 'google', models: { 'gemini-x': languageModel } },
+  xai: { id: 'xai', models: { 'grok-x': languageModel } },
+  mistral: { id: 'mistral', models: { 'mistral-x': languageModel } },
+});
+
+describe('assertPayloadIsUsable', () => {
+  it('accepts a payload carrying models for every vendor we ship', () => {
+    expect(() =>
+      assertPayloadIsUsable({ data: payload(), vendors: VENDORS }),
+    ).not.toThrow();
+  });
+
+  it('refuses an empty payload rather than generating an empty catalog', () => {
+    expect(() =>
+      assertPayloadIsUsable({ data: {} as ModelsDevData, vendors: VENDORS }),
+    ).toThrow(/refusing to overwrite the catalog/);
+  });
+
+  it('names the providers that came back empty', () => {
+    const data = payload();
+
+    data.anthropic.models = {};
+
+    expect(() => assertPayloadIsUsable({ data, vendors: VENDORS })).toThrow(
+      /anthropic/,
+    );
+  });
+});
+
+describe('buildCatalog', () => {
+  it('prices long context from the standard output rate when none is quoted', () => {
+    const data = payload();
+
+    data.openai.models['gpt-x'].cost = {
+      input: 2,
+      output: 6,
+      context_over_200k: { input: 4 },
+    };
+
+    const model = buildCatalog({ data, vendors: VENDORS }).openai.models[0];
+
+    // A missing long-context rate is unknown, not free.
+    expect(model.longContextCost?.outputCostPerMillionTokens).toBe(6);
+  });
+});

packages/twenty-server/scripts/ai-catalog-sync/__tests__/carry-over-committed-fields.spec.ts

@@ -0,0 +1,54 @@
+import { type GeneratedCatalog } from '../types/generated-catalog.type';
+import { type GeneratedModel } from '../types/generated-model.type';
+import { carryOverCommittedFields } from '../utils/carry-over-committed-fields.util';
+
+const openai = (models: GeneratedModel[]): GeneratedCatalog => ({
+  openai: { models },
+});
+
+describe('carryOverCommittedFields', () => {
+  it('restores the fields models.dev does not carry, and only those', () => {
+    const catalog = openai([
+      { name: 'gpt-x', label: 'GPT X', supportsReasoning: true },
+    ]);
+
+    carryOverCommittedFields({
+      catalog,
+      committedCatalog: openai([
+        {
+          name: 'gpt-x',
+          label: 'Old label',
+          inputCostPerMillionTokens: 1,
+          efforts: ['low', 'high'],
+          dataResidency: 'us',
+          zeroDataRetention: true,
+        },
+      ]),
+    });
+
+    expect(catalog.openai.models[0]).toEqual({
+      name: 'gpt-x',
+      label: 'GPT X',
+      supportsReasoning: true,
+      efforts: ['low', 'high'],
+      dataResidency: 'us',
+      zeroDataRetention: true,
+    });
+  });
+
+  it('leaves a model the committed catalog does not know untouched', () => {
+    const catalog = openai([{ name: 'gpt-new', label: 'GPT New' }]);
+
+    carryOverCommittedFields({
+      catalog,
+      committedCatalog: openai([
+        { name: 'gpt-x', label: 'GPT X', efforts: ['low'] },
+      ]),
+    });
+
+    expect(catalog.openai.models[0]).toEqual({
+      name: 'gpt-new',
+      label: 'GPT New',
+    });
+  });
+});

packages/twenty-server/scripts/ai-catalog-sync/__tests__/enrich-catalog.spec.ts

@@ -0,0 +1,131 @@
+import { type ModelsDevData } from 'src/engine/metadata-modules/ai/ai-models/types/models-dev-data.type';
+
+import { enrichCatalog } from '../utils/enrich-catalog.util';
+import { type BenchmarkIndex } from '../types/benchmark-index.type';
+import { type GeneratedCatalog } from '../types/generated-catalog.type';
+
+const MEASURED_AT = '2026-09-09';
+
+const catalogOf = (...modelNames: string[]): GeneratedCatalog => ({
+  openai: { models: modelNames.map((name) => ({ name, label: name })) },
+});
+
+const enrich = (catalog: GeneratedCatalog, benchmarkIndex: BenchmarkIndex) =>
+  enrichCatalog({
+    catalog,
+    modelsDevData: {} as ModelsDevData,
+    benchmarkIndex,
+    measuredAt: MEASURED_AT,
+  });
+
+describe('enrichCatalog', () => {
+  it('attaches a measurement to the model and mirrors it into the overlay', () => {
+    const catalog = catalogOf('gpt-x');
+    const overlay = enrich(
+      catalog,
+      new Map([['gptx', { intelligenceIndex: 41, aliases: ['gpt-x'] }]]),
+    );
+
+    expect(catalog.openai.models[0].benchmark).toEqual({
+      intelligenceIndex: 41,
+      outputTokensPerSecond: undefined,
+      costPerTask: undefined,
+      measuredAt: MEASURED_AT,
+    });
+    expect(overlay['gpt-x'].measuredAt).toBe(MEASURED_AT);
+  });
+
+  it('leaves a row the publisher measured nothing about out of both artifacts', () => {
+    const catalog = catalogOf('gpt-x');
+    const overlay = enrich(
+      catalog,
+      new Map([['gptx', { aliases: ['gpt-x'] }]]),
+    );
+
+    expect(overlay).toEqual({});
+    expect(catalog.openai.models[0].benchmark).toBeUndefined();
+  });
+
+  it('leaves a model no publisher covers untouched and out of the overlay', () => {
+    const catalog = catalogOf('gpt-x');
+    const overlay = enrich(catalog, new Map());
+
+    expect(catalog.openai.models[0].benchmark).toBeUndefined();
+    expect(overlay).toEqual({});
+  });
+
+  it('keeps isDeprecated last so the generated catalog stays stable', () => {
+    const catalog = catalogOf('gpt-x');
+
+    catalog.openai.models[0].isDeprecated = true;
+
+    enrich(
+      catalog,
+      new Map([['gptx', { intelligenceIndex: 41, aliases: ['gpt-x'] }]]),
+    );
+
+    expect(Object.keys(catalog.openai.models[0])).toEqual([
+      'name',
+      'label',
+      'benchmark',
+      'isDeprecated',
+    ]);
+  });
+
+  it('embeds per-effort readings in the catalog and mirrors them, with aliases, into the overlay', () => {
+    const catalog = catalogOf('gpt-x');
+
+    catalog.openai.models[0].efforts = ['low', 'high'];
+
+    const overlay = enrich(
+      catalog,
+      new Map([
+        [
+          'gptx',
+          { intelligenceIndex: 41, effort: 'high', aliases: ['GPT X (high)'] },
+        ],
+        [
+          'gptx@high',
+          { intelligenceIndex: 41, effort: 'high', aliases: ['GPT X (high)'] },
+        ],
+        [
+          'gptx@low',
+          { intelligenceIndex: 30, effort: 'low', aliases: ['GPT X (low)'] },
+        ],
+      ]),
+    );
+
+    expect(catalog.openai.models[0].benchmark?.effort).toBe('high');
+    expect(catalog.openai.models[0].benchmarkByEffort).toEqual({
+      low: { intelligenceIndex: 30, effort: 'low', measuredAt: MEASURED_AT },
+      high: { intelligenceIndex: 41, effort: 'high', measuredAt: MEASURED_AT },
+    });
+    expect(overlay['gpt-x'].benchmarkByEffort?.low).toEqual({
+      intelligenceIndex: 30,
+      effort: 'low',
+      measuredAt: MEASURED_AT,
+      aliases: ['GPT X (low)'],
+    });
+    expect(Object.keys(catalog.openai.models[0])).toEqual([
+      'name',
+      'label',
+      'efforts',
+      'benchmark',
+      'benchmarkByEffort',
+    ]);
+  });
+
+  it('writes no effort map for a model whose declared efforts have no rows', () => {
+    const catalog = catalogOf('gpt-x');
+
+    catalog.openai.models[0].efforts = ['low', 'high'];
+
+    const overlay = enrich(
+      catalog,
+      new Map([['gptx', { intelligenceIndex: 41, aliases: ['gpt-x'] }]]),
+    );
+
+    expect(catalog.openai.models[0].benchmarkByEffort).toBeUndefined();
+    expect(overlay['gpt-x'].benchmarkByEffort).toBeUndefined();
+  });
+});

packages/twenty-server/scripts/ai-catalog-sync/__tests__/fetch-artificial-analysis.spec.ts

@@ -0,0 +1,246 @@
+import { fetchArtificialAnalysisBenchmarks } from '../utils/fetch-artificial-analysis-benchmarks.util';
+
+const respondWith = (models: unknown[]): void => {
+  global.fetch = jest.fn().mockResolvedValue({
+    ok: true,
+    json: async () => ({ data: models }),
+  }) as unknown as typeof fetch;
+};
+
+const measuredModel = {
+  slug: 'claude-sonnet-5',
+  evaluations: { artificial_analysis_intelligence_index: 38.4 },
+  performance: { median_output_tokens_per_second: 92.1 },
+  artificial_analysis_intelligence_index_cost: {
+    cost_per_task: { total_cost: 0.42 },
+  },
+};
+
+describe('fetchArtificialAnalysisBenchmarks', () => {
+  it('reads the index, speed and cost per task for a measured model', async () => {
+    respondWith([measuredModel]);
+
+    const record = (await fetchArtificialAnalysisBenchmarks('key')).get(
+      'claudesonnet5',
+    );
+
+    expect(record?.intelligenceIndex).toBe(38.4);
+    expect(record?.outputTokensPerSecond).toBe(92.1);
+    expect(record?.costPerTask).toBe(0.42);
+  });
+
+  it('treats zero as not measured rather than as a measurement', async () => {
+    respondWith([
+      {
+        slug: 'grok-4-6',
+        evaluations: { artificial_analysis_intelligence_index: 44.1 },
+        performance: { median_output_tokens_per_second: 0 },
+      },
+    ]);
+
+    const record = (await fetchArtificialAnalysisBenchmarks('key')).get(
+      'grok46',
+    );
+
+    expect(record?.intelligenceIndex).toBe(44.1);
+    expect(record?.outputTokensPerSecond).toBeUndefined();
+  });
+
+  it.each([
+    ['best row first', true],
+    ['best row last', false],
+  ])(
+    'keeps the highest-scoring configuration whatever the order (%s)',
+    async (_label, bestFirst) => {
+      // The rows for one model are the same model at different reasoning
+      // efforts. Picking by how populated a row is scored Sonnet 4.6 at low
+      // effort against Sonnet 5 at max, and the gap read as capability.
+      const lowEffort = {
+        slug: 'claude-sonnet-4-6',
+        name: 'Claude Sonnet 4.6 (Non-reasoning, Low Effort)',
+        evaluations: { artificial_analysis_intelligence_index: 23.3 },
+        performance: { median_output_tokens_per_second: 120 },
+        artificial_analysis_intelligence_index_cost: {
+          cost_per_task: { total_cost: 0.1 },
+        },
+      };
+      const maxEffort = {
+        slug: 'claude-sonnet-4-6',
+        name: 'Claude Sonnet 4.6 (Adaptive Reasoning, Max Effort)',
+        evaluations: { artificial_analysis_intelligence_index: 36.2 },
+        performance: { median_output_tokens_per_second: 80 },
+      };
+
+      respondWith(bestFirst ? [maxEffort, lowEffort] : [lowEffort, maxEffort]);
+
+      const record = (await fetchArtificialAnalysisBenchmarks('key')).get(
+        'claudesonnet46',
+      );
+
+      expect(record?.intelligenceIndex).toBe(36.2);
+      // The whole row travels together, so speed and cost describe the same
+      // configuration the index was measured in.
+      expect(record?.outputTokensPerSecond).toBe(80);
+      expect(record?.costPerTask).toBeUndefined();
+    },
+  );
+
+  it.each([
+    ['unmeasured row first', true],
+    ['measured row first', false],
+  ])(
+    'keeps the measured row when an unmeasured duplicate shares its name (%s)',
+    async (_label, unmeasuredFirst) => {
+      const unmeasured = {
+        slug: 'claude-sonnet-5',
+        performance: { median_output_tokens_per_second: 0 },
+      };
+
+      respondWith(
+        unmeasuredFirst
+          ? [unmeasured, measuredModel]
+          : [measuredModel, unmeasured],
+      );
+
+      const record = (await fetchArtificialAnalysisBenchmarks('key')).get(
+        'claudesonnet5',
+      );
+
+      expect(record?.intelligenceIndex).toBe(38.4);
+    },
+  );
+
+  it('survives a null row rather than discarding the whole response', async () => {
+    respondWith([null, measuredModel]);
+
+    const record = (await fetchArtificialAnalysisBenchmarks('key')).get(
+      'claudesonnet5',
+    );
+
+    expect(record?.intelligenceIndex).toBe(38.4);
+  });
+
+  it('fails loudly when every row came back unmeasured', async () => {
+    respondWith([
+      {
+        slug: 'claude-sonnet-5',
+        performance: { median_output_tokens_per_second: 0 },
+      },
+    ]);
+
+    await expect(fetchArtificialAnalysisBenchmarks('key')).rejects.toThrow(
+      /none carried a measurement/,
+    );
+  });
+
+  it('ignores figures published at the top level instead of their container', async () => {
+    respondWith([
+      {
+        slug: 'claude-sonnet-5',
+        artificial_analysis_intelligence_index: 38.4,
+        performance: { median_output_tokens_per_second: 92.1 },
+      },
+    ]);
+
+    const record = (await fetchArtificialAnalysisBenchmarks('key')).get(
+      'claudesonnet5',
+    );
+
+    expect(record?.intelligenceIndex).toBeUndefined();
+    expect(record?.outputTokensPerSecond).toBe(92.1);
+  });
+
+  it('reads every page of the list before indexing', async () => {
+    const pages = [
+      {
+        data: [measuredModel],
+        pagination: { page: 1, has_more: true },
+      },
+      {
+        data: [
+          {
+            slug: 'gpt-5-6-luna',
+            evaluations: { artificial_analysis_intelligence_index: 37.5 },
+          },
+        ],
+        pagination: { page: 2, has_more: false },
+      },
+    ];
+    const fetchMock = jest.fn(async (url: URL) => ({
+      ok: true,
+      json: async () => pages[Number(url.searchParams.get('page')) - 1],
+    }));
+
+    global.fetch = fetchMock as unknown as typeof fetch;
+
+    const index = await fetchArtificialAnalysisBenchmarks('key');
+
+    expect(fetchMock).toHaveBeenCalledTimes(2);
+    expect(index.get('claudesonnet5')?.intelligenceIndex).toBe(38.4);
+    expect(index.get('gpt56luna')?.intelligenceIndex).toBe(37.5);
+  });
+
+  it('files a row under its effort as well as under the bare model name', async () => {
+    respondWith([
+      {
+        slug: 'gpt-5-6-sol',
+        name: 'GPT-5.6 Sol (max)',
+        evaluations: { artificial_analysis_intelligence_index: 47.1 },
+      },
+      {
+        slug: 'gpt-5-6-sol',
+        name: 'GPT-5.6 Sol (low)',
+        evaluations: { artificial_analysis_intelligence_index: 30.2 },
+        artificial_analysis_intelligence_index_cost: {
+          cost_per_task: { total_cost: 0.08 },
+        },
+      },
+    ]);
+
+    const index = await fetchArtificialAnalysisBenchmarks('key');
+
+    // The bare key still carries the ceiling, so nothing a consumer reads today
+    // changes.
+    expect(index.get('gpt56sol')?.intelligenceIndex).toBe(47.1);
+    expect(index.get('gpt56sol')?.effort).toBe('max');
+    expect(index.get('gpt56sol@max')?.intelligenceIndex).toBe(47.1);
+    expect(index.get('gpt56sol@low')?.intelligenceIndex).toBe(30.2);
+    expect(index.get('gpt56sol@low')?.costPerTask).toBe(0.08);
+    expect(index.get('gpt56sol@low')?.effort).toBe('low');
+  });
+
+  it('keeps the better-measured row when two rows share an effort', async () => {
+    respondWith([
+      {
+        slug: 'grok-4-6',
+        name: 'Grok 4.6 (high)',
+        evaluations: { artificial_analysis_intelligence_index: 44.4 },
+      },
+      {
+        slug: 'grok-4-6',
+        name: 'Grok 4.6 (high)',
+        evaluations: { artificial_analysis_intelligence_index: 44.4 },
+        performance: { median_output_tokens_per_second: 71 },
+      },
+    ]);
+
+    const index = await fetchArtificialAnalysisBenchmarks('key');
+
+    expect(index.get('grok46@high')?.outputTokensPerSecond).toBe(71);
+  });
+
+  it('files a row naming no effort under the bare name only', async () => {
+    respondWith([
+      {
+        slug: 'gemini-3-1-pro-preview',
+        name: 'Gemini 3.1 Pro Preview',
+        evaluations: { artificial_analysis_intelligence_index: 30.4 },
+      },
+    ]);
+
+    const index = await fetchArtificialAnalysisBenchmarks('key');
+
+    expect(index.get('gemini31propreview')?.effort).toBeUndefined();
+    expect([...index.keys()].some((key) => key.includes('@'))).toBe(false);
+  });
+});

packages/twenty-server/scripts/ai-catalog-sync/__tests__/match-benchmarks.spec.ts

@@ -0,0 +1,339 @@
+import { type ModelsDevModel } from 'src/engine/metadata-modules/ai/ai-models/types/models-dev-model.type';
+
+import { type BenchmarkIndex } from '../types/benchmark-index.type';
+import { type BenchmarkRecord } from '../types/benchmark-record.type';
+import { buildLookupCandidates } from '../utils/build-lookup-candidates.util';
+import { matchBenchmarks } from '../utils/match-benchmarks.util';
+
+const MEASURED_AT = '2026-09-09';
+
+const modelsDevModel = (
+  overrides: Partial<ModelsDevModel> = {},
+): ModelsDevModel => ({
+  id: 'model',
+  name: 'Model',
+  cost: { input: 2, output: 6 },
+  limit: { context: 128000, output: 8192 },
+  ...overrides,
+});
+
+const MISTRAL_MODELS: Record<string, ModelsDevModel> = {
+  'mistral-large-latest': modelsDevModel({ id: 'mistral-large-latest' }),
+  'mistral-large-2512': modelsDevModel({
+    id: 'mistral-large-2512',
+    release_date: '2025-12-01',
+  }),
+  'mistral-large-2411': modelsDevModel({
+    id: 'mistral-large-2411',
+    release_date: '2024-11-01',
+    cost: { input: 3, output: 9 },
+  }),
+};
+
+const indexOf = (entries: Record<string, BenchmarkRecord>): BenchmarkIndex =>
+  new Map(Object.entries(entries));
+
+const match = (modelName: string, benchmarkIndex: BenchmarkIndex) =>
+  matchBenchmarks({
+    modelName,
+    siblingModels: MISTRAL_MODELS,
+    benchmarkIndex,
+    measuredAt: MEASURED_AT,
+  });
+
+describe('buildLookupCandidates', () => {
+  it('resolves a rolling alias to the priced-identical release it points at', () => {
+    expect(
+      buildLookupCandidates({
+        modelName: 'mistral-large-latest',
+        siblingModels: MISTRAL_MODELS,
+      }),
+    ).toContain('mistral-large-2512');
+  });
+
+  it('prefers the newest release when several share the same price and limits', () => {
+    const candidates = buildLookupCandidates({
+      modelName: 'mistral-large-latest',
+      siblingModels: {
+        ...MISTRAL_MODELS,
+        'mistral-large-2506': modelsDevModel({ release_date: '2025-06-01' }),
+      },
+    });
+
+    expect(candidates).toContain('mistral-large-2512');
+    expect(candidates).not.toContain('mistral-large-2506');
+  });
+
+  it('resolves the same way whatever order an undated provider lists twins in', () => {
+    // Nothing in models.dev hits this today, but the resolution must not depend
+    // on the order a third party happens to serialize its models in.
+    const undated = {
+      'mistral-large-latest': modelsDevModel({ id: 'mistral-large-latest' }),
+      'mistral-large-2506': modelsDevModel({ id: 'mistral-large-2506' }),
+      'mistral-large-2512': modelsDevModel({ id: 'mistral-large-2512' }),
+    };
+
+    const resolve = (siblingModels: Record<string, ModelsDevModel>) =>
+      buildLookupCandidates({
+        modelName: 'mistral-large-latest',
+        siblingModels,
+      });
+
+    const reordered = {
+      'mistral-large-latest': undated['mistral-large-latest'],
+      'mistral-large-2512': undated['mistral-large-2512'],
+      'mistral-large-2506': undated['mistral-large-2506'],
+    };
+
+    [undated, reordered].forEach((siblingModels) => {
+      const candidates = resolve(siblingModels);
+
+      expect(candidates).toContain('mistral-large-2512');
+      expect(candidates).not.toContain('mistral-large-2506');
+    });
+  });
+
+  it('does not offer the undated name once the alias resolves to a release', () => {
+    // The leaderboard's bare `mistral-large` row is the Feb '24 model. Letting
+    // it stand in for the release `-latest` currently points at published a
+    // two-year-old score as the current one.
+    const candidates = buildLookupCandidates({
+      modelName: 'mistral-large-latest',
+      siblingModels: MISTRAL_MODELS,
+    });
+
+    expect(candidates).toContain('mistral-large-2512');
+    expect(candidates).not.toContain('mistral-large');
+  });
+
+  it('offers no undated spelling of a resolved full-date release either', () => {
+    // The publisher's undated row is whichever snapshot it last measured, which
+    // is the same trap as the bare rolling name in a longer date format.
+    const candidates = buildLookupCandidates({
+      modelName: 'claude-sonnet-5-latest',
+      siblingModels: {
+        'claude-sonnet-5-latest': modelsDevModel({
+          id: 'claude-sonnet-5-latest',
+        }),
+        'claude-sonnet-5-20260101': modelsDevModel({
+          id: 'claude-sonnet-5-20260101',
+          release_date: '2026-01-01',
+        }),
+      },
+    });
+
+    expect(candidates).toContain('claude-sonnet-5-20260101');
+    expect(candidates).not.toContain('claude-sonnet-5');
+  });
+
+  it('falls back to the undated name when no release can be resolved', () => {
+    const candidates = buildLookupCandidates({
+      modelName: 'mistral-large-latest',
+      siblingModels: {
+        'mistral-large-latest': modelsDevModel({ id: 'mistral-large-latest' }),
+      },
+    });
+
+    expect(candidates).toContain('mistral-large');
+  });
+
+  it('offers the undated name for a dated snapshot', () => {
+    expect(
+      buildLookupCandidates({
+        modelName: 'claude-sonnet-4-5-20250929',
+        siblingModels: {},
+      }),
+    ).toContain('claude-sonnet-4-5');
+  });
+});
+
+describe('matchBenchmarks', () => {
+  it('scores a rolling alias from the release it resolves to', () => {
+    const result = match(
+      'mistral-large-latest',
+      indexOf({ mistrallarge2512: { intelligenceIndex: 41, aliases: [] } }),
+    );
+
+    expect(result?.benchmark.intelligenceIndex).toBe(41);
+    expect(result?.benchmark.measuredAt).toBe(MEASURED_AT);
+  });
+
+  it('carries speed and cost per task through to the catalog', () => {
+    const result = match(
+      'mistral-large-2512',
+      indexOf({
+        mistrallarge2512: {
+          outputTokensPerSecond: 92,
+          costPerTask: 0.42,
+          aliases: [],
+        },
+      }),
+    );
+
+    expect(result?.benchmark.outputTokensPerSecond).toBe(92);
+    expect(result?.benchmark.costPerTask).toBe(0.42);
+  });
+
+  it('returns nothing for a model absent from the index', () => {
+    expect(match('mistral-large-2512', new Map())).toBeUndefined();
+  });
+
+  it('returns nothing for a matched row carrying no measurement at all', () => {
+    // The publisher lists models it has measured nothing about; matching one
+    // must not produce an entry holding only a timestamp.
+    expect(
+      match(
+        'mistral-large-2512',
+        indexOf({ mistrallarge2512: { aliases: ['mistral-large-2512'] } }),
+      ),
+    ).toBeUndefined();
+  });
+
+  it('prefers the release a rolling alias resolves to over an undated row', () => {
+    const result = match(
+      'mistral-large-latest',
+      indexOf({
+        mistrallarge: { intelligenceIndex: 20, aliases: [] },
+        mistrallarge2512: { intelligenceIndex: 41, aliases: [] },
+      }),
+    );
+
+    expect(result?.benchmark.intelligenceIndex).toBe(41);
+  });
+
+  it('collects the aliases the publisher gives so consumers can join on them', () => {
+    const result = match(
+      'mistral-large-2512',
+      indexOf({
+        mistrallarge2512: {
+          intelligenceIndex: 41,
+          aliases: ['mistral-large-2512', 'mistralai/mistral-large-2512'],
+        },
+      }),
+    );
+
+    expect(result?.aliases).toEqual([
+      'mistral-large-2512',
+      'mistralai/mistral-large-2512',
+    ]);
+  });
+
+  it('reads each declared effort from the row taken at that effort', () => {
+    const result = matchBenchmarks({
+      modelName: 'mistral-large-2512',
+      siblingModels: MISTRAL_MODELS,
+      benchmarkIndex: indexOf({
+        mistrallarge2512: {
+          intelligenceIndex: 41,
+          effort: 'high',
+          aliases: ['Mistral Large 3 (high)'],
+        },
+        'mistrallarge2512@high': {
+          intelligenceIndex: 41,
+          effort: 'high',
+          aliases: ['Mistral Large 3 (high)'],
+        },
+        'mistrallarge2512@none': {
+          intelligenceIndex: 22,
+          costPerTask: 0.1,
+          effort: 'none',
+          aliases: ['Mistral Large 3 (Non-reasoning)'],
+        },
+      }),
+      measuredAt: MEASURED_AT,
+      efforts: ['none', 'high'],
+    });
+
+    expect(result?.benchmark.effort).toBe('high');
+    expect(result?.benchmarkByEffort?.none).toEqual({
+      intelligenceIndex: 22,
+      costPerTask: 0.1,
+      effort: 'none',
+      measuredAt: MEASURED_AT,
+      aliases: ['Mistral Large 3 (Non-reasoning)'],
+    });
+    expect(result?.benchmarkByEffort?.high?.intelligenceIndex).toBe(41);
+  });
+
+  it('leaves a declared effort without its own row blank rather than lending it the ceiling', () => {
+    const result = matchBenchmarks({
+      modelName: 'mistral-large-2512',
+      siblingModels: MISTRAL_MODELS,
+      benchmarkIndex: indexOf({
+        mistrallarge2512: {
+          intelligenceIndex: 41,
+          effort: 'high',
+          aliases: [],
+        },
+        'mistrallarge2512@high': {
+          intelligenceIndex: 41,
+          effort: 'high',
+          aliases: [],
+        },
+      }),
+      measuredAt: MEASURED_AT,
+      efforts: ['none', 'high'],
+    });
+
+    expect(result?.benchmarkByEffort?.none).toBeUndefined();
+    expect(result?.benchmarkByEffort?.high?.intelligenceIndex).toBe(41);
+  });
+
+  it('carries no effort map at all when no declared effort has a row', () => {
+    const result = matchBenchmarks({
+      modelName: 'mistral-large-2512',
+      siblingModels: MISTRAL_MODELS,
+      benchmarkIndex: indexOf({
+        mistrallarge2512: { intelligenceIndex: 41, aliases: [] },
+      }),
+      measuredAt: MEASURED_AT,
+      efforts: ['none', 'high'],
+    });
+
+    expect(result?.benchmark.intelligenceIndex).toBe(41);
+    expect(result?.benchmarkByEffort).toBeUndefined();
+  });
+
+  it('resolves a rolling alias to its release for per-effort rows too', () => {
+    const result = matchBenchmarks({
+      modelName: 'mistral-large-latest',
+      siblingModels: MISTRAL_MODELS,
+      benchmarkIndex: indexOf({
+        mistrallarge2512: { intelligenceIndex: 41, aliases: [] },
+        'mistrallarge2512@high': {
+          intelligenceIndex: 41,
+          effort: 'high',
+          aliases: [],
+        },
+        'mistrallarge@high': {
+          intelligenceIndex: 20,
+          effort: 'high',
+          aliases: [],
+        },
+      }),
+      measuredAt: MEASURED_AT,
+      efforts: ['high'],
+    });
+
+    expect(result?.benchmarkByEffort?.high?.intelligenceIndex).toBe(41);
+  });
+
+  it('labels a per-effort reading with the key it was filed under', () => {
+    const result = matchBenchmarks({
+      modelName: 'mistral-large-2512',
+      siblingModels: MISTRAL_MODELS,
+      benchmarkIndex: indexOf({
+        mistrallarge2512: { intelligenceIndex: 41, aliases: [] },
+        'mistrallarge2512@low': {
+          intelligenceIndex: 30,
+          effort: 'high',
+          aliases: [],
+        },
+      }),
+      measuredAt: MEASURED_AT,
+      efforts: ['low'],
+    });
+
+    expect(result?.benchmarkByEffort?.low?.effort).toBe('low');
+  });
+});

packages/twenty-server/scripts/ai-catalog-sync/__tests__/parse-artificial-analysis-effort.spec.ts

@@ -0,0 +1,30 @@
+import { parseArtificialAnalysisEffort } from '../utils/parse-artificial-analysis-effort.util';
+
+describe('parseArtificialAnalysisEffort', () => {
+  it.each([
+    ['GPT-5.6 Sol (max)', 'max'],
+    ['GPT-5.4 (xhigh)', 'xhigh'],
+    ['Grok 4.6 (high)', 'high'],
+    ['Gemini 3.5 Flash (medium)', 'medium'],
+    ['GPT-5 nano (minimal)', 'minimal'],
+    ['Claude Opus 5 (Adaptive Reasoning, Max Effort)', 'max'],
+    [
+      'Claude Fable 5 (Adaptive Reasoning, Max Effort, Opus 4.8 Fallback)',
+      'max',
+    ],
+    ['Claude Sonnet 4.6 (Non-reasoning, Low Effort)', 'low'],
+    ['GPT-5.6 Luna (Non-reasoning)', 'none'],
+  ])('reads "%s" as %s', (displayName, effort) => {
+    expect(parseArtificialAnalysisEffort(displayName)).toBe(effort);
+  });
+
+  it.each([
+    'Claude Opus 4.5 (Reasoning)',
+    'Gemini 3.1 Pro Preview',
+    'GPT-4o (March 2025, chatgpt-4o-latest)',
+    "Llama 3.1 405B (May '24)",
+    'Mistral Large 3 (high) Turbo',
+  ])('names no effort for "%s"', (displayName) => {
+    expect(parseArtificialAnalysisEffort(displayName)).toBeUndefined();
+  });
+});

packages/twenty-server/scripts/ai-catalog-sync/__tests__/project-catalog.spec.ts

@@ -0,0 +1,292 @@
+import { type CatalogSpec } from '../types/catalog-spec.type';
+import {
+  type CanonicalCatalog,
+  projectCatalog,
+} from '../utils/project-catalog.util';
+
+const canonicalCatalog: CanonicalCatalog = {
+  openai: {
+    npm: '@ai-sdk/openai',
+    label: 'OpenAI',
+    apiKey: '{{OPENAI_API_KEY}}',
+    models: [
+      {
+        name: 'gpt-5.6-luna',
+        label: 'GPT-5.6 Luna',
+        inputCostPerMillionTokens: 0.2,
+        outputCostPerMillionTokens: 1.2,
+        cachedInputCostPerMillionTokens: 0.02,
+        modalities: ['image', 'pdf'],
+        efforts: ['low', 'medium', 'high'],
+        benchmark: { intelligenceIndex: 37.5, measuredAt: '2026-09-11' },
+        benchmarkByEffort: {
+          high: { intelligenceIndex: 32.4, measuredAt: '2026-09-11' },
+        },
+      },
+    ],
+  },
+  anthropic: {
+    npm: '@ai-sdk/anthropic',
+    label: 'Anthropic',
+    apiKey: '{{ANTHROPIC_API_KEY}}',
+    models: [{ name: 'claude-opus-4-7', label: 'Claude Opus 4.7' }],
+  },
+};
+
+describe('projectCatalog', () => {
+  it('describes a route from the catalog while keeping the route its own credentials', () => {
+    const spec: CatalogSpec = {
+      providers: [
+        {
+          name: 'azure-foundry',
+          npm: '@ai-sdk/azure',
+          label: 'Azure AI Foundry',
+          apiKey: '{{AZURE_FOUNDRY_API_KEY}}',
+          baseUrl: '{{AZURE_FOUNDRY_BASE_URL}}',
+          dataResidency: 'eu',
+          labelSuffix: ' (Azure)',
+          models: ['gpt-5.6-luna'],
+        },
+      ],
+    };
+
+    const projected = projectCatalog({ canonicalCatalog, spec });
+
+    expect(Object.keys(projected)).toEqual(['azure-foundry']);
+    expect(projected['azure-foundry']).toMatchObject({
+      npm: '@ai-sdk/azure',
+      label: 'Azure AI Foundry',
+      apiKey: '{{AZURE_FOUNDRY_API_KEY}}',
+      baseUrl: '{{AZURE_FOUNDRY_BASE_URL}}',
+    });
+    expect(projected['azure-foundry'].models?.[0]).toEqual({
+      name: 'gpt-5.6-luna',
+      label: 'GPT-5.6 Luna (Azure)',
+      inputCostPerMillionTokens: 0.2,
+      outputCostPerMillionTokens: 1.2,
+      cachedInputCostPerMillionTokens: 0.02,
+      modalities: ['image', 'pdf'],
+      efforts: ['low', 'medium', 'high'],
+      benchmark: { intelligenceIndex: 37.5, measuredAt: '2026-09-11' },
+      benchmarkByEffort: {
+        high: { intelligenceIndex: 32.4, measuredAt: '2026-09-11' },
+      },
+      dataResidency: 'eu',
+    });
+  });
+
+  it('serves a model under the id the route deploys it as', () => {
+    const spec: CatalogSpec = {
+      providers: [
+        {
+          name: 'amazon-bedrock',
+          npm: '@ai-sdk/amazon-bedrock',
+          region: 'eu-central-1',
+          models: [
+            { model: 'claude-opus-4-7', as: 'eu.anthropic.claude-opus-4-7' },
+          ],
+        },
+      ],
+    };
+
+    const projected = projectCatalog({ canonicalCatalog, spec });
+
+    expect(projected['amazon-bedrock'].models?.[0]).toMatchObject({
+      name: 'eu.anthropic.claude-opus-4-7',
+      label: 'Claude Opus 4.7',
+    });
+    expect(projected['amazon-bedrock'].region).toBe('eu-central-1');
+  });
+
+  it('takes a price the route negotiated and leaves the rest of the reading alone', () => {
+    const spec: CatalogSpec = {
+      providers: [
+        {
+          name: 'azure-foundry',
+          npm: '@ai-sdk/azure',
+          models: [
+            { model: 'gpt-5.6-luna', cachedInputCostPerMillionTokens: 0.5 },
+          ],
+        },
+      ],
+    };
+
+    const projected = projectCatalog({ canonicalCatalog, spec });
+
+    expect(projected['azure-foundry'].models?.[0]).toMatchObject({
+      cachedInputCostPerMillionTokens: 0.5,
+      inputCostPerMillionTokens: 0.2,
+      efforts: ['low', 'medium', 'high'],
+    });
+  });
+
+  it('takes a limit the route caps below the one the model allows', () => {
+    const spec: CatalogSpec = {
+      providers: [
+        {
+          name: 'amazon-bedrock',
+          npm: '@ai-sdk/amazon-bedrock',
+          models: [{ model: 'gpt-5.6-luna', maxOutputTokens: 64000 }],
+        },
+      ],
+    };
+
+    const projected = projectCatalog({ canonicalCatalog, spec });
+
+    expect(projected['amazon-bedrock'].models?.[0]).toMatchObject({
+      maxOutputTokens: 64000,
+    });
+  });
+
+  it('serves every model a vendor publishes when the spec names the vendor', () => {
+    const spec: CatalogSpec = {
+      providers: [
+        {
+          name: 'openai',
+          npm: '@ai-sdk/openai',
+          apiKey: '{{OPENAI_API_KEY}}',
+          models: { vendor: 'openai' },
+        },
+      ],
+    };
+
+    const projected = projectCatalog({ canonicalCatalog, spec });
+
+    expect(projected['openai'].models?.map((model) => model.name)).toEqual([
+      'gpt-5.6-luna',
+    ]);
+    expect(projected['openai'].models?.[0]).toMatchObject({
+      efforts: ['low', 'medium', 'high'],
+    });
+  });
+
+  it('refuses a spec naming a vendor the catalog does not carry', () => {
+    const spec: CatalogSpec = {
+      providers: [
+        {
+          name: 'openai',
+          npm: '@ai-sdk/openai',
+          models: { vendor: 'openai-imaginary' },
+        },
+      ],
+    };
+
+    expect(() => projectCatalog({ canonicalCatalog, spec })).toThrow(
+      'which the catalog does not carry',
+    );
+  });
+
+  it('refuses a spec naming a model the catalog does not carry', () => {
+    const spec: CatalogSpec = {
+      providers: [
+        {
+          name: 'azure-foundry',
+          npm: '@ai-sdk/azure',
+          models: ['gpt-5.6-luna', 'gpt-9-imaginary'],
+        },
+      ],
+    };
+
+    expect(() => projectCatalog({ canonicalCatalog, spec })).toThrow(
+      'azure-foundry/gpt-9-imaginary',
+    );
+  });
+
+  it('cannot restate what a model is, only which models a route serves', () => {
+    const spec = {
+      providers: [
+        {
+          name: 'azure-foundry',
+          npm: '@ai-sdk/azure',
+          models: [{ model: 'gpt-5.6-luna', modalities: ['image'] }],
+        },
+      ],
+    } as unknown as CatalogSpec;
+
+    const projected = projectCatalog({ canonicalCatalog, spec });
+
+    expect(projected['azure-foundry'].models?.[0]).toMatchObject({
+      modalities: ['image', 'pdf'],
+    });
+  });
+
+  it('refuses a repeated provider, which would drop the first one silently', () => {
+    const spec: CatalogSpec = {
+      providers: [
+        {
+          name: 'azure-foundry',
+          npm: '@ai-sdk/azure',
+          models: ['gpt-5.6-luna'],
+        },
+        {
+          name: 'azure-foundry',
+          npm: '@ai-sdk/azure',
+          models: ['claude-opus-4-7'],
+        },
+      ],
+    };
+
+    expect(() => projectCatalog({ canonicalCatalog, spec })).toThrow(
+      'repeated or reserved',
+    );
+  });
+
+  it('refuses an SDK package or residency the server would not parse', () => {
+    const withNpm = (npm: string): CatalogSpec => ({
+      providers: [{ name: 'gateway', npm, models: ['gpt-5.6-luna'] }],
+    });
+
+    expect(() =>
+      projectCatalog({ canonicalCatalog, spec: withNpm('@ai-sdk/imaginary') }),
+    ).toThrow('unsupported SDK package');
+
+    expect(() =>
+      projectCatalog({
+        canonicalCatalog,
+        spec: {
+          providers: [
+            {
+              name: 'gateway',
+              npm: '@ai-sdk/openai',
+              dataResidency: 'mars',
+              models: ['gpt-5.6-luna'],
+            },
+          ],
+        },
+      }),
+    ).toThrow('unsupported data residency');
+  });
+
+  it('refuses a negotiated price that is not a usable number', () => {
+    const spec: CatalogSpec = {
+      providers: [
+        {
+          name: 'azure-foundry',
+          npm: '@ai-sdk/azure',
+          models: [{ model: 'gpt-5.6-luna', inputCostPerMillionTokens: -1 }],
+        },
+      ],
+    };
+
+    expect(() => projectCatalog({ canonicalCatalog, spec })).toThrow(
+      'which is not a usable number',
+    );
+  });
+
+  it('marks the route on a label the spec overrides', () => {
+    const spec: CatalogSpec = {
+      providers: [
+        {
+          name: 'azure-foundry',
+          npm: '@ai-sdk/azure',
+          labelSuffix: ' (Azure)',
+          models: [{ model: 'gpt-5.6-luna', label: 'Luna' }],
+        },
+      ],
+    };
+
+    const projected = projectCatalog({ canonicalCatalog, spec });
+
+    expect(projected['azure-foundry'].models?.[0]?.label).toBe('Luna (Azure)');
+  });
+});

packages/twenty-server/scripts/ai-catalog-sync/__tests__/read-committed-benchmarks.spec.ts

@@ -0,0 +1,142 @@
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+
+import { matchBenchmarks } from '../utils/match-benchmarks.util';
+import { readCommittedBenchmarks } from '../utils/read-committed-benchmarks.util';
+
+const writeOverlay = (value: unknown): string => {
+  const filePath = path.join(
+    fs.mkdtempSync(path.join(os.tmpdir(), 'overlay-')),
+    'ai-model-benchmarks.json',
+  );
+
+  fs.writeFileSync(filePath, JSON.stringify(value), 'utf-8');
+
+  return filePath;
+};
+
+const PUBLISHED = {
+  source: 'artificialanalysis.ai',
+  measuredAt: '2026-09-01',
+  models: {
+    'claude-sonnet-5': {
+      intelligenceIndex: 38.4,
+      outputTokensPerSecond: 92.1,
+      measuredAt: '2026-09-01',
+      aliases: ['claude-sonnet-5', 'claude-sonnet-5-20260630'],
+    },
+  },
+};
+
+describe('readCommittedBenchmarks', () => {
+  it('returns nothing when no overlay has been published yet', () => {
+    expect(
+      readCommittedBenchmarks(path.join(os.tmpdir(), 'does-not-exist.json'))
+        .size,
+    ).toBe(0);
+  });
+
+  it('recovers every measurement', () => {
+    const index = readCommittedBenchmarks(writeOverlay(PUBLISHED));
+    const record = index.get('claudesonnet5');
+
+    expect(record?.intelligenceIndex).toBe(38.4);
+    expect(record?.outputTokensPerSecond).toBe(92.1);
+  });
+
+  it('indexes the published aliases so matching still resolves them', () => {
+    const index = readCommittedBenchmarks(writeOverlay(PUBLISHED));
+
+    expect(index.get('claudesonnet520260630')?.intelligenceIndex).toBe(38.4);
+  });
+
+  it('keeps a preserved measurement dated when it was taken, not today', () => {
+    // Restamping recovered data as measured today would present a stale
+    // benchmark as fresh on every failed run.
+    const index = readCommittedBenchmarks(writeOverlay(PUBLISHED));
+
+    const result = matchBenchmarks({
+      modelName: 'claude-sonnet-5',
+      siblingModels: {},
+      benchmarkIndex: index,
+      measuredAt: '2026-09-09',
+    });
+
+    expect(result?.benchmark.measuredAt).toBe('2026-09-01');
+  });
+
+  it('recovers per-effort readings under their effort keys, dated when they were taken', () => {
+    const index = readCommittedBenchmarks(
+      writeOverlay({
+        ...PUBLISHED,
+        models: {
+          'claude-sonnet-5': {
+            ...PUBLISHED.models['claude-sonnet-5'],
+            effort: 'max',
+            benchmarkByEffort: {
+              low: {
+                intelligenceIndex: 20.1,
+                measuredAt: '2026-09-01',
+                aliases: ['Claude Sonnet 5 (low)'],
+              },
+            },
+          },
+        },
+      }),
+    );
+
+    expect(index.get('claudesonnet5')?.effort).toBe('max');
+    expect(index.get('claudesonnet5@low')?.intelligenceIndex).toBe(20.1);
+    expect(index.get('claudesonnet5@low')?.effort).toBe('low');
+    expect(index.get('claudesonnet5@low')?.measuredAt).toBe('2026-09-01');
+    expect(index.get('claudesonnet5@max')).toBeUndefined();
+
+    const result = matchBenchmarks({
+      modelName: 'claude-sonnet-5',
+      siblingModels: {},
+      benchmarkIndex: index,
+      measuredAt: '2026-09-09',
+      efforts: ['low', 'max'],
+    });
+
+    expect(result?.benchmarkByEffort?.low?.measuredAt).toBe('2026-09-01');
+    expect(result?.benchmarkByEffort?.max).toBeUndefined();
+  });
+
+  it('ignores a per-effort key it does not recognise', () => {
+    const index = readCommittedBenchmarks(
+      writeOverlay({
+        ...PUBLISHED,
+        models: {
+          'claude-sonnet-5': {
+            ...PUBLISHED.models['claude-sonnet-5'],
+            benchmarkByEffort: {
+              turbo: { intelligenceIndex: 99, aliases: [] },
+            },
+          },
+        },
+      }),
+    );
+
+    expect([...index.keys()].some((key) => key.includes('@'))).toBe(false);
+  });
+
+  it('trusts the key over a nested reading that names another effort', () => {
+    const index = readCommittedBenchmarks(
+      writeOverlay({
+        ...PUBLISHED,
+        models: {
+          'claude-sonnet-5': {
+            ...PUBLISHED.models['claude-sonnet-5'],
+            benchmarkByEffort: {
+              low: { intelligenceIndex: 20.1, effort: 'high', aliases: [] },
+            },
+          },
+        },
+      }),
+    );
+
+    expect(index.get('claudesonnet5@low')?.effort).toBe('low');
+  });
+});

packages/twenty-server/scripts/ai-catalog-sync/__tests__/self-host-projection.spec.ts

@@ -0,0 +1,22 @@
+import canonicalCatalog from 'src/engine/metadata-modules/ai/ai-models/ai-models.json';
+import shippedCatalog from 'src/engine/metadata-modules/ai/ai-models/ai-providers.json';
+import selfHostSpec from 'src/engine/metadata-modules/ai/ai-models/ai-self-host-spec.json';
+
+import { type CatalogSpec } from '../types/catalog-spec.type';
+import {
+  type CanonicalCatalog,
+  projectCatalog,
+} from '../utils/project-catalog.util';
+
+// The shipped catalog is generated, and a hand edit to it is the drift this
+// pipeline exists to remove, so it has to stay what its spec produces.
+describe('the shipped catalog', () => {
+  it('is what the self-host spec projects from the model catalog', () => {
+    const projected = projectCatalog({
+      canonicalCatalog: canonicalCatalog as CanonicalCatalog,
+      spec: selfHostSpec as CatalogSpec,
+    });
+
+    expect(projected).toEqual(shippedCatalog);
+  });
+});

packages/twenty-server/scripts/ai-catalog-sync/index.ts

@@ -0,0 +1,211 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import * as prettier from 'prettier';
+
+import { isDefined, isNonEmptyArray } from 'twenty-shared/utils';
+
+import { MODELS_DEV_API_URL } from 'src/engine/metadata-modules/ai/ai-models/constants/models-dev.const';
+import { type ModelsDevData } from 'src/engine/metadata-modules/ai/ai-models/types/models-dev-data.type';
+
+import { assertPayloadIsUsable } from './utils/assert-payload-is-usable.util';
+import { buildCatalog } from './utils/build-catalog.util';
+import { carryOverCommittedFields } from './utils/carry-over-committed-fields.util';
+import { enrichCatalog } from './utils/enrich-catalog.util';
+import { fetchArtificialAnalysisBenchmarks } from './utils/fetch-artificial-analysis-benchmarks.util';
+import { projectCatalog } from './utils/project-catalog.util';
+import { readCommittedBenchmarks } from './utils/read-committed-benchmarks.util';
+import { buildCoverageReport } from './utils/build-coverage-report.util';
+import { renderCoverageReport } from './utils/render-coverage-report.util';
+import { type BenchmarkIndex } from './types/benchmark-index.type';
+import { type CatalogSpec } from './types/catalog-spec.type';
+import { type GeneratedCatalog } from './types/generated-catalog.type';
+
+const AI_MODELS_DIR = path.resolve(
+  __dirname,
+  '..',
+  '..',
+  'src',
+  'engine',
+  'metadata-modules',
+  'ai',
+  'ai-models',
+);
+
+const MODELS_PATH = path.join(AI_MODELS_DIR, 'ai-models.json');
+const SELF_HOST_SPEC_PATH = path.join(AI_MODELS_DIR, 'ai-self-host-spec.json');
+const CATALOG_PATH = path.join(AI_MODELS_DIR, 'ai-providers.json');
+const BENCHMARKS_PATH = path.join(AI_MODELS_DIR, 'ai-model-benchmarks.json');
+
+// oxlint-disable no-console
+const log = console.log;
+const warn = console.warn;
+// oxlint-enable no-console
+
+const readArgument = (flag: string): string | undefined => {
+  const index = process.argv.indexOf(flag);
+
+  return index === -1 ? undefined : process.argv[index + 1];
+};
+
+const readCommittedModels = (filePath: string): GeneratedCatalog =>
+  fs.existsSync(filePath)
+    ? (JSON.parse(fs.readFileSync(filePath, 'utf-8')) as GeneratedCatalog)
+    : {};
+
+// The vendors the shipped catalog carries are the ones the self-host spec
+// serves; nothing else needs to be fetched, checked or written.
+const readVendors = (spec: CatalogSpec): string[] => {
+  const vendors = [
+    ...new Set(
+      spec.providers.flatMap((provider) =>
+        Array.isArray(provider.models) ? [] : [provider.models.vendor],
+      ),
+    ),
+  ];
+
+  // An empty list would fetch nothing, assert nothing and write an empty
+  // catalog over the real one, and the sync PR automerges.
+  if (!isNonEmptyArray(vendors)) {
+    throw new Error(
+      `${SELF_HOST_SPEC_PATH} names no vendor to carry: every route lists models explicitly`,
+    );
+  }
+
+  return vendors;
+};
+
+const fetchModelsDev = async ({
+  vendors,
+}: {
+  vendors: string[];
+}): Promise<ModelsDevData> => {
+  const response = await fetch(MODELS_DEV_API_URL, {
+    signal: AbortSignal.timeout(120000),
+  });
+
+  if (!response.ok) {
+    throw new Error(`${response.status} ${response.statusText}`);
+  }
+
+  const data: ModelsDevData = await response.json();
+
+  assertPayloadIsUsable({ data, vendors });
+
+  return data;
+};
+
+// A leaderboard outage must never break the model catalog, so a failed fetch
+// degrades to an empty index and the catalog is written without benchmarks.
+const fetchBenchmarks = async (): Promise<BenchmarkIndex> => {
+  const apiKey = process.env.ARTIFICIAL_ANALYSIS_API_KEY;
+
+  if (!isDefined(apiKey) || apiKey.length === 0) {
+    warn('Skipping benchmarks: ARTIFICIAL_ANALYSIS_API_KEY is not set');
+
+    return new Map();
+  }
+
+  try {
+    const index = await fetchArtificialAnalysisBenchmarks(apiKey);
+
+    log(`Fetched ${index.size} model aliases from Artificial Analysis`);
+
+    return index;
+  } catch (error) {
+    warn(
+      `Skipping benchmarks: ${error instanceof Error ? error.message : String(error)}`,
+    );
+
+    return new Map();
+  }
+};
+
+const writeJson = async (filePath: string, value: unknown): Promise<void> => {
+  const prettierConfig = await prettier.resolveConfig(filePath);
+
+  const formatted = await prettier.format(
+    JSON.stringify(value, null, 2) + '\n',
+    { ...prettierConfig, filepath: filePath },
+  );
+
+  fs.writeFileSync(filePath, formatted, 'utf-8');
+  log(`Wrote ${filePath}`);
+};
+
+const main = async (): Promise<void> => {
+  const dryRun = process.argv.includes('--dry-run');
+  const reportPath = readArgument('--report');
+  const measuredAt = new Date().toISOString().slice(0, 10);
+
+  const selfHostSpec = JSON.parse(
+    fs.readFileSync(SELF_HOST_SPEC_PATH, 'utf-8'),
+  ) as CatalogSpec;
+  const vendors = readVendors(selfHostSpec);
+
+  log('Fetching models.dev API...');
+
+  const modelsDevData = await fetchModelsDev({ vendors });
+
+  log(`Fetched ${Object.keys(modelsDevData).length} providers from models.dev`);
+
+  const fetched = await fetchBenchmarks();
+
+  // A failed fetch must not delete measurements we already published: the
+  // catalog PR is automerged, so an empty index would silently strip every
+  // benchmark until the next healthy run.
+  const benchmarkIndex =
+    fetched.size > 0 ? fetched : readCommittedBenchmarks(BENCHMARKS_PATH);
+
+  if (fetched.size === 0 && benchmarkIndex.size > 0) {
+    warn(
+      `Reusing ${benchmarkIndex.size} committed benchmark aliases from a previous run`,
+    );
+  }
+
+  const catalog = buildCatalog({ data: modelsDevData, vendors });
+
+  carryOverCommittedFields({
+    catalog,
+    committedCatalog: readCommittedModels(MODELS_PATH),
+  });
+
+  const overlay = enrichCatalog({
+    catalog,
+    modelsDevData,
+    benchmarkIndex,
+    measuredAt,
+  });
+
+  const renderedReport = renderCoverageReport(buildCoverageReport(catalog));
+
+  log(`\n${renderedReport}\n`);
+
+  if (isDefined(reportPath)) {
+    fs.writeFileSync(reportPath, `${renderedReport}\n`, 'utf-8');
+  }
+
+  if (dryRun) {
+    log('[DRY RUN] Skipping writes');
+
+    return;
+  }
+
+  await writeJson(MODELS_PATH, catalog);
+  // Self-host runs the same projection cloud does, from a spec that names the
+  // five direct routes and lets each serve its whole vendor.
+  await writeJson(
+    CATALOG_PATH,
+    projectCatalog({ canonicalCatalog: catalog, spec: selfHostSpec }),
+  );
+  await writeJson(BENCHMARKS_PATH, {
+    source: 'artificialanalysis.ai',
+    measuredAt,
+    models: overlay,
+  });
+};
+
+main().catch((error) => {
+  // oxlint-disable-next-line no-console
+  console.error('AI catalog sync failed:', error);
+  process.exit(1);
+});

packages/twenty-server/scripts/ai-catalog-sync/project.ts

@@ -0,0 +1,134 @@
+// Projects the canonical catalog through a deployment spec. See README.md.
+//
+// This entry point and the files it imports resolve through node builtins and
+// source paths only, so a repository holding a private spec can run them
+// straight from a sparse checkout without installing the monorepo.
+
+import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { dirname, resolve } from 'node:path';
+
+import { type CatalogSpec } from './types/catalog-spec.type';
+import {
+  type CanonicalCatalog,
+  projectCatalog,
+} from './utils/project-catalog.util';
+
+const DEFAULT_CATALOG_PATH = resolve(
+  __dirname,
+  '..',
+  '..',
+  'src',
+  'engine',
+  'metadata-modules',
+  'ai',
+  'ai-models',
+  'ai-models.json',
+);
+
+// A flag whose value is missing would otherwise swallow the next flag and write
+// the catalog to a file named after it.
+const readArgument = (flag: string): string | undefined => {
+  const index = process.argv.indexOf(flag);
+
+  if (index === -1) {
+    return undefined;
+  }
+
+  const value = process.argv[index + 1];
+
+  if (value === undefined || value.startsWith('--')) {
+    throw new Error(`${flag} is missing its value`);
+  }
+
+  return value;
+};
+
+const readJson = <TValue>(path: string): TValue => {
+  try {
+    return JSON.parse(readFileSync(path, 'utf-8')) as TValue;
+  } catch (error) {
+    throw new Error(
+      `Could not read ${path}: ${error instanceof Error ? error.message : String(error)}`,
+    );
+  }
+};
+
+const assertSpecIsUsable = ({
+  spec,
+  path,
+}: {
+  spec: CatalogSpec;
+  path: string;
+}): void => {
+  if (!Array.isArray(spec.providers) || spec.providers.length === 0) {
+    throw new Error(`${path} declares no providers`);
+  }
+
+  for (const provider of spec.providers) {
+    if (typeof provider.name !== 'string' || provider.name.length === 0) {
+      throw new Error(`${path} has a provider without a name`);
+    }
+
+    if (typeof provider.npm !== 'string' || provider.npm.length === 0) {
+      throw new Error(
+        `${path}: provider "${provider.name}" has no npm package`,
+      );
+    }
+
+    const servesModels = Array.isArray(provider.models)
+      ? provider.models.length > 0
+      : typeof provider.models?.vendor === 'string' &&
+        provider.models.vendor.length > 0;
+
+    if (!servesModels) {
+      throw new Error(`${path}: provider "${provider.name}" serves no models`);
+    }
+  }
+};
+
+const describe = (catalog: CanonicalCatalog): string => {
+  const models = Object.values(catalog).flatMap(
+    (provider) => provider.models ?? [],
+  );
+  const withEfforts = models.filter((model) => model.efforts !== undefined);
+  const withBenchmark = models.filter((model) => model.benchmark !== undefined);
+
+  return `${Object.keys(catalog).length} providers, ${models.length} models, ${withEfforts.length} with efforts, ${withBenchmark.length} with a benchmark`;
+};
+
+const main = (): void => {
+  const specPath = readArgument('--spec');
+  const outPath = readArgument('--out');
+
+  if (specPath === undefined || outPath === undefined) {
+    throw new Error(
+      'Usage: project.ts --spec <spec.json> --out <catalog.json> [--catalog <ai-models.json>]',
+    );
+  }
+
+  const catalogPath = readArgument('--catalog') ?? DEFAULT_CATALOG_PATH;
+  const spec = readJson<CatalogSpec>(specPath);
+
+  assertSpecIsUsable({ spec, path: specPath });
+
+  const projected = projectCatalog({
+    canonicalCatalog: readJson<CanonicalCatalog>(catalogPath),
+    spec,
+  });
+
+  mkdirSync(dirname(resolve(outPath)), { recursive: true });
+  writeFileSync(outPath, `${JSON.stringify(projected, null, 2)}\n`, 'utf-8');
+
+  // oxlint-disable-next-line no-console
+  console.log(`Wrote ${outPath}: ${describe(projected)}`);
+};
+
+try {
+  main();
+} catch (error) {
+  // oxlint-disable-next-line no-console
+  console.error(
+    `AI catalog projection failed: ${error instanceof Error ? error.message : String(error)}`,
+  );
+  process.exit(1);
+}

packages/twenty-server/scripts/ai-catalog-sync/types/benchmark-index.type.ts

@@ -0,0 +1,5 @@
+import { type BenchmarkRecord } from './benchmark-record.type';
+
+// Keyed by normalized model name, so a single lookup resolves every alias a
+// source publishes for the same underlying model.
+export type BenchmarkIndex = Map<string, BenchmarkRecord>;

packages/twenty-server/scripts/ai-catalog-sync/types/benchmark-match.type.ts

@@ -0,0 +1,11 @@
+import { type AiModelEffort } from 'twenty-shared/ai';
+
+import { type AiModelBenchmark } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-benchmark.type';
+
+import { type BenchmarkOverlayReading } from './benchmark-overlay-entry.type';
+
+export type BenchmarkMatch = {
+  benchmark: AiModelBenchmark;
+  aliases: string[];
+  benchmarkByEffort?: Partial<Record<AiModelEffort, BenchmarkOverlayReading>>;
+};

packages/twenty-server/scripts/ai-catalog-sync/types/benchmark-overlay-entry.type.ts

@@ -0,0 +1,16 @@
+import { type AiModelEffort } from 'twenty-shared/ai';
+
+import { type AiModelBenchmark } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-benchmark.type';
+
+// A reading plus the alias set a joining consumer needs to find it under the
+// publisher's own spelling.
+export type BenchmarkOverlayReading = AiModelBenchmark & {
+  aliases: string[];
+};
+
+// The overlay is the cross-repo artifact: it carries what the catalog embeds
+// plus the aliases. Per-effort readings nest under the model's entry so a
+// consumer that reads only the top-level fields keeps working.
+export type BenchmarkOverlayEntry = BenchmarkOverlayReading & {
+  benchmarkByEffort?: Partial<Record<AiModelEffort, BenchmarkOverlayReading>>;
+};

packages/twenty-server/scripts/ai-catalog-sync/types/benchmark-record.type.ts

@@ -0,0 +1,13 @@
+import { type AiModelEffort } from 'twenty-shared/ai';
+
+export type BenchmarkRecord = {
+  intelligenceIndex?: number;
+  outputTokensPerSecond?: number;
+  costPerTask?: number;
+  // The effort the publisher's row was measured at, when its name says so.
+  effort?: AiModelEffort;
+  // Set only on a record recovered from the committed overlay, so preserved
+  // measurements keep the date they were actually taken.
+  measuredAt?: string;
+  aliases: string[];
+};

packages/twenty-server/scripts/ai-catalog-sync/types/catalog-spec.type.ts

@@ -0,0 +1,46 @@
+// Deliberately not extensible: a field a route cannot restate is a field it
+// cannot drift on, which is the property this pipeline is for.
+export type CatalogSpecModelOverrides = {
+  label?: string;
+  isDeprecated?: boolean;
+  contextWindowTokens?: number;
+  maxOutputTokens?: number;
+  inputCostPerMillionTokens?: number;
+  outputCostPerMillionTokens?: number;
+  cachedInputCostPerMillionTokens?: number;
+  cacheCreationCostPerMillionTokens?: number;
+};
+
+export type CatalogSpecModel = CatalogSpecModelOverrides & {
+  model: string;
+  // `eu.anthropic.claude-opus-4-7` for a Bedrock deployment of
+  // `claude-opus-4-7`.
+  as?: string;
+};
+
+// Every model the vendor publishes, so a route that serves a whole vendor
+// picks up new models with the daily sync instead of waiting for an edit here.
+export type CatalogSpecModelSelector = {
+  vendor: string;
+};
+
+export type CatalogSpecProvider = {
+  name: string;
+  npm: string;
+  label?: string;
+  apiKey?: string;
+  baseUrl?: string;
+  region?: string;
+  authType?: string;
+  accessKeyId?: string;
+  secretAccessKey?: string;
+  sessionToken?: string;
+  dataResidency?: string;
+  zeroDataRetention?: boolean;
+  labelSuffix?: string;
+  models: (string | CatalogSpecModel)[] | CatalogSpecModelSelector;
+};
+
+export type CatalogSpec = {
+  providers: CatalogSpecProvider[];
+};

packages/twenty-server/scripts/ai-catalog-sync/types/coverage-report.type.ts

@@ -0,0 +1,10 @@
+export type CoverageReport = {
+  generalPurposeModelCount: number;
+  scoredGeneralPurposeModelCount: number;
+  costedGeneralPurposeModelCount: number;
+  unscoredGeneralPurposeModelIds: string[];
+  specializedModelCount: number;
+  declaredEffortVariantCount: number;
+  scoredEffortVariantCount: number;
+  unscoredEffortVariantIds: string[];
+};

packages/twenty-server/scripts/ai-catalog-sync/types/enrich-catalog-args.type.ts

@@ -0,0 +1,11 @@
+import { type ModelsDevData } from 'src/engine/metadata-modules/ai/ai-models/types/models-dev-data.type';
+
+import { type BenchmarkIndex } from './benchmark-index.type';
+import { type GeneratedCatalog } from './generated-catalog.type';
+
+export type EnrichCatalogArgs = {
+  catalog: GeneratedCatalog;
+  modelsDevData: ModelsDevData;
+  benchmarkIndex: BenchmarkIndex;
+  measuredAt: string;
+};

packages/twenty-server/scripts/ai-catalog-sync/types/generated-catalog.type.ts

@@ -0,0 +1,3 @@
+import { type GeneratedProvider } from './generated-provider.type';
+
+export type GeneratedCatalog = Record<string, GeneratedProvider>;

packages/twenty-server/scripts/ai-catalog-sync/types/generated-model.type.ts

@@ -0,0 +1,27 @@
+import { type AiModelEffort, type DataResidency } from 'twenty-shared/ai';
+
+import { type AiModelBenchmark } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-benchmark.type';
+import { type LongContextCost } from 'src/engine/metadata-modules/ai/ai-models/types/long-context-cost.type';
+import { type ModelFamily } from 'src/engine/metadata-modules/ai/ai-models/types/model-family.enum';
+
+export type GeneratedModel = {
+  name: string;
+  label: string;
+  description?: string;
+  modelFamily?: ModelFamily;
+  inputCostPerMillionTokens?: number;
+  outputCostPerMillionTokens?: number;
+  cachedInputCostPerMillionTokens?: number;
+  cacheCreationCostPerMillionTokens?: number;
+  longContextCost?: LongContextCost;
+  contextWindowTokens?: number;
+  maxOutputTokens?: number;
+  modalities?: string[];
+  supportsReasoning?: boolean;
+  efforts?: AiModelEffort[];
+  dataResidency?: DataResidency;
+  zeroDataRetention?: boolean;
+  benchmark?: AiModelBenchmark;
+  benchmarkByEffort?: Partial<Record<AiModelEffort, AiModelBenchmark>>;
+  isDeprecated?: boolean;
+};

packages/twenty-server/scripts/ai-catalog-sync/types/generated-provider.type.ts

@@ -0,0 +1,5 @@
+import { type GeneratedModel } from './generated-model.type';
+
+export type GeneratedProvider = {
+  models: GeneratedModel[];
+};

packages/twenty-server/scripts/ai-catalog-sync/types/match-benchmarks-args.type.ts

@@ -0,0 +1,13 @@
+import { type AiModelEffort } from 'twenty-shared/ai';
+
+import { type ModelsDevModel } from 'src/engine/metadata-modules/ai/ai-models/types/models-dev-model.type';
+
+import { type BenchmarkIndex } from './benchmark-index.type';
+
+export type MatchBenchmarksArgs = {
+  modelName: string;
+  siblingModels: Record<string, ModelsDevModel>;
+  benchmarkIndex: BenchmarkIndex;
+  measuredAt: string;
+  efforts?: AiModelEffort[];
+};

packages/twenty-server/scripts/ai-catalog-sync/utils/assert-payload-is-usable.util.ts

@@ -0,0 +1,21 @@
+import { type ModelsDevData } from 'src/engine/metadata-modules/ai/ai-models/types/models-dev-data.type';
+
+// A 200 carrying an empty or reshaped payload would otherwise generate an
+// empty catalog, and the sync PR automerges over the real one.
+export const assertPayloadIsUsable = ({
+  data,
+  vendors,
+}: {
+  data: ModelsDevData;
+  vendors: string[];
+}): void => {
+  const missing = vendors.filter(
+    (vendor) => Object.keys(data[vendor]?.models ?? {}).length === 0,
+  );
+
+  if (missing.length > 0) {
+    throw new Error(
+      `models.dev payload is missing models for ${missing.join(', ')}; refusing to overwrite the catalog`,
+    );
+  }
+};

packages/twenty-server/scripts/ai-catalog-sync/utils/build-catalog.util.ts

@@ -0,0 +1,164 @@
+import { isDefined } from 'twenty-shared/utils';
+
+import { type ModelsDevData } from 'src/engine/metadata-modules/ai/ai-models/types/models-dev-data.type';
+import { type ModelsDevModel } from 'src/engine/metadata-modules/ai/ai-models/types/models-dev-model.type';
+import { inferModelFamily } from 'src/engine/metadata-modules/ai/ai-models/utils/infer-model-family.util';
+
+import { type GeneratedCatalog } from '../types/generated-catalog.type';
+import { type GeneratedModel } from '../types/generated-model.type';
+
+const EXCLUDED_MODEL_PREFIXES = [
+  'text-embedding',
+  'embedding',
+  'dall-e',
+  'tts-',
+  'whisper',
+  'moderation',
+  'davinci',
+  'babbage',
+  'ada',
+  'curie',
+  'text-search',
+  'text-similarity',
+  'code-search',
+  'text-davinci',
+  'text-curie',
+  'text-babbage',
+  'text-ada',
+  'ft:',
+  'canary',
+];
+
+const EXCLUDED_MODEL_SUFFIXES = ['-audio-preview', '-realtime-preview'];
+
+const LONG_CONTEXT_THRESHOLD_TOKENS = 200000;
+
+const isLanguageModel = (modelId: string): boolean => {
+  const lowerId = modelId.toLowerCase();
+
+  if (EXCLUDED_MODEL_PREFIXES.some((prefix) => lowerId.startsWith(prefix))) {
+    return false;
+  }
+
+  return !EXCLUDED_MODEL_SUFFIXES.some((suffix) => lowerId.endsWith(suffix));
+};
+
+const meetsInclusionCriteria = (modelData: ModelsDevModel): boolean =>
+  modelData.status !== 'beta' &&
+  (modelData.tool_call ?? false) &&
+  isDefined(modelData.cost?.input) &&
+  isDefined(modelData.limit?.context);
+
+const extractCost = ({
+  modelData,
+  model,
+}: {
+  modelData: ModelsDevModel;
+  model: GeneratedModel;
+}): void => {
+  const cost = modelData.cost;
+
+  if (!isDefined(cost)) {
+    return;
+  }
+
+  model.inputCostPerMillionTokens = cost.input;
+  model.outputCostPerMillionTokens = cost.output;
+  model.cachedInputCostPerMillionTokens = cost.cache_read;
+  model.cacheCreationCostPerMillionTokens = cost.cache_write;
+
+  const longContextCost = cost.context_over_200k;
+
+  if (!isDefined(longContextCost?.input)) {
+    return;
+  }
+
+  model.longContextCost = {
+    inputCostPerMillionTokens: longContextCost.input,
+    outputCostPerMillionTokens:
+      longContextCost.output ?? model.outputCostPerMillionTokens ?? 0,
+    thresholdTokens: LONG_CONTEXT_THRESHOLD_TOKENS,
+    cachedInputCostPerMillionTokens: longContextCost.cache_read,
+    cacheCreationCostPerMillionTokens: longContextCost.cache_write,
+  };
+};
+
+const buildModel = ({
+  providerName,
+  modelId,
+  modelData,
+}: {
+  providerName: string;
+  modelId: string;
+  modelData: ModelsDevModel;
+}): GeneratedModel => {
+  const model: GeneratedModel = {
+    name: modelId,
+    label: modelData.name ?? modelId,
+    modelFamily: inferModelFamily(providerName, modelId),
+  };
+
+  extractCost({ modelData, model });
+
+  model.contextWindowTokens = modelData.limit?.context;
+  model.maxOutputTokens = modelData.limit?.output;
+
+  const modalities = (modelData.modalities?.input ?? []).filter(
+    (modality) => modality !== 'text',
+  );
+
+  if (modalities.length > 0) {
+    model.modalities = modalities;
+  }
+
+  if (modelData.reasoning) {
+    model.supportsReasoning = true;
+  }
+
+  if (modelData.status === 'deprecated') {
+    model.isDeprecated = true;
+  }
+
+  return model;
+};
+
+// Which vendors we carry is a deployment decision, so it is read from the
+// spec that states it rather than inferred from how the SDK names a package.
+export const buildCatalog = ({
+  data,
+  vendors,
+}: {
+  data: ModelsDevData;
+  vendors: string[];
+}): GeneratedCatalog => {
+  const catalog: GeneratedCatalog = {};
+
+  for (const providerName of vendors) {
+    const providerData = data[providerName];
+
+    if (!isDefined(providerData)) {
+      // oxlint-disable-next-line no-console
+      console.warn(`Provider "${providerName}" not found in models.dev`);
+      continue;
+    }
+
+    const models = Object.entries(providerData.models)
+      .filter(
+        ([modelId, modelData]) =>
+          isLanguageModel(modelId) && meetsInclusionCriteria(modelData),
+      )
+      .map(([modelId, modelData]) =>
+        buildModel({ providerName, modelId, modelData }),
+      );
+
+    if (models.length === 0) {
+      // oxlint-disable-next-line no-console
+      console.warn(`No qualifying models for "${providerName}", skipping`);
+      continue;
+    }
+
+    catalog[providerName] = { models };
+  }
+
+  return catalog;
+};

packages/twenty-server/scripts/ai-catalog-sync/utils/build-coverage-report.util.ts

@@ -0,0 +1,67 @@
+import { isDefined } from 'twenty-shared/utils';
+
+import { type CoverageReport } from '../types/coverage-report.type';
+import { type GeneratedCatalog } from '../types/generated-catalog.type';
+
+// Models whose job is not general reasoning (speech, image, embeddings,
+// computer use, deep research). A capability index for them would be
+// meaningless, so they are excluded from the coverage denominator rather than
+// counted as gaps.
+const SPECIALIZED_MODEL_PATTERN =
+  /realtime|voxtral|tts|audio|-live|computer-use|deep-research|-image|pixtral|codestral|embed/;
+
+export const buildCoverageReport = (
+  catalog: GeneratedCatalog,
+): CoverageReport => {
+  const report: CoverageReport = {
+    generalPurposeModelCount: 0,
+    scoredGeneralPurposeModelCount: 0,
+    costedGeneralPurposeModelCount: 0,
+    unscoredGeneralPurposeModelIds: [],
+    specializedModelCount: 0,
+    declaredEffortVariantCount: 0,
+    scoredEffortVariantCount: 0,
+    unscoredEffortVariantIds: [],
+  };
+
+  for (const [providerName, provider] of Object.entries(catalog)) {
+    for (const model of provider.models) {
+      if (model.isDeprecated) {
+        continue;
+      }
+
+      if (SPECIALIZED_MODEL_PATTERN.test(model.name)) {
+        report.specializedModelCount += 1;
+        continue;
+      }
+
+      report.generalPurposeModelCount += 1;
+
+      if (isDefined(model.benchmark?.costPerTask)) {
+        report.costedGeneralPurposeModelCount += 1;
+      }
+
+      if (isDefined(model.benchmark?.intelligenceIndex)) {
+        report.scoredGeneralPurposeModelCount += 1;
+      } else {
+        report.unscoredGeneralPurposeModelIds.push(
+          `${providerName}/${model.name}`,
+        );
+      }
+
+      for (const effort of model.efforts ?? []) {
+        report.declaredEffortVariantCount += 1;
+
+        if (isDefined(model.benchmarkByEffort?.[effort]?.intelligenceIndex)) {
+          report.scoredEffortVariantCount += 1;
+        } else {
+          report.unscoredEffortVariantIds.push(
+            `${providerName}/${model.name}@${effort}`,
+          );
+        }
+      }
+    }
+  }
+
+  return report;
+};

packages/twenty-server/scripts/ai-catalog-sync/utils/build-effort-lookup-key.util.ts

@@ -0,0 +1,10 @@
+import { type AiModelEffort } from 'twenty-shared/ai';
+
+// A per-effort row is filed beside the bare model key, so the ceiling lookup the
+// catalog has always done keeps working while a variant can ask for the figure
+// taken at its own effort. `@` survives name normalization, so the two key
+// spaces cannot collide.
+export const buildEffortLookupKey = (
+  normalizedModelName: string,
+  effort: AiModelEffort,
+): string => `${normalizedModelName}@${effort}`;

packages/twenty-server/scripts/ai-catalog-sync/utils/build-lookup-candidates.util.ts

@@ -0,0 +1,69 @@
+import { isDefined } from 'twenty-shared/utils';
+
+import { type ModelsDevModel } from 'src/engine/metadata-modules/ai/ai-models/types/models-dev-model.type';
+
+const DATE_SUFFIX = /-\d{4}-?\d{2}-?\d{2}$/;
+const ROLLING_SUFFIX = /-(latest|preview|exp)$/;
+
+const fingerprint = (model: ModelsDevModel): string =>
+  JSON.stringify([model.cost ?? {}, model.limit ?? {}]);
+
+// `mistral-large-latest` is never benchmarked under that name, so it is
+// resolved to the dated release it currently points at by matching price and
+// limits against its non-rolling siblings. Re-resolves on its own when the
+// provider repoints the alias.
+const resolveRollingAlias = (
+  modelName: string,
+  siblingModels: Record<string, ModelsDevModel>,
+): string | undefined => {
+  const model = siblingModels[modelName];
+
+  if (!isDefined(model) || !ROLLING_SUFFIX.test(modelName)) {
+    return undefined;
+  }
+
+  const target = fingerprint(model);
+
+  const twins = Object.entries(siblingModels)
+    .filter(
+      ([siblingName, sibling]) =>
+        siblingName !== modelName &&
+        !ROLLING_SUFFIX.test(siblingName) &&
+        fingerprint(sibling) === target,
+    )
+    // Falling back to the name keeps this deterministic when a provider dates
+    // neither twin, and for the date-stamped ids providers actually use it
+    // still lands on the newer one.
+    .sort(
+      ([nameA, a], [nameB, b]) =>
+        (b.release_date ?? '').localeCompare(a.release_date ?? '') ||
+        nameB.localeCompare(nameA),
+    );
+
+  return twins[0]?.[0];
+};
+
+export const buildLookupCandidates = ({
+  modelName,
+  siblingModels,
+}: {
+  modelName: string;
+  siblingModels: Record<string, ModelsDevModel>;
+}): string[] => {
+  const candidates = [modelName, modelName.replace(DATE_SUFFIX, '')];
+
+  const resolved = resolveRollingAlias(modelName, siblingModels);
+
+  if (isDefined(resolved)) {
+    // Once the release an alias points at is known, only a row naming that
+    // release will do. Every undated spelling is some other release the
+    // publisher happens to have measured: `mistral-large` on the leaderboard is
+    // the Feb '24 model, and standing in for `mistral-large-2512` it published a
+    // two-year-old score as current.
+    return [...new Set([...candidates, resolved])];
+  }
+
+  // No dated release to point at, so the bare name is the only thing left and
+  // whichever release the publisher measured is the best answer available.
+  return [...new Set([...candidates, modelName.replace(ROLLING_SUFFIX, '')])];
+};

packages/twenty-server/scripts/ai-catalog-sync/utils/carry-over-committed-fields.util.ts

@@ -0,0 +1,43 @@
+import { isDefined } from 'twenty-shared/utils';
+
+import { type GeneratedCatalog } from '../types/generated-catalog.type';
+import { type GeneratedModel } from '../types/generated-model.type';
+
+// models.dev knows neither which effort levels a model takes nor its routing
+// facts, so a rebuild would drop them without this.
+const CARRIED_OVER_FIELDS = [
+  'efforts',
+  'dataResidency',
+  'zeroDataRetention',
+] as const satisfies readonly (keyof GeneratedModel)[];
+
+export const carryOverCommittedFields = ({
+  catalog,
+  committedCatalog,
+}: {
+  catalog: GeneratedCatalog;
+  committedCatalog: GeneratedCatalog;
+}): void => {
+  for (const [providerName, provider] of Object.entries(catalog)) {
+    const committedModels = new Map(
+      (committedCatalog[providerName]?.models ?? []).map((model) => [
+        model.name,
+        model,
+      ]),
+    );
+
+    for (const model of provider.models) {
+      const committedModel = committedModels.get(model.name);
+
+      if (!isDefined(committedModel)) {
+        continue;
+      }
+
+      for (const field of CARRIED_OVER_FIELDS) {
+        if (isDefined(committedModel[field])) {
+          Object.assign(model, { [field]: committedModel[field] });
+        }
+      }
+    }
+  }
+};

packages/twenty-server/scripts/ai-catalog-sync/utils/enrich-catalog.util.ts

@@ -0,0 +1,71 @@
+import { type AiModelEffort } from 'twenty-shared/ai';
+import { isDefined } from 'twenty-shared/utils';
+
+import { type AiModelBenchmark } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-benchmark.type';
+
+import {
+  type BenchmarkOverlayEntry,
+  type BenchmarkOverlayReading,
+} from '../types/benchmark-overlay-entry.type';
+import { type EnrichCatalogArgs } from '../types/enrich-catalog-args.type';
+
+import { matchBenchmarks } from './match-benchmarks.util';
+
+// The catalog embeds the figures only; the aliases stay in the overlay, which
+// is where a joining consumer looks for them.
+const toCatalogReadings = (
+  readings: Partial<Record<AiModelEffort, BenchmarkOverlayReading>>,
+): Partial<Record<AiModelEffort, AiModelBenchmark>> =>
+  Object.fromEntries(
+    Object.entries(readings).map(
+      ([effort, { aliases: _aliases, ...reading }]) => [effort, reading],
+    ),
+  );
+
+export const enrichCatalog = ({
+  catalog,
+  modelsDevData,
+  benchmarkIndex,
+  measuredAt,
+}: EnrichCatalogArgs): Record<string, BenchmarkOverlayEntry> => {
+  const overlay: Record<string, BenchmarkOverlayEntry> = {};
+
+  for (const [providerName, provider] of Object.entries(catalog)) {
+    const siblingModels = modelsDevData[providerName]?.models ?? {};
+
+    provider.models = provider.models.map((model) => {
+      const match = matchBenchmarks({
+        modelName: model.name,
+        siblingModels,
+        benchmarkIndex,
+        measuredAt,
+        efforts: model.efforts,
+      });
+
+      if (!isDefined(match)) {
+        return model;
+      }
+
+      const { benchmark, aliases, benchmarkByEffort } = match;
+
+      overlay[model.name] = {
+        ...benchmark,
+        aliases,
+        ...(isDefined(benchmarkByEffort) ? { benchmarkByEffort } : {}),
+      };
+
+      const { isDeprecated, ...rest } = model;
+
+      return {
+        ...rest,
+        benchmark,
+        ...(isDefined(benchmarkByEffort)
+          ? { benchmarkByEffort: toCatalogReadings(benchmarkByEffort) }
+          : {}),
+        ...(isDefined(isDeprecated) ? { isDeprecated } : {}),
+      };
+    });
+  }
+
+  return overlay;
+};

packages/twenty-server/scripts/ai-catalog-sync/utils/fetch-artificial-analysis-benchmarks.util.ts

@@ -0,0 +1,219 @@
+import { isDefined } from 'twenty-shared/utils';
+
+import { buildEffortLookupKey } from './build-effort-lookup-key.util';
+import { normalizeModelName } from './normalize-model-name.util';
+import { parseArtificialAnalysisEffort } from './parse-artificial-analysis-effort.util';
+import { type BenchmarkIndex } from '../types/benchmark-index.type';
+import { type BenchmarkRecord } from '../types/benchmark-record.type';
+
+const DEFAULT_ARTIFICIAL_ANALYSIS_API_URL =
+  'https://artificialanalysis.ai/api/v2/language/models/free';
+
+const PAGE_SIZE = 200;
+
+type ArtificialAnalysisModel = Record<string, unknown> & {
+  id?: string;
+  slug?: string;
+  name?: string;
+};
+
+type ArtificialAnalysisPage = {
+  data?: unknown;
+  pagination?: { has_more?: unknown };
+};
+
+// This endpoint returns 0 for "not measured", and a zero tokens-per-second is a
+// lie rather than a datum.
+const readPositiveNumber = (value: unknown): number | undefined =>
+  typeof value === 'number' && Number.isFinite(value) && value > 0
+    ? value
+    : undefined;
+
+const readNested = ({
+  model,
+  path,
+}: {
+  model: ArtificialAnalysisModel;
+  path: string[];
+}): number | undefined => {
+  let current: unknown = model;
+
+  for (const key of path) {
+    if (typeof current !== 'object' || current === null) {
+      return undefined;
+    }
+
+    current = (current as Record<string, unknown>)[key];
+  }
+
+  return readPositiveNumber(current);
+};
+
+const readModels = (page: ArtificialAnalysisPage): ArtificialAnalysisModel[] =>
+  Array.isArray(page.data) ? (page.data as ArtificialAnalysisModel[]) : [];
+
+// How much a row actually says. The endpoint carries several rows per model and
+// some hosts publish no measurements at all, so an unmeasured row must never
+// displace a measured one under the same key.
+const informationScore = (record: BenchmarkRecord): number =>
+  [
+    record.intelligenceIndex,
+    record.outputTokensPerSecond,
+    record.costPerTask,
+  ].filter(isDefined).length;
+
+// The rows for one model are the same model at different reasoning efforts, and
+// the publisher spells the effort only inside a display name. Taking the
+// best-scoring row gives every model its ceiling, measured the same way, which
+// is the whole reason for standing on one publisher. Picking by how populated a
+// row happened to be scored Sonnet 4.6 at low effort against Sonnet 5 at max,
+// and the gap read as a capability difference.
+const preferOver = (
+  candidate: BenchmarkRecord,
+  existing: BenchmarkRecord,
+): boolean => {
+  const candidateIndex = candidate.intelligenceIndex ?? -1;
+  const existingIndex = existing.intelligenceIndex ?? -1;
+
+  if (candidateIndex !== existingIndex) {
+    return candidateIndex > existingIndex;
+  }
+
+  return informationScore(candidate) > informationScore(existing);
+};
+
+const fetchPage = async ({
+  url,
+  apiKey,
+  page,
+}: {
+  url: string;
+  apiKey: string;
+  page: number;
+}): Promise<ArtificialAnalysisPage> => {
+  const pageUrl = new URL(url);
+
+  pageUrl.searchParams.set('page', String(page));
+  pageUrl.searchParams.set('page_size', String(PAGE_SIZE));
+
+  const response = await fetch(pageUrl, {
+    headers: { 'x-api-key': apiKey },
+    signal: AbortSignal.timeout(60000),
+  });
+
+  if (!response.ok) {
+    throw new Error(`${response.status} ${response.statusText}`);
+  }
+
+  return (await response.json()) as ArtificialAnalysisPage;
+};
+
+// The list is paginated and the free tier allows ten requests a day, so every
+// page is read in one run rather than spread across runs.
+const fetchAllModels = async ({
+  url,
+  apiKey,
+}: {
+  url: string;
+  apiKey: string;
+}): Promise<ArtificialAnalysisModel[]> => {
+  const models: ArtificialAnalysisModel[] = [];
+
+  for (let page = 1; ; page += 1) {
+    const response = await fetchPage({ url, apiKey, page });
+
+    models.push(...readModels(response));
+
+    if (response.pagination?.has_more !== true) {
+      return models;
+    }
+  }
+};
+
+export const fetchArtificialAnalysisBenchmarks = async (
+  apiKey: string,
+): Promise<BenchmarkIndex> => {
+  const url =
+    process.env.ARTIFICIAL_ANALYSIS_API_URL ??
+    DEFAULT_ARTIFICIAL_ANALYSIS_API_URL;
+
+  const models = await fetchAllModels({ url, apiKey });
+
+  if (models.length === 0) {
+    throw new Error(`No models in response from ${url}`);
+  }
+
+  const index: BenchmarkIndex = new Map();
+
+  const file = (key: string, record: BenchmarkRecord): void => {
+    const existing = index.get(key);
+
+    if (!isDefined(existing) || preferOver(record, existing)) {
+      index.set(key, record);
+    }
+  };
+
+  for (const model of models) {
+    // One malformed row must not cost us every measurement in the response.
+    if (typeof model !== 'object' || model === null) {
+      continue;
+    }
+
+    const aliases = [model.slug, model.id, model.name].filter(isDefined);
+
+    // Cost per task is the price of one task of the intelligence index run at
+    // this row's effort, so it reflects reasoning volume where per-token
+    // pricing cannot.
+    const record: BenchmarkRecord = {
+      intelligenceIndex: readNested({
+        model,
+        path: ['evaluations', 'artificial_analysis_intelligence_index'],
+      }),
+      outputTokensPerSecond: readNested({
+        model,
+        path: ['performance', 'median_output_tokens_per_second'],
+      }),
+      costPerTask: readNested({
+        model,
+        path: [
+          'artificial_analysis_intelligence_index_cost',
+          'cost_per_task',
+          'total_cost',
+        ],
+      }),
+      effort:
+        typeof model.name === 'string'
+          ? parseArtificialAnalysisEffort(model.name)
+          : undefined,
+      aliases,
+    };
+
+    if (informationScore(record) === 0) {
+      continue;
+    }
+
+    for (const alias of aliases) {
+      const key = normalizeModelName(alias);
+
+      if (key.length === 0) {
+        continue;
+      }
+
+      // The bare key keeps the ceiling; the effort key lets a pinned variant
+      // read the figure taken at its own effort.
+      file(key, record);
+
+      if (isDefined(record.effort)) {
+        file(buildEffortLookupKey(key, record.effort), record);
+      }
+    }
+  }
+
+  if (index.size === 0) {
+    throw new Error(
+      `${models.length} models returned from ${url} but none carried a measurement`,
+    );
+  }
+
+  return index;
+};

packages/twenty-server/scripts/ai-catalog-sync/utils/match-benchmarks.util.ts

@@ -0,0 +1,94 @@
+import { type AiModelEffort } from 'twenty-shared/ai';
+import { isDefined } from 'twenty-shared/utils';
+
+import { type AiModelBenchmark } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-benchmark.type';
+
+import { type BenchmarkIndex } from '../types/benchmark-index.type';
+import { type BenchmarkMatch } from '../types/benchmark-match.type';
+import { type BenchmarkOverlayReading } from '../types/benchmark-overlay-entry.type';
+import { type BenchmarkRecord } from '../types/benchmark-record.type';
+import { type MatchBenchmarksArgs } from '../types/match-benchmarks-args.type';
+
+import { buildEffortLookupKey } from './build-effort-lookup-key.util';
+import { buildLookupCandidates } from './build-lookup-candidates.util';
+import { normalizeModelName } from './normalize-model-name.util';
+
+const lookup = (
+  index: BenchmarkIndex,
+  keys: string[],
+): BenchmarkRecord | undefined => {
+  for (const key of keys) {
+    const record = index.get(key);
+
+    if (isDefined(record)) {
+      return record;
+    }
+  }
+
+  return undefined;
+};
+
+// The publisher lists plenty of models it has measured nothing about, and a row
+// of pure aliases says nothing worth carrying into either artifact.
+const hasMeasurement = (record: BenchmarkRecord): boolean =>
+  [
+    record.intelligenceIndex,
+    record.outputTokensPerSecond,
+    record.costPerTask,
+  ].some(isDefined);
+
+const toBenchmark = (
+  record: BenchmarkRecord,
+  measuredAt: string,
+): AiModelBenchmark => ({
+  intelligenceIndex: record.intelligenceIndex,
+  outputTokensPerSecond: record.outputTokensPerSecond,
+  costPerTask: record.costPerTask,
+  effort: record.effort,
+  measuredAt: record.measuredAt ?? measuredAt,
+});
+
+export const matchBenchmarks = ({
+  modelName,
+  siblingModels,
+  benchmarkIndex,
+  measuredAt,
+  efforts,
+}: MatchBenchmarksArgs): BenchmarkMatch | undefined => {
+  const keys = buildLookupCandidates({ modelName, siblingModels }).map(
+    normalizeModelName,
+  );
+
+  const record = lookup(benchmarkIndex, keys);
+
+  if (!isDefined(record) || !hasMeasurement(record)) {
+    return undefined;
+  }
+
+  const benchmarkByEffort: Partial<
+    Record<AiModelEffort, BenchmarkOverlayReading>
+  > = {};
+
+  // A declared effort without a row of its own stays blank: the ceiling was
+  // measured at another effort and would overstate it.
+  for (const effort of efforts ?? []) {
+    const effortRecord = lookup(
+      benchmarkIndex,
+      keys.map((key) => buildEffortLookupKey(key, effort)),
+    );
+
+    if (isDefined(effortRecord) && hasMeasurement(effortRecord)) {
+      benchmarkByEffort[effort] = {
+        ...toBenchmark(effortRecord, measuredAt),
+        effort,
+        aliases: [...new Set(effortRecord.aliases)],
+      };
+    }
+  }
+
+  return {
+    benchmark: toBenchmark(record, measuredAt),
+    aliases: [...new Set([modelName, ...record.aliases])],
+    ...(Object.keys(benchmarkByEffort).length > 0 ? { benchmarkByEffort } : {}),
+  };
+};

packages/twenty-server/scripts/ai-catalog-sync/utils/normalize-model-name.util.ts

@@ -0,0 +1,12 @@
+// Benchmark publishers, providers and the AI SDK all spell the same model
+// differently (`claude-sonnet-4-6`, `claude-sonnet-4.6`, `Claude Sonnet 4.6`),
+// so every lookup key goes through this before it is compared. The trailing
+// `-v1:0` strip lets a Bedrock deployment id join the same row as the model it
+// serves.
+export const normalizeModelName = (modelName: string): string =>
+  modelName
+    .toLowerCase()
+    .replace(/:batch$/, '')
+    .replace(/\(.*\)/g, '')
+    .replace(/-v\d+(:\d+)?$/, '')
+    .replace(/[\s.\-_/]/g, '');

packages/twenty-server/scripts/ai-catalog-sync/utils/parse-artificial-analysis-effort.util.ts

@@ -0,0 +1,35 @@
+import { type AiModelEffort, isAiModelEffort } from 'twenty-shared/ai';
+import { isDefined } from 'twenty-shared/utils';
+
+// A row run with reasoning switched off carries no effort word at all.
+const NO_REASONING_MARKERS = new Set(['non-reasoning', 'nonreasoning']);
+
+// The publisher spells the configuration only inside the trailing parentheses
+// of a display name: `GPT-5.6 Sol (max)`, `Gemini 3.8 Flash (high)`,
+// `Claude Opus 5 (Adaptive Reasoning, Max Effort)`. Dates, fallbacks and a
+// bare "Reasoning" in the same place name no effort.
+export const parseArtificialAnalysisEffort = (
+  displayName: string,
+): AiModelEffort | undefined => {
+  const configuration = displayName.match(/\(([^()]*)\)\s*$/)?.[1];
+
+  if (!isDefined(configuration)) {
+    return undefined;
+  }
+
+  const segments = configuration
+    .split(',')
+    .map((segment) => segment.trim().toLowerCase());
+
+  const effort = segments
+    .map((segment) => segment.replace(/\s+effort$/, ''))
+    .find(isAiModelEffort);
+
+  if (isDefined(effort)) {
+    return effort;
+  }
+
+  return segments.some((segment) => NO_REASONING_MARKERS.has(segment))
+    ? 'none'
+    : undefined;
+};

packages/twenty-server/scripts/ai-catalog-sync/utils/project-catalog.util.ts

@@ -0,0 +1,242 @@
+// twenty-shared by source path rather than by package name: the consuming
+// repository runs this from a sparse checkout with no workspace install, so
+// 'twenty-shared/ai' would not resolve there.
+import { isAiSdkPackage } from '../../../../twenty-shared/src/ai/utils/is-ai-sdk-package.util';
+import { isDataResidency } from '../../../../twenty-shared/src/ai/utils/is-data-residency.util';
+
+import {
+  type CatalogSpec,
+  type CatalogSpecModel,
+  type CatalogSpecModelOverrides,
+  type CatalogSpecProvider,
+} from '../types/catalog-spec.type';
+
+// Structural, so the catalog needs no schema here and this file can run from a
+// checkout that has not installed the monorepo.
+type CanonicalModel = { name: string; label?: string } & Record<
+  string,
+  unknown
+>;
+type CanonicalProvider = { models?: CanonicalModel[] } & Record<
+  string,
+  unknown
+>;
+export type CanonicalCatalog = Record<string, CanonicalProvider>;
+
+const OVERRIDABLE_FIELDS = [
+  'label',
+  'isDeprecated',
+  'contextWindowTokens',
+  'maxOutputTokens',
+  'inputCostPerMillionTokens',
+  'outputCostPerMillionTokens',
+  'cachedInputCostPerMillionTokens',
+  'cacheCreationCostPerMillionTokens',
+] as const satisfies readonly (keyof CatalogSpecModelOverrides)[];
+
+const NUMERIC_FIELDS = [
+  'contextWindowTokens',
+  'maxOutputTokens',
+  'inputCostPerMillionTokens',
+  'outputCostPerMillionTokens',
+  'cachedInputCostPerMillionTokens',
+  'cacheCreationCostPerMillionTokens',
+] as const satisfies readonly (keyof CatalogSpecModelOverrides)[];
+
+const PROVIDER_CREDENTIAL_FIELDS = [
+  'label',
+  'apiKey',
+  'baseUrl',
+  'region',
+  'authType',
+  'accessKeyId',
+  'secretAccessKey',
+  'sessionToken',
+] as const;
+
+const asSpecModel = (entry: string | CatalogSpecModel): CatalogSpecModel =>
+  typeof entry === 'string' ? { model: entry } : entry;
+
+const indexCanonicalModels = (
+  catalog: CanonicalCatalog,
+): Map<string, CanonicalModel> => {
+  const models = new Map<string, CanonicalModel>();
+
+  for (const provider of Object.values(catalog)) {
+    for (const model of provider.models ?? []) {
+      models.set(model.name, model);
+    }
+  }
+
+  return models;
+};
+
+const projectModel = ({
+  canonicalModel,
+  specModel,
+  provider,
+}: {
+  canonicalModel: CanonicalModel;
+  specModel: CatalogSpecModel;
+  provider: CatalogSpecProvider;
+}): CanonicalModel => {
+  const overrides = Object.fromEntries(
+    OVERRIDABLE_FIELDS.filter((field) => specModel[field] !== undefined).map(
+      (field) => [field, specModel[field]],
+    ),
+  );
+
+  const routeFields = {
+    ...(provider.dataResidency === undefined
+      ? {}
+      : { dataResidency: provider.dataResidency }),
+    ...(provider.zeroDataRetention === undefined
+      ? {}
+      : { zeroDataRetention: provider.zeroDataRetention }),
+  };
+
+  return {
+    ...canonicalModel,
+    ...routeFields,
+    ...overrides,
+    name: specModel.as ?? canonicalModel.name,
+    // The suffix marks the route, so it is appended to whichever name the model
+    // ends up with rather than to the catalog's.
+    label: `${specModel.label ?? canonicalModel.label ?? canonicalModel.name}${provider.labelSuffix ?? ''}`,
+  };
+};
+
+const assertProviderIsUsable = ({
+  provider,
+  seenNames,
+}: {
+  provider: CatalogSpecProvider;
+  seenNames: Set<string>;
+}): void => {
+  // Providers key an object, so a repeat would drop the first one's credentials
+  // and models on the floor, and `__proto__` would drop its own.
+  if (seenNames.has(provider.name) || provider.name === '__proto__') {
+    throw new Error(`Provider "${provider.name}" is repeated or reserved`);
+  }
+
+  if (!isAiSdkPackage(provider.npm)) {
+    throw new Error(
+      `Provider "${provider.name}" names an unsupported SDK package: ${provider.npm}`,
+    );
+  }
+
+  if (
+    provider.dataResidency !== undefined &&
+    !isDataResidency(provider.dataResidency)
+  ) {
+    throw new Error(
+      `Provider "${provider.name}" names an unsupported data residency: ${provider.dataResidency}`,
+    );
+  }
+};
+
+const assertNumbersAreUsable = ({
+  provider,
+  specModel,
+}: {
+  provider: CatalogSpecProvider;
+  specModel: CatalogSpecModel;
+}): void => {
+  for (const field of NUMERIC_FIELDS) {
+    const value = specModel[field];
+
+    if (value !== undefined && (!Number.isFinite(value) || value < 0)) {
+      throw new Error(
+        `${provider.name}/${specModel.model} sets ${field} to ${value}, which is not a usable number`,
+      );
+    }
+  }
+};
+
+// A spec naming a model the catalog does not carry is the failure this whole
+// pipeline exists to prevent: it publishes a route to nothing, and the tier
+// chains then fall through to a neighbouring rung in silence.
+const resolveModels = ({
+  provider,
+  canonicalCatalog,
+  canonicalModels,
+}: {
+  provider: CatalogSpecProvider;
+  canonicalCatalog: CanonicalCatalog;
+  canonicalModels: Map<string, CanonicalModel>;
+}): { canonicalModel: CanonicalModel; specModel: CatalogSpecModel }[] => {
+  if (!Array.isArray(provider.models)) {
+    const { vendor } = provider.models;
+    const vendorModels = canonicalCatalog[vendor]?.models ?? [];
+
+    if (vendorModels.length === 0) {
+      throw new Error(
+        `Provider "${provider.name}" serves every model from "${vendor}", which the catalog does not carry`,
+      );
+    }
+
+    return vendorModels.map((canonicalModel) => ({
+      canonicalModel,
+      specModel: { model: canonicalModel.name },
+    }));
+  }
+
+  const specModels = provider.models.map(asSpecModel);
+  const unknown = specModels
+    .filter(({ model }) => !canonicalModels.has(model))
+    .map(({ model }) => `${provider.name}/${model}`);
+
+  if (unknown.length > 0) {
+    throw new Error(
+      `The catalog does not carry: ${unknown.join(', ')}. Check the model names, or wait for the catalog sync to pick them up.`,
+    );
+  }
+
+  return specModels.map((specModel) => {
+    assertNumbersAreUsable({ provider, specModel });
+
+    const canonicalModel = canonicalModels.get(specModel.model);
+
+    if (canonicalModel === undefined) {
+      throw new Error(`The catalog does not carry ${specModel.model}`);
+    }
+
+    return { canonicalModel, specModel };
+  });
+};
+
+export const projectCatalog = ({
+  canonicalCatalog,
+  spec,
+}: {
+  canonicalCatalog: CanonicalCatalog;
+  spec: CatalogSpec;
+}): CanonicalCatalog => {
+  const canonicalModels = indexCanonicalModels(canonicalCatalog);
+  const projected: CanonicalCatalog = {};
+  const seenNames = new Set<string>();
+
+  for (const provider of spec.providers) {
+    assertProviderIsUsable({ provider, seenNames });
+    seenNames.add(provider.name);
+
+    const credentials = Object.fromEntries(
+      PROVIDER_CREDENTIAL_FIELDS.filter(
+        (field) => provider[field] !== undefined,
+      ).map((field) => [field, provider[field]]),
+    );
+
+    projected[provider.name] = {
+      npm: provider.npm,
+      label: provider.label ?? provider.name,
+      ...credentials,
+      models: resolveModels({
+        provider,
+        canonicalCatalog,
+        canonicalModels,
+      }).map((resolved) => projectModel({ ...resolved, provider })),
+    };
+  }
+
+  return projected;
+};

packages/twenty-server/scripts/ai-catalog-sync/utils/read-committed-benchmarks.util.ts

@@ -0,0 +1,103 @@
+import * as fs from 'fs';
+
+import { type AiModelEffort, isAiModelEffort } from 'twenty-shared/ai';
+
+import { buildEffortLookupKey } from './build-effort-lookup-key.util';
+import { normalizeModelName } from './normalize-model-name.util';
+import { type BenchmarkIndex } from '../types/benchmark-index.type';
+import { type BenchmarkRecord } from '../types/benchmark-record.type';
+
+type CommittedReading = Partial<BenchmarkRecord> & {
+  measuredAt?: string;
+  aliases?: string[];
+};
+
+type CommittedEntry = CommittedReading & {
+  benchmarkByEffort?: Partial<Record<AiModelEffort, CommittedReading>>;
+};
+
+const toRecord = ({
+  reading,
+  modelName,
+  effort,
+}: {
+  reading: CommittedReading;
+  modelName: string;
+  effort?: AiModelEffort;
+}): BenchmarkRecord => ({
+  intelligenceIndex: reading.intelligenceIndex,
+  outputTokensPerSecond: reading.outputTokensPerSecond,
+  costPerTask: reading.costPerTask,
+  // The key a reading is filed under is the authority on its effort.
+  effort: effort ?? reading.effort,
+  measuredAt: reading.measuredAt,
+  aliases: reading.aliases ?? [modelName],
+});
+
+const file = ({
+  index,
+  record,
+  modelName,
+  toKey,
+}: {
+  index: BenchmarkIndex;
+  record: BenchmarkRecord;
+  modelName: string;
+  toKey: (normalizedName: string) => string;
+}): void => {
+  for (const alias of record.aliases) {
+    const key = normalizeModelName(alias);
+
+    if (key.length > 0 && !index.has(toKey(key))) {
+      index.set(toKey(key), record);
+    }
+  }
+
+  // An entry always claims its own name, whatever a sibling's alias list says.
+  const ownKey = normalizeModelName(modelName);
+
+  if (ownKey.length > 0) {
+    index.set(toKey(ownKey), record);
+  }
+};
+
+// The last published measurements, read back so a failed fetch degrades to
+// stale data rather than deleting the catalog's benchmarks. Each record keeps
+// its own measuredAt, so a stale entry stays visibly stale.
+export const readCommittedBenchmarks = (filePath: string): BenchmarkIndex => {
+  if (!fs.existsSync(filePath)) {
+    return new Map();
+  }
+
+  const parsed = JSON.parse(fs.readFileSync(filePath, 'utf-8')) as {
+    models?: Record<string, CommittedEntry>;
+  };
+
+  const index: BenchmarkIndex = new Map();
+
+  for (const [modelName, entry] of Object.entries(parsed.models ?? {})) {
+    file({
+      index,
+      record: toRecord({ reading: entry, modelName }),
+      modelName,
+      toKey: (normalizedName) => normalizedName,
+    });
+
+    for (const [effort, reading] of Object.entries(
+      entry.benchmarkByEffort ?? {},
+    )) {
+      if (!isAiModelEffort(effort)) {
+        continue;
+      }
+
+      file({
+        index,
+        record: toRecord({ reading, modelName, effort }),
+        modelName,
+        toKey: (normalizedName) => buildEffortLookupKey(normalizedName, effort),
+      });
+    }
+  }
+
+  return index;
+};

packages/twenty-server/scripts/ai-catalog-sync/utils/render-coverage-report.util.ts

@@ -0,0 +1,55 @@
+import { type CoverageReport } from '../types/coverage-report.type';
+
+const asPercentage = ({
+  numerator,
+  denominator,
+}: {
+  numerator: number;
+  denominator: number;
+}): string =>
+  denominator === 0 ? 'n/a' : `${Math.round((numerator / denominator) * 100)}%`;
+
+const renderGapList = ({
+  summary,
+  modelIds,
+}: {
+  summary: string;
+  modelIds: string[];
+}): string[] =>
+  modelIds.length === 0
+    ? []
+    : [
+        '',
+        `<details><summary>${summary}</summary>`,
+        '',
+        ...modelIds.map((modelId) => `- ${modelId}`),
+        '',
+        '</details>',
+      ];
+
+export const renderCoverageReport = (report: CoverageReport): string => {
+  const {
+    generalPurposeModelCount: total,
+    scoredGeneralPurposeModelCount: scored,
+    costedGeneralPurposeModelCount: costed,
+    declaredEffortVariantCount: declaredVariants,
+    scoredEffortVariantCount: scoredVariants,
+  } = report;
+
+  return [
+    '### Benchmark coverage',
+    '',
+    `- General-purpose models with an intelligence index: **${scored}/${total}** (${asPercentage({ numerator: scored, denominator: total })})`,
+    `- Declared effort variants with an index measured at their own effort: **${scoredVariants}/${declaredVariants}** (${asPercentage({ numerator: scoredVariants, denominator: declaredVariants })})`,
+    `- General-purpose models with a cost per task: **${costed}/${total}** (${asPercentage({ numerator: costed, denominator: total })})`,
+    `- Specialized models excluded from the denominator: ${report.specializedModelCount}`,
+    ...renderGapList({
+      summary: 'General-purpose models with no matched intelligence index',
+      modelIds: report.unscoredGeneralPurposeModelIds,
+    }),
+    ...renderGapList({
+      summary: 'Effort variants with no reading at their own effort',
+      modelIds: report.unscoredEffortVariantIds,
+    }),
+  ].join('\n');
+};