snapshot of https://github.com/twentyhq/twenty.git (main, 2026-09-17; blobs > 131072B omitted)

packages/twenty-server/scripts/ai-catalog-sync/__tests__/build-catalog.spec.ts

@@ -0,0 +1,63 @@
+import { type ModelsDevData } from 'src/engine/metadata-modules/ai/ai-models/types/models-dev-data.type';
+
+import { assertPayloadIsUsable } from '../utils/assert-payload-is-usable.util';
+import { buildCatalog } from '../utils/build-catalog.util';
+
+const languageModel = {
+  id: 'model',
+  name: 'Model',
+  tool_call: true,
+  cost: { input: 2, output: 6 },
+  limit: { context: 128000, output: 8192 },
+};
+
+const VENDORS = ['openai', 'anthropic', 'google', 'xai', 'mistral'];
+
+const payload = (): ModelsDevData => ({
+  openai: { id: 'openai', models: { 'gpt-x': languageModel } },
+  anthropic: { id: 'anthropic', models: { 'claude-x': languageModel } },
+  google: { id: 'google', models: { 'gemini-x': languageModel } },
+  xai: { id: 'xai', models: { 'grok-x': languageModel } },
+  mistral: { id: 'mistral', models: { 'mistral-x': languageModel } },
+});
+
+describe('assertPayloadIsUsable', () => {
+  it('accepts a payload carrying models for every vendor we ship', () => {
+    expect(() =>
+      assertPayloadIsUsable({ data: payload(), vendors: VENDORS }),
+    ).not.toThrow();
+  });
+
+  it('refuses an empty payload rather than generating an empty catalog', () => {
+    expect(() =>
+      assertPayloadIsUsable({ data: {} as ModelsDevData, vendors: VENDORS }),
+    ).toThrow(/refusing to overwrite the catalog/);
+  });
+
+  it('names the providers that came back empty', () => {
+    const data = payload();
+
+    data.anthropic.models = {};
+
+    expect(() => assertPayloadIsUsable({ data, vendors: VENDORS })).toThrow(
+      /anthropic/,
+    );
+  });
+});
+
+describe('buildCatalog', () => {
+  it('prices long context from the standard output rate when none is quoted', () => {
+    const data = payload();
+
+    data.openai.models['gpt-x'].cost = {
+      input: 2,
+      output: 6,
+      context_over_200k: { input: 4 },
+    };
+
+    const model = buildCatalog({ data, vendors: VENDORS }).openai.models[0];
+
+    // A missing long-context rate is unknown, not free.
+    expect(model.longContextCost?.outputCostPerMillionTokens).toBe(6);
+  });
+});

packages/twenty-server/scripts/ai-catalog-sync/__tests__/carry-over-committed-fields.spec.ts

@@ -0,0 +1,54 @@
+import { type GeneratedCatalog } from '../types/generated-catalog.type';
+import { type GeneratedModel } from '../types/generated-model.type';
+import { carryOverCommittedFields } from '../utils/carry-over-committed-fields.util';
+
+const openai = (models: GeneratedModel[]): GeneratedCatalog => ({
+  openai: { models },
+});
+
+describe('carryOverCommittedFields', () => {
+  it('restores the fields models.dev does not carry, and only those', () => {
+    const catalog = openai([
+      { name: 'gpt-x', label: 'GPT X', supportsReasoning: true },
+    ]);
+
+    carryOverCommittedFields({
+      catalog,
+      committedCatalog: openai([
+        {
+          name: 'gpt-x',
+          label: 'Old label',
+          inputCostPerMillionTokens: 1,
+          efforts: ['low', 'high'],
+          dataResidency: 'us',
+          zeroDataRetention: true,
+        },
+      ]),
+    });
+
+    expect(catalog.openai.models[0]).toEqual({
+      name: 'gpt-x',
+      label: 'GPT X',
+      supportsReasoning: true,
+      efforts: ['low', 'high'],
+      dataResidency: 'us',
+      zeroDataRetention: true,
+    });
+  });
+
+  it('leaves a model the committed catalog does not know untouched', () => {
+    const catalog = openai([{ name: 'gpt-new', label: 'GPT New' }]);
+
+    carryOverCommittedFields({
+      catalog,
+      committedCatalog: openai([
+        { name: 'gpt-x', label: 'GPT X', efforts: ['low'] },
+      ]),
+    });
+
+    expect(catalog.openai.models[0]).toEqual({
+      name: 'gpt-new',
+      label: 'GPT New',
+    });
+  });
+});

packages/twenty-server/scripts/ai-catalog-sync/__tests__/enrich-catalog.spec.ts

@@ -0,0 +1,131 @@
+import { type ModelsDevData } from 'src/engine/metadata-modules/ai/ai-models/types/models-dev-data.type';
+
+import { enrichCatalog } from '../utils/enrich-catalog.util';
+import { type BenchmarkIndex } from '../types/benchmark-index.type';
+import { type GeneratedCatalog } from '../types/generated-catalog.type';
+
+const MEASURED_AT = '2026-09-09';
+
+const catalogOf = (...modelNames: string[]): GeneratedCatalog => ({
+  openai: { models: modelNames.map((name) => ({ name, label: name })) },
+});
+
+const enrich = (catalog: GeneratedCatalog, benchmarkIndex: BenchmarkIndex) =>
+  enrichCatalog({
+    catalog,
+    modelsDevData: {} as ModelsDevData,
+    benchmarkIndex,
+    measuredAt: MEASURED_AT,
+  });
+
+describe('enrichCatalog', () => {
+  it('attaches a measurement to the model and mirrors it into the overlay', () => {
+    const catalog = catalogOf('gpt-x');
+    const overlay = enrich(
+      catalog,
+      new Map([['gptx', { intelligenceIndex: 41, aliases: ['gpt-x'] }]]),
+    );
+
+    expect(catalog.openai.models[0].benchmark).toEqual({
+      intelligenceIndex: 41,
+      outputTokensPerSecond: undefined,
+      costPerTask: undefined,
+      measuredAt: MEASURED_AT,
+    });
+    expect(overlay['gpt-x'].measuredAt).toBe(MEASURED_AT);
+  });
+
+  it('leaves a row the publisher measured nothing about out of both artifacts', () => {
+    const catalog = catalogOf('gpt-x');
+    const overlay = enrich(
+      catalog,
+      new Map([['gptx', { aliases: ['gpt-x'] }]]),
+    );
+
+    expect(overlay).toEqual({});
+    expect(catalog.openai.models[0].benchmark).toBeUndefined();
+  });
+
+  it('leaves a model no publisher covers untouched and out of the overlay', () => {
+    const catalog = catalogOf('gpt-x');
+    const overlay = enrich(catalog, new Map());
+
+    expect(catalog.openai.models[0].benchmark).toBeUndefined();
+    expect(overlay).toEqual({});
+  });
+
+  it('keeps isDeprecated last so the generated catalog stays stable', () => {
+    const catalog = catalogOf('gpt-x');
+
+    catalog.openai.models[0].isDeprecated = true;
+
+    enrich(
+      catalog,
+      new Map([['gptx', { intelligenceIndex: 41, aliases: ['gpt-x'] }]]),
+    );
+
+    expect(Object.keys(catalog.openai.models[0])).toEqual([
+      'name',
+      'label',
+      'benchmark',
+      'isDeprecated',
+    ]);
+  });
+
+  it('embeds per-effort readings in the catalog and mirrors them, with aliases, into the overlay', () => {
+    const catalog = catalogOf('gpt-x');
+
+    catalog.openai.models[0].efforts = ['low', 'high'];
+
+    const overlay = enrich(
+      catalog,
+      new Map([
+        [
+          'gptx',
+          { intelligenceIndex: 41, effort: 'high', aliases: ['GPT X (high)'] },
+        ],
+        [
+          'gptx@high',
+          { intelligenceIndex: 41, effort: 'high', aliases: ['GPT X (high)'] },
+        ],
+        [
+          'gptx@low',
+          { intelligenceIndex: 30, effort: 'low', aliases: ['GPT X (low)'] },
+        ],
+      ]),
+    );
+
+    expect(catalog.openai.models[0].benchmark?.effort).toBe('high');
+    expect(catalog.openai.models[0].benchmarkByEffort).toEqual({
+      low: { intelligenceIndex: 30, effort: 'low', measuredAt: MEASURED_AT },
+      high: { intelligenceIndex: 41, effort: 'high', measuredAt: MEASURED_AT },
+    });
+    expect(overlay['gpt-x'].benchmarkByEffort?.low).toEqual({
+      intelligenceIndex: 30,
+      effort: 'low',
+      measuredAt: MEASURED_AT,
+      aliases: ['GPT X (low)'],
+    });
+    expect(Object.keys(catalog.openai.models[0])).toEqual([
+      'name',
+      'label',
+      'efforts',
+      'benchmark',
+      'benchmarkByEffort',
+    ]);
+  });
+
+  it('writes no effort map for a model whose declared efforts have no rows', () => {
+    const catalog = catalogOf('gpt-x');
+
+    catalog.openai.models[0].efforts = ['low', 'high'];
+
+    const overlay = enrich(
+      catalog,
+      new Map([['gptx', { intelligenceIndex: 41, aliases: ['gpt-x'] }]]),
+    );
+
+    expect(catalog.openai.models[0].benchmarkByEffort).toBeUndefined();
+    expect(overlay['gpt-x'].benchmarkByEffort).toBeUndefined();
+  });
+});

packages/twenty-server/scripts/ai-catalog-sync/__tests__/fetch-artificial-analysis.spec.ts

@@ -0,0 +1,246 @@
+import { fetchArtificialAnalysisBenchmarks } from '../utils/fetch-artificial-analysis-benchmarks.util';
+
+const respondWith = (models: unknown[]): void => {
+  global.fetch = jest.fn().mockResolvedValue({
+    ok: true,
+    json: async () => ({ data: models }),
+  }) as unknown as typeof fetch;
+};
+
+const measuredModel = {
+  slug: 'claude-sonnet-5',
+  evaluations: { artificial_analysis_intelligence_index: 38.4 },
+  performance: { median_output_tokens_per_second: 92.1 },
+  artificial_analysis_intelligence_index_cost: {
+    cost_per_task: { total_cost: 0.42 },
+  },
+};
+
+describe('fetchArtificialAnalysisBenchmarks', () => {
+  it('reads the index, speed and cost per task for a measured model', async () => {
+    respondWith([measuredModel]);
+
+    const record = (await fetchArtificialAnalysisBenchmarks('key')).get(
+      'claudesonnet5',
+    );
+
+    expect(record?.intelligenceIndex).toBe(38.4);
+    expect(record?.outputTokensPerSecond).toBe(92.1);
+    expect(record?.costPerTask).toBe(0.42);
+  });
+
+  it('treats zero as not measured rather than as a measurement', async () => {
+    respondWith([
+      {
+        slug: 'grok-4-6',
+        evaluations: { artificial_analysis_intelligence_index: 44.1 },
+        performance: { median_output_tokens_per_second: 0 },
+      },
+    ]);
+
+    const record = (await fetchArtificialAnalysisBenchmarks('key')).get(
+      'grok46',
+    );
+
+    expect(record?.intelligenceIndex).toBe(44.1);
+    expect(record?.outputTokensPerSecond).toBeUndefined();
+  });
+
+  it.each([
+    ['best row first', true],
+    ['best row last', false],
+  ])(
+    'keeps the highest-scoring configuration whatever the order (%s)',
+    async (_label, bestFirst) => {
+      // The rows for one model are the same model at different reasoning
+      // efforts. Picking by how populated a row is scored Sonnet 4.6 at low
+      // effort against Sonnet 5 at max, and the gap read as capability.
+      const lowEffort = {
+        slug: 'claude-sonnet-4-6',
+        name: 'Claude Sonnet 4.6 (Non-reasoning, Low Effort)',
+        evaluations: { artificial_analysis_intelligence_index: 23.3 },
+        performance: { median_output_tokens_per_second: 120 },
+        artificial_analysis_intelligence_index_cost: {
+          cost_per_task: { total_cost: 0.1 },
+        },
+      };
+      const maxEffort = {
+        slug: 'claude-sonnet-4-6',
+        name: 'Claude Sonnet 4.6 (Adaptive Reasoning, Max Effort)',
+        evaluations: { artificial_analysis_intelligence_index: 36.2 },
+        performance: { median_output_tokens_per_second: 80 },
+      };
+
+      respondWith(bestFirst ? [maxEffort, lowEffort] : [lowEffort, maxEffort]);
+
+      const record = (await fetchArtificialAnalysisBenchmarks('key')).get(
+        'claudesonnet46',
+      );
+
+      expect(record?.intelligenceIndex).toBe(36.2);
+      // The whole row travels together, so speed and cost describe the same
+      // configuration the index was measured in.
+      expect(record?.outputTokensPerSecond).toBe(80);
+      expect(record?.costPerTask).toBeUndefined();
+    },
+  );
+
+  it.each([
+    ['unmeasured row first', true],
+    ['measured row first', false],
+  ])(
+    'keeps the measured row when an unmeasured duplicate shares its name (%s)',
+    async (_label, unmeasuredFirst) => {
+      const unmeasured = {
+        slug: 'claude-sonnet-5',
+        performance: { median_output_tokens_per_second: 0 },
+      };
+
+      respondWith(
+        unmeasuredFirst
+          ? [unmeasured, measuredModel]
+          : [measuredModel, unmeasured],
+      );
+
+      const record = (await fetchArtificialAnalysisBenchmarks('key')).get(
+        'claudesonnet5',
+      );
+
+      expect(record?.intelligenceIndex).toBe(38.4);
+    },
+  );
+
+  it('survives a null row rather than discarding the whole response', async () => {
+    respondWith([null, measuredModel]);
+
+    const record = (await fetchArtificialAnalysisBenchmarks('key')).get(
+      'claudesonnet5',
+    );
+
+    expect(record?.intelligenceIndex).toBe(38.4);
+  });
+
+  it('fails loudly when every row came back unmeasured', async () => {
+    respondWith([
+      {
+        slug: 'claude-sonnet-5',
+        performance: { median_output_tokens_per_second: 0 },
+      },
+    ]);
+
+    await expect(fetchArtificialAnalysisBenchmarks('key')).rejects.toThrow(
+      /none carried a measurement/,
+    );
+  });
+
+  it('ignores figures published at the top level instead of their container', async () => {
+    respondWith([
+      {
+        slug: 'claude-sonnet-5',
+        artificial_analysis_intelligence_index: 38.4,
+        performance: { median_output_tokens_per_second: 92.1 },
+      },
+    ]);
+
+    const record = (await fetchArtificialAnalysisBenchmarks('key')).get(
+      'claudesonnet5',
+    );
+
+    expect(record?.intelligenceIndex).toBeUndefined();
+    expect(record?.outputTokensPerSecond).toBe(92.1);
+  });
+
+  it('reads every page of the list before indexing', async () => {
+    const pages = [
+      {
+        data: [measuredModel],
+        pagination: { page: 1, has_more: true },
+      },
+      {
+        data: [
+          {
+            slug: 'gpt-5-6-luna',
+            evaluations: { artificial_analysis_intelligence_index: 37.5 },
+          },
+        ],
+        pagination: { page: 2, has_more: false },
+      },
+    ];
+    const fetchMock = jest.fn(async (url: URL) => ({
+      ok: true,
+      json: async () => pages[Number(url.searchParams.get('page')) - 1],
+    }));
+
+    global.fetch = fetchMock as unknown as typeof fetch;
+
+    const index = await fetchArtificialAnalysisBenchmarks('key');
+
+    expect(fetchMock).toHaveBeenCalledTimes(2);
+    expect(index.get('claudesonnet5')?.intelligenceIndex).toBe(38.4);
+    expect(index.get('gpt56luna')?.intelligenceIndex).toBe(37.5);
+  });
+
+  it('files a row under its effort as well as under the bare model name', async () => {
+    respondWith([
+      {
+        slug: 'gpt-5-6-sol',
+        name: 'GPT-5.6 Sol (max)',
+        evaluations: { artificial_analysis_intelligence_index: 47.1 },
+      },
+      {
+        slug: 'gpt-5-6-sol',
+        name: 'GPT-5.6 Sol (low)',
+        evaluations: { artificial_analysis_intelligence_index: 30.2 },
+        artificial_analysis_intelligence_index_cost: {
+          cost_per_task: { total_cost: 0.08 },
+        },
+      },
+    ]);
+
+    const index = await fetchArtificialAnalysisBenchmarks('key');
+
+    // The bare key still carries the ceiling, so nothing a consumer reads today
+    // changes.
+    expect(index.get('gpt56sol')?.intelligenceIndex).toBe(47.1);
+    expect(index.get('gpt56sol')?.effort).toBe('max');
+    expect(index.get('gpt56sol@max')?.intelligenceIndex).toBe(47.1);
+    expect(index.get('gpt56sol@low')?.intelligenceIndex).toBe(30.2);
+    expect(index.get('gpt56sol@low')?.costPerTask).toBe(0.08);
+    expect(index.get('gpt56sol@low')?.effort).toBe('low');
+  });
+
+  it('keeps the better-measured row when two rows share an effort', async () => {
+    respondWith([
+      {
+        slug: 'grok-4-6',
+        name: 'Grok 4.6 (high)',
+        evaluations: { artificial_analysis_intelligence_index: 44.4 },
+      },
+      {
+        slug: 'grok-4-6',
+        name: 'Grok 4.6 (high)',
+        evaluations: { artificial_analysis_intelligence_index: 44.4 },
+        performance: { median_output_tokens_per_second: 71 },
+      },
+    ]);
+
+    const index = await fetchArtificialAnalysisBenchmarks('key');
+
+    expect(index.get('grok46@high')?.outputTokensPerSecond).toBe(71);
+  });
+
+  it('files a row naming no effort under the bare name only', async () => {
+    respondWith([
+      {
+        slug: 'gemini-3-1-pro-preview',
+        name: 'Gemini 3.1 Pro Preview',
+        evaluations: { artificial_analysis_intelligence_index: 30.4 },
+      },
+    ]);
+
+    const index = await fetchArtificialAnalysisBenchmarks('key');
+
+    expect(index.get('gemini31propreview')?.effort).toBeUndefined();
+    expect([...index.keys()].some((key) => key.includes('@'))).toBe(false);
+  });
+});

packages/twenty-server/scripts/ai-catalog-sync/__tests__/match-benchmarks.spec.ts

@@ -0,0 +1,339 @@
+import { type ModelsDevModel } from 'src/engine/metadata-modules/ai/ai-models/types/models-dev-model.type';
+
+import { type BenchmarkIndex } from '../types/benchmark-index.type';
+import { type BenchmarkRecord } from '../types/benchmark-record.type';
+import { buildLookupCandidates } from '../utils/build-lookup-candidates.util';
+import { matchBenchmarks } from '../utils/match-benchmarks.util';
+
+const MEASURED_AT = '2026-09-09';
+
+const modelsDevModel = (
+  overrides: Partial<ModelsDevModel> = {},
+): ModelsDevModel => ({
+  id: 'model',
+  name: 'Model',
+  cost: { input: 2, output: 6 },
+  limit: { context: 128000, output: 8192 },
+  ...overrides,
+});
+
+const MISTRAL_MODELS: Record<string, ModelsDevModel> = {
+  'mistral-large-latest': modelsDevModel({ id: 'mistral-large-latest' }),
+  'mistral-large-2512': modelsDevModel({
+    id: 'mistral-large-2512',
+    release_date: '2025-12-01',
+  }),
+  'mistral-large-2411': modelsDevModel({
+    id: 'mistral-large-2411',
+    release_date: '2024-11-01',
+    cost: { input: 3, output: 9 },
+  }),
+};
+
+const indexOf = (entries: Record<string, BenchmarkRecord>): BenchmarkIndex =>
+  new Map(Object.entries(entries));
+
+const match = (modelName: string, benchmarkIndex: BenchmarkIndex) =>
+  matchBenchmarks({
+    modelName,
+    siblingModels: MISTRAL_MODELS,
+    benchmarkIndex,
+    measuredAt: MEASURED_AT,
+  });
+
+describe('buildLookupCandidates', () => {
+  it('resolves a rolling alias to the priced-identical release it points at', () => {
+    expect(
+      buildLookupCandidates({
+        modelName: 'mistral-large-latest',
+        siblingModels: MISTRAL_MODELS,
+      }),
+    ).toContain('mistral-large-2512');
+  });
+
+  it('prefers the newest release when several share the same price and limits', () => {
+    const candidates = buildLookupCandidates({
+      modelName: 'mistral-large-latest',
+      siblingModels: {
+        ...MISTRAL_MODELS,
+        'mistral-large-2506': modelsDevModel({ release_date: '2025-06-01' }),
+      },
+    });
+
+    expect(candidates).toContain('mistral-large-2512');
+    expect(candidates).not.toContain('mistral-large-2506');
+  });
+
+  it('resolves the same way whatever order an undated provider lists twins in', () => {
+    // Nothing in models.dev hits this today, but the resolution must not depend
+    // on the order a third party happens to serialize its models in.
+    const undated = {
+      'mistral-large-latest': modelsDevModel({ id: 'mistral-large-latest' }),
+      'mistral-large-2506': modelsDevModel({ id: 'mistral-large-2506' }),
+      'mistral-large-2512': modelsDevModel({ id: 'mistral-large-2512' }),
+    };
+
+    const resolve = (siblingModels: Record<string, ModelsDevModel>) =>
+      buildLookupCandidates({
+        modelName: 'mistral-large-latest',
+        siblingModels,
+      });
+
+    const reordered = {
+      'mistral-large-latest': undated['mistral-large-latest'],
+      'mistral-large-2512': undated['mistral-large-2512'],
+      'mistral-large-2506': undated['mistral-large-2506'],
+    };
+
+    [undated, reordered].forEach((siblingModels) => {
+      const candidates = resolve(siblingModels);
+
+      expect(candidates).toContain('mistral-large-2512');
+      expect(candidates).not.toContain('mistral-large-2506');
+    });
+  });
+
+  it('does not offer the undated name once the alias resolves to a release', () => {
+    // The leaderboard's bare `mistral-large` row is the Feb '24 model. Letting
+    // it stand in for the release `-latest` currently points at published a
+    // two-year-old score as the current one.
+    const candidates = buildLookupCandidates({
+      modelName: 'mistral-large-latest',
+      siblingModels: MISTRAL_MODELS,
+    });
+
+    expect(candidates).toContain('mistral-large-2512');
+    expect(candidates).not.toContain('mistral-large');
+  });
+
+  it('offers no undated spelling of a resolved full-date release either', () => {
+    // The publisher's undated row is whichever snapshot it last measured, which
+    // is the same trap as the bare rolling name in a longer date format.
+    const candidates = buildLookupCandidates({
+      modelName: 'claude-sonnet-5-latest',
+      siblingModels: {
+        'claude-sonnet-5-latest': modelsDevModel({
+          id: 'claude-sonnet-5-latest',
+        }),
+        'claude-sonnet-5-20260101': modelsDevModel({
+          id: 'claude-sonnet-5-20260101',
+          release_date: '2026-01-01',
+        }),
+      },
+    });
+
+    expect(candidates).toContain('claude-sonnet-5-20260101');
+    expect(candidates).not.toContain('claude-sonnet-5');
+  });
+
+  it('falls back to the undated name when no release can be resolved', () => {
+    const candidates = buildLookupCandidates({
+      modelName: 'mistral-large-latest',
+      siblingModels: {
+        'mistral-large-latest': modelsDevModel({ id: 'mistral-large-latest' }),
+      },
+    });
+
+    expect(candidates).toContain('mistral-large');
+  });
+
+  it('offers the undated name for a dated snapshot', () => {
+    expect(
+      buildLookupCandidates({
+        modelName: 'claude-sonnet-4-5-20250929',
+        siblingModels: {},
+      }),
+    ).toContain('claude-sonnet-4-5');
+  });
+});
+
+describe('matchBenchmarks', () => {
+  it('scores a rolling alias from the release it resolves to', () => {
+    const result = match(
+      'mistral-large-latest',
+      indexOf({ mistrallarge2512: { intelligenceIndex: 41, aliases: [] } }),
+    );
+
+    expect(result?.benchmark.intelligenceIndex).toBe(41);
+    expect(result?.benchmark.measuredAt).toBe(MEASURED_AT);
+  });
+
+  it('carries speed and cost per task through to the catalog', () => {
+    const result = match(
+      'mistral-large-2512',
+      indexOf({
+        mistrallarge2512: {
+          outputTokensPerSecond: 92,
+          costPerTask: 0.42,
+          aliases: [],
+        },
+      }),
+    );
+
+    expect(result?.benchmark.outputTokensPerSecond).toBe(92);
+    expect(result?.benchmark.costPerTask).toBe(0.42);
+  });
+
+  it('returns nothing for a model absent from the index', () => {
+    expect(match('mistral-large-2512', new Map())).toBeUndefined();
+  });
+
+  it('returns nothing for a matched row carrying no measurement at all', () => {
+    // The publisher lists models it has measured nothing about; matching one
+    // must not produce an entry holding only a timestamp.
+    expect(
+      match(
+        'mistral-large-2512',
+        indexOf({ mistrallarge2512: { aliases: ['mistral-large-2512'] } }),
+      ),
+    ).toBeUndefined();
+  });
+
+  it('prefers the release a rolling alias resolves to over an undated row', () => {
+    const result = match(
+      'mistral-large-latest',
+      indexOf({
+        mistrallarge: { intelligenceIndex: 20, aliases: [] },
+        mistrallarge2512: { intelligenceIndex: 41, aliases: [] },
+      }),
+    );
+
+    expect(result?.benchmark.intelligenceIndex).toBe(41);
+  });
+
+  it('collects the aliases the publisher gives so consumers can join on them', () => {
+    const result = match(
+      'mistral-large-2512',
+      indexOf({
+        mistrallarge2512: {
+          intelligenceIndex: 41,
+          aliases: ['mistral-large-2512', 'mistralai/mistral-large-2512'],
+        },
+      }),
+    );
+
+    expect(result?.aliases).toEqual([
+      'mistral-large-2512',
+      'mistralai/mistral-large-2512',
+    ]);
+  });
+
+  it('reads each declared effort from the row taken at that effort', () => {
+    const result = matchBenchmarks({
+      modelName: 'mistral-large-2512',
+      siblingModels: MISTRAL_MODELS,
+      benchmarkIndex: indexOf({
+        mistrallarge2512: {
+          intelligenceIndex: 41,
+          effort: 'high',
+          aliases: ['Mistral Large 3 (high)'],
+        },
+        'mistrallarge2512@high': {
+          intelligenceIndex: 41,
+          effort: 'high',
+          aliases: ['Mistral Large 3 (high)'],
+        },
+        'mistrallarge2512@none': {
+          intelligenceIndex: 22,
+          costPerTask: 0.1,
+          effort: 'none',
+          aliases: ['Mistral Large 3 (Non-reasoning)'],
+        },
+      }),
+      measuredAt: MEASURED_AT,
+      efforts: ['none', 'high'],
+    });
+
+    expect(result?.benchmark.effort).toBe('high');
+    expect(result?.benchmarkByEffort?.none).toEqual({
+      intelligenceIndex: 22,
+      costPerTask: 0.1,
+      effort: 'none',
+      measuredAt: MEASURED_AT,
+      aliases: ['Mistral Large 3 (Non-reasoning)'],
+    });
+    expect(result?.benchmarkByEffort?.high?.intelligenceIndex).toBe(41);
+  });
+
+  it('leaves a declared effort without its own row blank rather than lending it the ceiling', () => {
+    const result = matchBenchmarks({
+      modelName: 'mistral-large-2512',
+      siblingModels: MISTRAL_MODELS,
+      benchmarkIndex: indexOf({
+        mistrallarge2512: {
+          intelligenceIndex: 41,
+          effort: 'high',
+          aliases: [],
+        },
+        'mistrallarge2512@high': {
+          intelligenceIndex: 41,
+          effort: 'high',
+          aliases: [],
+        },
+      }),
+      measuredAt: MEASURED_AT,
+      efforts: ['none', 'high'],
+    });
+
+    expect(result?.benchmarkByEffort?.none).toBeUndefined();
+    expect(result?.benchmarkByEffort?.high?.intelligenceIndex).toBe(41);
+  });
+
+  it('carries no effort map at all when no declared effort has a row', () => {
+    const result = matchBenchmarks({
+      modelName: 'mistral-large-2512',
+      siblingModels: MISTRAL_MODELS,
+      benchmarkIndex: indexOf({
+        mistrallarge2512: { intelligenceIndex: 41, aliases: [] },
+      }),
+      measuredAt: MEASURED_AT,
+      efforts: ['none', 'high'],
+    });
+
+    expect(result?.benchmark.intelligenceIndex).toBe(41);
+    expect(result?.benchmarkByEffort).toBeUndefined();
+  });
+
+  it('resolves a rolling alias to its release for per-effort rows too', () => {
+    const result = matchBenchmarks({
+      modelName: 'mistral-large-latest',
+      siblingModels: MISTRAL_MODELS,
+      benchmarkIndex: indexOf({
+        mistrallarge2512: { intelligenceIndex: 41, aliases: [] },
+        'mistrallarge2512@high': {
+          intelligenceIndex: 41,
+          effort: 'high',
+          aliases: [],
+        },
+        'mistrallarge@high': {
+          intelligenceIndex: 20,
+          effort: 'high',
+          aliases: [],
+        },
+      }),
+      measuredAt: MEASURED_AT,
+      efforts: ['high'],
+    });
+
+    expect(result?.benchmarkByEffort?.high?.intelligenceIndex).toBe(41);
+  });
+
+  it('labels a per-effort reading with the key it was filed under', () => {
+    const result = matchBenchmarks({
+      modelName: 'mistral-large-2512',
+      siblingModels: MISTRAL_MODELS,
+      benchmarkIndex: indexOf({
+        mistrallarge2512: { intelligenceIndex: 41, aliases: [] },
+        'mistrallarge2512@low': {
+          intelligenceIndex: 30,
+          effort: 'high',
+          aliases: [],
+        },
+      }),
+      measuredAt: MEASURED_AT,
+      efforts: ['low'],
+    });
+
+    expect(result?.benchmarkByEffort?.low?.effort).toBe('low');
+  });
+});

packages/twenty-server/scripts/ai-catalog-sync/__tests__/parse-artificial-analysis-effort.spec.ts

@@ -0,0 +1,30 @@
+import { parseArtificialAnalysisEffort } from '../utils/parse-artificial-analysis-effort.util';
+
+describe('parseArtificialAnalysisEffort', () => {
+  it.each([
+    ['GPT-5.6 Sol (max)', 'max'],
+    ['GPT-5.4 (xhigh)', 'xhigh'],
+    ['Grok 4.6 (high)', 'high'],
+    ['Gemini 3.5 Flash (medium)', 'medium'],
+    ['GPT-5 nano (minimal)', 'minimal'],
+    ['Claude Opus 5 (Adaptive Reasoning, Max Effort)', 'max'],
+    [
+      'Claude Fable 5 (Adaptive Reasoning, Max Effort, Opus 4.8 Fallback)',
+      'max',
+    ],
+    ['Claude Sonnet 4.6 (Non-reasoning, Low Effort)', 'low'],
+    ['GPT-5.6 Luna (Non-reasoning)', 'none'],
+  ])('reads "%s" as %s', (displayName, effort) => {
+    expect(parseArtificialAnalysisEffort(displayName)).toBe(effort);
+  });
+
+  it.each([
+    'Claude Opus 4.5 (Reasoning)',
+    'Gemini 3.1 Pro Preview',
+    'GPT-4o (March 2025, chatgpt-4o-latest)',
+    "Llama 3.1 405B (May '24)",
+    'Mistral Large 3 (high) Turbo',
+  ])('names no effort for "%s"', (displayName) => {
+    expect(parseArtificialAnalysisEffort(displayName)).toBeUndefined();
+  });
+});

packages/twenty-server/scripts/ai-catalog-sync/__tests__/project-catalog.spec.ts

@@ -0,0 +1,292 @@
+import { type CatalogSpec } from '../types/catalog-spec.type';
+import {
+  type CanonicalCatalog,
+  projectCatalog,
+} from '../utils/project-catalog.util';
+
+const canonicalCatalog: CanonicalCatalog = {
+  openai: {
+    npm: '@ai-sdk/openai',
+    label: 'OpenAI',
+    apiKey: '{{OPENAI_API_KEY}}',
+    models: [
+      {
+        name: 'gpt-5.6-luna',
+        label: 'GPT-5.6 Luna',
+        inputCostPerMillionTokens: 0.2,
+        outputCostPerMillionTokens: 1.2,
+        cachedInputCostPerMillionTokens: 0.02,
+        modalities: ['image', 'pdf'],
+        efforts: ['low', 'medium', 'high'],
+        benchmark: { intelligenceIndex: 37.5, measuredAt: '2026-09-11' },
+        benchmarkByEffort: {
+          high: { intelligenceIndex: 32.4, measuredAt: '2026-09-11' },
+        },
+      },
+    ],
+  },
+  anthropic: {
+    npm: '@ai-sdk/anthropic',
+    label: 'Anthropic',
+    apiKey: '{{ANTHROPIC_API_KEY}}',
+    models: [{ name: 'claude-opus-4-7', label: 'Claude Opus 4.7' }],
+  },
+};
+
+describe('projectCatalog', () => {
+  it('describes a route from the catalog while keeping the route its own credentials', () => {
+    const spec: CatalogSpec = {
+      providers: [
+        {
+          name: 'azure-foundry',
+          npm: '@ai-sdk/azure',
+          label: 'Azure AI Foundry',
+          apiKey: '{{AZURE_FOUNDRY_API_KEY}}',
+          baseUrl: '{{AZURE_FOUNDRY_BASE_URL}}',
+          dataResidency: 'eu',
+          labelSuffix: ' (Azure)',
+          models: ['gpt-5.6-luna'],
+        },
+      ],
+    };
+
+    const projected = projectCatalog({ canonicalCatalog, spec });
+
+    expect(Object.keys(projected)).toEqual(['azure-foundry']);
+    expect(projected['azure-foundry']).toMatchObject({
+      npm: '@ai-sdk/azure',
+      label: 'Azure AI Foundry',
+      apiKey: '{{AZURE_FOUNDRY_API_KEY}}',
+      baseUrl: '{{AZURE_FOUNDRY_BASE_URL}}',
+    });
+    expect(projected['azure-foundry'].models?.[0]).toEqual({
+      name: 'gpt-5.6-luna',
+      label: 'GPT-5.6 Luna (Azure)',
+      inputCostPerMillionTokens: 0.2,
+      outputCostPerMillionTokens: 1.2,
+      cachedInputCostPerMillionTokens: 0.02,
+      modalities: ['image', 'pdf'],
+      efforts: ['low', 'medium', 'high'],
+      benchmark: { intelligenceIndex: 37.5, measuredAt: '2026-09-11' },
+      benchmarkByEffort: {
+        high: { intelligenceIndex: 32.4, measuredAt: '2026-09-11' },
+      },
+      dataResidency: 'eu',
+    });
+  });
+
+  it('serves a model under the id the route deploys it as', () => {
+    const spec: CatalogSpec = {
+      providers: [
+        {
+          name: 'amazon-bedrock',
+          npm: '@ai-sdk/amazon-bedrock',
+          region: 'eu-central-1',
+          models: [
+            { model: 'claude-opus-4-7', as: 'eu.anthropic.claude-opus-4-7' },
+          ],
+        },
+      ],
+    };
+
+    const projected = projectCatalog({ canonicalCatalog, spec });
+
+    expect(projected['amazon-bedrock'].models?.[0]).toMatchObject({
+      name: 'eu.anthropic.claude-opus-4-7',
+      label: 'Claude Opus 4.7',
+    });
+    expect(projected['amazon-bedrock'].region).toBe('eu-central-1');
+  });
+
+  it('takes a price the route negotiated and leaves the rest of the reading alone', () => {
+    const spec: CatalogSpec = {
+      providers: [
+        {
+          name: 'azure-foundry',
+          npm: '@ai-sdk/azure',
+          models: [
+            { model: 'gpt-5.6-luna', cachedInputCostPerMillionTokens: 0.5 },
+          ],
+        },
+      ],
+    };
+
+    const projected = projectCatalog({ canonicalCatalog, spec });
+
+    expect(projected['azure-foundry'].models?.[0]).toMatchObject({
+      cachedInputCostPerMillionTokens: 0.5,
+      inputCostPerMillionTokens: 0.2,
+      efforts: ['low', 'medium', 'high'],
+    });
+  });
+
+  it('takes a limit the route caps below the one the model allows', () => {
+    const spec: CatalogSpec = {
+      providers: [
+        {
+          name: 'amazon-bedrock',
+          npm: '@ai-sdk/amazon-bedrock',
+          models: [{ model: 'gpt-5.6-luna', maxOutputTokens: 64000 }],
+        },
+      ],
+    };
+
+    const projected = projectCatalog({ canonicalCatalog, spec });
+
+    expect(projected['amazon-bedrock'].models?.[0]).toMatchObject({
+      maxOutputTokens: 64000,
+    });
+  });
+
+  it('serves every model a vendor publishes when the spec names the vendor', () => {
+    const spec: CatalogSpec = {
+      providers: [
+        {
+          name: 'openai',
+          npm: '@ai-sdk/openai',
+          apiKey: '{{OPENAI_API_KEY}}',
+          models: { vendor: 'openai' },
+        },
+      ],
+    };
+
+    const projected = projectCatalog({ canonicalCatalog, spec });
+
+    expect(projected['openai'].models?.map((model) => model.name)).toEqual([
+      'gpt-5.6-luna',
+    ]);
+    expect(projected['openai'].models?.[0]).toMatchObject({
+      efforts: ['low', 'medium', 'high'],
+    });
+  });
+
+  it('refuses a spec naming a vendor the catalog does not carry', () => {
+    const spec: CatalogSpec = {
+      providers: [
+        {
+          name: 'openai',
+          npm: '@ai-sdk/openai',
+          models: { vendor: 'openai-imaginary' },
+        },
+      ],
+    };
+
+    expect(() => projectCatalog({ canonicalCatalog, spec })).toThrow(
+      'which the catalog does not carry',
+    );
+  });
+
+  it('refuses a spec naming a model the catalog does not carry', () => {
+    const spec: CatalogSpec = {
+      providers: [
+        {
+          name: 'azure-foundry',
+          npm: '@ai-sdk/azure',
+          models: ['gpt-5.6-luna', 'gpt-9-imaginary'],
+        },
+      ],
+    };
+
+    expect(() => projectCatalog({ canonicalCatalog, spec })).toThrow(
+      'azure-foundry/gpt-9-imaginary',
+    );
+  });
+
+  it('cannot restate what a model is, only which models a route serves', () => {
+    const spec = {
+      providers: [
+        {
+          name: 'azure-foundry',
+          npm: '@ai-sdk/azure',
+          models: [{ model: 'gpt-5.6-luna', modalities: ['image'] }],
+        },
+      ],
+    } as unknown as CatalogSpec;
+
+    const projected = projectCatalog({ canonicalCatalog, spec });
+
+    expect(projected['azure-foundry'].models?.[0]).toMatchObject({
+      modalities: ['image', 'pdf'],
+    });
+  });
+
+  it('refuses a repeated provider, which would drop the first one silently', () => {
+    const spec: CatalogSpec = {
+      providers: [
+        {
+          name: 'azure-foundry',
+          npm: '@ai-sdk/azure',
+          models: ['gpt-5.6-luna'],
+        },
+        {
+          name: 'azure-foundry',
+          npm: '@ai-sdk/azure',
+          models: ['claude-opus-4-7'],
+        },
+      ],
+    };
+
+    expect(() => projectCatalog({ canonicalCatalog, spec })).toThrow(
+      'repeated or reserved',
+    );
+  });
+
+  it('refuses an SDK package or residency the server would not parse', () => {
+    const withNpm = (npm: string): CatalogSpec => ({
+      providers: [{ name: 'gateway', npm, models: ['gpt-5.6-luna'] }],
+    });
+
+    expect(() =>
+      projectCatalog({ canonicalCatalog, spec: withNpm('@ai-sdk/imaginary') }),
+    ).toThrow('unsupported SDK package');
+
+    expect(() =>
+      projectCatalog({
+        canonicalCatalog,
+        spec: {
+          providers: [
+            {
+              name: 'gateway',
+              npm: '@ai-sdk/openai',
+              dataResidency: 'mars',
+              models: ['gpt-5.6-luna'],
+            },
+          ],
+        },
+      }),
+    ).toThrow('unsupported data residency');
+  });
+
+  it('refuses a negotiated price that is not a usable number', () => {
+    const spec: CatalogSpec = {
+      providers: [
+        {
+          name: 'azure-foundry',
+          npm: '@ai-sdk/azure',
+          models: [{ model: 'gpt-5.6-luna', inputCostPerMillionTokens: -1 }],
+        },
+      ],
+    };
+
+    expect(() => projectCatalog({ canonicalCatalog, spec })).toThrow(
+      'which is not a usable number',
+    );
+  });
+
+  it('marks the route on a label the spec overrides', () => {
+    const spec: CatalogSpec = {
+      providers: [
+        {
+          name: 'azure-foundry',
+          npm: '@ai-sdk/azure',
+          labelSuffix: ' (Azure)',
+          models: [{ model: 'gpt-5.6-luna', label: 'Luna' }],
+        },
+      ],
+    };
+
+    const projected = projectCatalog({ canonicalCatalog, spec });
+
+    expect(projected['azure-foundry'].models?.[0]?.label).toBe('Luna (Azure)');
+  });
+});

packages/twenty-server/scripts/ai-catalog-sync/__tests__/read-committed-benchmarks.spec.ts

@@ -0,0 +1,142 @@
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+
+import { matchBenchmarks } from '../utils/match-benchmarks.util';
+import { readCommittedBenchmarks } from '../utils/read-committed-benchmarks.util';
+
+const writeOverlay = (value: unknown): string => {
+  const filePath = path.join(
+    fs.mkdtempSync(path.join(os.tmpdir(), 'overlay-')),
+    'ai-model-benchmarks.json',
+  );
+
+  fs.writeFileSync(filePath, JSON.stringify(value), 'utf-8');
+
+  return filePath;
+};
+
+const PUBLISHED = {
+  source: 'artificialanalysis.ai',
+  measuredAt: '2026-09-01',
+  models: {
+    'claude-sonnet-5': {
+      intelligenceIndex: 38.4,
+      outputTokensPerSecond: 92.1,
+      measuredAt: '2026-09-01',
+      aliases: ['claude-sonnet-5', 'claude-sonnet-5-20260630'],
+    },
+  },
+};
+
+describe('readCommittedBenchmarks', () => {
+  it('returns nothing when no overlay has been published yet', () => {
+    expect(
+      readCommittedBenchmarks(path.join(os.tmpdir(), 'does-not-exist.json'))
+        .size,
+    ).toBe(0);
+  });
+
+  it('recovers every measurement', () => {
+    const index = readCommittedBenchmarks(writeOverlay(PUBLISHED));
+    const record = index.get('claudesonnet5');
+
+    expect(record?.intelligenceIndex).toBe(38.4);
+    expect(record?.outputTokensPerSecond).toBe(92.1);
+  });
+
+  it('indexes the published aliases so matching still resolves them', () => {
+    const index = readCommittedBenchmarks(writeOverlay(PUBLISHED));
+
+    expect(index.get('claudesonnet520260630')?.intelligenceIndex).toBe(38.4);
+  });
+
+  it('keeps a preserved measurement dated when it was taken, not today', () => {
+    // Restamping recovered data as measured today would present a stale
+    // benchmark as fresh on every failed run.
+    const index = readCommittedBenchmarks(writeOverlay(PUBLISHED));
+
+    const result = matchBenchmarks({
+      modelName: 'claude-sonnet-5',
+      siblingModels: {},
+      benchmarkIndex: index,
+      measuredAt: '2026-09-09',
+    });
+
+    expect(result?.benchmark.measuredAt).toBe('2026-09-01');
+  });
+
+  it('recovers per-effort readings under their effort keys, dated when they were taken', () => {
+    const index = readCommittedBenchmarks(
+      writeOverlay({
+        ...PUBLISHED,
+        models: {
+          'claude-sonnet-5': {
+            ...PUBLISHED.models['claude-sonnet-5'],
+            effort: 'max',
+            benchmarkByEffort: {
+              low: {
+                intelligenceIndex: 20.1,
+                measuredAt: '2026-09-01',
+                aliases: ['Claude Sonnet 5 (low)'],
+              },
+            },
+          },
+        },
+      }),
+    );
+
+    expect(index.get('claudesonnet5')?.effort).toBe('max');
+    expect(index.get('claudesonnet5@low')?.intelligenceIndex).toBe(20.1);
+    expect(index.get('claudesonnet5@low')?.effort).toBe('low');
+    expect(index.get('claudesonnet5@low')?.measuredAt).toBe('2026-09-01');
+    expect(index.get('claudesonnet5@max')).toBeUndefined();
+
+    const result = matchBenchmarks({
+      modelName: 'claude-sonnet-5',
+      siblingModels: {},
+      benchmarkIndex: index,
+      measuredAt: '2026-09-09',
+      efforts: ['low', 'max'],
+    });
+
+    expect(result?.benchmarkByEffort?.low?.measuredAt).toBe('2026-09-01');
+    expect(result?.benchmarkByEffort?.max).toBeUndefined();
+  });
+
+  it('ignores a per-effort key it does not recognise', () => {
+    const index = readCommittedBenchmarks(
+      writeOverlay({
+        ...PUBLISHED,
+        models: {
+          'claude-sonnet-5': {
+            ...PUBLISHED.models['claude-sonnet-5'],
+            benchmarkByEffort: {
+              turbo: { intelligenceIndex: 99, aliases: [] },
+            },
+          },
+        },
+      }),
+    );
+
+    expect([...index.keys()].some((key) => key.includes('@'))).toBe(false);
+  });
+
+  it('trusts the key over a nested reading that names another effort', () => {
+    const index = readCommittedBenchmarks(
+      writeOverlay({
+        ...PUBLISHED,
+        models: {
+          'claude-sonnet-5': {
+            ...PUBLISHED.models['claude-sonnet-5'],
+            benchmarkByEffort: {
+              low: { intelligenceIndex: 20.1, effort: 'high', aliases: [] },
+            },
+          },
+        },
+      }),
+    );
+
+    expect(index.get('claudesonnet5@low')?.effort).toBe('low');
+  });
+});

packages/twenty-server/scripts/ai-catalog-sync/__tests__/self-host-projection.spec.ts

@@ -0,0 +1,22 @@
+import canonicalCatalog from 'src/engine/metadata-modules/ai/ai-models/ai-models.json';
+import shippedCatalog from 'src/engine/metadata-modules/ai/ai-models/ai-providers.json';
+import selfHostSpec from 'src/engine/metadata-modules/ai/ai-models/ai-self-host-spec.json';
+
+import { type CatalogSpec } from '../types/catalog-spec.type';
+import {
+  type CanonicalCatalog,
+  projectCatalog,
+} from '../utils/project-catalog.util';
+
+// The shipped catalog is generated, and a hand edit to it is the drift this
+// pipeline exists to remove, so it has to stay what its spec produces.
+describe('the shipped catalog', () => {
+  it('is what the self-host spec projects from the model catalog', () => {
+    const projected = projectCatalog({
+      canonicalCatalog: canonicalCatalog as CanonicalCatalog,
+      spec: selfHostSpec as CatalogSpec,
+    });
+
+    expect(projected).toEqual(shippedCatalog);
+  });
+});