snapshot of https://github.com/tldraw/tldraw.git (main, 2026-09-23; blobs > 131072B omitted)

templates/agent/shared/schema/AgentActionSchemaRegistry.ts

@@ -0,0 +1,95 @@
+import z from 'zod'
+
+/**
+ * Options for registering an action schema.
+ */
+export interface RegisterActionSchemaOptions {
+	/**
+	 * If specified, this schema will only be used when the agent is in one of these modes.
+	 * Otherwise, it will be the default schema for this action type.
+	 */
+	forModes?: string[]
+}
+
+// Default registry: actionType -> schema (used when no mode-specific schema is registered)
+const defaultSchemaRegistry = new Map<string, z.ZodType>()
+
+// Mode registry: actionType -> (mode -> schema) (mode-specific overrides)
+const modeSchemaRegistry = new Map<string, Map<string, z.ZodType>>()
+
+/**
+ * Register an action schema. Call this for each action type.
+ *
+ * @param type - The action type (the `_type` literal value).
+ * @param schema - The Zod schema for this action.
+ * @param options - Optional configuration for mode-specific registration.
+ * @returns The schema (for chaining).
+ */
+export function registerActionSchema<T extends z.ZodType>(
+	type: string,
+	schema: T,
+	options?: RegisterActionSchemaOptions
+): T {
+	const { forModes } = options ?? {}
+
+	if (forModes && forModes.length > 0) {
+		// Mode-specific registration
+		if (!modeSchemaRegistry.has(type)) {
+			modeSchemaRegistry.set(type, new Map())
+		}
+		const modeMap = modeSchemaRegistry.get(type)!
+		for (const mode of forModes) {
+			if (modeMap.has(mode)) {
+				throw new Error(`Action schema for ${type} already registered for mode ${mode}`)
+			}
+			modeMap.set(mode, schema)
+		}
+	} else {
+		// Default registration
+		if (defaultSchemaRegistry.has(type)) {
+			throw new Error(`Action schema already registered: ${type}`)
+		}
+		defaultSchemaRegistry.set(type, schema)
+	}
+
+	return schema
+}
+
+/**
+ * Get an action schema for a specific type and mode.
+ * Returns the mode-specific schema if one exists, otherwise the default.
+ *
+ * @param type - The action type to look up.
+ * @param mode - The mode to resolve the schema for.
+ * @returns The schema, or undefined if not found.
+ */
+export function getActionSchemaForMode(type: string, mode: string): z.ZodType | undefined {
+	// Check for mode-specific schema first
+	const modeMap = modeSchemaRegistry.get(type)
+	if (modeMap) {
+		const modeSchema = modeMap.get(mode)
+		if (modeSchema) {
+			return modeSchema
+		}
+	}
+
+	// Fall back to default schema
+	return defaultSchemaRegistry.get(type)
+}
+
+/**
+ * Get the default schema for an action type (ignoring mode).
+ *
+ * @param type - The action type to look up.
+ * @returns The default schema, or undefined if not found.
+ */
+export function getDefaultActionSchema(type: string): z.ZodType | undefined {
+	return defaultSchemaRegistry.get(type)
+}
+
+/**
+ * Check if a default schema is already registered for a type.
+ */
+export function hasDefaultActionSchema(type: string): boolean {
+	return defaultSchemaRegistry.has(type)
+}

templates/agent/shared/schema/AgentActionSchemas.ts

@@ -0,0 +1,385 @@
+import z from 'zod'
+import { FocusedColor } from '../format/FocusedColor'
+import { FocusedFillSchema } from '../format/FocusedFill'
+import { FocusedShapeSchema, FocusedTextAnchorSchema } from '../format/FocusedShape'
+import { SimpleShapeIdSchema, TodoIdSchema } from '../types/ids-schema'
+
+/**
+ * `_systemPromptCategory` is used for system prompt generation
+ * but is stripped from the JSON schema sent to the model.
+ *
+ * See `SystemPromptCategory.ts` for available values.
+ */
+
+// Add Detail Action
+export const AddDetailAction = z
+	.object({
+		_type: z.literal('add-detail'),
+		intent: z.string(),
+	})
+	.meta({
+		title: 'Add Detail',
+		description: 'The AI plans further work so that it can add detail to its work.',
+	})
+
+export type AddDetailAction = z.infer<typeof AddDetailAction>
+
+// Align Action
+export const AlignAction = z
+	.object({
+		_type: z.literal('align'),
+		alignment: z.enum(['top', 'bottom', 'left', 'right', 'center-horizontal', 'center-vertical']),
+		gap: z.number(),
+		intent: z.string(),
+		shapeIds: z.array(SimpleShapeIdSchema),
+	})
+	.meta({
+		title: 'Align',
+		description: 'The AI aligns shapes to each other on an axis.',
+		_systemPromptCategory: 'edit',
+	})
+
+export type AlignAction = z.infer<typeof AlignAction>
+
+// Bring to Front Action
+export const BringToFrontAction = z
+	.object({
+		_type: z.literal('bringToFront'),
+		intent: z.string(),
+		shapeIds: z.array(SimpleShapeIdSchema),
+	})
+	.meta({
+		title: 'Bring to Front',
+		description:
+			'The AI brings one or more shapes to the front so that they appear in front of everything else.',
+		_systemPromptCategory: 'edit',
+	})
+
+export type BringToFrontAction = z.infer<typeof BringToFrontAction>
+
+// Clear Action
+export const ClearAction = z
+	.object({
+		_type: z.literal('clear'),
+	})
+	.meta({
+		title: 'Clear',
+		description: 'The agent deletes all shapes on the canvas.',
+	})
+
+export type ClearAction = z.infer<typeof ClearAction>
+
+// Count Shapes Action
+export const CountShapesAction = z
+	.object({
+		_type: z.literal('count'),
+		expression: z.string(),
+	})
+	.meta({
+		title: 'Count',
+		description:
+			'The AI requests to count the number of shapes in the canvas. The answer will be provided to the AI in a follow-up request.',
+	})
+
+export type CountShapesAction = z.infer<typeof CountShapesAction>
+
+// Country Info Action
+export const CountryInfoAction = z
+	.object({
+		_type: z.literal('countryInfo'),
+		code: z.string(),
+	})
+	.meta({
+		title: 'Country info',
+		description:
+			'The AI gets information about a country by providing its country code, eg: "de" for Germany.',
+	})
+
+export type CountryInfoAction = z.infer<typeof CountryInfoAction>
+
+// Create Action
+export const CreateAction = z
+	.object({
+		_type: z.literal('create'),
+		intent: z.string(),
+		shape: FocusedShapeSchema,
+	})
+	.meta({ title: 'Create', description: 'The AI creates a new shape.' })
+
+export type CreateAction = z.infer<typeof CreateAction>
+
+// Delete Action
+export const DeleteAction = z
+	.object({
+		_type: z.literal('delete'),
+		intent: z.string(),
+		shapeId: SimpleShapeIdSchema,
+	})
+	.meta({ title: 'Delete', description: 'The AI deletes a shape.', _systemPromptCategory: 'edit' })
+
+export type DeleteAction = z.infer<typeof DeleteAction>
+
+// Distribute Action
+export const DistributeAction = z
+	.object({
+		_type: z.literal('distribute'),
+		direction: z.enum(['horizontal', 'vertical']),
+		intent: z.string(),
+		shapeIds: z.array(SimpleShapeIdSchema),
+	})
+	.meta({
+		title: 'Distribute',
+		description: 'The AI distributes shapes horizontally or vertically.',
+		_systemPromptCategory: 'edit',
+	})
+
+export type DistributeAction = z.infer<typeof DistributeAction>
+
+// Label Action
+export const LabelAction = z
+	.object({
+		_type: z.literal('label'),
+		intent: z.string(),
+		shapeId: SimpleShapeIdSchema,
+		text: z.string(),
+	})
+	.meta({
+		title: 'Label',
+		description: "The AI changes a shape's text.",
+		_systemPromptCategory: 'edit',
+	})
+
+export type LabelAction = z.infer<typeof LabelAction>
+
+// Message Action
+export const MessageAction = z
+	.object({
+		_type: z.literal('message'),
+		text: z.string(),
+	})
+	.meta({ title: 'Message', description: 'The AI sends a message to the user.' })
+
+export type MessageAction = z.infer<typeof MessageAction>
+
+// Move Action
+export const MoveAction = z
+	.object({
+		_type: z.literal('move'),
+		intent: z.string(),
+		anchor: FocusedTextAnchorSchema,
+		shapeId: SimpleShapeIdSchema,
+		x: z.number(),
+		y: z.number(),
+	})
+	.meta({
+		title: 'Move',
+		description: 'The agent moves a shape to a new position.',
+		_systemPromptCategory: 'edit',
+	})
+
+export type MoveAction = z.infer<typeof MoveAction>
+
+// Pen Action
+export const PenAction = z
+	.object({
+		_type: z.literal('pen'),
+		shapeId: SimpleShapeIdSchema,
+		color: FocusedColor,
+		closed: z.boolean(),
+		fill: FocusedFillSchema,
+		intent: z.string(),
+		points: z.array(
+			z.object({
+				x: z.number(),
+				y: z.number(),
+			})
+		),
+		style: z.enum(['smooth', 'straight']),
+	})
+	.meta({
+		title: 'Pen',
+		description:
+			'The AI draws a freeform line with a pen. This is useful for drawing custom paths that are not available with the other available shapes. The "smooth" style will automatically smooth the line between points. The "straight" style will render a straight line between points. The "closed" property will determine if the drawn line gets automatically closed to form a complete shape or not. Remember that the pen will be *down* until the action is over. If you want to lift up the pen, start a new pen action.',
+	})
+
+export type PenAction = z.infer<typeof PenAction>
+
+// Place Action
+export const PlaceAction = z
+	.object({
+		_type: z.literal('place'),
+		align: z.enum(['start', 'center', 'end']),
+		alignOffset: z.number(),
+		intent: z.string(),
+		referenceShapeId: SimpleShapeIdSchema,
+		side: z.enum(['top', 'bottom', 'left', 'right']),
+		sideOffset: z.number(),
+		shapeId: SimpleShapeIdSchema,
+	})
+	.meta({
+		title: 'Place',
+		description: 'The AI places a shape relative to another shape.',
+		_systemPromptCategory: 'edit',
+	})
+
+export type PlaceAction = z.infer<typeof PlaceAction>
+
+// Resize Action
+export const ResizeAction = z
+	.object({
+		_type: z.literal('resize'),
+		intent: z.string(),
+		originX: z.number(),
+		originY: z.number(),
+		scaleX: z.number(),
+		scaleY: z.number(),
+		shapeIds: z.array(SimpleShapeIdSchema),
+	})
+	.meta({
+		title: 'Resize',
+		description:
+			'The AI resizes one or more shapes, with the resize operation being performed relative to an origin point.',
+		_systemPromptCategory: 'edit',
+	})
+
+export type ResizeAction = z.infer<typeof ResizeAction>
+
+// Review Action
+export const ReviewAction = z
+	.object({
+		_type: z.literal('review'),
+		intent: z.string(),
+		x: z.number(),
+		y: z.number(),
+		w: z.number(),
+		h: z.number(),
+	})
+	.meta({
+		title: 'Review',
+		description:
+			'The AI schedules further work or a review so that it can look at the results of its work so far and take further action, such as reviewing what it has done or taking further steps that would benefit from seeing the results of its work so far.',
+	})
+
+export type ReviewAction = z.infer<typeof ReviewAction>
+
+// Rotate Action
+export const RotateAction = z
+	.object({
+		_type: z.literal('rotate'),
+		centerY: z.number(),
+		degrees: z.number(),
+		intent: z.string(),
+		originX: z.number(),
+		originY: z.number(),
+		shapeIds: z.array(SimpleShapeIdSchema),
+	})
+	.meta({
+		title: 'Rotate',
+		description: 'The AI rotates one or more shapes around an origin point.',
+		_systemPromptCategory: 'edit',
+	})
+
+export type RotateAction = z.infer<typeof RotateAction>
+
+// Send to Back Action
+export const SendToBackAction = z
+	.object({
+		_type: z.literal('sendToBack'),
+		intent: z.string(),
+		shapeIds: z.array(SimpleShapeIdSchema),
+	})
+	.meta({
+		title: 'Send to Back',
+		description:
+			'The AI sends one or more shapes to the back so that they appear behind everything else.',
+		_systemPromptCategory: 'edit',
+	})
+
+export type SendToBackAction = z.infer<typeof SendToBackAction>
+
+// Set My View Action
+export const SetMyViewAction = z
+	.object({
+		_type: z.literal('setMyView'),
+		intent: z.string(),
+		x: z.number(),
+		y: z.number(),
+		w: z.number(),
+		h: z.number(),
+	})
+	.meta({
+		title: 'Set My View',
+		description:
+			'The AI changes the bounds of its own viewport to navigate to other areas of the canvas if needed. `x` and `y` are the top-left corner of the area you want to see, and `w` and `h` are its width and height. To center your view on a point, set `x` and `y` to that point minus half the width and height.',
+	})
+
+export type SetMyViewAction = z.infer<typeof SetMyViewAction>
+
+// Stack Action
+export const StackAction = z
+	.object({
+		_type: z.literal('stack'),
+		direction: z.enum(['vertical', 'horizontal']),
+		gap: z.number(),
+		intent: z.string(),
+		shapeIds: z.array(SimpleShapeIdSchema),
+	})
+	.meta({
+		title: 'Stack',
+		description:
+			"The AI stacks shapes horizontally or vertically. Note that this doesn't align shapes, it only stacks them along one axis.",
+		_systemPromptCategory: 'edit',
+	})
+
+export type StackAction = z.infer<typeof StackAction>
+
+// Think Action
+export const ThinkAction = z
+	.object({
+		_type: z.literal('think'),
+		text: z.string(),
+	})
+	.meta({ title: 'Think', description: 'The AI describes its intent or reasoning.' })
+
+export type ThinkAction = z.infer<typeof ThinkAction>
+
+// Todo List Action
+export const UpsertPersonalTodoItemAction = z
+	.object({
+		_type: z.literal('update-todo-list'),
+		id: TodoIdSchema,
+		status: z.enum(['todo', 'in-progress', 'done']),
+		text: z.string().optional(),
+	})
+	.meta({
+		title: 'Update Todo List',
+		description: 'The AI updates a current todo list item or creates a new one',
+	})
+
+export type UpsertPersonalTodoItemAction = z.infer<typeof UpsertPersonalTodoItemAction>
+
+// Update Action
+export const UpdateAction = z
+	.object({
+		_type: z.literal('update'),
+		intent: z.string(),
+		update: FocusedShapeSchema,
+	})
+	.meta({
+		title: 'Update',
+		description: 'The AI updates an existing shape.',
+		_systemPromptCategory: 'edit',
+	})
+
+export type UpdateAction = z.infer<typeof UpdateAction>
+// Unknown Action (catch-all for unrecognized actions)
+export const UnknownAction = z
+	.object({
+		_type: z.literal('unknown'),
+	})
+	.meta({
+		title: 'Unknown',
+		description: 'An action with an unknown or unrecognized type.',
+	})
+
+export type UnknownAction = z.infer<typeof UnknownAction>

templates/agent/shared/schema/PromptPartDefinitions.ts

@@ -0,0 +1,528 @@
+import { BoxModel, JsonValue } from 'tldraw'
+import { BlurryShape } from '../format/BlurryShape'
+import { FocusedShape } from '../format/FocusedShape'
+import { PeripheralShapeCluster } from '../format/PeripheralShapesCluster'
+import { AgentModelName } from '../models'
+import type { AgentAction } from '../types/AgentAction'
+import { AgentCanvasLint } from '../types/AgentCanvasLint'
+import { AgentMessage, AgentMessageContent } from '../types/AgentMessage'
+import { AgentRequest } from '../types/AgentRequest'
+import { ChatHistoryItem } from '../types/ChatHistoryItem'
+import { ContextItem } from '../types/ContextItem'
+import { SimpleShapeId } from '../types/ids-schema'
+import type { PromptPart, PromptPartDefinition } from '../types/PromptPart'
+import { TodoItem } from '../types/TodoItem'
+
+// ============================================================================
+// Prompt Part Type Interfaces
+// ============================================================================
+
+export interface BlurryShapesPart {
+	type: 'blurryShapes'
+	shapes: BlurryShape[]
+}
+
+export interface CanvasLintsPart {
+	type: 'canvasLints'
+	lints: AgentCanvasLint[]
+}
+
+export interface ChatHistoryPart {
+	type: 'chatHistory'
+	history: ChatHistoryItem[]
+}
+
+export interface ContextItemsPart {
+	type: 'contextItems'
+	items: ContextItem[]
+	requestSource: AgentRequest['source']
+}
+
+export interface DataPart {
+	type: 'data'
+	data: JsonValue[]
+}
+
+export interface MessagesPart {
+	type: 'messages'
+	agentMessages: string[]
+	requestSource: AgentRequest['source']
+}
+
+export interface ModelNamePart {
+	type: 'modelName'
+	modelName: AgentModelName
+}
+
+export interface PeripheralShapesPart {
+	type: 'peripheralShapes'
+	clusters: PeripheralShapeCluster[]
+}
+
+export interface ScreenshotPart {
+	type: 'screenshot'
+	screenshot: string
+}
+
+export interface SelectedShapesPart {
+	type: 'selectedShapes'
+	shapeIds: SimpleShapeId[]
+}
+
+export interface TimePart {
+	type: 'time'
+	time: string
+}
+
+export interface TodoListPart {
+	type: 'todoList'
+	items: TodoItem[]
+}
+
+export interface UserActionHistoryPart {
+	type: 'userActionHistory'
+	added: {
+		shapeId: string
+		type: FocusedShape['_type']
+	}[]
+	removed: {
+		shapeId: string
+		type: FocusedShape['_type']
+	}[]
+	updated: {
+		shapeId: string
+		type: FocusedShape['_type']
+		before: Partial<FocusedShape>
+		after: Partial<FocusedShape>
+	}[]
+}
+
+export interface UserViewportBoundsPart {
+	type: 'userViewportBounds'
+	userBounds: BoxModel | null
+}
+
+export interface AgentViewportBoundsPart {
+	type: 'agentViewportBounds'
+	agentBounds: BoxModel | null
+}
+
+export interface ModePart {
+	type: 'mode'
+	modeType: string
+	partTypes: PromptPart['type'][]
+	actionTypes: AgentAction['_type'][]
+}
+
+export interface DebugPart {
+	type: 'debug'
+	logSystemPrompt: boolean
+	logMessages: boolean
+}
+
+// ============================================================================
+// Prompt Part Definitions
+// ============================================================================
+
+// BlurryShapes
+export const BlurryShapesPartDefinition: PromptPartDefinition<BlurryShapesPart> = {
+	type: 'blurryShapes',
+	priority: -70,
+	buildContent: ({ shapes }) => {
+		if (shapes.length === 0) return ['There are no shapes in your view at the moment.']
+		return [`These are the shapes you can currently see:`, JSON.stringify(shapes)]
+	},
+}
+
+// CanvasLints
+export const CanvasLintsPartDefinition: PromptPartDefinition<CanvasLintsPart> = {
+	type: 'canvasLints',
+	priority: -50,
+	buildContent({ lints }: CanvasLintsPart) {
+		if (!lints || lints.length === 0) {
+			return []
+		}
+
+		const messages: string[] = []
+
+		// Group lints by type
+		const growYLints = lints.filter((l) => l.type === 'growY-on-shape')
+		const overlappingTextLints = lints.filter((l) => l.type === 'overlapping-text')
+		const friendlessArrowLints = lints.filter((l) => l.type === 'friendless-arrow')
+
+		messages.push(
+			"[LINTER]: The following potential visual problems have been detected in the canvas. You should decide if you want to address them. Defer to your view of the canvas to decide if you need to make changes; it's very possible that you don't need to make any changes."
+		)
+
+		if (growYLints.length > 0) {
+			const shapeIds = growYLints.flatMap((l) => l.shapeIds)
+			const lines = [
+				'Text overflow: These shapes have text that caused their containers to grow past the size that they were intended to be, potentially breaking out of their container. If you decide to fix: you need to set the height back to what you originally intended after increasing the width.',
+				...shapeIds.map((id) => `  - ${id}`),
+			]
+			messages.push(lines.join('\n'))
+		}
+
+		if (overlappingTextLints.length > 0) {
+			const lines = [
+				'Overlapping text: The shapes in each group have text and overlap each other, which may make text hard to read. If you decide to fix this, you may need to increase the size of any shapes containing the text.',
+				...overlappingTextLints.map((lint) => `  - ${lint.shapeIds.join(', ')}`),
+			]
+			messages.push(lines.join('\n'))
+		}
+
+		if (friendlessArrowLints.length > 0) {
+			const shapeIds = friendlessArrowLints.flatMap((l) => l.shapeIds)
+			const lines = [
+				"Unconnected arrows: These arrows aren't fully connected to other shapes.",
+				...shapeIds.map((id) => `  - ${id}`),
+			]
+			messages.push(lines.join('\n'))
+		}
+
+		return messages
+	},
+}
+
+// ChatHistory
+const CHAT_HISTORY_PRIORITY = -Infinity // history should appear first in the prompt (low priority)
+
+export const ChatHistoryPartDefinition: PromptPartDefinition<ChatHistoryPart> = {
+	type: 'chatHistory',
+	priority: CHAT_HISTORY_PRIORITY,
+	buildMessages: ({ history }) => {
+		if (history.length === 0) return []
+
+		const messages: AgentMessage[] = []
+
+		// If the last message is from the user, skip it
+		const lastIndex = history.length - 1
+		let end = history.length
+		if (end > 0 && history[lastIndex].type === 'prompt') {
+			end = lastIndex
+		}
+
+		for (let i = 0; i < end; i++) {
+			const item = history[i]
+			const message = buildHistoryItemMessage(item, CHAT_HISTORY_PRIORITY)
+			if (message) messages.push(message)
+		}
+
+		return messages
+	},
+}
+
+function buildHistoryItemMessage(item: ChatHistoryItem, priority: number): AgentMessage | null {
+	switch (item.type) {
+		case 'prompt': {
+			const content: AgentMessageContent[] = []
+
+			if (item.agentFacingMessage.trim() !== '') {
+				content.push({
+					type: 'text',
+					text: item.agentFacingMessage,
+				})
+			}
+
+			if (item.contextItems.length > 0) {
+				for (const contextItem of item.contextItems) {
+					switch (contextItem.type) {
+						case 'shape': {
+							const focusedShape = contextItem.shape
+							content.push({
+								type: 'text',
+								text: `[CONTEXT]: ${JSON.stringify(focusedShape)}`,
+							})
+							break
+						}
+						case 'shapes': {
+							const focusedShapes = contextItem.shapes
+							content.push({
+								type: 'text',
+								text: `[CONTEXT]: ${JSON.stringify(focusedShapes)}`,
+							})
+							break
+						}
+						default: {
+							content.push({
+								type: 'text',
+								text: `[CONTEXT]: ${JSON.stringify(contextItem)}`,
+							})
+							break
+						}
+					}
+				}
+			}
+
+			if (content.length === 0) {
+				return null
+			}
+
+			const role =
+				item.promptSource === 'user' || item.promptSource === 'other-agent' ? 'user' : 'assistant'
+			return {
+				role,
+				content,
+				priority,
+			}
+		}
+		case 'continuation': {
+			if (item.data.length === 0) {
+				return null
+			}
+			const text = `[DATA RETRIEVED]: ${JSON.stringify(item.data)}`
+			return {
+				role: 'assistant',
+				content: [{ type: 'text', text }],
+				priority,
+			}
+		}
+		case 'action': {
+			const { action } = item
+			let text: string
+			switch (action._type) {
+				case 'message': {
+					text = action.text || '<message data lost>'
+					break
+				}
+				case 'think': {
+					text = '[THOUGHT]: ' + (action.text || '<thought data lost>')
+					break
+				}
+				default: {
+					const { complete: _complete, time: _time, ...rawAction } = action || {}
+					text = '[ACTION]: ' + JSON.stringify(rawAction)
+					break
+				}
+			}
+			return {
+				role: 'assistant',
+				content: [{ type: 'text', text }],
+				priority,
+			}
+		}
+	}
+}
+
+// ContextItems
+export const ContextItemsPartDefinition: PromptPartDefinition<ContextItemsPart> = {
+	type: 'contextItems',
+	priority: -55, // context items in middle
+	buildContent: ({ items, requestSource }) => {
+		const messages: string[] = []
+
+		const shapeItems = items.filter((item) => item.type === 'shape')
+		const shapesItems = items.filter((item) => item.type === 'shapes')
+		const areaItems = items.filter((item) => item.type === 'area')
+		const pointItems = items.filter((item) => item.type === 'point')
+
+		// Handle area context items
+		if (areaItems.length > 0) {
+			const isSelf = requestSource === 'self'
+			const areas = areaItems.map((item) => item.bounds)
+			messages.push(
+				isSelf
+					? 'You have decided to focus your view on the following area. Make sure to focus your task here.'
+					: 'The user has specifically brought your attention to the following areas in this request. The user might refer to them as the "area(s)" or perhaps "here" or "there", but either way, it\'s implied that you should focus on these areas in both your reasoning and actions. Make sure to focus your task on these areas:'
+			)
+			for (const area of areas) {
+				messages.push(JSON.stringify(area))
+			}
+		}
+
+		// Handle point context items
+		if (pointItems.length > 0) {
+			const points = pointItems.map((item) => item.point)
+			messages.push(
+				'The user has specifically brought your attention to the following points in this request. The user might refer to them as the "point(s)" or perhaps "here" or "there", but either way, it\'s implied that you should focus on these points in both your reasoning and actions. Make sure to focus your task on these points:'
+			)
+			for (const point of points) {
+				messages.push(JSON.stringify(point))
+			}
+		}
+
+		// Handle individual shape context items
+		if (shapeItems.length > 0) {
+			const shapes = shapeItems.map((item) => item.shape)
+			messages.push(
+				`The user has specifically brought your attention to these ${shapes.length} shapes individually in this request. Make sure to focus your task on these shapes where applicable:`
+			)
+			for (const shape of shapes) {
+				messages.push(JSON.stringify(shape))
+			}
+		}
+
+		// Handle groups of shapes context items
+		for (const contextItem of shapesItems) {
+			const shapes = contextItem.shapes
+			if (shapes.length > 0) {
+				messages.push(
+					`The user has specifically brought your attention to the following group of ${shapes.length} shapes in this request. Make sure to focus your task on these shapes where applicable:`
+				)
+				messages.push(shapes.map((shape) => JSON.stringify(shape)).join('\n'))
+			}
+		}
+
+		return messages
+	},
+}
+
+// Data
+export const DataPartDefinition: PromptPartDefinition<DataPart> = {
+	type: 'data',
+	priority: 200, // API data should come right before the user message but after most other parts
+	buildContent: ({ data }) => {
+		if (data.length === 0) return []
+
+		const formattedData = data.map((item) => {
+			return `${JSON.stringify(item)}`
+		})
+
+		return ["Here's the data you requested:", ...formattedData]
+	},
+}
+
+// Messages
+export const MessagesPartDefinition: PromptPartDefinition<MessagesPart> = {
+	type: 'messages',
+	priority: Infinity, // user message should be last (highest priority)
+	buildContent: ({ agentMessages, requestSource }) => {
+		switch (requestSource) {
+			// we treat all sources the same for the messages part, but you don't have to!
+			case 'user':
+			case 'self':
+			case 'other-agent':
+				return agentMessages
+		}
+	},
+}
+
+// ModelName
+export const ModelNamePartDefinition: PromptPartDefinition<ModelNamePart> = {
+	type: 'modelName',
+	getModelName: (part) => {
+		return part.modelName
+	},
+}
+
+// PeripheralShapes
+export const PeripheralShapesPartDefinition: PromptPartDefinition<PeripheralShapesPart> = {
+	type: 'peripheralShapes',
+	priority: -65, // peripheral content after viewport shapes
+	buildContent: ({ clusters }) => {
+		if (clusters.length === 0) {
+			return []
+		}
+
+		return [
+			"There are some groups of shapes in your peripheral vision, outside the your main view. You can't make out their details or content. If you want to see their content, you need to get closer. The groups are as follows",
+			JSON.stringify(clusters),
+		]
+	},
+}
+
+// Screenshot
+export const ScreenshotPartDefinition: PromptPartDefinition<ScreenshotPart> = {
+	type: 'screenshot',
+	priority: -40, // screenshot after text content
+	buildContent: ({ screenshot }) => {
+		if (screenshot === '') return []
+
+		return [
+			'Here is the part of the canvas that you can currently see at this moment. It is not a reference image.',
+			screenshot,
+		]
+	},
+}
+
+export const SelectedShapesPartDefinition: PromptPartDefinition<SelectedShapesPart> = {
+	type: 'selectedShapes',
+	priority: -55,
+	buildContent({ shapeIds }: SelectedShapesPart) {
+		if (!shapeIds || shapeIds.length === 0) {
+			return []
+		}
+
+		if (shapeIds.length === 1) {
+			return [`The user has this shape selected: ${shapeIds[0]}`]
+		}
+
+		return [`The user has these shapes selected: ${shapeIds.join(', ')}`]
+	},
+}
+
+// Time
+export const TimePartDefinition: PromptPartDefinition<TimePart> = {
+	type: 'time',
+	priority: -100,
+	buildContent({ time }: TimePart) {
+		return [`The user's current time is: ${time}`]
+	},
+}
+
+// TodoList
+export const TodoListPartDefinition: PromptPartDefinition<TodoListPart> = {
+	type: 'todoList',
+	priority: 10,
+	buildContent: ({ items }) => {
+		if (items.length === 0) return ['You have no todos yet.']
+		return [`Here is your current todo list:`, JSON.stringify(items)]
+	},
+}
+
+// UserActionHistory
+export const UserActionHistoryPartDefinition: PromptPartDefinition<UserActionHistoryPart> = {
+	type: 'userActionHistory',
+	priority: -40,
+	buildContent: (part) => {
+		const { updated, removed, added } = part
+		if (updated.length === 0 && removed.length === 0 && added.length === 0) {
+			return []
+		}
+
+		return [
+			'Since the previous request, the user has made the following changes to the canvas:',
+			JSON.stringify(part),
+		]
+	},
+}
+
+// UserViewportBounds
+export const UserViewportBoundsPartDefinition: PromptPartDefinition<UserViewportBoundsPart> = {
+	type: 'userViewportBounds',
+	priority: -80,
+	buildContent({ userBounds }: UserViewportBoundsPart) {
+		if (!userBounds) {
+			return []
+		}
+		return [
+			`The bounds of the part of the canvas that the user can currently see are: ${JSON.stringify(userBounds)}`,
+		]
+	},
+}
+
+// AgentViewportBounds
+export const AgentViewportBoundsPartDefinition: PromptPartDefinition<AgentViewportBoundsPart> = {
+	type: 'agentViewportBounds',
+	priority: -80,
+	buildContent({ agentBounds }: AgentViewportBoundsPart) {
+		if (!agentBounds) {
+			return []
+		}
+		return [
+			`The bounds of the part of the canvas that you can currently see are: ${JSON.stringify(agentBounds)}`,
+		]
+	},
+}
+
+// Mode - sends mode metadata to worker for prompt construction
+export const ModePartDefinition: PromptPartDefinition<ModePart> = {
+	type: 'mode',
+	// No buildContent - this is metadata for the worker, not prompt content for the model
+}
+
+// Debug - sends debug flags to worker for logging
+export const DebugPartDefinition: PromptPartDefinition<DebugPart> = {
+	type: 'debug',
+	// No buildContent - this is metadata for the worker, not prompt content for the model
+}

templates/agent/shared/schema/buildResponseSchema.ts

@@ -0,0 +1,45 @@
+import z from 'zod'
+import { ActionMeta, AgentAction, getActionSchemaForMode } from '../types/AgentAction'
+
+/**
+ * Internal meta keys that should be stripped from the JSON schema.
+ * These are added via .meta() in action schemas but shouldn't be sent to the model.
+ */
+const INTERNAL_META_KEYS: Set<string> = new Set([
+	'_systemPromptCategory',
+] satisfies (keyof ActionMeta)[])
+
+/**
+ * Recursively strips internal meta fields from a JSON schema.
+ * Only removes explicitly listed internal keys, preserving all other fields.
+ */
+export function stripInternalMeta(obj: object): object {
+	if (Array.isArray(obj)) {
+		return obj.map(stripInternalMeta)
+	}
+	if (obj && typeof obj === 'object') {
+		return Object.fromEntries(
+			Object.entries(obj)
+				.filter(([key]) => !INTERNAL_META_KEYS.has(key))
+				.map(([key, value]) => [key, stripInternalMeta(value)])
+		)
+	}
+	return obj
+}
+
+export function buildResponseSchema(actionTypes: AgentAction['_type'][], mode: string) {
+	const actionSchemas = actionTypes
+		.map((type) => getActionSchemaForMode(type, mode))
+		.filter((schema) => schema !== undefined)
+
+	if (actionSchemas.length === 0) {
+		throw new Error('No action schemas found for the provided action types')
+	}
+
+	const actionSchema = z.union(actionSchemas)
+	const schema = z.object({
+		actions: z.array(actionSchema),
+	})
+
+	return stripInternalMeta(z.toJSONSchema(schema, { reused: 'ref' }))
+}