snapshot of https://github.com/tldraw/tldraw.git (main, 2026-09-23; blobs > 131072B omitted)

templates/agent/shared/format/BlurryShape.ts

@@ -0,0 +1,11 @@
+import { FocusedShape } from './FocusedShape'
+
+export interface BlurryShape {
+	shapeId: string
+	text?: string
+	type: FocusedShape['_type']
+	x: number
+	y: number
+	w: number
+	h: number
+}

templates/agent/shared/format/FocusedColor.ts

@@ -0,0 +1,36 @@
+import z from 'zod'
+
+export const FocusedColor = z.enum([
+	'red',
+	'light-red',
+	'green',
+	'light-green',
+	'blue',
+	'light-blue',
+	'orange',
+	'yellow',
+	'black',
+	'violet',
+	'light-violet',
+	'grey',
+	'white',
+])
+
+export type IFocusedColor = z.infer<typeof FocusedColor>
+
+export function asColor(color: string): IFocusedColor {
+	if (FocusedColor.safeParse(color).success) {
+		return color as IFocusedColor
+	}
+
+	switch (color) {
+		case 'pink': {
+			return 'light-violet'
+		}
+		case 'light-pink': {
+			return 'light-violet'
+		}
+	}
+
+	return 'black'
+}

templates/agent/shared/format/FocusedFill.ts

@@ -0,0 +1,31 @@
+import { TLDefaultFillStyle } from 'tldraw'
+import z from 'zod'
+
+export const FocusedFillSchema = z.enum(['none', 'tint', 'background', 'solid', 'pattern'])
+
+export type FocusedFill = z.infer<typeof FocusedFillSchema>
+
+const FOCUSED_TO_SHAPE_FILLS: Record<FocusedFill, TLDefaultFillStyle> = {
+	none: 'none',
+	solid: 'lined-fill',
+	background: 'semi',
+	tint: 'solid',
+	pattern: 'pattern',
+}
+
+const SHAPE_TO_FOCUSED_FILLS: Record<TLDefaultFillStyle, FocusedFill> = {
+	none: 'none',
+	fill: 'solid',
+	'lined-fill': 'solid',
+	semi: 'background',
+	solid: 'tint',
+	pattern: 'pattern',
+}
+
+export function convertFocusedFillToTldrawFill(fill: FocusedFill): TLDefaultFillStyle {
+	return FOCUSED_TO_SHAPE_FILLS[fill]
+}
+
+export function convertTldrawFillToFocusedFill(fill: TLDefaultFillStyle): FocusedFill {
+	return SHAPE_TO_FOCUSED_FILLS[fill]
+}

templates/agent/shared/format/FocusedFontSize.ts

@@ -0,0 +1,86 @@
+import {
+	createShapeId,
+	Editor,
+	getDisplayValues,
+	IndexKey,
+	TLDefaultSizeStyle,
+	TLTextShape,
+	TextShapeUtil,
+	TextShapeUtilDisplayValues,
+} from 'tldraw'
+import { z } from 'zod'
+
+const TEXT_SIZE_STYLES = ['s', 'm', 'l', 'xl'] as const satisfies readonly TLDefaultSizeStyle[]
+
+export const FocusedFontSize = z.number()
+
+/**
+ * Calculates the closest predefined font size and scale combination to achieve a target font size
+ * @param editor - The tldraw editor instance
+ * @param targetFontSize - The desired font size in pixels
+ * @param textProps - The text shape props to use when resolving display values
+ * @returns An object containing the closest predefined font size key and the scale factor
+ */
+export function convertFocusedFontSizeToTldrawFontSizeAndScale(
+	editor: Editor,
+	targetFontSize: number,
+	textProps?: Partial<TLTextShape['props']>
+) {
+	const fontSizeEntries = TEXT_SIZE_STYLES.map(
+		(size) => [size, getTextShapeDisplayFontSize(editor, { ...textProps, size })] as const
+	)
+	let closestSize = fontSizeEntries[0]
+	let closestPixelSize = closestSize[1]
+	let minDifference = Math.abs(targetFontSize - closestPixelSize)
+
+	for (const [size, pixelSize] of fontSizeEntries) {
+		const difference = Math.abs(targetFontSize - pixelSize)
+		if (difference < minDifference) {
+			minDifference = difference
+			closestSize = [size, pixelSize]
+			closestPixelSize = pixelSize
+		}
+	}
+
+	const textSize = closestSize[0] as TLDefaultSizeStyle
+	const scale = targetFontSize / closestPixelSize
+
+	return { textSize, scale }
+}
+
+/**
+ * Converts a tldraw font size and scale to a focused font size
+ * @param editor - The tldraw editor instance
+ * @param shape - The text shape to convert
+ * @returns The focused font size
+ */
+export function convertTldrawFontSizeAndScaleToFocusedFontSize(editor: Editor, shape: TLTextShape) {
+	const util = editor.getShapeUtil<TextShapeUtil>('text')
+	const displayValues = getDisplayValues<TLTextShape, TextShapeUtilDisplayValues>(util, shape)
+	return Math.round(displayValues.fontSize * shape.props.scale)
+}
+
+function getTextShapeDisplayFontSize(
+	editor: Editor,
+	textProps?: Partial<TLTextShape['props']>
+): number {
+	const util = editor.getShapeUtil<TextShapeUtil>('text')
+	const shape: TLTextShape = {
+		id: createShapeId('agent-font-size'),
+		typeName: 'shape',
+		type: 'text',
+		x: 0,
+		y: 0,
+		rotation: 0,
+		index: 'a1' as IndexKey,
+		parentId: editor.getCurrentPageId(),
+		isLocked: false,
+		opacity: 1,
+		props: {
+			...util.getDefaultProps(),
+			...textProps,
+		},
+		meta: {},
+	}
+	return getDisplayValues<TLTextShape, TextShapeUtilDisplayValues>(util, shape).fontSize
+}

templates/agent/shared/format/FocusedGeoShapeType.ts

@@ -0,0 +1,26 @@
+import { z } from 'zod'
+
+export const FocusedGeoShapeTypeSchema = z.enum([
+	'rectangle',
+	'ellipse',
+	'triangle',
+	'diamond',
+	'hexagon',
+	'pill',
+	'cloud',
+	'x-box',
+	'check-box',
+	'heart',
+	'pentagon',
+	'octagon',
+	'star',
+	'parallelogram-right',
+	'parallelogram-left',
+	'trapezoid',
+	'fat-arrow-right',
+	'fat-arrow-left',
+	'fat-arrow-up',
+	'fat-arrow-down',
+])
+
+export type FocusedGeoShapeType = z.infer<typeof FocusedGeoShapeTypeSchema>

templates/agent/shared/format/FocusedShape.ts

@@ -0,0 +1,179 @@
+import { z } from 'zod'
+import { SimpleShapeIdSchema } from '../types/ids-schema'
+import { FocusedColor } from './FocusedColor'
+import { FocusedFillSchema } from './FocusedFill'
+import { FocusedFontSize } from './FocusedFontSize'
+import { FocusedGeoShapeTypeSchema } from './FocusedGeoShapeType'
+
+const FocusedLabel = z.string()
+
+export const FocusedTextAnchorSchema = z.enum([
+	'bottom-center',
+	'bottom-left',
+	'bottom-right',
+	'center-left',
+	'center-right',
+	'center',
+	'top-center',
+	'top-left',
+	'top-right',
+])
+
+export type FocusedTextAnchor = z.infer<typeof FocusedTextAnchorSchema>
+
+export const FocusedGeoShape = z.object({
+	_type: FocusedGeoShapeTypeSchema,
+	color: FocusedColor,
+	fill: FocusedFillSchema,
+	h: z.number(),
+	note: z.string(),
+	shapeId: SimpleShapeIdSchema,
+	text: FocusedLabel.optional(),
+	textAlign: z.enum(['start', 'middle', 'end']).optional(),
+	w: z.number(),
+	x: z.number(),
+	y: z.number(),
+})
+
+export type FocusedGeoShape = z.infer<typeof FocusedGeoShape>
+
+export const FocusedGeoShapePartialSchema = FocusedGeoShape.partial()
+export type FocusedGeoShapePartial = z.infer<typeof FocusedGeoShapePartialSchema>
+
+const FocusedLineShape = z.object({
+	_type: z.literal('line'),
+	color: FocusedColor,
+	note: z.string(),
+	shapeId: SimpleShapeIdSchema,
+	x1: z.number(),
+	x2: z.number(),
+	y1: z.number(),
+	y2: z.number(),
+})
+
+export type FocusedLineShape = z.infer<typeof FocusedLineShape>
+
+const FocusedNoteShape = z.object({
+	_type: z.literal('note'),
+	color: FocusedColor,
+	note: z.string(),
+	shapeId: SimpleShapeIdSchema,
+	text: FocusedLabel.optional(),
+	x: z.number(),
+	y: z.number(),
+})
+
+export type FocusedNoteShape = z.infer<typeof FocusedNoteShape>
+
+const FocusedTextShape = z
+	.object({
+		_type: z.literal('text'),
+		anchor: FocusedTextAnchorSchema,
+		color: FocusedColor,
+		fontSize: FocusedFontSize.optional(),
+		maxWidth: z.number().nullable(),
+		note: z.string(),
+		shapeId: SimpleShapeIdSchema,
+		text: FocusedLabel,
+		x: z.number(),
+		y: z.number(),
+	})
+	.meta({
+		title: 'Text Shape',
+		description:
+			'A text shape is a shape that contains text. The `anchor` property indicates how the text shape is positioned and aligned. For example, the "top-left" anchor means the text shape\'s x and y coordinates are the top left corner of the text shape, and the text gets left aligned. A shape with the "bottom-center" anchor means the text shape\'s x and y coordinates are the bottom center of the text shape, and the text gets center aligned on the horizontal axis. By default, text shapes auto-size to fit their content. If you provide a `maxWidth`, the text will automatically wrap to the next line if it exceeds that width (there is no need to add manual line breaks for word wrapping).',
+	})
+
+export type FocusedTextShape = z.infer<typeof FocusedTextShape>
+
+export const FocusedTextShapePartialSchema = FocusedTextShape.partial()
+export type FocusedTextShapePartial = z.infer<typeof FocusedTextShapePartialSchema>
+
+const FocusedArrowShape = z.object({
+	_type: z.literal('arrow'),
+	color: FocusedColor,
+	fromId: SimpleShapeIdSchema.nullable(),
+	note: z.string(),
+	shapeId: SimpleShapeIdSchema,
+	text: z.string().optional(),
+	toId: SimpleShapeIdSchema.nullable(),
+	x1: z.number(),
+	x2: z.number(),
+	y1: z.number(),
+	y2: z.number(),
+	bend: z.number().optional(),
+})
+
+export type FocusedArrowShape = z.infer<typeof FocusedArrowShape>
+
+const FocusedDrawShape = z
+	.object({
+		_type: z.literal('draw'),
+		color: FocusedColor,
+		fill: FocusedFillSchema.optional(),
+		note: z.string(),
+		shapeId: SimpleShapeIdSchema,
+	})
+	.meta({
+		title: 'Draw Shape',
+		description:
+			'A draw shape is a freeform shape that was drawn by the pen tool. IMPORTANT: Do not create draw shapes with the "create" action. To create new draw shapes, the AI must use the pen event because it gives more control.',
+	})
+
+export type FocusedDrawShape = z.infer<typeof FocusedDrawShape>
+
+const FocusedUnknownShape = z
+	.object({
+		_type: z.literal('unknown'),
+		note: z.string(),
+		shapeId: SimpleShapeIdSchema,
+		subType: z.string(),
+		x: z.number(),
+		y: z.number(),
+	})
+	.meta({
+		title: 'Unknown Shape',
+		description:
+			"A special shape that is not represented by one of the canvas's core shape types. The AI cannot create these shapes, but it *can* interact with them. eg: The AI can move these shapes. The `subType` property contains the internal name of the shape's type.",
+	})
+
+export type FocusedUnknownShape = z.infer<typeof FocusedUnknownShape>
+
+const FOCUSED_SHAPES = [
+	FocusedDrawShape,
+	FocusedGeoShape,
+	FocusedLineShape,
+	FocusedTextShape,
+	FocusedArrowShape,
+	FocusedNoteShape,
+	FocusedUnknownShape,
+] as const
+export const FocusedShapeSchema = z.union(FOCUSED_SHAPES)
+
+export type FocusedShape = z.infer<typeof FocusedShapeSchema>
+
+export type FocusedShapePartial = Partial<FocusedShape>
+
+/**
+ * Extract all shape type names from the schema
+ */
+export function getFocusedShapeSchemaNames() {
+	const typeNames: FocusedShape['_type'][] = []
+
+	for (const shapeSchema of FOCUSED_SHAPES) {
+		const typeField = shapeSchema.shape._type
+
+		if (typeField) {
+			// Handle ZodLiterals (like FocusedDrawShape)
+			if ('value' in typeField && typeof typeField.value === 'string') {
+				typeNames.push(typeField.value)
+			}
+			// Handle ZodEnums (like FocusedGeoShape)
+			else if ('options' in typeField && Array.isArray(typeField.options)) {
+				typeNames.push(...typeField.options)
+			}
+		}
+	}
+
+	return typeNames
+}

templates/agent/shared/format/PeripheralShapesCluster.ts

@@ -0,0 +1,6 @@
+import { BoxModel } from 'tldraw'
+
+export interface PeripheralShapeCluster {
+	bounds: BoxModel
+	numberOfShapes: number
+}

templates/agent/shared/format/convertFocusedShapeToTldrawShape.ts

@@ -0,0 +1,817 @@
+import {
+	Box,
+	createShapeId,
+	Editor,
+	IndexKey,
+	reverseRecordsDiff,
+	TLArrowShape,
+	TLBindingCreate,
+	TLDefaultShape,
+	TLDefaultSizeStyle,
+	TLDrawShape,
+	TLGeoShape,
+	TLGeoShapeGeoStyle,
+	TLLineShape,
+	TLNoteShape,
+	TLShape,
+	TLShapeId,
+	TLTextShape,
+	toRichText,
+	Vec,
+	VecLike,
+} from 'tldraw'
+import { asColor } from './FocusedColor'
+import { convertFocusedFillToTldrawFill } from './FocusedFill'
+import { convertFocusedFontSizeToTldrawFontSizeAndScale } from './FocusedFontSize'
+import { FocusedGeoShapeType } from './FocusedGeoShapeType'
+import {
+	FocusedArrowShape,
+	FocusedDrawShape,
+	FocusedGeoShape,
+	FocusedGeoShapePartial,
+	FocusedLineShape,
+	FocusedNoteShape,
+	FocusedShape,
+	FocusedShapePartial,
+	FocusedTextShape,
+	FocusedTextShapePartial,
+	FocusedUnknownShape,
+} from './FocusedShape'
+
+/**
+ * Convert a FocusedShape to a shape object to a tldraw shape using defaultShape for fallback values
+ * @param editor - The tldraw editor instance
+ * @param focusedShape - The focused shape to convert
+ * @param defaultShape - The default shape to use for fallback values
+ * @returns The converted shape and bindings
+ */
+export function convertFocusedShapeToTldrawShape(
+	editor: Editor,
+	focusedShape: FocusedShape,
+	{ defaultShape }: { defaultShape: Partial<TLShape> }
+): { shape: TLShape; bindings?: TLBindingCreate[] } {
+	switch (focusedShape._type) {
+		case 'text': {
+			return convertTextShapeToTldrawShape(editor, focusedShape, { defaultShape })
+		}
+		case 'line': {
+			return convertLineShapeToTldrawShape(editor, focusedShape, { defaultShape })
+		}
+		case 'arrow': {
+			return convertArrowShapeToTldrawShape(editor, focusedShape, { defaultShape })
+		}
+		case 'cloud':
+		case 'rectangle':
+		case 'triangle':
+		case 'diamond':
+		case 'hexagon':
+		case 'pill':
+		case 'x-box':
+		case 'pentagon':
+		case 'octagon':
+		case 'star':
+		case 'parallelogram-right':
+		case 'parallelogram-left':
+		case 'trapezoid':
+		case 'fat-arrow-right':
+		case 'fat-arrow-left':
+		case 'fat-arrow-up':
+		case 'fat-arrow-down':
+		case 'check-box':
+		case 'heart':
+		case 'ellipse': {
+			return convertGeoShapeToTldrawShape(editor, focusedShape, { defaultShape })
+		}
+		case 'note': {
+			return convertNoteShapeToTldrawShape(editor, focusedShape, { defaultShape })
+		}
+		case 'draw': {
+			return convertDrawShapeToTldrawShape(editor, focusedShape, { defaultShape })
+		}
+		case 'unknown': {
+			return convertUnknownShapeToTldrawShape(editor, focusedShape, { defaultShape })
+		}
+	}
+}
+
+export function convertSimpleIdToTldrawId(id: string): TLShapeId {
+	return ('shape:' + id) as TLShapeId
+}
+
+export function convertFocusedTypeToTldrawType(
+	type: FocusedShape['_type']
+): TLGeoShapeGeoStyle | TLDefaultShape['type'] | 'unknown' {
+	if (type in FOCUSED_TO_GEO_TYPES) {
+		return convertFocusedGeoTypeToTldrawGeoGeoType(
+			type as FocusedGeoShapeType
+		) as TLGeoShapeGeoStyle
+	}
+	return type as TLDefaultShape['type'] | 'unknown'
+}
+
+export function convertFocusedGeoTypeToTldrawGeoGeoType(
+	type: FocusedGeoShapeType
+): TLGeoShapeGeoStyle {
+	return FOCUSED_TO_GEO_TYPES[type]
+}
+
+export const FOCUSED_TO_GEO_TYPES: Record<FocusedGeoShapeType, TLGeoShapeGeoStyle> = {
+	rectangle: 'rectangle',
+	ellipse: 'ellipse',
+	triangle: 'triangle',
+	diamond: 'diamond',
+	hexagon: 'hexagon',
+	pill: 'oval',
+	cloud: 'cloud',
+	'x-box': 'x-box',
+	'check-box': 'check-box',
+	heart: 'heart',
+	pentagon: 'pentagon',
+	octagon: 'octagon',
+	star: 'star',
+	'parallelogram-right': 'rhombus',
+	'parallelogram-left': 'rhombus-2',
+	trapezoid: 'trapezoid',
+	'fat-arrow-right': 'arrow-right',
+	'fat-arrow-left': 'arrow-left',
+	'fat-arrow-up': 'arrow-up',
+	'fat-arrow-down': 'arrow-down',
+} as const
+
+function convertTextShapeToTldrawShape(
+	editor: Editor,
+	focusedShape: FocusedTextShape,
+	{ defaultShape }: { defaultShape: Partial<TLShape> }
+): { shape: TLTextShape } {
+	const shapeId = convertSimpleIdToTldrawId(focusedShape.shapeId)
+	const defaultTextShape = defaultShape as TLTextShape
+
+	// Determine the base font size and scale - focusedShape takes priority
+	let textSize: TLDefaultSizeStyle = 's'
+	let scale = 1
+	const font = defaultTextShape.props?.font ?? 'draw'
+
+	if (focusedShape.fontSize) {
+		const { textSize: calculatedTextSize, scale: calculatedScale } =
+			convertFocusedFontSizeToTldrawFontSizeAndScale(editor, focusedShape.fontSize, {
+				...defaultTextShape.props,
+				font,
+			})
+		textSize = calculatedTextSize
+		scale = calculatedScale
+	} else if (defaultTextShape.props?.size) {
+		textSize = defaultTextShape.props.size
+		scale = defaultTextShape.props.scale ?? 1
+	}
+
+	// If maxWidth is provided as a number, enable wrapping (autoSize = false)
+	// Otherwise (undefined or null), preserve existing autoSize behavior
+	// Check for undefined first to distinguish "not provided" from "explicitly null"
+	const autoSize =
+		focusedShape.maxWidth !== undefined && focusedShape.maxWidth !== null
+			? false
+			: (defaultTextShape.props?.autoSize ?? true)
+
+	let richText
+	if (focusedShape.text !== undefined) {
+		richText = toRichText(focusedShape.text)
+	} else if (defaultTextShape.props?.richText) {
+		richText = defaultTextShape.props.richText
+	} else {
+		richText = toRichText('')
+	}
+
+	let textAlign: TLTextShape['props']['textAlign'] = defaultTextShape.props?.textAlign ?? 'start'
+	switch (focusedShape.anchor) {
+		case 'top-left':
+		case 'bottom-left':
+		case 'center-left':
+			textAlign = 'start'
+			break
+		case 'top-center':
+		case 'bottom-center':
+		case 'center':
+			textAlign = 'middle'
+			break
+		case 'top-right':
+		case 'bottom-right':
+		case 'center-right':
+			textAlign = 'end'
+			break
+	}
+
+	const unpositionedShape: TLTextShape = {
+		id: shapeId,
+		type: 'text',
+		typeName: 'shape',
+		x: 0,
+		y: 0,
+		rotation: defaultTextShape.rotation ?? 0,
+		index: defaultTextShape.index ?? editor.getHighestIndexForParent(editor.getCurrentPageId()),
+		parentId: defaultTextShape.parentId ?? editor.getCurrentPageId(),
+		isLocked: defaultTextShape.isLocked ?? false,
+		opacity: defaultTextShape.opacity ?? 1,
+		props: {
+			size: textSize,
+			scale,
+			richText,
+			color: asColor(focusedShape.color ?? defaultTextShape.props?.color ?? 'black'),
+			textAlign,
+			autoSize,
+			w:
+				focusedShape.maxWidth !== undefined && focusedShape.maxWidth !== null
+					? focusedShape.maxWidth
+					: (defaultTextShape.props?.w ?? 100),
+			font,
+		},
+		meta: {
+			note: focusedShape.note ?? defaultTextShape.meta?.note ?? '',
+		},
+	}
+
+	const unpositionedBounds = getDummyBounds(editor, unpositionedShape)
+
+	const position = new Vec(defaultTextShape.x ?? 0, defaultTextShape.y ?? 0)
+	const x = focusedShape.x ?? defaultTextShape.x ?? 0
+	const y = focusedShape.y ?? defaultTextShape.y ?? 0
+	switch (focusedShape.anchor) {
+		case 'top-left': {
+			position.x = x
+			position.y = y
+			break
+		}
+		case 'top-center': {
+			position.x = x - unpositionedBounds.w / 2
+			position.y = y
+			break
+		}
+		case 'top-right': {
+			position.x = x - unpositionedBounds.w
+			position.y = y
+			break
+		}
+		case 'bottom-left': {
+			position.x = x
+			position.y = y - unpositionedBounds.h
+			break
+		}
+		case 'bottom-center': {
+			position.x = x - unpositionedBounds.w / 2
+			position.y = y - unpositionedBounds.h
+			break
+		}
+		case 'bottom-right': {
+			position.x = x - unpositionedBounds.w
+			position.y = y - unpositionedBounds.h
+			break
+		}
+		case 'center-left': {
+			position.x = x
+			position.y = y - unpositionedBounds.h / 2
+			break
+		}
+		case 'center-right': {
+			position.x = x - unpositionedBounds.w
+			position.y = y - unpositionedBounds.h / 2
+			break
+		}
+		case 'center': {
+			position.x = focusedShape.x - unpositionedBounds.w / 2
+			position.y = focusedShape.y - unpositionedBounds.h / 2
+			break
+		}
+	}
+	const positionedShape: TLTextShape = {
+		...unpositionedShape,
+		x: position.x,
+		y: position.y,
+	}
+
+	return {
+		shape: positionedShape,
+	}
+}
+
+function convertLineShapeToTldrawShape(
+	editor: Editor,
+	focusedShape: FocusedLineShape,
+	{ defaultShape }: { defaultShape: Partial<TLShape> }
+): { shape: TLShape } {
+	const shapeId = convertSimpleIdToTldrawId(focusedShape.shapeId)
+	const defaultLineShape = defaultShape as TLLineShape
+
+	const x1 = focusedShape.x1 ?? 0
+	const y1 = focusedShape.y1 ?? 0
+	const x2 = focusedShape.x2 ?? 0
+	const y2 = focusedShape.y2 ?? 0
+	const minX = Math.min(x1, x2)
+	const minY = Math.min(y1, y2)
+
+	return {
+		shape: {
+			id: shapeId,
+			type: 'line',
+			typeName: 'shape',
+			x: minX,
+			y: minY,
+			rotation: defaultLineShape.rotation ?? 0,
+			index: defaultLineShape.index ?? editor.getHighestIndexForParent(editor.getCurrentPageId()),
+			parentId: defaultLineShape.parentId ?? editor.getCurrentPageId(),
+			isLocked: defaultLineShape.isLocked ?? false,
+			opacity: defaultLineShape.opacity ?? 1,
+			props: {
+				size: defaultLineShape.props?.size ?? 's',
+				points: {
+					a1: {
+						id: 'a1',
+						index: 'a1' as IndexKey,
+						x: x1 - minX,
+						y: y1 - minY,
+					},
+					a2: {
+						id: 'a2',
+						index: 'a2' as IndexKey,
+						x: x2 - minX,
+						y: y2 - minY,
+					},
+				},
+				color: asColor(focusedShape.color ?? defaultLineShape.props?.color ?? 'black'),
+				dash: defaultLineShape.props?.dash ?? 'draw',
+				scale: defaultLineShape.props?.scale ?? 1,
+				spline: defaultLineShape.props?.spline ?? 'line',
+			},
+			meta: {
+				note: focusedShape.note ?? defaultLineShape.meta?.note ?? '',
+			},
+		},
+	}
+}
+
+function convertArrowShapeToTldrawShape(
+	editor: Editor,
+	focusedShape: FocusedArrowShape,
+	{ defaultShape }: { defaultShape: Partial<TLShape> }
+): { shape: TLShape; bindings?: TLBindingCreate[] } {
+	const shapeId = convertSimpleIdToTldrawId(focusedShape.shapeId)
+	const defaultArrowShape = defaultShape as TLArrowShape
+
+	const x1 = focusedShape.x1 ?? defaultArrowShape.props?.start?.x ?? 0
+	const y1 = focusedShape.y1 ?? defaultArrowShape.props?.start?.y ?? 0
+	const x2 = focusedShape.x2 ?? defaultArrowShape.props?.end?.x ?? 0
+	const y2 = focusedShape.y2 ?? defaultArrowShape.props?.end?.y ?? 0
+	const minX = Math.min(x1, x2)
+	const minY = Math.min(y1, y2)
+
+	// Handle richText properly - focusedShape takes priority
+	let richText
+	if (focusedShape.text !== undefined) {
+		richText = toRichText(focusedShape.text)
+	} else if (defaultArrowShape.props?.richText) {
+		richText = defaultArrowShape.props.richText
+	} else {
+		richText = toRichText('')
+	}
+
+	const shape = {
+		id: shapeId,
+		type: 'arrow' as const,
+		typeName: 'shape' as const,
+		x: minX,
+		y: minY,
+		rotation: defaultArrowShape.rotation ?? 0,
+		index: defaultArrowShape.index ?? editor.getHighestIndexForParent(editor.getCurrentPageId()),
+		parentId: defaultArrowShape.parentId ?? editor.getCurrentPageId(),
+		isLocked: defaultArrowShape.isLocked ?? false,
+		opacity: defaultArrowShape.opacity ?? 1,
+		props: {
+			arrowheadEnd: defaultArrowShape.props?.arrowheadEnd ?? 'arrow',
+			arrowheadStart: defaultArrowShape.props?.arrowheadStart ?? 'none',
+			bend: (focusedShape.bend ?? (defaultArrowShape.props?.bend ?? 0) * -1) * -1,
+			color: asColor(focusedShape.color ?? defaultArrowShape.props?.color ?? 'black'),
+			dash: defaultArrowShape.props?.dash ?? 'draw',
+			elbowMidPoint: defaultArrowShape.props?.elbowMidPoint ?? 0.5,
+			end: { x: x2 - minX, y: y2 - minY },
+			fill: defaultArrowShape.props?.fill ?? 'none',
+			font: defaultArrowShape.props?.font ?? 'draw',
+			kind: defaultArrowShape.props?.kind ?? 'arc',
+			labelColor: defaultArrowShape.props?.labelColor ?? 'black',
+			labelPosition: defaultArrowShape.props?.labelPosition ?? 0.5,
+			richText,
+			scale: defaultArrowShape.props?.scale ?? 1,
+			size: defaultArrowShape.props?.size ?? 's',
+			start: { x: x1 - minX, y: y1 - minY },
+		},
+		meta: {
+			note: focusedShape.note ?? defaultArrowShape.meta?.note ?? '',
+		},
+	}
+
+	// Handle arrow bindings if fromId or toId are provided
+	const bindings: TLBindingCreate[] = []
+
+	if (focusedShape.fromId) {
+		const fromId = convertSimpleIdToTldrawId(focusedShape.fromId)
+		const startShape = editor.getShape(fromId)
+		if (startShape) {
+			const targetPoint = { x: x1, y: y1 }
+			const finalNormalizedAnchor = calculateArrowBindingAnchor(editor, startShape, targetPoint)
+			bindings.push({
+				type: 'arrow',
+				typeName: 'binding',
+				fromId: shapeId,
+				toId: startShape.id,
+				props: {
+					normalizedAnchor: finalNormalizedAnchor,
+					isExact: false,
+					isPrecise: true,
+					terminal: 'start',
+				},
+				meta: {},
+			})
+		}
+	}
+
+	if (focusedShape.toId) {
+		const toId = convertSimpleIdToTldrawId(focusedShape.toId)
+		const endShape = editor.getShape(toId)
+		if (endShape) {
+			const targetPoint = { x: x2, y: y2 }
+			const finalNormalizedAnchor = calculateArrowBindingAnchor(editor, endShape, targetPoint)
+			bindings.push({
+				type: 'arrow',
+				typeName: 'binding',
+				fromId: shapeId,
+				toId: endShape.id,
+				props: {
+					normalizedAnchor: finalNormalizedAnchor,
+					isExact: false,
+					isPrecise: true,
+					terminal: 'end',
+				},
+				meta: {},
+			})
+		}
+	}
+
+	return {
+		shape,
+		bindings: bindings.length > 0 ? bindings : undefined,
+	}
+}
+
+function convertGeoShapeToTldrawShape(
+	editor: Editor,
+	focusedShape: FocusedGeoShape,
+	{ defaultShape }: { defaultShape: Partial<TLShape> }
+): { shape: TLShape } {
+	const shapeId = convertSimpleIdToTldrawId(focusedShape.shapeId)
+	const shapeType = convertFocusedGeoTypeToTldrawGeoGeoType(focusedShape._type)
+	const defaultGeoShape = defaultShape as TLGeoShape
+
+	// Handle richText properly - focusedShape takes priority
+	let richText
+	if (focusedShape.text !== undefined) {
+		richText = toRichText(focusedShape.text)
+	} else if (defaultGeoShape.props?.richText) {
+		richText = defaultGeoShape.props.richText
+	} else {
+		richText = toRichText('')
+	}
+
+	// Handle fill properly - focusedShape takes priority
+	let fill
+	if (focusedShape.fill !== undefined) {
+		fill = convertFocusedFillToTldrawFill(focusedShape.fill) ?? 'none'
+	} else if (defaultGeoShape.props?.fill) {
+		fill = defaultGeoShape.props.fill
+	} else {
+		fill = convertFocusedFillToTldrawFill('none')
+	}
+
+	return {
+		shape: {
+			id: shapeId,
+			type: 'geo',
+			typeName: 'shape',
+			x: focusedShape.x ?? defaultGeoShape.x ?? 0,
+			y: focusedShape.y ?? defaultGeoShape.y ?? 0,
+			rotation: defaultGeoShape.rotation ?? 0,
+			index: defaultGeoShape.index ?? editor.getHighestIndexForParent(editor.getCurrentPageId()),
+			parentId: defaultGeoShape.parentId ?? editor.getCurrentPageId(),
+			isLocked: defaultGeoShape.isLocked ?? false,
+			opacity: defaultGeoShape.opacity ?? 1,
+			props: {
+				align: focusedShape.textAlign ?? defaultGeoShape.props?.align ?? 'middle',
+				color: asColor(focusedShape.color ?? defaultGeoShape.props?.color ?? 'black'),
+				dash: defaultGeoShape.props?.dash ?? 'draw',
+				fill,
+				font: defaultGeoShape.props?.font ?? 'draw',
+				geo: shapeType,
+				growY: defaultGeoShape.props?.growY ?? 0,
+				h: focusedShape.h ?? defaultGeoShape.props?.h ?? 100,
+				labelColor: defaultGeoShape.props?.labelColor ?? 'black',
+				richText,
+				scale: defaultGeoShape.props?.scale ?? 1,
+				size: defaultGeoShape.props?.size ?? 's',
+				url: defaultGeoShape.props?.url ?? '',
+				verticalAlign: defaultGeoShape.props?.verticalAlign ?? 'middle',
+				w: focusedShape.w ?? defaultGeoShape.props?.w ?? 100,
+				flipX: defaultGeoShape.props?.flipX ?? false,
+				flipY: defaultGeoShape.props?.flipY ?? false,
+			},
+			meta: {
+				note: focusedShape.note ?? defaultGeoShape.meta?.note ?? '',
+			},
+		},
+	}
+}
+
+function convertNoteShapeToTldrawShape(
+	editor: Editor,
+	focusedShape: FocusedNoteShape,
+	{ defaultShape }: { defaultShape: Partial<TLShape> }
+): { shape: TLShape } {
+	const shapeId = convertSimpleIdToTldrawId(focusedShape.shapeId)
+
+	const defaultNoteShape = defaultShape as TLNoteShape
+
+	// Handle richText properly - focusedShape takes priority
+	let richText
+	if (focusedShape.text !== undefined) {
+		richText = toRichText(focusedShape.text)
+	} else if (defaultNoteShape.props?.richText) {
+		richText = defaultNoteShape.props.richText
+	} else {
+		richText = toRichText('')
+	}
+
+	return {
+		shape: {
+			id: shapeId,
+			type: 'note',
+			typeName: 'shape',
+			x: focusedShape.x ?? defaultNoteShape.x ?? 0,
+			y: focusedShape.y ?? defaultNoteShape.y ?? 0,
+			rotation: defaultNoteShape.rotation ?? 0,
+			index: defaultNoteShape.index ?? editor.getHighestIndexForParent(editor.getCurrentPageId()),
+			parentId: defaultNoteShape.parentId ?? editor.getCurrentPageId(),
+			isLocked: defaultNoteShape.isLocked ?? false,
+			opacity: defaultNoteShape.opacity ?? 1,
+			props: {
+				color: asColor(focusedShape.color ?? defaultNoteShape.props?.color ?? 'black'),
+				richText,
+				size: defaultNoteShape.props?.size ?? 's',
+				align: defaultNoteShape.props?.align ?? 'middle',
+				font: defaultNoteShape.props?.font ?? 'draw',
+				fontSizeAdjustment: defaultNoteShape.props?.fontSizeAdjustment ?? 1,
+				growY: defaultNoteShape.props?.growY ?? 0,
+				labelColor: defaultNoteShape.props?.labelColor ?? 'black',
+				scale: defaultNoteShape.props?.scale ?? 1,
+				url: defaultNoteShape.props?.url ?? '',
+				verticalAlign: defaultNoteShape.props?.verticalAlign ?? 'middle',
+				textLastEditedBy: defaultNoteShape.props?.textLastEditedBy ?? null,
+			},
+			meta: {
+				note: focusedShape.note ?? defaultNoteShape.meta?.note ?? '',
+			},
+		},
+	}
+}
+
+function convertDrawShapeToTldrawShape(
+	editor: Editor,
+	focusedShape: FocusedDrawShape,
+	{ defaultShape }: { defaultShape: Partial<TLShape> }
+): { shape: TLShape } {
+	const shapeId = convertSimpleIdToTldrawId(focusedShape.shapeId)
+	const defaultDrawShape = defaultShape as TLDrawShape
+
+	// Handle fill properly - focusedShape takes priority
+	let fill
+	if (focusedShape.fill !== undefined) {
+		fill = convertFocusedFillToTldrawFill(focusedShape.fill)
+	} else if (defaultDrawShape.props?.fill) {
+		fill = defaultDrawShape.props.fill
+	} else {
+		fill = convertFocusedFillToTldrawFill('none')
+	}
+
+	return {
+		shape: {
+			id: shapeId,
+			type: 'draw',
+			typeName: 'shape',
+			x: defaultDrawShape.x ?? 0,
+			y: defaultDrawShape.y ?? 0,
+			rotation: defaultDrawShape.rotation ?? 0,
+			index: defaultDrawShape.index ?? editor.getHighestIndexForParent(editor.getCurrentPageId()),
+			parentId: defaultDrawShape.parentId ?? editor.getCurrentPageId(),
+			isLocked: defaultDrawShape.isLocked ?? false,
+			opacity: defaultDrawShape.opacity ?? 1,
+			props: {
+				...editor.getShapeUtil('draw').getDefaultProps(),
+				color: asColor(focusedShape.color ?? defaultDrawShape.props?.color ?? 'black'),
+				fill,
+			},
+			meta: {
+				note: focusedShape.note ?? defaultDrawShape.meta?.note ?? '',
+			},
+		},
+	}
+}
+
+function convertUnknownShapeToTldrawShape(
+	editor: Editor,
+	focusedShape: FocusedUnknownShape,
+	{ defaultShape }: { defaultShape: Partial<TLShape> }
+): { shape: TLShape } {
+	const shapeId = convertSimpleIdToTldrawId(focusedShape.shapeId)
+
+	return {
+		shape: {
+			id: shapeId,
+			type: defaultShape.type ?? 'geo',
+			typeName: 'shape',
+			x: focusedShape.x ?? defaultShape.x ?? 0,
+			y: focusedShape.y ?? defaultShape.y ?? 0,
+			rotation: defaultShape.rotation ?? 0,
+			index: defaultShape.index ?? editor.getHighestIndexForParent(editor.getCurrentPageId()),
+			parentId: defaultShape.parentId ?? editor.getCurrentPageId(),
+			isLocked: defaultShape.isLocked ?? false,
+			opacity: defaultShape.opacity ?? 1,
+			props: defaultShape.props ?? ({} as any),
+			meta: {
+				note: focusedShape.note ?? defaultShape.meta?.note ?? '',
+			},
+		},
+	}
+}
+
+/**
+ * Helper function to calculate the best normalized anchor point for arrow binding
+ * @param editor - The tldraw editor instance
+ * @param targetShape - The shape to bind to
+ * @param targetPoint - The desired point in page space
+ * @returns The normalized anchor point (0-1 range within shape bounds)
+ */
+function calculateArrowBindingAnchor(
+	editor: Editor,
+	targetShape: TLShape,
+	targetPoint: VecLike
+): VecLike {
+	const targetShapePageBounds = editor.getShapePageBounds(targetShape)
+	const targetShapeGeometry = editor.getShapeGeometry(targetShape)
+
+	if (!targetShapePageBounds || !targetShapeGeometry) {
+		return { x: 0.5, y: 0.5 } // Fall back to center
+	}
+
+	// transforms the target shape's geometry to page space for calculations
+	const pageTransform = editor.getShapePageTransform(targetShape)
+	const targetShapeGeometryInPageSpace = targetShapeGeometry.transform(pageTransform)
+
+	// Step 1: Find the best anchor point on the shape
+	// If the target point is inside the shape, use it; otherwise use the nearest point on the shape
+	const anchorPoint = targetShapeGeometryInPageSpace.hitTestPoint(targetPoint, 0, true)
+		? targetPoint
+		: targetShapeGeometryInPageSpace.nearestPoint(targetPoint)
+
+	// Step 2: Convert anchor point to normalized coordinates (0-1 range within shape bounds)
+	const normalizedAnchor = {
+		x: (anchorPoint.x - targetShapePageBounds.x) / targetShapePageBounds.w,
+		y: (anchorPoint.y - targetShapePageBounds.y) / targetShapePageBounds.h,
+	}
+
+	// Step 3: Clamp normalized coordinates to valid range [0, 1]
+	const clampedNormalizedAnchor = {
+		x: Math.max(0.1, Math.min(0.9, normalizedAnchor.x)),
+		y: Math.max(0.1, Math.min(0.9, normalizedAnchor.y)),
+	}
+
+	// Step 4: Validate that the clamped anchor point is still within the shape geometry. This is necessary because the above logic sometimes fails for some reason?
+	const clampedAnchorInPageSpace = {
+		x: targetShapePageBounds.x + clampedNormalizedAnchor.x * targetShapePageBounds.w,
+		y: targetShapePageBounds.y + clampedNormalizedAnchor.y * targetShapePageBounds.h,
+	}
+
+	const finalNormalizedAnchor = targetShapeGeometryInPageSpace.hitTestPoint(
+		clampedAnchorInPageSpace,
+		0,
+		true
+	)
+		? clampedNormalizedAnchor
+		: { x: 0.5, y: 0.5 } // Fall back to center
+
+	return finalNormalizedAnchor
+}
+
+/**
+ * Get the bounds of a shape by temporarily creating it in the editor.
+ * This is useful when we need to know the bounds before the shape is actually created.
+ */
+function getDummyBounds(editor: Editor, shape: TLShape): Box {
+	const bounds = editor.getShapePageBounds(shape)
+	if (bounds) {
+		return bounds
+	}
+
+	// Create a dummy shape and get the bounds, then reverse the creation of the dummy shape
+	let dummyBounds: Box | undefined
+	const diff = editor.store.extractingChanges(() => {
+		editor.run(
+			() => {
+				const dummyId = createShapeId()
+				editor.createShape({
+					...shape,
+					id: dummyId,
+				})
+				dummyBounds = editor.getShapePageBounds(dummyId)
+			},
+			{ ignoreShapeLock: false, history: 'ignore' }
+		)
+	})
+	const reverseDiff = reverseRecordsDiff(diff)
+	editor.store.applyDiff(reverseDiff)
+
+	if (!dummyBounds) {
+		throw new Error('Failed to get bounds for shape')
+	}
+	return dummyBounds
+}
+
+/**
+ * Convert a partial FocusedShape to a tldraw shape.
+ * This is used for streaming shapes that are still being generated.
+ *
+ * @param editor - The tldraw editor instance
+ * @param focusedShape - The partial focused shape to convert
+ * @param defaultShape - The default shape to use for fallback values
+ * @param complete - Whether the shape is complete
+ * @returns The converted shape (or null if essential fields are missing), bindings, and position
+ */
+export function convertPartialFocusedShapeToTldrawShape(
+	editor: Editor,
+	focusedShape: FocusedShapePartial,
+	{ defaultShape, complete }: { defaultShape: Partial<TLShape>; complete: boolean }
+): { shape: TLShape | null; bindings: TLBindingCreate[] | null; position: VecLike | null } {
+	// For text shapes, require: x, y, text
+	if (focusedShape._type === 'text') {
+		const partial = focusedShape as FocusedTextShapePartial
+		if (partial.x === undefined || partial.y === undefined || partial.text === undefined) {
+			return { shape: null, bindings: null, position: null }
+		}
+		// Fill in minimal required fields, let converter handle the rest with defaults
+		const fullShape: FocusedTextShape = {
+			...partial,
+			_type: 'text',
+			shapeId: partial.shapeId ?? ('streaming-shape' as any),
+			note: partial.note ?? '',
+			anchor: partial.anchor ?? 'top-left',
+			color: partial.color ?? 'black',
+			maxWidth: partial.maxWidth ?? null,
+		} as FocusedTextShape
+		const result = convertTextShapeToTldrawShape(editor, fullShape, { defaultShape })
+		return { shape: result.shape, bindings: null, position: { x: partial.x, y: partial.y } }
+	}
+
+	// For geo shapes, require: _type, x, y, w, h
+	if (focusedShape._type && focusedShape._type in FOCUSED_TO_GEO_TYPES) {
+		const partial = focusedShape as FocusedGeoShapePartial
+		if (
+			partial.x === undefined ||
+			partial.y === undefined ||
+			partial.w === undefined ||
+			partial.h === undefined
+		) {
+			return { shape: null, bindings: null, position: null }
+		}
+		// Fill in minimal required fields, let converter handle the rest with defaults
+		const fullShape: FocusedGeoShape = {
+			...partial,
+			_type: focusedShape._type as FocusedGeoShape['_type'],
+			shapeId: partial.shapeId ?? ('streaming-shape' as any),
+			note: partial.note ?? '',
+			color: partial.color ?? 'black',
+			textAlign: partial.textAlign || 'middle',
+		} as FocusedGeoShape
+		const result = convertGeoShapeToTldrawShape(editor, fullShape, { defaultShape })
+		return { shape: result.shape, bindings: null, position: { x: partial.x, y: partial.y } }
+	}
+
+	// For all other shapes, require complete: true
+	if (!complete) {
+		return { shape: null, bindings: null, position: null }
+	}
+
+	// If complete, use the full converter
+	const result = convertFocusedShapeToTldrawShape(editor, focusedShape as FocusedShape, {
+		defaultShape,
+	})
+	const position =
+		'x' in focusedShape && 'y' in focusedShape
+			? { x: focusedShape.x as number, y: focusedShape.y as number }
+			: 'x1' in focusedShape && 'y1' in focusedShape
+				? { x: focusedShape.x1 as number, y: focusedShape.y1 as number }
+				: null
+	return { shape: result.shape, bindings: result.bindings ?? null, position }
+}

templates/agent/shared/format/convertTldrawShapeToBlurryShape.ts

@@ -0,0 +1,33 @@
+import { Editor, TLShape } from 'tldraw'
+import { BlurryShape } from './BlurryShape'
+import {
+	convertTldrawIdToSimpleId,
+	convertTldrawShapeToFocusedType,
+} from './convertTldrawShapeToFocusedShape'
+import { FocusedShape } from './FocusedShape'
+
+/**
+ * Convert a tldraw shape to the blurry shape format
+ */
+export function convertTldrawShapeToBlurryShape(
+	editor: Editor,
+	shape: TLShape
+): BlurryShape | null {
+	const bounds = editor.getShapeMaskedPageBounds(shape)
+	if (!bounds) return null
+
+	const util = editor.getShapeUtil(shape)
+	const text = util.getText(shape)
+
+	const shapeType = convertTldrawShapeToFocusedType(shape)
+
+	return {
+		x: Math.round(bounds.x),
+		y: Math.round(bounds.y),
+		w: Math.round(bounds.w),
+		h: Math.round(bounds.h),
+		type: shapeType as FocusedShape['_type'],
+		shapeId: convertTldrawIdToSimpleId(shape.id),
+		text,
+	}
+}

templates/agent/shared/format/convertTldrawShapeToFocusedShape.ts

@@ -0,0 +1,315 @@
+import {
+	Box,
+	createShapeId,
+	Editor,
+	isPageId,
+	reverseRecordsDiff,
+	TLArrowBinding,
+	TLArrowShape,
+	TLDrawShape,
+	TLGeoShape,
+	TLGeoShapeGeoStyle,
+	TLLineShape,
+	TLNoteShape,
+	TLShape,
+	TLShapeId,
+	TLTextShape,
+	Vec,
+} from 'tldraw'
+import { SimpleShapeId } from '../types/ids-schema'
+import { convertTldrawFillToFocusedFill } from './FocusedFill'
+import { convertTldrawFontSizeAndScaleToFocusedFontSize } from './FocusedFontSize'
+import { FocusedGeoShapeType } from './FocusedGeoShapeType'
+import {
+	FocusedArrowShape,
+	FocusedDrawShape,
+	FocusedGeoShape,
+	FocusedLineShape,
+	FocusedNoteShape,
+	FocusedShape,
+	FocusedTextAnchor,
+	FocusedTextShape,
+	FocusedUnknownShape,
+} from './FocusedShape'
+
+/**
+ * Convert a tldraw shape to a focused shape
+ */
+export function convertTldrawShapeToFocusedShape(editor: Editor, shape: TLShape): FocusedShape {
+	switch (shape.type) {
+		case 'text':
+			return convertTextShapeToFocused(editor, shape as TLTextShape)
+		case 'geo':
+			return convertGeoShapeToFocused(editor, shape as TLGeoShape)
+		case 'line':
+			return convertLineShapeToFocused(editor, shape as TLLineShape)
+		case 'arrow':
+			return convertArrowShapeToFocused(editor, shape as TLArrowShape)
+		case 'note':
+			return convertNoteShapeToFocused(editor, shape as TLNoteShape)
+		case 'draw':
+			return convertDrawShapeToFocused(editor, shape as TLDrawShape)
+		default:
+			return convertUnknownShapeToFocused(editor, shape)
+	}
+}
+
+export function convertTldrawShapeToFocusedType(shape: TLShape): FocusedShape['_type'] {
+	switch (shape.type) {
+		case 'geo': {
+			const geoShape = shape as TLGeoShape
+			return GEO_TO_FOCUSED_TYPES[geoShape.props.geo]
+		}
+		case 'text':
+		case 'line':
+		case 'arrow':
+		case 'note':
+		case 'draw':
+			return shape.type
+		default:
+			return 'unknown'
+	}
+}
+
+const GEO_TO_FOCUSED_TYPES: Record<TLGeoShapeGeoStyle, FocusedGeoShapeType> = {
+	rectangle: 'rectangle',
+	ellipse: 'ellipse',
+	triangle: 'triangle',
+	diamond: 'diamond',
+	hexagon: 'hexagon',
+	oval: 'pill',
+	cloud: 'cloud',
+	'x-box': 'x-box',
+	'check-box': 'check-box',
+	heart: 'heart',
+	pentagon: 'pentagon',
+	octagon: 'octagon',
+	star: 'star',
+	rhombus: 'parallelogram-right',
+	'rhombus-2': 'parallelogram-left',
+	trapezoid: 'trapezoid',
+	'arrow-right': 'fat-arrow-right',
+	'arrow-left': 'fat-arrow-left',
+	'arrow-up': 'fat-arrow-up',
+	'arrow-down': 'fat-arrow-down',
+} as const
+
+export function convertTldrawIdToSimpleId(id: TLShapeId): SimpleShapeId {
+	return id.slice(6) as SimpleShapeId
+}
+
+function convertDrawShapeToFocused(editor: Editor, shape: TLDrawShape): FocusedDrawShape {
+	return {
+		_type: 'draw',
+		color: shape.props.color,
+		fill: convertTldrawFillToFocusedFill(shape.props.fill),
+		note: (shape.meta.note as string) ?? '',
+		shapeId: convertTldrawIdToSimpleId(shape.id),
+	}
+}
+
+function convertTextShapeToFocused(editor: Editor, shape: TLTextShape): FocusedTextShape {
+	const util = editor.getShapeUtil(shape)
+	const text = util.getText(shape) ?? ''
+	const bounds = getSimpleBounds(editor, shape)
+
+	const position = new Vec()
+	let anchor: FocusedTextAnchor = 'top-left'
+	switch (shape.props.textAlign) {
+		case 'middle': {
+			anchor = 'top-center'
+			position.x = bounds.center.x
+			position.y = bounds.top
+			break
+		}
+		case 'end': {
+			anchor = 'top-right'
+			position.x = bounds.right
+			position.y = bounds.top
+			break
+		}
+		case 'start': {
+			anchor = 'top-left'
+			position.x = bounds.left
+			position.y = bounds.top
+			break
+		}
+	}
+
+	return {
+		_type: 'text',
+		anchor,
+		color: shape.props.color,
+		fontSize: convertTldrawFontSizeAndScaleToFocusedFontSize(editor, shape),
+		maxWidth: shape.props.autoSize ? null : shape.props.w,
+		note: (shape.meta.note as string) ?? '',
+		shapeId: convertTldrawIdToSimpleId(shape.id),
+		text: text,
+		x: position.x,
+		y: position.y,
+	}
+}
+
+function convertGeoShapeToFocused(editor: Editor, shape: TLGeoShape): FocusedGeoShape {
+	const util = editor.getShapeUtil(shape)
+	const text = util.getText(shape)
+	const bounds = getSimpleBounds(editor, shape)
+	const shapeTextAlign = shape.props.align
+
+	let newTextAlign: FocusedGeoShape['textAlign']
+	switch (shapeTextAlign) {
+		case 'start-legacy':
+			newTextAlign = 'start'
+			break
+		case 'middle-legacy':
+			newTextAlign = 'middle'
+			break
+		case 'end-legacy':
+			newTextAlign = 'end'
+			break
+		default:
+			newTextAlign = shapeTextAlign
+			break
+	}
+
+	return {
+		_type: GEO_TO_FOCUSED_TYPES[shape.props.geo],
+		color: shape.props.color,
+		fill: convertTldrawFillToFocusedFill(shape.props.fill),
+		h: shape.props.h,
+		note: (shape.meta.note as string) ?? '',
+		shapeId: convertTldrawIdToSimpleId(shape.id),
+		text: text ?? '',
+		textAlign: newTextAlign,
+		w: shape.props.w,
+		x: bounds.x,
+		y: bounds.y,
+	}
+}
+
+function convertLineShapeToFocused(editor: Editor, shape: TLLineShape): FocusedLineShape {
+	const bounds = getSimpleBounds(editor, shape)
+	const points = Object.values(shape.props.points).sort((a, b) => a.index.localeCompare(b.index))
+	return {
+		_type: 'line',
+		color: shape.props.color,
+		note: (shape.meta.note as string) ?? '',
+		shapeId: convertTldrawIdToSimpleId(shape.id),
+		x1: points[0].x + bounds.x,
+		x2: points[1].x + bounds.x,
+		y1: points[0].y + bounds.y,
+		y2: points[1].y + bounds.y,
+	}
+}
+
+function convertArrowShapeToFocused(editor: Editor, shape: TLArrowShape): FocusedArrowShape {
+	const bounds = getSimpleBounds(editor, shape)
+	const bindings = editor.store.query.records('binding').get()
+	const arrowBindings = bindings.filter(
+		(b) => b.type === 'arrow' && b.fromId === shape.id
+	) as TLArrowBinding[]
+	const startBinding = arrowBindings.find((b) => b.props.terminal === 'start')
+	const endBinding = arrowBindings.find((b) => b.props.terminal === 'end')
+
+	return {
+		_type: 'arrow',
+		bend: shape.props.bend * -1,
+		color: shape.props.color,
+		fromId: startBinding ? convertTldrawIdToSimpleId(startBinding.toId) : null,
+		note: (shape.meta.note as string) ?? '',
+		shapeId: convertTldrawIdToSimpleId(shape.id),
+		text: (shape.meta.text as string) ?? '',
+		toId: endBinding ? convertTldrawIdToSimpleId(endBinding.toId) : null,
+		x1: shape.props.start.x + bounds.x,
+		x2: shape.props.end.x + bounds.x,
+		y1: shape.props.start.y + bounds.y,
+		y2: shape.props.end.y + bounds.y,
+	}
+}
+
+function convertNoteShapeToFocused(editor: Editor, shape: TLNoteShape): FocusedNoteShape {
+	const util = editor.getShapeUtil(shape)
+	const text = util.getText(shape)
+	const bounds = getSimpleBounds(editor, shape)
+	return {
+		_type: 'note',
+		color: shape.props.color,
+		note: (shape.meta.note as string) ?? '',
+		shapeId: convertTldrawIdToSimpleId(shape.id),
+		text: text ?? '',
+		x: bounds.x,
+		y: bounds.y,
+	}
+}
+
+function convertUnknownShapeToFocused(editor: Editor, shape: TLShape): FocusedUnknownShape {
+	const bounds = getSimpleBounds(editor, shape)
+	return {
+		_type: 'unknown',
+		note: (shape.meta.note as string) ?? '',
+		shapeId: convertTldrawIdToSimpleId(shape.id),
+		subType: shape.type,
+		x: bounds.x,
+		y: bounds.y,
+	}
+}
+
+function getSimpleBounds(editor: Editor, shape: TLShape): Box {
+	// Compute page position from the shape record's own x/y, not the editor's cached bounds.
+	// This is critical for diffing historical shape records where the editor's
+	// current state differs from the shape record being converted.
+	const pagePoint = getShapePagePoint(editor, shape)
+
+	// Try to get dimensions from shape props first
+	const props = shape.props as { w?: number; h?: number }
+	if (props.w !== undefined && props.h !== undefined) {
+		return new Box(pagePoint.x, pagePoint.y, props.w, props.h)
+	}
+
+	// Fall back to editor bounds for dimensions only (position comes from shape record)
+	const bounds = editor.getShapePageBounds(shape)
+	if (bounds) {
+		return new Box(pagePoint.x, pagePoint.y, bounds.w, bounds.h)
+	}
+
+	// Create a mock shape and get the bounds, then reverse the creation of the mock shape
+	let mockBounds: Box | undefined
+	const diff = editor.store.extractingChanges(() => {
+		editor.run(
+			() => {
+				const mockId = createShapeId()
+				editor.createShape({
+					...shape,
+					id: mockId,
+				})
+				mockBounds = editor.getShapePageBounds(mockId)
+			},
+			{ ignoreShapeLock: false, history: 'ignore' }
+		)
+	})
+	const reverseDiff = reverseRecordsDiff(diff)
+	editor.store.applyDiff(reverseDiff)
+
+	if (!mockBounds) {
+		throw new Error('Failed to get bounds for shape')
+	}
+	return new Box(pagePoint.x, pagePoint.y, mockBounds.w, mockBounds.h)
+}
+
+/**
+ * Get the page-space position of a shape from its record's x/y values.
+ * For shapes at the root level, this is just shape.x/y.
+ * For shapes inside frames/groups, we transform through the parent's page transform.
+ */
+function getShapePagePoint(editor: Editor, shape: TLShape): Vec {
+	// If the shape is at the root level (parent is a page), use x/y directly
+	if (isPageId(shape.parentId)) {
+		return new Vec(shape.x, shape.y)
+	}
+
+	// For shapes inside parents, get the parent's page transform and apply it
+	// Note: We use the editor's current parent transform, which is correct as long
+	// as the parent itself hasn't moved in the same diff (uncommon case)
+	const parentTransform = editor.getShapePageTransform(shape.parentId)
+	return parentTransform.applyToPoint(new Vec(shape.x, shape.y))
+}

templates/agent/shared/format/convertTldrawShapesToPeripheralShapes.ts

@@ -0,0 +1,54 @@
+import { Box, Editor, TLShape } from 'tldraw'
+import { PeripheralShapeCluster } from './PeripheralShapesCluster'
+
+export function convertTldrawShapesToPeripheralShapes(
+	editor: Editor,
+	shapes: TLShape[],
+	{ padding = 0 }: { padding?: number } = {}
+): PeripheralShapeCluster[] {
+	if (shapes.length === 0) return []
+	const groups: { shapes: TLShape[]; bounds: Box; numberOfShapes: number }[] = []
+
+	const expandedBounds = shapes.map((shape) => {
+		return {
+			shape,
+			bounds: editor.getShapeMaskedPageBounds(shape)!.clone().expandBy(padding),
+		}
+	})
+	for (let i = 0; i < expandedBounds.length; i++) {
+		const item = expandedBounds[i]
+		if (i === 0) {
+			groups[0] = {
+				shapes: [item.shape],
+				bounds: item.bounds,
+				numberOfShapes: 1,
+			}
+			continue
+		}
+		let didLand = false
+		for (const group of groups) {
+			if (group.bounds.includes(item.bounds)) {
+				group.shapes.push(item.shape)
+				group.bounds.expand(item.bounds)
+				group.numberOfShapes++
+				didLand = true
+				break
+			}
+		}
+		if (!didLand) {
+			groups.push({
+				shapes: [item.shape],
+				bounds: item.bounds,
+				numberOfShapes: 1,
+			})
+		}
+	}
+
+	return groups.map((group) => {
+		const shrunkBounds = group.bounds.clone().expandBy(-padding)
+		return {
+			bounds: shrunkBounds,
+			numberOfShapes: group.numberOfShapes,
+		}
+	})
+}

templates/agent/shared/icons/AgentIcon.tsx

@@ -0,0 +1,39 @@
+import { AtIcon } from './AtIcon'
+import { BrainIcon } from './BrainIcon'
+import { ChevronDownIcon } from './ChevronDownIcon'
+import { ChevronRightIcon } from './ChevronRightIcon'
+import { CommentIcon } from './CommentIcon'
+import { CrossIcon } from './CrossIcon'
+import { CursorIcon } from './CursorIcon'
+import { EllipsisIcon } from './EllipsisIcon'
+import { EyeIcon } from './EyeIcon'
+import { NoteIcon } from './NoteIcon'
+import { PencilIcon } from './PencilIcon'
+import { RefreshIcon } from './RefreshIcon'
+import { SearchIcon } from './SearchIcon'
+import { TargetIcon } from './TargetIcon'
+import { TrashIcon } from './TrashIcon'
+
+const AGENT_ICONS = {
+	'chevron-down': <ChevronDownIcon />,
+	'chevron-right': <ChevronRightIcon />,
+	at: <AtIcon />,
+	brain: <BrainIcon />,
+	comment: <CommentIcon />,
+	cross: <CrossIcon />,
+	cursor: <CursorIcon />,
+	ellipsis: <EllipsisIcon />,
+	eye: <EyeIcon />,
+	note: <NoteIcon />,
+	pencil: <PencilIcon />,
+	refresh: <RefreshIcon />,
+	search: <SearchIcon />,
+	target: <TargetIcon />,
+	trash: <TrashIcon />,
+}
+
+export type AgentIconType = keyof typeof AGENT_ICONS
+
+export function AgentIcon({ type }: { type: AgentIconType }) {
+	return AGENT_ICONS[type]
+}

templates/agent/shared/icons/AtIcon.tsx

@@ -0,0 +1,19 @@
+export function AtIcon() {
+	return (
+		<svg
+			xmlns="http://www.w3.org/2000/svg"
+			width="24"
+			height="24"
+			viewBox="0 0 24 24"
+			fill="none"
+			stroke="currentColor"
+			strokeWidth="2"
+			strokeLinecap="round"
+			strokeLinejoin="round"
+			className="chat-icon"
+		>
+			<circle cx="12" cy="12" r="4" />
+			<path d="M16 8v5a3 3 0 0 0 6 0v-1a10 10 0 1 0-4 8" />
+		</svg>
+	)
+}

templates/agent/shared/icons/BrainIcon.tsx

@@ -0,0 +1,25 @@
+export function BrainIcon() {
+	return (
+		<svg
+			xmlns="http://www.w3.org/2000/svg"
+			width="24"
+			height="24"
+			viewBox="0 0 24 24"
+			fill="none"
+			stroke="currentColor"
+			strokeWidth="2"
+			strokeLinecap="round"
+			strokeLinejoin="round"
+			className="chat-icon"
+		>
+			<path d="M12 18V5" />
+			<path d="M15 13a4.17 4.17 0 0 1-3-4 4.17 4.17 0 0 1-3 4" />
+			<path d="M17.598 6.5A3 3 0 1 0 12 5a3 3 0 1 0-5.598 1.5" />
+			<path d="M17.997 5.125a4 4 0 0 1 2.526 5.77" />
+			<path d="M18 18a4 4 0 0 0 2-7.464" />
+			<path d="M19.967 17.483A4 4 0 1 1 12 18a4 4 0 1 1-7.967-.517" />
+			<path d="M6 18a4 4 0 0 1-2-7.464" />
+			<path d="M6.003 5.125a4 4 0 0 0-2.526 5.77" />
+		</svg>
+	)
+}

templates/agent/shared/icons/ChevronDownIcon.tsx

@@ -0,0 +1,18 @@
+export function ChevronDownIcon() {
+	return (
+		<svg
+			xmlns="http://www.w3.org/2000/svg"
+			width="24"
+			height="24"
+			viewBox="0 0 24 24"
+			fill="none"
+			stroke="currentColor"
+			strokeWidth="2"
+			strokeLinecap="round"
+			strokeLinejoin="round"
+			className="chat-icon"
+		>
+			<path d="m6 9 6 6 6-6" />
+		</svg>
+	)
+}

templates/agent/shared/icons/ChevronRightIcon.tsx

@@ -0,0 +1,18 @@
+export function ChevronRightIcon() {
+	return (
+		<svg
+			xmlns="http://www.w3.org/2000/svg"
+			width="24"
+			height="24"
+			viewBox="0 0 24 24"
+			fill="none"
+			stroke="currentColor"
+			strokeWidth="2"
+			strokeLinecap="round"
+			strokeLinejoin="round"
+			className="chat-icon"
+		>
+			<path d="m9 18 6-6-6-6" />
+		</svg>
+	)
+}

templates/agent/shared/icons/CommentIcon.tsx

@@ -0,0 +1,18 @@
+export function CommentIcon() {
+	return (
+		<svg
+			xmlns="http://www.w3.org/2000/svg"
+			width="24"
+			height="24"
+			viewBox="0 0 24 24"
+			fill="none"
+			stroke="currentColor"
+			strokeWidth="2"
+			strokeLinecap="round"
+			strokeLinejoin="round"
+			className="chat-icon"
+		>
+			<path d="M21 15a2 2 0 0 1-2 2H7l-4 4V5a2 2 0 0 1 2-2h14a2 2 0 0 1 2 2z" />
+		</svg>
+	)
+}

templates/agent/shared/icons/CrossIcon.tsx

@@ -0,0 +1,19 @@
+export function CrossIcon() {
+	return (
+		<svg
+			xmlns="http://www.w3.org/2000/svg"
+			width="24"
+			height="24"
+			viewBox="0 0 24 24"
+			fill="none"
+			stroke="currentColor"
+			strokeWidth="2"
+			strokeLinecap="round"
+			strokeLinejoin="round"
+			className="chat-icon"
+		>
+			<path d="M18 6 6 18" />
+			<path d="m6 6 12 12" />
+		</svg>
+	)
+}

templates/agent/shared/icons/CursorIcon.tsx

@@ -0,0 +1,18 @@
+export function CursorIcon() {
+	return (
+		<svg
+			xmlns="http://www.w3.org/2000/svg"
+			width="24"
+			height="24"
+			viewBox="0 0 20 24"
+			fill="none"
+			stroke="currentColor"
+			strokeWidth="2"
+			strokeLinecap="round"
+			strokeLinejoin="round"
+			className="chat-icon"
+		>
+			<path d="M4.037 4.688a.495.495 0 0 1 .651-.651l16 6.5a.5.5 0 0 1-.063.947l-6.124 1.58a2 2 0 0 0-1.438 1.435l-1.579 6.126a.5.5 0 0 1-.947.063z" />
+		</svg>
+	)
+}

templates/agent/shared/icons/EllipsisIcon.tsx

@@ -0,0 +1,20 @@
+export function EllipsisIcon() {
+	return (
+		<svg
+			xmlns="http://www.w3.org/2000/svg"
+			width="24"
+			height="24"
+			viewBox="0 0 24 24"
+			fill="none"
+			stroke="currentColor"
+			strokeWidth="2"
+			strokeLinecap="round"
+			strokeLinejoin="round"
+			className="chat-icon "
+		>
+			<path d="M12 12m-1 0a1 1 0 1 0 2 0a1 1 0 1 0 -2 0" />
+			<path d="M19 12m-1 0a1 1 0 1 0 2 0a1 1 0 1 0 -2 0" />
+			<path d="M5 12m-1 0a1 1 0 1 0 2 0a1 1 0 1 0 -2 0" />
+		</svg>
+	)
+}

templates/agent/shared/icons/EyeIcon.tsx

@@ -0,0 +1,19 @@
+export function EyeIcon() {
+	return (
+		<svg
+			xmlns="http://www.w3.org/2000/svg"
+			width="24"
+			height="24"
+			viewBox="0 0 24 24"
+			fill="none"
+			stroke="currentColor"
+			strokeWidth="2"
+			strokeLinecap="round"
+			strokeLinejoin="round"
+			className="chat-icon"
+		>
+			<path d="M2.062 12.348a1 1 0 0 1 0-.696 10.75 10.75 0 0 1 19.876 0 1 1 0 0 1 0 .696 10.75 10.75 0 0 1-19.876 0" />
+			<circle cx="12" cy="12" r="3" />
+		</svg>
+	)
+}

templates/agent/shared/icons/NoteIcon.tsx

@@ -0,0 +1,23 @@
+export function NoteIcon() {
+	return (
+		<svg
+			xmlns="http://www.w3.org/2000/svg"
+			width="24"
+			height="24"
+			viewBox="0 0 24 24"
+			fill="none"
+			stroke="currentColor"
+			strokeWidth="2"
+			strokeLinecap="round"
+			strokeLinejoin="round"
+			className="chat-icon"
+		>
+			<path d="M13.4 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2v-7.4" />
+			<path d="M2 6h4" />
+			<path d="M2 10h4" />
+			<path d="M2 14h4" />
+			<path d="M2 18h4" />
+			<path d="M21.378 5.626a1 1 0 1 0-3.004-3.004l-5.01 5.012a2 2 0 0 0-.506.854l-.837 2.87a.5.5 0 0 0 .62.62l2.87-.837a2 2 0 0 0 .854-.506z" />
+		</svg>
+	)
+}

templates/agent/shared/icons/PencilIcon.tsx

@@ -0,0 +1,19 @@
+export function PencilIcon() {
+	return (
+		<svg
+			xmlns="http://www.w3.org/2000/svg"
+			width="24"
+			height="24"
+			viewBox="0 0 24 24"
+			fill="none"
+			stroke="currentColor"
+			strokeWidth="2"
+			strokeLinecap="round"
+			strokeLinejoin="round"
+			className="chat-icon"
+		>
+			<path d="M21.174 6.812a1 1 0 0 0-3.986-3.987L3.842 16.174a2 2 0 0 0-.5.83l-1.321 4.352a.5.5 0 0 0 .623.622l4.353-1.32a2 2 0 0 0 .83-.497z" />
+			<path d="m15 5 4 4" />
+		</svg>
+	)
+}

templates/agent/shared/icons/RefreshIcon.tsx

@@ -0,0 +1,21 @@
+export function RefreshIcon() {
+	return (
+		<svg
+			xmlns="http://www.w3.org/2000/svg"
+			width="24"
+			height="24"
+			viewBox="0 0 24 24"
+			fill="none"
+			stroke="currentColor"
+			strokeWidth="2"
+			strokeLinecap="round"
+			strokeLinejoin="round"
+			className="chat-icon"
+		>
+			<path d="M21 12a9 9 0 0 0-9-9 9.75 9.75 0 0 0-6.74 2.74L3 8" />
+			<path d="M3 3v5h5" />
+			<path d="M3 12a9 9 0 0 0 9 9 9.75 9.75 0 0 0 6.74-2.74L21 16" />
+			<path d="M16 16h5v5" />
+		</svg>
+	)
+}

templates/agent/shared/icons/SearchIcon.tsx

@@ -0,0 +1,19 @@
+export function SearchIcon() {
+	return (
+		<svg
+			xmlns="http://www.w3.org/2000/svg"
+			width="24"
+			height="24"
+			viewBox="0 0 24 24"
+			fill="none"
+			stroke="currentColor"
+			strokeWidth="2"
+			strokeLinecap="round"
+			strokeLinejoin="round"
+			className="chat-icon"
+		>
+			<path d="m21 21-4.34-4.34" />
+			<circle cx="11" cy="11" r="8" />
+		</svg>
+	)
+}

templates/agent/shared/icons/SmallSpinner.tsx

@@ -0,0 +1,17 @@
+export function SmallSpinner(props: React.SVGProps<SVGSVGElement>) {
+	return (
+		<svg
+			width={11}
+			height={11}
+			viewBox="0 0 11 11"
+			aria-hidden="false"
+			{...props}
+			className="tl-spinner"
+		>
+			<g strokeWidth={2} fill="none" fillRule="evenodd">
+				<circle strokeOpacity={0.25} cx={5.5} cy={5.5} r={4.5} stroke="currentColor" />
+				<path strokeLinecap="round" d="M10.5 5.5c0-2.5-2.5-4.5-4.5-4.5" stroke="currentColor" />
+			</g>
+		</svg>
+	)
+}

templates/agent/shared/icons/TargetIcon.tsx

@@ -0,0 +1,22 @@
+export function TargetIcon() {
+	return (
+		<svg
+			xmlns="http://www.w3.org/2000/svg"
+			width="24"
+			height="24"
+			viewBox="0 0 24 24"
+			fill="none"
+			stroke="currentColor"
+			strokeWidth="2"
+			strokeLinecap="round"
+			strokeLinejoin="round"
+			className="chat-icon"
+		>
+			<circle cx="12" cy="12" r="10" />
+			<line x1="12" y1="2" x2="12" y2="8" />
+			<line x1="12" y1="16" x2="12" y2="22" />
+			<line x1="2" y1="12" x2="8" y2="12" />
+			<line x1="16" y1="12" x2="22" y2="12" />
+		</svg>
+	)
+}

templates/agent/shared/icons/TickIcon.tsx

@@ -0,0 +1,18 @@
+export function TickIcon() {
+	return (
+		<svg
+			xmlns="http://www.w3.org/2000/svg"
+			width="24"
+			height="24"
+			viewBox="0 0 24 24"
+			fill="none"
+			stroke="currentColor"
+			strokeWidth="2"
+			strokeLinecap="round"
+			strokeLinejoin="round"
+			className="chat-icon"
+		>
+			<path d="M20 6 9 17l-5-5" />
+		</svg>
+	)
+}

templates/agent/shared/icons/TrashIcon.tsx

@@ -0,0 +1,22 @@
+export function TrashIcon() {
+	return (
+		<svg
+			xmlns="http://www.w3.org/2000/svg"
+			width="24"
+			height="24"
+			viewBox="0 0 24 24"
+			fill="none"
+			stroke="currentColor"
+			strokeWidth="2"
+			strokeLinecap="round"
+			strokeLinejoin="round"
+			className="chat-icon"
+		>
+			<path d="M10 11v6" />
+			<path d="M14 11v6" />
+			<path d="M19 6v14a2 2 0 0 1-2 2H7a2 2 0 0 1-2-2V6" />
+			<path d="M3 6h18" />
+			<path d="M8 6V4a2 2 0 0 1 2-2h4a2 2 0 0 1 2 2v2" />
+		</svg>
+	)
+}

templates/agent/shared/models.ts

@@ -0,0 +1,145 @@
+export type AgentModelName = keyof typeof AGENT_MODEL_DEFINITIONS
+export type AgentModelProvider = 'openai' | 'anthropic' | 'google'
+
+/** Adaptive-thinking mode passed to the Anthropic provider. */
+export type AnthropicThinking = 'adaptive' | 'disabled'
+
+/** Effort level passed to the Anthropic provider (Opus 4.6+/Sonnet 4.6+; not supported on Haiku 4.5). */
+export type AnthropicEffort = 'low' | 'medium' | 'high' | 'xhigh' | 'max'
+
+/** Reasoning effort passed to the OpenAI provider. */
+export type OpenAIReasoningEffort = 'none' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh' | 'max'
+
+/** Thinking level passed to the Google provider. */
+export type GeminiThinkingLevel = 'low' | 'medium' | 'high'
+
+interface BaseAgentModelDefinition {
+	name: AgentModelName
+	id: string
+
+	/**
+	 * Whether the model accepts a prefilled assistant turn to force the JSON start.
+	 * Opus 4.6+ and Sonnet 4.6+ reject last-assistant-turn prefills (400).
+	 */
+	supportsPrefill: boolean
+
+	/**
+	 * Whether the model accepts a `temperature` sampling parameter.
+	 * Opus 4.7+ and Sonnet 5 removed `temperature`/`top_p`/`top_k` (sending them is a 400).
+	 */
+	supportsTemperature: boolean
+}
+
+export interface AnthropicModelDefinition extends BaseAgentModelDefinition {
+	provider: 'anthropic'
+	thinking: AnthropicThinking
+	/** Effort level; omit for models that don't support it (e.g. Haiku 4.5). */
+	effort?: AnthropicEffort
+}
+
+export interface GoogleModelDefinition extends BaseAgentModelDefinition {
+	provider: 'google'
+	thinkingLevel: GeminiThinkingLevel
+}
+
+export interface OpenAIModelDefinition extends BaseAgentModelDefinition {
+	provider: 'openai'
+	reasoningEffort: OpenAIReasoningEffort
+}
+
+export type AgentModelDefinition =
+	| AnthropicModelDefinition
+	| GoogleModelDefinition
+	| OpenAIModelDefinition
+
+export const AGENT_MODEL_DEFINITIONS = {
+	// Anthropic models
+	'claude-opus-5': {
+		name: 'claude-opus-5',
+		id: 'claude-opus-5',
+		provider: 'anthropic',
+		supportsPrefill: false,
+		supportsTemperature: false,
+		thinking: 'adaptive',
+		effort: 'medium',
+	},
+
+	'claude-sonnet-5': {
+		name: 'claude-sonnet-5',
+		id: 'claude-sonnet-5',
+		provider: 'anthropic',
+		supportsPrefill: false,
+		supportsTemperature: false,
+		thinking: 'adaptive',
+		effort: 'low',
+	},
+
+	'claude-haiku-4-5': {
+		name: 'claude-haiku-4-5',
+		id: 'claude-haiku-4-5',
+		provider: 'anthropic',
+		supportsPrefill: true,
+		supportsTemperature: true,
+		thinking: 'disabled',
+	},
+
+	// Google models
+	'gemini-3.8-flash': {
+		name: 'gemini-3.8-flash',
+		id: 'gemini-3.8-flash',
+		provider: 'google',
+		supportsPrefill: false,
+		supportsTemperature: false,
+		thinkingLevel: 'low',
+	},
+
+	// OpenAI models
+	'gpt-5.6-sol': {
+		name: 'gpt-5.6-sol',
+		id: 'gpt-5.6-sol',
+		provider: 'openai',
+		supportsPrefill: false,
+		supportsTemperature: false,
+		reasoningEffort: 'medium',
+	},
+
+	'gpt-5.6-terra': {
+		name: 'gpt-5.6-terra',
+		id: 'gpt-5.6-terra',
+		provider: 'openai',
+		supportsPrefill: false,
+		supportsTemperature: false,
+		reasoningEffort: 'high',
+	},
+
+	'gpt-5.6-luna': {
+		name: 'gpt-5.6-luna',
+		id: 'gpt-5.6-luna',
+		provider: 'openai',
+		supportsPrefill: false,
+		supportsTemperature: false,
+		reasoningEffort: 'max',
+	},
+} as const
+
+export const DEFAULT_MODEL_NAME: AgentModelName = 'claude-sonnet-5'
+
+/**
+ * Check if a string is a valid AgentModelName.
+ */
+export function isValidModelName(value: string | undefined): value is AgentModelName {
+	return !!value && value in AGENT_MODEL_DEFINITIONS
+}
+
+/**
+ * Get the full information about a model from its name.
+ * @param modelName - The name of the model.
+ * @returns The full definition of the model.
+ */
+export function getAgentModelDefinition(modelName: AgentModelName): AgentModelDefinition {
+	const definition = AGENT_MODEL_DEFINITIONS[modelName]
+	if (!definition) {
+		throw new Error(`Model ${modelName} not found`)
+	}
+	return definition
+}

templates/agent/shared/schema/AgentActionSchemaRegistry.ts

@@ -0,0 +1,95 @@
+import z from 'zod'
+
+/**
+ * Options for registering an action schema.
+ */
+export interface RegisterActionSchemaOptions {
+	/**
+	 * If specified, this schema will only be used when the agent is in one of these modes.
+	 * Otherwise, it will be the default schema for this action type.
+	 */
+	forModes?: string[]
+}
+
+// Default registry: actionType -> schema (used when no mode-specific schema is registered)
+const defaultSchemaRegistry = new Map<string, z.ZodType>()
+
+// Mode registry: actionType -> (mode -> schema) (mode-specific overrides)
+const modeSchemaRegistry = new Map<string, Map<string, z.ZodType>>()
+
+/**
+ * Register an action schema. Call this for each action type.
+ *
+ * @param type - The action type (the `_type` literal value).
+ * @param schema - The Zod schema for this action.
+ * @param options - Optional configuration for mode-specific registration.
+ * @returns The schema (for chaining).
+ */
+export function registerActionSchema<T extends z.ZodType>(
+	type: string,
+	schema: T,
+	options?: RegisterActionSchemaOptions
+): T {
+	const { forModes } = options ?? {}
+
+	if (forModes && forModes.length > 0) {
+		// Mode-specific registration
+		if (!modeSchemaRegistry.has(type)) {
+			modeSchemaRegistry.set(type, new Map())
+		}
+		const modeMap = modeSchemaRegistry.get(type)!
+		for (const mode of forModes) {
+			if (modeMap.has(mode)) {
+				throw new Error(`Action schema for ${type} already registered for mode ${mode}`)
+			}
+			modeMap.set(mode, schema)
+		}
+	} else {
+		// Default registration
+		if (defaultSchemaRegistry.has(type)) {
+			throw new Error(`Action schema already registered: ${type}`)
+		}
+		defaultSchemaRegistry.set(type, schema)
+	}
+
+	return schema
+}
+
+/**
+ * Get an action schema for a specific type and mode.
+ * Returns the mode-specific schema if one exists, otherwise the default.
+ *
+ * @param type - The action type to look up.
+ * @param mode - The mode to resolve the schema for.
+ * @returns The schema, or undefined if not found.
+ */
+export function getActionSchemaForMode(type: string, mode: string): z.ZodType | undefined {
+	// Check for mode-specific schema first
+	const modeMap = modeSchemaRegistry.get(type)
+	if (modeMap) {
+		const modeSchema = modeMap.get(mode)
+		if (modeSchema) {
+			return modeSchema
+		}
+	}
+
+	// Fall back to default schema
+	return defaultSchemaRegistry.get(type)
+}
+
+/**
+ * Get the default schema for an action type (ignoring mode).
+ *
+ * @param type - The action type to look up.
+ * @returns The default schema, or undefined if not found.
+ */
+export function getDefaultActionSchema(type: string): z.ZodType | undefined {
+	return defaultSchemaRegistry.get(type)
+}
+
+/**
+ * Check if a default schema is already registered for a type.
+ */
+export function hasDefaultActionSchema(type: string): boolean {
+	return defaultSchemaRegistry.has(type)
+}

templates/agent/shared/schema/AgentActionSchemas.ts

@@ -0,0 +1,385 @@
+import z from 'zod'
+import { FocusedColor } from '../format/FocusedColor'
+import { FocusedFillSchema } from '../format/FocusedFill'
+import { FocusedShapeSchema, FocusedTextAnchorSchema } from '../format/FocusedShape'
+import { SimpleShapeIdSchema, TodoIdSchema } from '../types/ids-schema'
+
+/**
+ * `_systemPromptCategory` is used for system prompt generation
+ * but is stripped from the JSON schema sent to the model.
+ *
+ * See `SystemPromptCategory.ts` for available values.
+ */
+
+// Add Detail Action
+export const AddDetailAction = z
+	.object({
+		_type: z.literal('add-detail'),
+		intent: z.string(),
+	})
+	.meta({
+		title: 'Add Detail',
+		description: 'The AI plans further work so that it can add detail to its work.',
+	})
+
+export type AddDetailAction = z.infer<typeof AddDetailAction>
+
+// Align Action
+export const AlignAction = z
+	.object({
+		_type: z.literal('align'),
+		alignment: z.enum(['top', 'bottom', 'left', 'right', 'center-horizontal', 'center-vertical']),
+		gap: z.number(),
+		intent: z.string(),
+		shapeIds: z.array(SimpleShapeIdSchema),
+	})
+	.meta({
+		title: 'Align',
+		description: 'The AI aligns shapes to each other on an axis.',
+		_systemPromptCategory: 'edit',
+	})
+
+export type AlignAction = z.infer<typeof AlignAction>
+
+// Bring to Front Action
+export const BringToFrontAction = z
+	.object({
+		_type: z.literal('bringToFront'),
+		intent: z.string(),
+		shapeIds: z.array(SimpleShapeIdSchema),
+	})
+	.meta({
+		title: 'Bring to Front',
+		description:
+			'The AI brings one or more shapes to the front so that they appear in front of everything else.',
+		_systemPromptCategory: 'edit',
+	})
+
+export type BringToFrontAction = z.infer<typeof BringToFrontAction>
+
+// Clear Action
+export const ClearAction = z
+	.object({
+		_type: z.literal('clear'),
+	})
+	.meta({
+		title: 'Clear',
+		description: 'The agent deletes all shapes on the canvas.',
+	})
+
+export type ClearAction = z.infer<typeof ClearAction>
+
+// Count Shapes Action
+export const CountShapesAction = z
+	.object({
+		_type: z.literal('count'),
+		expression: z.string(),
+	})
+	.meta({
+		title: 'Count',
+		description:
+			'The AI requests to count the number of shapes in the canvas. The answer will be provided to the AI in a follow-up request.',
+	})
+
+export type CountShapesAction = z.infer<typeof CountShapesAction>
+
+// Country Info Action
+export const CountryInfoAction = z
+	.object({
+		_type: z.literal('countryInfo'),
+		code: z.string(),
+	})
+	.meta({
+		title: 'Country info',
+		description:
+			'The AI gets information about a country by providing its country code, eg: "de" for Germany.',
+	})
+
+export type CountryInfoAction = z.infer<typeof CountryInfoAction>
+
+// Create Action
+export const CreateAction = z
+	.object({
+		_type: z.literal('create'),
+		intent: z.string(),
+		shape: FocusedShapeSchema,
+	})
+	.meta({ title: 'Create', description: 'The AI creates a new shape.' })
+
+export type CreateAction = z.infer<typeof CreateAction>
+
+// Delete Action
+export const DeleteAction = z
+	.object({
+		_type: z.literal('delete'),
+		intent: z.string(),
+		shapeId: SimpleShapeIdSchema,
+	})
+	.meta({ title: 'Delete', description: 'The AI deletes a shape.', _systemPromptCategory: 'edit' })
+
+export type DeleteAction = z.infer<typeof DeleteAction>
+
+// Distribute Action
+export const DistributeAction = z
+	.object({
+		_type: z.literal('distribute'),
+		direction: z.enum(['horizontal', 'vertical']),
+		intent: z.string(),
+		shapeIds: z.array(SimpleShapeIdSchema),
+	})
+	.meta({
+		title: 'Distribute',
+		description: 'The AI distributes shapes horizontally or vertically.',
+		_systemPromptCategory: 'edit',
+	})
+
+export type DistributeAction = z.infer<typeof DistributeAction>
+
+// Label Action
+export const LabelAction = z
+	.object({
+		_type: z.literal('label'),
+		intent: z.string(),
+		shapeId: SimpleShapeIdSchema,
+		text: z.string(),
+	})
+	.meta({
+		title: 'Label',
+		description: "The AI changes a shape's text.",
+		_systemPromptCategory: 'edit',
+	})
+
+export type LabelAction = z.infer<typeof LabelAction>
+
+// Message Action
+export const MessageAction = z
+	.object({
+		_type: z.literal('message'),
+		text: z.string(),
+	})
+	.meta({ title: 'Message', description: 'The AI sends a message to the user.' })
+
+export type MessageAction = z.infer<typeof MessageAction>
+
+// Move Action
+export const MoveAction = z
+	.object({
+		_type: z.literal('move'),
+		intent: z.string(),
+		anchor: FocusedTextAnchorSchema,
+		shapeId: SimpleShapeIdSchema,
+		x: z.number(),
+		y: z.number(),
+	})
+	.meta({
+		title: 'Move',
+		description: 'The agent moves a shape to a new position.',
+		_systemPromptCategory: 'edit',
+	})
+
+export type MoveAction = z.infer<typeof MoveAction>
+
+// Pen Action
+export const PenAction = z
+	.object({
+		_type: z.literal('pen'),
+		shapeId: SimpleShapeIdSchema,
+		color: FocusedColor,
+		closed: z.boolean(),
+		fill: FocusedFillSchema,
+		intent: z.string(),
+		points: z.array(
+			z.object({
+				x: z.number(),
+				y: z.number(),
+			})
+		),
+		style: z.enum(['smooth', 'straight']),
+	})
+	.meta({
+		title: 'Pen',
+		description:
+			'The AI draws a freeform line with a pen. This is useful for drawing custom paths that are not available with the other available shapes. The "smooth" style will automatically smooth the line between points. The "straight" style will render a straight line between points. The "closed" property will determine if the drawn line gets automatically closed to form a complete shape or not. Remember that the pen will be *down* until the action is over. If you want to lift up the pen, start a new pen action.',
+	})
+
+export type PenAction = z.infer<typeof PenAction>
+
+// Place Action
+export const PlaceAction = z
+	.object({
+		_type: z.literal('place'),
+		align: z.enum(['start', 'center', 'end']),
+		alignOffset: z.number(),
+		intent: z.string(),
+		referenceShapeId: SimpleShapeIdSchema,
+		side: z.enum(['top', 'bottom', 'left', 'right']),
+		sideOffset: z.number(),
+		shapeId: SimpleShapeIdSchema,
+	})
+	.meta({
+		title: 'Place',
+		description: 'The AI places a shape relative to another shape.',
+		_systemPromptCategory: 'edit',
+	})
+
+export type PlaceAction = z.infer<typeof PlaceAction>
+
+// Resize Action
+export const ResizeAction = z
+	.object({
+		_type: z.literal('resize'),
+		intent: z.string(),
+		originX: z.number(),
+		originY: z.number(),
+		scaleX: z.number(),
+		scaleY: z.number(),
+		shapeIds: z.array(SimpleShapeIdSchema),
+	})
+	.meta({
+		title: 'Resize',
+		description:
+			'The AI resizes one or more shapes, with the resize operation being performed relative to an origin point.',
+		_systemPromptCategory: 'edit',
+	})
+
+export type ResizeAction = z.infer<typeof ResizeAction>
+
+// Review Action
+export const ReviewAction = z
+	.object({
+		_type: z.literal('review'),
+		intent: z.string(),
+		x: z.number(),
+		y: z.number(),
+		w: z.number(),
+		h: z.number(),
+	})
+	.meta({
+		title: 'Review',
+		description:
+			'The AI schedules further work or a review so that it can look at the results of its work so far and take further action, such as reviewing what it has done or taking further steps that would benefit from seeing the results of its work so far.',
+	})
+
+export type ReviewAction = z.infer<typeof ReviewAction>
+
+// Rotate Action
+export const RotateAction = z
+	.object({
+		_type: z.literal('rotate'),
+		centerY: z.number(),
+		degrees: z.number(),
+		intent: z.string(),
+		originX: z.number(),
+		originY: z.number(),
+		shapeIds: z.array(SimpleShapeIdSchema),
+	})
+	.meta({
+		title: 'Rotate',
+		description: 'The AI rotates one or more shapes around an origin point.',
+		_systemPromptCategory: 'edit',
+	})
+
+export type RotateAction = z.infer<typeof RotateAction>
+
+// Send to Back Action
+export const SendToBackAction = z
+	.object({
+		_type: z.literal('sendToBack'),
+		intent: z.string(),
+		shapeIds: z.array(SimpleShapeIdSchema),
+	})
+	.meta({
+		title: 'Send to Back',
+		description:
+			'The AI sends one or more shapes to the back so that they appear behind everything else.',
+		_systemPromptCategory: 'edit',
+	})
+
+export type SendToBackAction = z.infer<typeof SendToBackAction>
+
+// Set My View Action
+export const SetMyViewAction = z
+	.object({
+		_type: z.literal('setMyView'),
+		intent: z.string(),
+		x: z.number(),
+		y: z.number(),
+		w: z.number(),
+		h: z.number(),
+	})
+	.meta({
+		title: 'Set My View',
+		description:
+			'The AI changes the bounds of its own viewport to navigate to other areas of the canvas if needed. `x` and `y` are the top-left corner of the area you want to see, and `w` and `h` are its width and height. To center your view on a point, set `x` and `y` to that point minus half the width and height.',
+	})
+
+export type SetMyViewAction = z.infer<typeof SetMyViewAction>
+
+// Stack Action
+export const StackAction = z
+	.object({
+		_type: z.literal('stack'),
+		direction: z.enum(['vertical', 'horizontal']),
+		gap: z.number(),
+		intent: z.string(),
+		shapeIds: z.array(SimpleShapeIdSchema),
+	})
+	.meta({
+		title: 'Stack',
+		description:
+			"The AI stacks shapes horizontally or vertically. Note that this doesn't align shapes, it only stacks them along one axis.",
+		_systemPromptCategory: 'edit',
+	})
+
+export type StackAction = z.infer<typeof StackAction>
+
+// Think Action
+export const ThinkAction = z
+	.object({
+		_type: z.literal('think'),
+		text: z.string(),
+	})
+	.meta({ title: 'Think', description: 'The AI describes its intent or reasoning.' })
+
+export type ThinkAction = z.infer<typeof ThinkAction>
+
+// Todo List Action
+export const UpsertPersonalTodoItemAction = z
+	.object({
+		_type: z.literal('update-todo-list'),
+		id: TodoIdSchema,
+		status: z.enum(['todo', 'in-progress', 'done']),
+		text: z.string().optional(),
+	})
+	.meta({
+		title: 'Update Todo List',
+		description: 'The AI updates a current todo list item or creates a new one',
+	})
+
+export type UpsertPersonalTodoItemAction = z.infer<typeof UpsertPersonalTodoItemAction>
+
+// Update Action
+export const UpdateAction = z
+	.object({
+		_type: z.literal('update'),
+		intent: z.string(),
+		update: FocusedShapeSchema,
+	})
+	.meta({
+		title: 'Update',
+		description: 'The AI updates an existing shape.',
+		_systemPromptCategory: 'edit',
+	})
+
+export type UpdateAction = z.infer<typeof UpdateAction>
+// Unknown Action (catch-all for unrecognized actions)
+export const UnknownAction = z
+	.object({
+		_type: z.literal('unknown'),
+	})
+	.meta({
+		title: 'Unknown',
+		description: 'An action with an unknown or unrecognized type.',
+	})
+
+export type UnknownAction = z.infer<typeof UnknownAction>

templates/agent/shared/schema/PromptPartDefinitions.ts

@@ -0,0 +1,528 @@
+import { BoxModel, JsonValue } from 'tldraw'
+import { BlurryShape } from '../format/BlurryShape'
+import { FocusedShape } from '../format/FocusedShape'
+import { PeripheralShapeCluster } from '../format/PeripheralShapesCluster'
+import { AgentModelName } from '../models'
+import type { AgentAction } from '../types/AgentAction'
+import { AgentCanvasLint } from '../types/AgentCanvasLint'
+import { AgentMessage, AgentMessageContent } from '../types/AgentMessage'
+import { AgentRequest } from '../types/AgentRequest'
+import { ChatHistoryItem } from '../types/ChatHistoryItem'
+import { ContextItem } from '../types/ContextItem'
+import { SimpleShapeId } from '../types/ids-schema'
+import type { PromptPart, PromptPartDefinition } from '../types/PromptPart'
+import { TodoItem } from '../types/TodoItem'
+
+// ============================================================================
+// Prompt Part Type Interfaces
+// ============================================================================
+
+export interface BlurryShapesPart {
+	type: 'blurryShapes'
+	shapes: BlurryShape[]
+}
+
+export interface CanvasLintsPart {
+	type: 'canvasLints'
+	lints: AgentCanvasLint[]
+}
+
+export interface ChatHistoryPart {
+	type: 'chatHistory'
+	history: ChatHistoryItem[]
+}
+
+export interface ContextItemsPart {
+	type: 'contextItems'
+	items: ContextItem[]
+	requestSource: AgentRequest['source']
+}
+
+export interface DataPart {
+	type: 'data'
+	data: JsonValue[]
+}
+
+export interface MessagesPart {
+	type: 'messages'
+	agentMessages: string[]
+	requestSource: AgentRequest['source']
+}
+
+export interface ModelNamePart {
+	type: 'modelName'
+	modelName: AgentModelName
+}
+
+export interface PeripheralShapesPart {
+	type: 'peripheralShapes'
+	clusters: PeripheralShapeCluster[]
+}
+
+export interface ScreenshotPart {
+	type: 'screenshot'
+	screenshot: string
+}
+
+export interface SelectedShapesPart {
+	type: 'selectedShapes'
+	shapeIds: SimpleShapeId[]
+}
+
+export interface TimePart {
+	type: 'time'
+	time: string
+}
+
+export interface TodoListPart {
+	type: 'todoList'
+	items: TodoItem[]
+}
+
+export interface UserActionHistoryPart {
+	type: 'userActionHistory'
+	added: {
+		shapeId: string
+		type: FocusedShape['_type']
+	}[]
+	removed: {
+		shapeId: string
+		type: FocusedShape['_type']
+	}[]
+	updated: {
+		shapeId: string
+		type: FocusedShape['_type']
+		before: Partial<FocusedShape>
+		after: Partial<FocusedShape>
+	}[]
+}
+
+export interface UserViewportBoundsPart {
+	type: 'userViewportBounds'
+	userBounds: BoxModel | null
+}
+
+export interface AgentViewportBoundsPart {
+	type: 'agentViewportBounds'
+	agentBounds: BoxModel | null
+}
+
+export interface ModePart {
+	type: 'mode'
+	modeType: string
+	partTypes: PromptPart['type'][]
+	actionTypes: AgentAction['_type'][]
+}
+
+export interface DebugPart {
+	type: 'debug'
+	logSystemPrompt: boolean
+	logMessages: boolean
+}
+
+// ============================================================================
+// Prompt Part Definitions
+// ============================================================================
+
+// BlurryShapes
+export const BlurryShapesPartDefinition: PromptPartDefinition<BlurryShapesPart> = {
+	type: 'blurryShapes',
+	priority: -70,
+	buildContent: ({ shapes }) => {
+		if (shapes.length === 0) return ['There are no shapes in your view at the moment.']
+		return [`These are the shapes you can currently see:`, JSON.stringify(shapes)]
+	},
+}
+
+// CanvasLints
+export const CanvasLintsPartDefinition: PromptPartDefinition<CanvasLintsPart> = {
+	type: 'canvasLints',
+	priority: -50,
+	buildContent({ lints }: CanvasLintsPart) {
+		if (!lints || lints.length === 0) {
+			return []
+		}
+
+		const messages: string[] = []
+
+		// Group lints by type
+		const growYLints = lints.filter((l) => l.type === 'growY-on-shape')
+		const overlappingTextLints = lints.filter((l) => l.type === 'overlapping-text')
+		const friendlessArrowLints = lints.filter((l) => l.type === 'friendless-arrow')
+
+		messages.push(
+			"[LINTER]: The following potential visual problems have been detected in the canvas. You should decide if you want to address them. Defer to your view of the canvas to decide if you need to make changes; it's very possible that you don't need to make any changes."
+		)
+
+		if (growYLints.length > 0) {
+			const shapeIds = growYLints.flatMap((l) => l.shapeIds)
+			const lines = [
+				'Text overflow: These shapes have text that caused their containers to grow past the size that they were intended to be, potentially breaking out of their container. If you decide to fix: you need to set the height back to what you originally intended after increasing the width.',
+				...shapeIds.map((id) => `  - ${id}`),
+			]
+			messages.push(lines.join('\n'))
+		}
+
+		if (overlappingTextLints.length > 0) {
+			const lines = [
+				'Overlapping text: The shapes in each group have text and overlap each other, which may make text hard to read. If you decide to fix this, you may need to increase the size of any shapes containing the text.',
+				...overlappingTextLints.map((lint) => `  - ${lint.shapeIds.join(', ')}`),
+			]
+			messages.push(lines.join('\n'))
+		}
+
+		if (friendlessArrowLints.length > 0) {
+			const shapeIds = friendlessArrowLints.flatMap((l) => l.shapeIds)
+			const lines = [
+				"Unconnected arrows: These arrows aren't fully connected to other shapes.",
+				...shapeIds.map((id) => `  - ${id}`),
+			]
+			messages.push(lines.join('\n'))
+		}
+
+		return messages
+	},
+}
+
+// ChatHistory
+const CHAT_HISTORY_PRIORITY = -Infinity // history should appear first in the prompt (low priority)
+
+export const ChatHistoryPartDefinition: PromptPartDefinition<ChatHistoryPart> = {
+	type: 'chatHistory',
+	priority: CHAT_HISTORY_PRIORITY,
+	buildMessages: ({ history }) => {
+		if (history.length === 0) return []
+
+		const messages: AgentMessage[] = []
+
+		// If the last message is from the user, skip it
+		const lastIndex = history.length - 1
+		let end = history.length
+		if (end > 0 && history[lastIndex].type === 'prompt') {
+			end = lastIndex
+		}
+
+		for (let i = 0; i < end; i++) {
+			const item = history[i]
+			const message = buildHistoryItemMessage(item, CHAT_HISTORY_PRIORITY)
+			if (message) messages.push(message)
+		}
+
+		return messages
+	},
+}
+
+function buildHistoryItemMessage(item: ChatHistoryItem, priority: number): AgentMessage | null {
+	switch (item.type) {
+		case 'prompt': {
+			const content: AgentMessageContent[] = []
+
+			if (item.agentFacingMessage.trim() !== '') {
+				content.push({
+					type: 'text',
+					text: item.agentFacingMessage,
+				})
+			}
+
+			if (item.contextItems.length > 0) {
+				for (const contextItem of item.contextItems) {
+					switch (contextItem.type) {
+						case 'shape': {
+							const focusedShape = contextItem.shape
+							content.push({
+								type: 'text',
+								text: `[CONTEXT]: ${JSON.stringify(focusedShape)}`,
+							})
+							break
+						}
+						case 'shapes': {
+							const focusedShapes = contextItem.shapes
+							content.push({
+								type: 'text',
+								text: `[CONTEXT]: ${JSON.stringify(focusedShapes)}`,
+							})
+							break
+						}
+						default: {
+							content.push({
+								type: 'text',
+								text: `[CONTEXT]: ${JSON.stringify(contextItem)}`,
+							})
+							break
+						}
+					}
+				}
+			}
+
+			if (content.length === 0) {
+				return null
+			}
+
+			const role =
+				item.promptSource === 'user' || item.promptSource === 'other-agent' ? 'user' : 'assistant'
+			return {
+				role,
+				content,
+				priority,
+			}
+		}
+		case 'continuation': {
+			if (item.data.length === 0) {
+				return null
+			}
+			const text = `[DATA RETRIEVED]: ${JSON.stringify(item.data)}`
+			return {
+				role: 'assistant',
+				content: [{ type: 'text', text }],
+				priority,
+			}
+		}
+		case 'action': {
+			const { action } = item
+			let text: string
+			switch (action._type) {
+				case 'message': {
+					text = action.text || '<message data lost>'
+					break
+				}
+				case 'think': {
+					text = '[THOUGHT]: ' + (action.text || '<thought data lost>')
+					break
+				}
+				default: {
+					const { complete: _complete, time: _time, ...rawAction } = action || {}
+					text = '[ACTION]: ' + JSON.stringify(rawAction)
+					break
+				}
+			}
+			return {
+				role: 'assistant',
+				content: [{ type: 'text', text }],
+				priority,
+			}
+		}
+	}
+}
+
+// ContextItems
+export const ContextItemsPartDefinition: PromptPartDefinition<ContextItemsPart> = {
+	type: 'contextItems',
+	priority: -55, // context items in middle
+	buildContent: ({ items, requestSource }) => {
+		const messages: string[] = []
+
+		const shapeItems = items.filter((item) => item.type === 'shape')
+		const shapesItems = items.filter((item) => item.type === 'shapes')
+		const areaItems = items.filter((item) => item.type === 'area')
+		const pointItems = items.filter((item) => item.type === 'point')
+
+		// Handle area context items
+		if (areaItems.length > 0) {
+			const isSelf = requestSource === 'self'
+			const areas = areaItems.map((item) => item.bounds)
+			messages.push(
+				isSelf
+					? 'You have decided to focus your view on the following area. Make sure to focus your task here.'
+					: 'The user has specifically brought your attention to the following areas in this request. The user might refer to them as the "area(s)" or perhaps "here" or "there", but either way, it\'s implied that you should focus on these areas in both your reasoning and actions. Make sure to focus your task on these areas:'
+			)
+			for (const area of areas) {
+				messages.push(JSON.stringify(area))
+			}
+		}
+
+		// Handle point context items
+		if (pointItems.length > 0) {
+			const points = pointItems.map((item) => item.point)
+			messages.push(
+				'The user has specifically brought your attention to the following points in this request. The user might refer to them as the "point(s)" or perhaps "here" or "there", but either way, it\'s implied that you should focus on these points in both your reasoning and actions. Make sure to focus your task on these points:'
+			)
+			for (const point of points) {
+				messages.push(JSON.stringify(point))
+			}
+		}
+
+		// Handle individual shape context items
+		if (shapeItems.length > 0) {
+			const shapes = shapeItems.map((item) => item.shape)
+			messages.push(
+				`The user has specifically brought your attention to these ${shapes.length} shapes individually in this request. Make sure to focus your task on these shapes where applicable:`
+			)
+			for (const shape of shapes) {
+				messages.push(JSON.stringify(shape))
+			}
+		}
+
+		// Handle groups of shapes context items
+		for (const contextItem of shapesItems) {
+			const shapes = contextItem.shapes
+			if (shapes.length > 0) {
+				messages.push(
+					`The user has specifically brought your attention to the following group of ${shapes.length} shapes in this request. Make sure to focus your task on these shapes where applicable:`
+				)
+				messages.push(shapes.map((shape) => JSON.stringify(shape)).join('\n'))
+			}
+		}
+
+		return messages
+	},
+}
+
+// Data
+export const DataPartDefinition: PromptPartDefinition<DataPart> = {
+	type: 'data',
+	priority: 200, // API data should come right before the user message but after most other parts
+	buildContent: ({ data }) => {
+		if (data.length === 0) return []
+
+		const formattedData = data.map((item) => {
+			return `${JSON.stringify(item)}`
+		})
+
+		return ["Here's the data you requested:", ...formattedData]
+	},
+}
+
+// Messages
+export const MessagesPartDefinition: PromptPartDefinition<MessagesPart> = {
+	type: 'messages',
+	priority: Infinity, // user message should be last (highest priority)
+	buildContent: ({ agentMessages, requestSource }) => {
+		switch (requestSource) {
+			// we treat all sources the same for the messages part, but you don't have to!
+			case 'user':
+			case 'self':
+			case 'other-agent':
+				return agentMessages
+		}
+	},
+}
+
+// ModelName
+export const ModelNamePartDefinition: PromptPartDefinition<ModelNamePart> = {
+	type: 'modelName',
+	getModelName: (part) => {
+		return part.modelName
+	},
+}
+
+// PeripheralShapes
+export const PeripheralShapesPartDefinition: PromptPartDefinition<PeripheralShapesPart> = {
+	type: 'peripheralShapes',
+	priority: -65, // peripheral content after viewport shapes
+	buildContent: ({ clusters }) => {
+		if (clusters.length === 0) {
+			return []
+		}
+
+		return [
+			"There are some groups of shapes in your peripheral vision, outside the your main view. You can't make out their details or content. If you want to see their content, you need to get closer. The groups are as follows",
+			JSON.stringify(clusters),
+		]
+	},
+}
+
+// Screenshot
+export const ScreenshotPartDefinition: PromptPartDefinition<ScreenshotPart> = {
+	type: 'screenshot',
+	priority: -40, // screenshot after text content
+	buildContent: ({ screenshot }) => {
+		if (screenshot === '') return []
+
+		return [
+			'Here is the part of the canvas that you can currently see at this moment. It is not a reference image.',
+			screenshot,
+		]
+	},
+}
+
+export const SelectedShapesPartDefinition: PromptPartDefinition<SelectedShapesPart> = {
+	type: 'selectedShapes',
+	priority: -55,
+	buildContent({ shapeIds }: SelectedShapesPart) {
+		if (!shapeIds || shapeIds.length === 0) {
+			return []
+		}
+
+		if (shapeIds.length === 1) {
+			return [`The user has this shape selected: ${shapeIds[0]}`]
+		}
+
+		return [`The user has these shapes selected: ${shapeIds.join(', ')}`]
+	},
+}
+
+// Time
+export const TimePartDefinition: PromptPartDefinition<TimePart> = {
+	type: 'time',
+	priority: -100,
+	buildContent({ time }: TimePart) {
+		return [`The user's current time is: ${time}`]
+	},
+}
+
+// TodoList
+export const TodoListPartDefinition: PromptPartDefinition<TodoListPart> = {
+	type: 'todoList',
+	priority: 10,
+	buildContent: ({ items }) => {
+		if (items.length === 0) return ['You have no todos yet.']
+		return [`Here is your current todo list:`, JSON.stringify(items)]
+	},
+}
+
+// UserActionHistory
+export const UserActionHistoryPartDefinition: PromptPartDefinition<UserActionHistoryPart> = {
+	type: 'userActionHistory',
+	priority: -40,
+	buildContent: (part) => {
+		const { updated, removed, added } = part
+		if (updated.length === 0 && removed.length === 0 && added.length === 0) {
+			return []
+		}
+
+		return [
+			'Since the previous request, the user has made the following changes to the canvas:',
+			JSON.stringify(part),
+		]
+	},
+}
+
+// UserViewportBounds
+export const UserViewportBoundsPartDefinition: PromptPartDefinition<UserViewportBoundsPart> = {
+	type: 'userViewportBounds',
+	priority: -80,
+	buildContent({ userBounds }: UserViewportBoundsPart) {
+		if (!userBounds) {
+			return []
+		}
+		return [
+			`The bounds of the part of the canvas that the user can currently see are: ${JSON.stringify(userBounds)}`,
+		]
+	},
+}
+
+// AgentViewportBounds
+export const AgentViewportBoundsPartDefinition: PromptPartDefinition<AgentViewportBoundsPart> = {
+	type: 'agentViewportBounds',
+	priority: -80,
+	buildContent({ agentBounds }: AgentViewportBoundsPart) {
+		if (!agentBounds) {
+			return []
+		}
+		return [
+			`The bounds of the part of the canvas that you can currently see are: ${JSON.stringify(agentBounds)}`,
+		]
+	},
+}
+
+// Mode - sends mode metadata to worker for prompt construction
+export const ModePartDefinition: PromptPartDefinition<ModePart> = {
+	type: 'mode',
+	// No buildContent - this is metadata for the worker, not prompt content for the model
+}
+
+// Debug - sends debug flags to worker for logging
+export const DebugPartDefinition: PromptPartDefinition<DebugPart> = {
+	type: 'debug',
+	// No buildContent - this is metadata for the worker, not prompt content for the model
+}

templates/agent/shared/schema/buildResponseSchema.ts

@@ -0,0 +1,45 @@
+import z from 'zod'
+import { ActionMeta, AgentAction, getActionSchemaForMode } from '../types/AgentAction'
+
+/**
+ * Internal meta keys that should be stripped from the JSON schema.
+ * These are added via .meta() in action schemas but shouldn't be sent to the model.
+ */
+const INTERNAL_META_KEYS: Set<string> = new Set([
+	'_systemPromptCategory',
+] satisfies (keyof ActionMeta)[])
+
+/**
+ * Recursively strips internal meta fields from a JSON schema.
+ * Only removes explicitly listed internal keys, preserving all other fields.
+ */
+export function stripInternalMeta(obj: object): object {
+	if (Array.isArray(obj)) {
+		return obj.map(stripInternalMeta)
+	}
+	if (obj && typeof obj === 'object') {
+		return Object.fromEntries(
+			Object.entries(obj)
+				.filter(([key]) => !INTERNAL_META_KEYS.has(key))
+				.map(([key, value]) => [key, stripInternalMeta(value)])
+		)
+	}
+	return obj
+}
+
+export function buildResponseSchema(actionTypes: AgentAction['_type'][], mode: string) {
+	const actionSchemas = actionTypes
+		.map((type) => getActionSchemaForMode(type, mode))
+		.filter((schema) => schema !== undefined)
+
+	if (actionSchemas.length === 0) {
+		throw new Error('No action schemas found for the provided action types')
+	}
+
+	const actionSchema = z.union(actionSchemas)
+	const schema = z.object({
+		actions: z.array(actionSchema),
+	})
+
+	return stripInternalMeta(z.toJSONSchema(schema, { reused: 'ref' }))
+}

templates/agent/shared/types/AgentAction.ts

@@ -0,0 +1,113 @@
+import z from 'zod'
+import {
+	getDefaultActionSchema,
+	hasDefaultActionSchema,
+	registerActionSchema,
+} from '../schema/AgentActionSchemaRegistry'
+
+// Re-export mode-aware schema lookup
+export { getActionSchemaForMode } from '../schema/AgentActionSchemaRegistry'
+
+// ============================================================================
+// Type Derivation
+// ============================================================================
+
+/**
+ * Import all schemas as a namespace for type extraction.
+ */
+import type * as AllSchemasType from '../schema/AgentActionSchemas'
+
+/** Extract inferred type from a Zod schema */
+type ExtractZodType<T> = T extends z.ZodType<infer U> ? U : never
+
+/** Get all values from the schemas module that are ZodTypes */
+type SchemaExports = (typeof AllSchemasType)[keyof typeof AllSchemasType]
+
+/** Filter to only Zod types (excludes the type aliases) */
+export type AgentActionSchema = Extract<SchemaExports, z.ZodType>
+
+/**
+ * Union of all agent action types, automatically derived from exported schemas.
+ * When you add a new action schema export to AgentActionSchemas.ts, this type
+ * automatically includes it - no arrays or manual type updates needed!
+ */
+export type AgentAction = ExtractZodType<AgentActionSchema>
+
+// ============================================================================
+// Runtime Lookup (built from exports and registered with schema registry)
+// ============================================================================
+
+/** Runtime import of all schemas for building lookup */
+import * as AllSchemas from '../schema/AgentActionSchemas'
+
+/** Type guard to check if a value is a Zod object schema with _type */
+function isActionSchema(value: unknown): value is z.ZodObject<{ _type: z.ZodLiteral<string> }> {
+	if (!(value instanceof z.ZodObject)) return false
+	const shape = (value as z.ZodObject<z.ZodRawShape>).shape
+	if (!('_type' in shape)) return false
+	const typeField = shape._type
+	return typeField instanceof z.ZodLiteral
+}
+
+/** Extract the _type literal value from a schema */
+function getSchemaType(schema: z.ZodObject<{ _type: z.ZodLiteral<string> }>): string {
+	return schema.shape._type.value
+}
+
+/** Build lookup object from all exported schemas and register them as defaults */
+const schemasByType: Record<string, AgentActionSchema> = {}
+for (const value of Object.values(AllSchemas)) {
+	if (!isActionSchema(value)) continue
+	const type = getSchemaType(value)
+	// Skip if a default schema is already registered for this type
+	// (handles cases where multiple schemas share the same _type, e.g., mode-specific variants)
+	if (hasDefaultActionSchema(type)) continue
+	registerActionSchema(type, value)
+	schemasByType[type] = value as AgentActionSchema
+}
+
+/**
+ * Get all action schemas (default schemas only).
+ */
+export function getAllActionSchemas(): AgentActionSchema[] {
+	return Object.values(schemasByType)
+}
+
+/**
+ * Get an action schema by its _type value.
+ * Returns the default schema (ignoring mode).
+ */
+export function getActionSchema(type: string): AgentActionSchema | undefined {
+	return getDefaultActionSchema(type) as AgentActionSchema | undefined
+}
+
+/**
+ * Check if an action schema exists for a given type.
+ */
+export function hasActionSchema(type: string): boolean {
+	return type in schemasByType
+}
+
+// ============================================================================
+// Schema Metadata
+// ============================================================================
+
+import type { SystemPromptCategory } from './SystemPromptCategory'
+
+/**
+ * Metadata that can be attached to action schemas via .meta().
+ */
+export interface ActionMeta {
+	title?: string
+	description?: string
+	_systemPromptCategory?: SystemPromptCategory
+}
+
+/**
+ * Get the metadata for an action schema.
+ */
+export function getActionMeta(type: AgentAction['_type']): ActionMeta | undefined {
+	const schema = getActionSchema(type)
+	if (!schema) return undefined
+	return schema.meta() as ActionMeta | undefined
+}

templates/agent/shared/types/AgentCanvasLint.ts

@@ -0,0 +1,8 @@
+import { SimpleShapeId } from './ids-schema'
+
+export interface AgentCanvasLint {
+	type: AgentCanvasLintType
+	shapeIds: SimpleShapeId[]
+}
+
+export type AgentCanvasLintType = 'growY-on-shape' | 'overlapping-text' | 'friendless-arrow'

templates/agent/shared/types/AgentInput.ts

@@ -0,0 +1,21 @@
+import { AgentRequest } from './AgentRequest'
+
+/**
+ * A user input, used to form an agent request.
+ * This object is more flexible than the AgentRequest object it gets turned into.
+ *
+ * ```ts
+ * const agent = useAgent()
+ * agent.prompt('Draw a cat')
+ * agent.prompt(['Draw a cat', 'Draw a dog'])
+ * agent.prompt({ agentMessages: ['Draw a cat', 'Draw a dog'] })
+ * ```
+ *
+ * The `message` property is a shortcut for providing a single agent message and user message.
+ * You can't specify `agentMessages` and `userMessages` when using the `message` property.
+ */
+export type AgentInput =
+	| Partial<AgentRequest>
+	| (Partial<Omit<AgentRequest, 'agentMessages' | 'userMessages'>> & { message: string })
+	| string
+	| string[]

templates/agent/shared/types/AgentMessage.ts

@@ -0,0 +1,11 @@
+export interface AgentMessage {
+	role: 'user' | 'assistant'
+	content: AgentMessageContent[]
+	priority: number // higher priority (lower numbers) appear later in the prompt
+}
+
+export interface AgentMessageContent {
+	type: 'text' | 'image'
+	text?: string
+	image?: string
+}

templates/agent/shared/types/AgentPrompt.ts

@@ -0,0 +1,24 @@
+import { Editor } from 'tldraw'
+import { AgentRequest } from './AgentRequest'
+import { BasePromptPart } from './BasePromptPart'
+import { PromptPart } from './PromptPart'
+
+/**
+ * AgentPromptOptions contains the information needed to construct all the parts of the prompt.
+ */
+export interface AgentPromptOptions {
+	editor: Editor
+	request: AgentRequest
+}
+
+/**
+ * An AgentPrompt contains all the parts created from prompt options.
+ */
+export type BaseAgentPrompt<T extends BasePromptPart = BasePromptPart> = {
+	[P in T as P['type']]: P
+}
+
+/**
+ * An object with all parts of the prompt, defined by PROMPT_PART_DEFINITIONS in shared/schema.
+ */
+export type AgentPrompt = BaseAgentPrompt<PromptPart>

templates/agent/shared/types/AgentRequest.ts

@@ -0,0 +1,49 @@
+import { BoxModel, JsonValue } from 'tldraw'
+import { ContextItem } from './ContextItem'
+
+/**
+ * A request that we send to the agent.
+ */
+export interface AgentRequest {
+	/**
+	 * Messages associated with the request.
+	 * These are the agent-facing messages that will be sent to the model.
+	 */
+	agentMessages: string[]
+
+	/**
+	 * Optional user-facing messages that will be displayed in the UI.
+	 * Each index corresponds to the same index in the `agentMessages` array.
+	 * If a user-facing message is not provided for a particular message (null),
+	 * the UI may fall back to displaying the agent-facing message.
+	 * If this array is shorter than `agentMessages`, missing entries are treated as null.
+	 */
+	userMessages: string[]
+
+	/**
+	 * The bounds of the request.
+	 */
+	bounds: BoxModel
+
+	/**
+	 * Any extra data that has been retrieved as part of this request.
+	 * All promises in this array will be resolved before the request is sent.
+	 */
+	data: (JsonValue | Promise<JsonValue>)[]
+
+	/**
+	 * Where the request came from.
+	 * - 'user' is a request from the user.
+	 * - 'self' is a request from the agent itself.
+	 * - 'other-agent' is a request from another agent.
+	 */
+	source: 'user' | 'self' | 'other-agent'
+
+	/**
+	 * Context items that were active when this request was created.
+	 * This is a snapshot of the context at request creation time.
+	 */
+	contextItems: ContextItem[]
+}
+
+export type AgentRequestSource = AgentRequest['source']

templates/agent/shared/types/BaseAgentAction.ts

@@ -0,0 +1,6 @@
+/**
+ * This type represents the bare minimum properties that all agent actions must have.
+ */
+export interface BaseAgentAction<T extends string = string> {
+	_type: T
+}

templates/agent/shared/types/BasePromptPart.ts

@@ -0,0 +1,6 @@
+/**
+ * This type represents the bare minimum properties that all prompt parts must have.
+ */
+export interface BasePromptPart<T extends string = string> {
+	type: T
+}

templates/agent/shared/types/ChatHistoryInfo.ts

@@ -0,0 +1,35 @@
+import { AgentIconType } from '../icons/AgentIcon'
+import { BaseAgentAction } from './BaseAgentAction'
+import { Streaming } from './Streaming'
+
+/**
+ * Information on how the action should be displayed in chat history.
+ */
+export interface ChatHistoryInfo {
+	/**
+	 * The icon to display in chat history.
+	 * Null to not show an icon.
+	 */
+	icon: AgentIconType | null
+
+	/**
+	 * The description to display in chat history.
+	 * Null to not show a description.
+	 * Defaults to the stringified action if not set.
+	 */
+	description: string | null
+
+	/**
+	 * A short summary that can be shown when the action is collapsed.
+	 * Null to disable collapsing for this action.
+	 */
+	summary: string | null
+
+	/**
+	 * A function that determines whether the action can be grouped with another action.
+	 * By default, the action will automatically group with everything.
+	 * @param other - The other action
+	 * @returns Whether the action can be grouped with the other action
+	 */
+	canGroup(other: Streaming<BaseAgentAction>): boolean
+}

templates/agent/shared/types/ChatHistoryItem.ts

@@ -0,0 +1,41 @@
+import { JsonValue, RecordsDiff, TLRecord } from 'tldraw'
+import { FocusedShape } from '../format/FocusedShape'
+import { AgentAction } from './AgentAction'
+import { AgentRequestSource } from './AgentRequest'
+import { ContextItem } from './ContextItem'
+import { Streaming } from './Streaming'
+
+export type ChatHistoryItem =
+	| ChatHistoryActionItem
+	| ChatHistoryPromptItem
+	| ChatHistoryContinuationItem
+
+/**
+ * A prompt from a user, another agent, or the agent itself.
+ */
+export interface ChatHistoryPromptItem {
+	type: 'prompt'
+	promptSource: AgentRequestSource
+	agentFacingMessage: string
+	userFacingMessage: string | null
+	contextItems: ContextItem[]
+	selectedShapes: FocusedShape[]
+}
+
+/**
+ * An action done by the agent.
+ */
+export interface ChatHistoryActionItem {
+	type: 'action'
+	action: Streaming<AgentAction>
+	diff: RecordsDiff<TLRecord>
+	acceptance: 'pending' | 'accepted' | 'rejected'
+}
+
+/**
+ * A follow-up request from the agent, with data retrieved from the previous request.
+ */
+export interface ChatHistoryContinuationItem {
+	type: 'continuation'
+	data: JsonValue[]
+}

templates/agent/shared/types/ContextItem.ts

@@ -0,0 +1,70 @@
+import { BoxModel, Editor, VecModel } from 'tldraw'
+import { FocusedShape } from '../format/FocusedShape'
+import { AgentIconType } from '../icons/AgentIcon'
+
+export type ContextItem = ShapeContextItem | AreaContextItem | PointContextItem | ShapesContextItem
+
+export interface ShapeContextItem {
+	type: 'shape'
+	shape: FocusedShape
+	source: 'agent' | 'user'
+}
+
+export interface ShapesContextItem {
+	type: 'shapes'
+	shapes: FocusedShape[]
+	source: 'agent' | 'user'
+}
+
+export interface AreaContextItem {
+	type: 'area'
+	bounds: BoxModel
+	source: 'agent' | 'user'
+}
+
+export interface PointContextItem {
+	type: 'point'
+	point: VecModel
+	source: 'agent' | 'user'
+}
+
+export const CONTEXT_TYPE_DEFINITIONS: Record<
+	ContextItem['type'],
+	{
+		icon: AgentIconType
+		name(item: ContextItem, editor: Editor): string
+	}
+> = {
+	shape: {
+		icon: 'target',
+		name: (item: ShapeContextItem) => {
+			let name = item.shape.note
+			if (!name) {
+				name = item.shape._type
+				if (item.shape._type === 'draw') {
+					name = 'drawing'
+				} else if (item.shape._type === 'unknown') {
+					name = item.shape.subType
+				}
+			}
+
+			return name[0].toUpperCase() + name.slice(1)
+		},
+	},
+	area: {
+		icon: 'target',
+		name: () => 'Area',
+	},
+	point: {
+		icon: 'target',
+		name: () => 'Point',
+	},
+	shapes: {
+		icon: 'target',
+		name: (item: ShapesContextItem, editor: Editor) => {
+			const count = item.shapes.length
+			if (count === 1) return CONTEXT_TYPE_DEFINITIONS['shape'].name(item, editor)
+			return `${count} shapes`
+		},
+	},
+}

templates/agent/shared/types/PromptPart.ts

@@ -0,0 +1,126 @@
+import type { AgentModelName } from '../models'
+import type { AgentMessage } from './AgentMessage'
+import type { BasePromptPart } from './BasePromptPart'
+
+// ============================================================================
+// Prompt Part Definition Interface
+// ============================================================================
+
+/**
+ * A prompt part definition contains pure transformation functions for converting
+ * prompt part data into messages, content, or system prompts.
+ */
+export interface PromptPartDefinition<T extends BasePromptPart> {
+	/**
+	 * The unique type of the prompt part.
+	 */
+	type: T['type']
+
+	/**
+	 * The priority of the prompt part. Higher priority = later in the prompt.
+	 */
+	priority?: number
+
+	/**
+	 * Override the default buildContent function to choose which content to send to the model.
+	 * Gives you less control than buildMessages, but is much simpler to use.
+	 */
+	buildContent?(part: T): string[]
+
+	/**
+	 * Override the default buildMessages function to choose how to turn content into AgentMessages.
+	 */
+	buildMessages?(part: T): AgentMessage[]
+
+	/**
+	 * Override the default getModelName function to choose which model to use for the prompt part.
+	 */
+	getModelName?(part: T): AgentModelName | null
+}
+
+// ============================================================================
+// Type Derivation
+// ============================================================================
+
+/**
+ * Import all definitions as a namespace for type extraction.
+ */
+import type * as AllDefinitionsType from '../schema/PromptPartDefinitions'
+
+/** Extract the prompt part type from a definition */
+type ExtractPromptPart<T> = T extends PromptPartDefinition<infer U> ? U : never
+
+/** Get all values from the definitions module that are PromptPartDefinitions */
+type DefinitionExports = (typeof AllDefinitionsType)[keyof typeof AllDefinitionsType]
+
+/** Filter to only PromptPartDefinition types (excludes type aliases and other exports) */
+export type PromptPartDefinitionType = Extract<
+	DefinitionExports,
+	PromptPartDefinition<BasePromptPart>
+>
+
+/**
+ * Union of all prompt part types, automatically derived from exported definitions.
+ * When you add a new definition export to PromptPartDefinitions.ts, this type
+ * automatically includes it - no arrays or manual type updates needed!
+ */
+export type PromptPart = ExtractPromptPart<PromptPartDefinitionType>
+
+// ============================================================================
+// Runtime Lookup (built from exports)
+// ============================================================================
+
+/** Runtime import of all definitions for building lookup */
+import * as AllDefinitions from '../schema/PromptPartDefinitions'
+
+/** Type guard to check if a value is a PromptPartDefinition */
+function isPromptPartDefinition(value: unknown): value is PromptPartDefinition<BasePromptPart> {
+	return (
+		typeof value === 'object' &&
+		value !== null &&
+		typeof value !== 'function' &&
+		'type' in value &&
+		typeof (value as PromptPartDefinition<BasePromptPart>).type === 'string'
+	)
+}
+
+/** Build lookup object from all exported definitions */
+const definitionsByType: Record<string, PromptPartDefinition<BasePromptPart>> = Object.fromEntries(
+	(
+		Object.values(AllDefinitions).filter(
+			isPromptPartDefinition
+		) as PromptPartDefinition<BasePromptPart>[]
+	).map((def) => [def.type, def])
+)
+
+/**
+ * Get a prompt part definition by its type.
+ */
+export function getPromptPartDefinition(type: string): PromptPartDefinition<PromptPart> {
+	const definition = definitionsByType[type]
+	if (!definition) {
+		throw new Error(`No prompt part definition found for type: ${type}`)
+	}
+	return definition as PromptPartDefinition<PromptPart>
+}
+
+/**
+ * Check if a prompt part definition exists for a given type.
+ */
+export function hasPromptPartDefinition(type: string): boolean {
+	return type in definitionsByType
+}
+
+/**
+ * Get all prompt part definitions.
+ */
+export function getAllPromptPartDefinitions(): PromptPartDefinition<PromptPart>[] {
+	return Object.values(definitionsByType) as PromptPartDefinition<PromptPart>[]
+}
+
+/**
+ * Get all available prompt part types.
+ */
+export function getAllPromptPartTypes(): string[] {
+	return Object.keys(definitionsByType)
+}

templates/agent/shared/types/Streaming.ts

@@ -0,0 +1,10 @@
+/**
+ * An object that is currently being streamed to the user, so may or may not be complete.
+ * If it's not yet complete, the object will be a partial and `complete` will be set to false.
+ * If it's complete, the object will be the full object and `complete` will be set to true.
+ *
+ * The object also has a property for how many milliseconds have passed since the streaming started.
+ */
+export type Streaming<T> =
+	| (Partial<T> & { complete: false; time: number })
+	| (T & { complete: true; time: number })

templates/agent/shared/types/SystemPromptCategory.ts

@@ -0,0 +1,8 @@
+/**
+ * Categories used for deriving flags in getSystemPromptFlags.ts.
+ *
+ * These categories can be applied to actions (via _systemPromptCategory in .meta())
+ *
+ * If you like, you could extend the PromptPartDefinition interface to include a _systemPromptCategory field as well.
+ */
+export type SystemPromptCategory = 'edit' // Actions that modify shapes (used to derive `canEdit` flag)

templates/agent/shared/types/TodoItem.ts

@@ -0,0 +1,7 @@
+import { TodoId } from './ids-schema'
+
+export interface TodoItem {
+	id: TodoId
+	text: string
+	status: 'todo' | 'in-progress' | 'done'
+}

templates/agent/shared/types/ids-schema.ts

@@ -0,0 +1,21 @@
+import z from 'zod'
+
+export const SimpleShapeIdSchema = z.string().brand('simpleShapeId')
+export type SimpleShapeId = z.infer<typeof SimpleShapeIdSchema>
+
+export type SimpleShapeIdKey = `${SimpleShapeId}_${string}`
+
+export const TldrawShapeIdSchema = z.string().brand('tldrawShapeId')
+export type TldrawShapeId = z.infer<typeof TldrawShapeIdSchema>
+
+export function toTldrawShapeId(id: SimpleShapeId): TldrawShapeId {
+	return `shape:${id}` as TldrawShapeId
+}
+
+export function toSimpleShapeId(id: string): SimpleShapeId {
+	if (id.startsWith('shape:')) throw new Error('Invalid simple shape id')
+	return id as SimpleShapeId
+}
+
+export const TodoIdSchema = z.number().brand('todoId')
+export type TodoId = z.infer<typeof TodoIdSchema>