snapshot of https://github.com/tldraw/tldraw.git (main, 2026-09-23; blobs > 131072B omitted)

templates/agent/client/actions/AddDetailActionUtil.ts

@@ -0,0 +1,25 @@
+import { AddDetailAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const AddDetailActionUtil = registerActionUtil(
+	class AddDetailActionUtil extends AgentActionUtil<AddDetailAction> {
+		static override type = 'add-detail' as const
+
+		override getInfo(action: Streaming<AddDetailAction>) {
+			const label = 'Adding detail'
+			const text = action.intent?.startsWith('#') ? `\n\n${action.intent}` : action.intent
+			const description = `**${label}:** ${text ?? ''}`
+
+			return {
+				icon: 'pencil' as const,
+				description,
+			}
+		}
+
+		override applyAction(action: Streaming<AddDetailAction>) {
+			if (!action.complete) return
+			this.agent.schedule('Add detail to the canvas.')
+		}
+	}
+)

templates/agent/client/actions/AgentActionUtil.ts

@@ -0,0 +1,167 @@
+import { Editor } from 'tldraw'
+import { AgentAction } from '../../shared/types/AgentAction'
+import { BaseAgentAction } from '../../shared/types/BaseAgentAction'
+import { ChatHistoryInfo } from '../../shared/types/ChatHistoryInfo'
+import { Streaming } from '../../shared/types/Streaming'
+import { TldrawAgent } from '../agent/TldrawAgent'
+import { AgentHelpers } from '../AgentHelpers'
+
+// ============================================================================
+// Registry
+// ============================================================================
+
+/**
+ * Options for registering an action util.
+ */
+export interface RegisterActionUtilOptions {
+	/**
+	 * If specified, this util will only be used when the agent is in one of these modes.
+	 * Otherwise, it will be the default util for this action type.
+	 */
+	forModes?: string[]
+}
+
+// Default registry: actionType -> util (used when no mode-specific util is registered)
+const defaultRegistry = new Map<string, AgentActionUtilConstructor<BaseAgentAction>>()
+
+// Mode registry: actionType -> (mode -> util) (mode-specific overrides)
+const modeRegistry = new Map<string, Map<string, AgentActionUtilConstructor<BaseAgentAction>>>()
+
+/**
+ * Register an agent action util class. Call this after defining each util class.
+ *
+ * @param util - The action util class to register.
+ * @param options - Optional configuration for mode-specific registration.
+ * @returns The registered util class.
+ */
+export function registerActionUtil<T extends AgentActionUtilConstructor<BaseAgentAction>>(
+	util: T,
+	options?: RegisterActionUtilOptions
+): T {
+	const { forModes } = options ?? {}
+
+	if (forModes && forModes.length > 0) {
+		// Mode-specific registration
+		if (!modeRegistry.has(util.type)) {
+			modeRegistry.set(util.type, new Map())
+		}
+		const modeMap = modeRegistry.get(util.type)!
+		for (const mode of forModes) {
+			if (modeMap.has(mode)) {
+				throw new Error(`Action util for ${util.type} already registered for mode ${mode}`)
+			}
+			modeMap.set(mode, util)
+		}
+	} else {
+		// Default registration (existing behavior)
+		if (defaultRegistry.has(util.type)) {
+			throw new Error(`Agent action util already registered: ${util.type}`)
+		}
+		defaultRegistry.set(util.type, util)
+	}
+
+	return util
+}
+
+/**
+ * Get all registered agent action util classes.
+ * Returns both default and mode-specific utils (deduplicated).
+ */
+export function getAllActionUtils(): AgentActionUtilConstructor<AgentAction>[] {
+	const allUtils = new Set<AgentActionUtilConstructor<BaseAgentAction>>()
+
+	// Add all default utils
+	for (const util of defaultRegistry.values()) {
+		allUtils.add(util)
+	}
+
+	// Add all mode-specific utils
+	for (const modeMap of modeRegistry.values()) {
+		for (const util of modeMap.values()) {
+			allUtils.add(util)
+		}
+	}
+
+	return Array.from(allUtils) as AgentActionUtilConstructor<AgentAction>[]
+}
+
+/**
+ * Get an object containing instantiated agent action utils for an agent,
+ * resolved for a specific mode. Mode-specific utils override defaults.
+ *
+ * @param agent - The agent to create utils for.
+ * @param mode - The mode to resolve utils for.
+ * @returns A record of action utils keyed by action type.
+ */
+export function getAgentActionUtilsRecordForMode(agent: TldrawAgent, mode: string) {
+	const object = {} as Record<AgentAction['_type'], AgentActionUtil<AgentAction>>
+
+	// Start with defaults
+	for (const [type, util] of defaultRegistry.entries()) {
+		object[type as AgentAction['_type']] = new util(agent) as AgentActionUtil<AgentAction>
+	}
+
+	// Override with mode-specific utils
+	for (const [type, modeMap] of modeRegistry.entries()) {
+		const modeUtil = modeMap.get(mode)
+		if (modeUtil) {
+			object[type as AgentAction['_type']] = new modeUtil(agent) as AgentActionUtil<AgentAction>
+		}
+	}
+
+	return object
+}
+
+// ============================================================================
+// Base Class
+// ============================================================================
+
+export abstract class AgentActionUtil<T extends BaseAgentAction = BaseAgentAction> {
+	static type: string
+
+	agent: TldrawAgent
+	editor: Editor
+
+	constructor(agent: TldrawAgent) {
+		this.agent = agent
+		this.editor = agent?.editor
+	}
+
+	/**
+	 * Get information about the action to display within the chat history UI.
+	 * Return null to not show anything.
+	 * Defaults to the stringified action if not set.
+	 */
+	getInfo(_action: Streaming<T>): Partial<ChatHistoryInfo> | null {
+		return {}
+	}
+
+	/**
+	 * Transforms the action before saving it to chat history.
+	 * Useful for sanitizing or correcting actions.
+	 * @returns The transformed action, or null to reject the action
+	 */
+	sanitizeAction(action: Streaming<T>, _helpers: AgentHelpers): Streaming<T> | null {
+		return action
+	}
+
+	/**
+	 * Apply the action to the editor.
+	 * Any changes that happen during this function will be displayed as a diff.
+	 */
+	applyAction(_action: Streaming<T>, _helpers: AgentHelpers): Promise<void> | void {
+		// Do nothing by default
+	}
+
+	/**
+	 * Whether the action gets saved to history.
+	 */
+	savesToHistory(): boolean {
+		return true
+	}
+}
+
+export interface AgentActionUtilConstructor<T extends BaseAgentAction = BaseAgentAction> {
+	new (agent: TldrawAgent): AgentActionUtil<T>
+	type: T['_type']
+}

templates/agent/client/actions/AlignActionUtil.ts

@@ -0,0 +1,32 @@
+import { TLShapeId } from 'tldraw'
+import { AlignAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentHelpers } from '../AgentHelpers'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const AlignActionUtil = registerActionUtil(
+	class AlignActionUtil extends AgentActionUtil<AlignAction> {
+		static override type = 'align' as const
+
+		override getInfo(action: Streaming<AlignAction>) {
+			return {
+				icon: 'cursor' as const,
+				description: action.intent ?? '',
+			}
+		}
+
+		override sanitizeAction(action: Streaming<AlignAction>, helpers: AgentHelpers) {
+			action.shapeIds = helpers.ensureShapeIdsExist(action.shapeIds ?? [])
+			return action
+		}
+
+		override applyAction(action: Streaming<AlignAction>) {
+			if (!action.complete) return
+
+			this.editor.alignShapes(
+				action.shapeIds.map((id) => `shape:${id}` as TLShapeId),
+				action.alignment
+			)
+		}
+	}
+)

templates/agent/client/actions/BringToFrontActionUtil.ts

@@ -0,0 +1,28 @@
+import { TLShapeId } from 'tldraw'
+import { BringToFrontAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentHelpers } from '../AgentHelpers'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const BringToFrontActionUtil = registerActionUtil(
+	class BringToFrontActionUtil extends AgentActionUtil<BringToFrontAction> {
+		static override type = 'bringToFront' as const
+
+		override getInfo(action: Streaming<BringToFrontAction>) {
+			return {
+				icon: 'cursor' as const,
+				description: action.intent ?? '',
+			}
+		}
+
+		override sanitizeAction(action: Streaming<BringToFrontAction>, helpers: AgentHelpers) {
+			action.shapeIds = helpers.ensureShapeIdsExist(action.shapeIds ?? [])
+			return action
+		}
+
+		override applyAction(action: Streaming<BringToFrontAction>) {
+			if (!action.shapeIds) return
+			this.editor.bringToFront(action.shapeIds.map((shapeId) => `shape:${shapeId}` as TLShapeId))
+		}
+	}
+)

templates/agent/client/actions/ClearActionUtil.ts

@@ -0,0 +1,37 @@
+import { ClearAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const ClearActionUtil = registerActionUtil(
+	class ClearActionUtil extends AgentActionUtil<ClearAction> {
+		static override type = 'clear' as const
+
+		/**
+		 * Tell the model what the action's schema is
+		 */
+
+		/**
+		 * Tell the model how to display this action in the chat history UI
+		 */
+		override getInfo() {
+			return {
+				icon: 'trash' as const,
+				description: 'Cleared the canvas',
+			}
+		}
+
+		/**
+		 * Tell the model how to apply the action
+		 */
+		override applyAction(action: Streaming<ClearAction>) {
+			// Don't do anything if the action hasn't finished streaming
+			if (!action.complete) return
+
+			// Delete all shapes on the page
+			const { editor } = this
+
+			const allShapes = editor.getCurrentPageShapes()
+			editor.deleteShapes(allShapes)
+		}
+	}
+)

templates/agent/client/actions/CountShapesActionUtil.ts

@@ -0,0 +1,28 @@
+import { CountShapesAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentHelpers } from '../AgentHelpers'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const CountShapesActionUtil = registerActionUtil(
+	class CountShapesActionUtil extends AgentActionUtil<CountShapesAction> {
+		static override type = 'count' as const
+
+		override getInfo(action: Streaming<CountShapesAction>) {
+			const description = action.complete ? 'Counted shapes' : 'Counting shapes'
+			return {
+				icon: 'search' as const,
+				description,
+			}
+		}
+
+		override async applyAction(action: Streaming<CountShapesAction>, helpers: AgentHelpers) {
+			if (!action.complete) return
+			const { agent, editor } = helpers
+
+			// Add the shape count to the next request
+			agent.schedule({
+				data: [`Number of shapes on the canvas: ${editor.getCurrentPageShapes().length}`],
+			})
+		}
+	}
+)

templates/agent/client/actions/CountryInfoActionUtil.ts

@@ -0,0 +1,41 @@
+import { JsonValue } from 'tldraw'
+import { CountryInfoAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const CountryInfoActionUtil = registerActionUtil(
+	class CountryInfoActionUtil extends AgentActionUtil<CountryInfoAction> {
+		static override type = 'countryInfo' as const
+
+		override getInfo(action: Streaming<CountryInfoAction>) {
+			const description = action.complete
+				? 'Searched for country info'
+				: 'Searching for country info'
+			return {
+				icon: 'search' as const,
+				description,
+			}
+		}
+
+		override async applyAction(action: Streaming<CountryInfoAction>) {
+			// Wait until the action has finished streaming
+			if (!action.complete) return
+			const data = await fetchCountryInfo(action.code)
+			this.agent.schedule({ data: [data] })
+		}
+	}
+)
+
+export async function fetchCountryInfo(code: string) {
+	const response = await fetch(`https://restcountries.com/v3.1/alpha/${code}`)
+
+	if (!response.ok) {
+		throw new Error(`Country API returned status ${response.status}, ${response.statusText}`)
+	}
+
+	const json = await response.json()
+	if (Array.isArray(json)) {
+		return json[0] as JsonValue
+	}
+	return json as JsonValue
+}

templates/agent/client/actions/CreateActionUtil.ts

@@ -0,0 +1,230 @@
+import { IndexKey, TLShape, TLShapeId, toRichText } from 'tldraw'
+import {
+	convertPartialFocusedShapeToTldrawShape,
+	FOCUSED_TO_GEO_TYPES,
+} from '../../shared/format/convertFocusedShapeToTldrawShape'
+import { FocusedShape } from '../../shared/format/FocusedShape'
+import { CreateAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentHelpers } from '../AgentHelpers'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const CreateActionUtil = registerActionUtil(
+	class CreateActionUtil extends AgentActionUtil<CreateAction> {
+		static override type = 'create' as const
+
+		override getInfo(action: Streaming<CreateAction>) {
+			return {
+				icon: 'pencil' as const,
+				description: action.intent ?? '',
+			}
+		}
+
+		override sanitizeAction(action: Streaming<CreateAction>, helpers: AgentHelpers) {
+			const { shape } = action
+
+			// If there's no shape yet, return action (will be filtered in applyAction)
+			if (!shape) return action
+
+			// Ensure the created shape has a unique ID (only if shapeId is present)
+			if (shape.shapeId) {
+				shape.shapeId = helpers.ensureShapeIdIsUnique(shape.shapeId)
+			}
+
+			// If the shape is an arrow and complete, ensure the from and to IDs are real shapes
+			if (action.complete && shape._type === 'arrow') {
+				if (shape.fromId) {
+					shape.fromId = helpers.ensureShapeIdExists(shape.fromId)
+				}
+				if (shape.toId) {
+					shape.toId = helpers.ensureShapeIdExists(shape.toId)
+				}
+				if ('x1' in shape) {
+					shape.x1 = helpers.ensureValueIsNumber(shape.x1) ?? 0
+				}
+				if ('y1' in shape) {
+					shape.y1 = helpers.ensureValueIsNumber(shape.y1) ?? 0
+				}
+				if ('x2' in shape) {
+					shape.x2 = helpers.ensureValueIsNumber(shape.x2) ?? 0
+				}
+				if ('y2' in shape) {
+					shape.y2 = helpers.ensureValueIsNumber(shape.y2) ?? 0
+				}
+				if ('bend' in shape) {
+					shape.bend = helpers.ensureValueIsNumber(shape.bend) ?? 0
+				}
+			}
+
+			return action
+		}
+
+		override applyAction(action: Streaming<CreateAction>, helpers: AgentHelpers) {
+			const { editor } = this
+			const { shape } = action
+
+			// If there's no shape yet, return early
+			if (!shape || !shape._type) return
+
+			// Translate the shape back to the chat's position
+			const shapePartial = helpers.removeOffsetFromShapePartial(shape)
+
+			const result = convertPartialFocusedShapeToTldrawShape(editor, shapePartial, {
+				defaultShape: getDefaultShape(shape._type, action.complete),
+				complete: action.complete,
+			})
+
+			if (!result.shape) return
+
+			editor.createShape(result.shape)
+
+			// Handle arrow bindings if they exist
+			if (result.bindings) {
+				for (const binding of result.bindings) {
+					editor.createBinding({
+						type: binding.type,
+						fromId: binding.fromId,
+						toId: binding.toId,
+						props: binding.props,
+						meta: binding.meta,
+					})
+				}
+			}
+		}
+	}
+)
+
+function getDefaultShape(shapeType: FocusedShape['_type'], complete: boolean): Partial<TLShape> {
+	const isGeo = shapeType in FOCUSED_TO_GEO_TYPES
+	const defaultShape = isGeo
+		? SHAPE_DEFAULTS.geo
+		: (SHAPE_DEFAULTS[shapeType as keyof typeof SHAPE_DEFAULTS] ?? SHAPE_DEFAULTS.unknown)
+	return complete ? defaultShape : { ...defaultShape, isLocked: true }
+}
+
+type BaseShapeDefaults = Pick<TLShape, 'id' | 'isLocked' | 'opacity' | 'rotation' | 'meta'>
+
+const SHARED_DEFAULTS: BaseShapeDefaults = {
+	isLocked: false,
+	opacity: 1,
+	rotation: 0,
+	meta: {},
+	id: 'shape:shape' as TLShapeId,
+}
+
+type ShapeDefaults<T extends TLShape = TLShape> = BaseShapeDefaults & {
+	props?: Partial<T['props']>
+}
+
+type SupportedShapeTypes = 'text' | 'line' | 'arrow' | 'geo' | 'note' | 'draw'
+
+type ShapeDefaultsMap = {
+	[K in SupportedShapeTypes]: ShapeDefaults<TLShape<K>>
+} & {
+	unknown: ShapeDefaults
+}
+
+const SHAPE_DEFAULTS: ShapeDefaultsMap = {
+	text: {
+		...SHARED_DEFAULTS,
+		props: {
+			autoSize: true,
+			color: 'black',
+			font: 'draw',
+			richText: toRichText(''),
+			scale: 1,
+			size: 's',
+			textAlign: 'start',
+			w: 100,
+		},
+	},
+	line: {
+		...SHARED_DEFAULTS,
+		props: {
+			size: 's',
+			color: 'black',
+			dash: 'draw',
+			points: {
+				a1: {
+					id: 'a1',
+					index: 'a1' as IndexKey,
+					x: 0,
+					y: 0,
+				},
+				a2: {
+					id: 'a2',
+					index: 'a2' as IndexKey,
+					x: 100,
+					y: 0,
+				},
+			},
+			scale: 1,
+			spline: 'line',
+		},
+	},
+	arrow: {
+		...SHARED_DEFAULTS,
+		props: {
+			arrowheadEnd: 'arrow',
+			arrowheadStart: 'none',
+			bend: 0,
+			color: 'black',
+			dash: 'draw',
+			elbowMidPoint: 0.5,
+			end: { x: 100, y: 0 },
+			fill: 'none',
+			font: 'draw',
+			kind: 'arc',
+			labelColor: 'black',
+			labelPosition: 0.5,
+			richText: toRichText(''),
+			scale: 1,
+			size: 's',
+			start: { x: 0, y: 0 },
+		},
+	},
+	geo: {
+		...SHARED_DEFAULTS,
+		props: {
+			align: 'middle',
+			color: 'black',
+			dash: 'draw',
+			fill: 'none',
+			font: 'draw',
+			geo: 'rectangle',
+			growY: 0,
+			h: 100,
+			labelColor: 'black',
+			richText: toRichText(''),
+			scale: 1,
+			size: 's',
+			url: '',
+			verticalAlign: 'middle',
+			w: 100,
+		},
+	},
+	note: {
+		...SHARED_DEFAULTS,
+		props: {
+			color: 'black',
+			richText: toRichText(''),
+			size: 's',
+			align: 'middle',
+			font: 'draw',
+			fontSizeAdjustment: 1,
+			growY: 0,
+			labelColor: 'black',
+			scale: 1,
+			url: '',
+			verticalAlign: 'middle',
+		},
+	},
+	draw: {
+		...SHARED_DEFAULTS,
+		props: {},
+	},
+	unknown: {
+		...SHARED_DEFAULTS,
+		props: {},
+	},
+}

templates/agent/client/actions/DeleteActionUtil.ts

@@ -0,0 +1,36 @@
+import { TLShapeId } from 'tldraw'
+import { DeleteAction } from '../../shared/schema/AgentActionSchemas'
+import { BaseAgentAction } from '../../shared/types/BaseAgentAction'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentHelpers } from '../AgentHelpers'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const DeleteActionUtil = registerActionUtil(
+	class DeleteActionUtil extends AgentActionUtil<DeleteAction> {
+		static override type = 'delete' as const
+
+		override getInfo(action: Streaming<DeleteAction>) {
+			return {
+				icon: 'trash' as const,
+				description: action.intent ?? '',
+				canGroup: (other: Streaming<BaseAgentAction>) => other._type === 'delete',
+			}
+		}
+
+		override sanitizeAction(action: Streaming<DeleteAction>, helpers: AgentHelpers) {
+			if (!action.complete) return action
+
+			const shapeId = helpers.ensureShapeIdExists(action.shapeId)
+			if (!shapeId) return null
+
+			action.shapeId = shapeId
+			return action
+		}
+
+		override applyAction(action: Streaming<DeleteAction>) {
+			if (!action.complete) return
+
+			this.editor.deleteShape(`shape:${action.shapeId}` as TLShapeId)
+		}
+	}
+)

templates/agent/client/actions/DistributeActionUtil.ts

@@ -0,0 +1,32 @@
+import { TLShapeId } from 'tldraw'
+import { DistributeAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentHelpers } from '../AgentHelpers'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const DistributeActionUtil = registerActionUtil(
+	class DistributeActionUtil extends AgentActionUtil<DistributeAction> {
+		static override type = 'distribute' as const
+
+		override getInfo(action: Streaming<DistributeAction>) {
+			return {
+				icon: 'cursor' as const,
+				description: action.intent ?? '',
+			}
+		}
+
+		override sanitizeAction(action: Streaming<DistributeAction>, helpers: AgentHelpers) {
+			action.shapeIds = helpers.ensureShapeIdsExist(action.shapeIds ?? [])
+			return action
+		}
+
+		override applyAction(action: Streaming<DistributeAction>) {
+			if (!action.complete) return
+
+			this.editor.distributeShapes(
+				action.shapeIds.map((id) => `shape:${id}` as TLShapeId),
+				action.direction
+			)
+		}
+	}
+)

templates/agent/client/actions/LabelActionUtil.ts

@@ -0,0 +1,66 @@
+import { TLRichText, TLShape, TLShapeId, toRichText } from 'tldraw'
+import { LabelAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentHelpers } from '../AgentHelpers'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+type ShapeWithRichText = Extract<TLShape, { props: { richText: TLRichText } }>
+
+function isShapeWithRichText(shape: TLShape | null | undefined): shape is ShapeWithRichText {
+	return !!(shape && 'richText' in shape.props)
+}
+
+function assertShapeWithRichText(
+	shape: TLShape | null | undefined
+): asserts shape is ShapeWithRichText {
+	if (!isShapeWithRichText(shape)) {
+		throw new Error('Shape is not a valid ShapeWithRichText')
+	}
+}
+
+export const LabelActionUtil = registerActionUtil(
+	class LabelActionUtil extends AgentActionUtil<LabelAction> {
+		static override type = 'label' as const
+
+		override getInfo(action: Streaming<LabelAction>) {
+			return {
+				icon: 'pencil' as const,
+				description: action.intent ?? '',
+			}
+		}
+
+		override sanitizeAction(action: Streaming<LabelAction>, helpers: AgentHelpers) {
+			if (!action.complete) return action
+
+			const shapeId = helpers.ensureShapeIdExists(action.shapeId)
+			if (!shapeId) {
+				return null
+			}
+			const shape = this.editor.getShape(`shape:${shapeId}` as TLShapeId)
+			if (!shape) {
+				return null
+			}
+			if (!isShapeWithRichText(shape)) {
+				console.warn(`Shape type "${shape.type}" does not support richText labels`)
+				return null
+			}
+			action.shapeId = shapeId
+			return action
+		}
+
+		override applyAction(action: Streaming<LabelAction>) {
+			if (!action.complete) return
+			const { editor } = this
+
+			const shapeId = `shape:${action.shapeId}` as TLShapeId
+			const shape = editor.getShape(shapeId)
+			assertShapeWithRichText(shape)
+
+			editor.updateShape({
+				id: shapeId,
+				type: shape.type,
+				props: { richText: toRichText(action.text ?? '') },
+			})
+		}
+	}
+)

templates/agent/client/actions/MessageActionUtil.ts

@@ -0,0 +1,16 @@
+import { MessageAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const MessageActionUtil = registerActionUtil(
+	class MessageActionUtil extends AgentActionUtil<MessageAction> {
+		static override type = 'message' as const
+
+		override getInfo(action: Streaming<MessageAction>) {
+			return {
+				description: action.text ?? '',
+				canGroup: () => false,
+			}
+		}
+	}
+)

templates/agent/client/actions/MoveActionUtil.ts

@@ -0,0 +1,128 @@
+import { TLShapeId, Vec } from 'tldraw'
+import { MoveAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentHelpers } from '../AgentHelpers'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const MoveActionUtil = registerActionUtil(
+	class MoveActionUtil extends AgentActionUtil<MoveAction> {
+		static override type = 'move' as const
+
+		override getInfo(action: Streaming<MoveAction>) {
+			return {
+				icon: 'cursor' as const,
+				description: action.intent ?? '',
+			}
+		}
+
+		override sanitizeAction(action: Streaming<MoveAction>, helpers: AgentHelpers) {
+			if (!action.complete) return action
+
+			// Make sure the shape ID refers to a real shape
+			const shapeId = helpers.ensureShapeIdExists(action.shapeId)
+			if (!shapeId) return null
+			action.shapeId = shapeId
+
+			// Make sure the x and y values are numbers
+			const floatX = helpers.ensureValueIsNumber(action.x)
+			const floatY = helpers.ensureValueIsNumber(action.y)
+			if (floatX === null || floatY === null) return null
+			action.x = floatX
+			action.y = floatY
+
+			return action
+		}
+
+		override applyAction(action: Streaming<MoveAction>, helpers: AgentHelpers) {
+			if (!action.complete) return
+			const { editor } = this
+
+			// Translate the position back to the chat's position
+			const { x, y } = helpers.removeOffsetFromVec({ x: action.x, y: action.y })
+
+			const shapeId = `shape:${action.shapeId}` as TLShapeId
+			const shape = editor.getShape(shapeId)
+			if (!shape) return
+
+			const shapeBounds = editor.getShapePageBounds(shapeId)
+			if (!shapeBounds) return
+
+			const moveTarget = new Vec(x, y)
+			const shapeOrigin = new Vec(shape.x, shape.y)
+			const shapeBoundsOrigin = new Vec(shapeBounds.minX, shapeBounds.minY)
+
+			// Calculate the offset from the shape bounds origin to the shape origin
+			const shapeOriginDelta = shapeOrigin.sub(shapeBoundsOrigin)
+
+			// Adjust the target position based on the anchor point
+			const boundsWidth = shapeBounds.w
+			const boundsHeight = shapeBounds.h
+
+			// Calculate the anchor point offset from the bounds origin
+			let anchorOffsetX = 0
+			let anchorOffsetY = 0
+
+			switch (action.anchor) {
+				case 'top-left': {
+					anchorOffsetX = 0
+					anchorOffsetY = 0
+					break
+				}
+				case 'top-center': {
+					anchorOffsetX = boundsWidth / 2
+					anchorOffsetY = 0
+					break
+				}
+				case 'top-right': {
+					anchorOffsetX = boundsWidth
+					anchorOffsetY = 0
+					break
+				}
+				case 'bottom-left': {
+					anchorOffsetX = 0
+					anchorOffsetY = boundsHeight
+					break
+				}
+				case 'bottom-center': {
+					anchorOffsetX = boundsWidth / 2
+					anchorOffsetY = boundsHeight
+					break
+				}
+				case 'bottom-right': {
+					anchorOffsetX = boundsWidth
+					anchorOffsetY = boundsHeight
+					break
+				}
+				case 'center-left': {
+					anchorOffsetX = 0
+					anchorOffsetY = boundsHeight / 2
+					break
+				}
+				case 'center-right': {
+					anchorOffsetX = boundsWidth
+					anchorOffsetY = boundsHeight / 2
+					break
+				}
+				case 'center': {
+					anchorOffsetX = boundsWidth / 2
+					anchorOffsetY = boundsHeight / 2
+					break
+				}
+			}
+
+			// Adjust the target to account for the anchor point
+			// The target x,y should be where the anchor point is positioned
+			// So we subtract the anchor offset to get the bounds origin position
+			const adjustedTarget = moveTarget.sub(new Vec(anchorOffsetX, anchorOffsetY))
+
+			const newTarget = adjustedTarget.add(shapeOriginDelta)
+
+			editor.updateShape({
+				id: shapeId,
+				type: shape.type,
+				x: newTarget.x,
+				y: newTarget.y,
+			})
+		}
+	}
+)

templates/agent/client/actions/PenActionUtil.ts

@@ -0,0 +1,115 @@
+import { b64Vecs, TLDrawShapeSegment, TLShapeId, Vec, VecModel } from 'tldraw'
+import { asColor } from '../../shared/format/FocusedColor'
+import { convertFocusedFillToTldrawFill } from '../../shared/format/FocusedFill'
+import { PenAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentHelpers } from '../AgentHelpers'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const PenActionUtil = registerActionUtil(
+	class PenActionUtil extends AgentActionUtil<PenAction> {
+		static override type = 'pen' as const
+
+		override getInfo(action: Streaming<PenAction>) {
+			return {
+				icon: 'pencil' as const,
+				description: action.intent ?? '',
+			}
+		}
+
+		override sanitizeAction(action: Streaming<PenAction>, helpers: AgentHelpers) {
+			if (!action.points) return action
+
+			// Ensure the shape has a unique ID
+			action.shapeId = helpers.ensureShapeIdIsUnique(action.shapeId)
+
+			// Don't include the final point if we're still streaming.
+			// Its numbers might be incomplete.
+			const points = action.complete ? action.points : action.points.slice(0, -1)
+
+			// This is a complex action for the model, so validate the data it gives us
+			const validPoints = points
+				.map((point) => helpers.ensureValueIsVec(point))
+				.filter((v) => v !== null)
+
+			action.points = validPoints
+			action.closed = helpers.ensureValueIsBoolean(action.closed) ?? false
+			action.fill = helpers.ensureValueIsFocusedFill(action.fill) ?? 'none'
+
+			return action
+		}
+
+		override applyAction(action: Streaming<PenAction>, helpers: AgentHelpers) {
+			if (!action.points) return
+			if (action.points.length === 0) return
+
+			if (!action.shapeId) return
+			const shapeId = `shape:${action.shapeId}` as TLShapeId
+
+			action.points = action.points.map((point) => helpers.removeOffsetFromVec(point))
+
+			if (action.closed) {
+				const firstPoint = action.points[0]
+				action.points.push(firstPoint)
+			}
+
+			const minX = Math.min(...action.points.map((p) => p.x))
+			const minY = Math.min(...action.points.map((p) => p.y))
+
+			const points: VecModel[] = []
+			const maxDistanceBetweenPoints = action.style === 'smooth' ? 10 : 2
+			for (let i = 0; i < action.points.length - 1; i++) {
+				const point = action.points[i]
+				points.push(point)
+
+				const nextPoint = action.points[i + 1]
+				if (!nextPoint) continue
+
+				const distance = Vec.Dist(point, nextPoint)
+				const numPointsToAdd = Math.floor(distance / maxDistanceBetweenPoints)
+				const pointsToAdd = Array.from({ length: numPointsToAdd }, (_, j) => {
+					const t = (j + 1) / (numPointsToAdd + 1)
+					return Vec.Lrp(point, nextPoint, t)
+				})
+				points.push(...pointsToAdd)
+			}
+
+			// Require at least 2 points to draw a line
+			if (points.length <= 1) {
+				return
+			}
+
+			const segmentPoints = points.map((point) => ({
+				x: point.x - minX,
+				y: point.y - minY,
+				z: 0.75,
+			}))
+			const base64Points = b64Vecs.encodePoints(segmentPoints)
+
+			const segments: TLDrawShapeSegment[] = [
+				{
+					type: 'free',
+					path: base64Points,
+				},
+			]
+
+			this.editor.createShape({
+				id: shapeId,
+				type: 'draw',
+				x: minX,
+				y: minY,
+				isLocked: !action.complete,
+				props: {
+					color: asColor(action.color ?? 'black'),
+					fill: convertFocusedFillToTldrawFill(action.fill ?? 'none'),
+					dash: 'draw',
+					size: 's',
+					segments,
+					isComplete: action.complete,
+					isClosed: action.closed,
+					isPen: true,
+				},
+			})
+		}
+	}
+)

templates/agent/client/actions/PlaceActionUtil.ts

@@ -0,0 +1,135 @@
+import { TLShapeId } from 'tldraw'
+import { PlaceAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentHelpers } from '../AgentHelpers'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const PlaceActionUtil = registerActionUtil(
+	class PlaceActionUtil extends AgentActionUtil<PlaceAction> {
+		static override type = 'place' as const
+
+		override getInfo(action: Streaming<PlaceAction>) {
+			return {
+				icon: 'target' as const,
+				description: action.intent ?? '',
+			}
+		}
+
+		override sanitizeAction(action: Streaming<PlaceAction>, helpers: AgentHelpers) {
+			if (!action.complete) return action
+
+			const shapeId = helpers.ensureShapeIdExists(action.shapeId)
+			if (!shapeId) return null
+			action.shapeId = shapeId
+
+			const referenceShapeId = helpers.ensureShapeIdExists(action.referenceShapeId)
+			if (!referenceShapeId) return null
+			action.referenceShapeId = referenceShapeId
+
+			return action
+		}
+
+		override applyAction(action: Streaming<PlaceAction>) {
+			if (!action.complete) return
+			const { editor } = this
+
+			const { side, sideOffset = 0, align, alignOffset = 0 } = action
+			const referenceShapeId = `shape:${action.referenceShapeId}` as TLShapeId
+			const shapeId = `shape:${action.shapeId}` as TLShapeId
+
+			const shape = editor.getShape(shapeId)
+			const referenceShape = editor.getShape(referenceShapeId)
+			if (!shape || !referenceShape) return
+
+			const bbA = editor.getShapePageBounds(shape)!
+			const bbR = editor.getShapePageBounds(referenceShape)!
+			if (side === 'top' && align === 'start') {
+				editor.updateShape({
+					id: shapeId,
+					type: shape.type,
+					x: bbR.minX + alignOffset,
+					y: bbR.minY - bbA.height - sideOffset,
+				})
+			} else if (side === 'top' && align === 'center') {
+				editor.updateShape({
+					id: shapeId,
+					type: shape.type,
+					x: bbR.midX - bbA.width / 2 + alignOffset,
+					y: bbR.minY - bbA.height - sideOffset,
+				})
+			} else if (side === 'top' && align === 'end') {
+				editor.updateShape({
+					id: shapeId,
+					type: shape.type,
+					x: bbR.maxX - bbA.width - alignOffset,
+					y: bbR.minY - bbA.height - sideOffset,
+				})
+			} else if (side === 'bottom' && align === 'start') {
+				editor.updateShape({
+					id: shapeId,
+					type: shape.type,
+					x: bbR.minX + alignOffset,
+					y: bbR.maxY + sideOffset,
+				})
+			} else if (side === 'bottom' && align === 'center') {
+				editor.updateShape({
+					id: shapeId,
+					type: shape.type,
+					x: bbR.midX - bbA.width / 2 + alignOffset,
+					y: bbR.maxY + sideOffset,
+				})
+			} else if (side === 'bottom' && align === 'end') {
+				editor.updateShape({
+					id: shapeId,
+					type: shape.type,
+					x: bbR.maxX - bbA.width - alignOffset,
+					y: bbR.maxY + sideOffset,
+				})
+				// LEFT SIDE (corrected)
+			} else if (side === 'left' && align === 'start') {
+				editor.updateShape({
+					id: shapeId,
+					type: shape.type,
+					x: bbR.minX - bbA.width - sideOffset,
+					y: bbR.minY + alignOffset,
+				})
+			} else if (side === 'left' && align === 'center') {
+				editor.updateShape({
+					id: shapeId,
+					type: shape.type,
+					x: bbR.minX - bbA.width - sideOffset,
+					y: bbR.midY - bbA.height / 2 + alignOffset,
+				})
+			} else if (side === 'left' && align === 'end') {
+				editor.updateShape({
+					id: shapeId,
+					type: shape.type,
+					x: bbR.minX - bbA.width - sideOffset,
+					y: bbR.maxY - bbA.height - alignOffset,
+				})
+				// RIGHT SIDE (corrected)
+			} else if (side === 'right' && align === 'start') {
+				editor.updateShape({
+					id: shapeId,
+					type: shape.type,
+					x: bbR.maxX + sideOffset,
+					y: bbR.minY + alignOffset,
+				})
+			} else if (side === 'right' && align === 'center') {
+				editor.updateShape({
+					id: shapeId,
+					type: shape.type,
+					x: bbR.maxX + sideOffset,
+					y: bbR.midY - bbA.height / 2 + alignOffset,
+				})
+			} else if (side === 'right' && align === 'end') {
+				editor.updateShape({
+					id: shapeId,
+					type: shape.type,
+					x: bbR.maxX + sideOffset,
+					y: bbR.maxY - bbA.height - alignOffset,
+				})
+			}
+		}
+	}
+)

templates/agent/client/actions/ResizeActionUtil.ts

@@ -0,0 +1,49 @@
+import { TLShapeId } from 'tldraw'
+import { ResizeAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentHelpers } from '../AgentHelpers'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const ResizeActionUtil = registerActionUtil(
+	class ResizeActionUtil extends AgentActionUtil<ResizeAction> {
+		static override type = 'resize' as const
+
+		override getInfo(action: Streaming<ResizeAction>) {
+			return {
+				icon: 'cursor' as const,
+				description: action.intent ?? '',
+			}
+		}
+
+		override sanitizeAction(action: Streaming<ResizeAction>, helpers: AgentHelpers) {
+			const shapeIds = helpers.ensureShapeIdsExist(action.shapeIds ?? [])
+			if (shapeIds.length === 0) return null
+
+			action.shapeIds = shapeIds
+			return action
+		}
+
+		override applyAction(action: Streaming<ResizeAction>, helpers: AgentHelpers) {
+			if (
+				!action.shapeIds ||
+				!action.scaleX ||
+				!action.scaleY ||
+				!action.originX ||
+				!action.originY
+			) {
+				return
+			}
+
+			const origin = helpers.removeOffsetFromVec({ x: action.originX, y: action.originY })
+			const shapeIds = action.shapeIds.map((shapeId) => `shape:${shapeId}` as TLShapeId)
+
+			for (const shapeId of shapeIds) {
+				this.editor.resizeShape(
+					shapeId,
+					{ x: action.scaleX, y: action.scaleY },
+					{ scaleOrigin: origin }
+				)
+			}
+		}
+	}
+)

templates/agent/client/actions/ReviewActionUtil.ts

@@ -0,0 +1,61 @@
+import { Box } from 'tldraw'
+import { ReviewAction } from '../../shared/schema/AgentActionSchemas'
+import { AreaContextItem } from '../../shared/types/ContextItem'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentHelpers } from '../AgentHelpers'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const ReviewActionUtil = registerActionUtil(
+	class ReviewActionUtil extends AgentActionUtil<ReviewAction> {
+		static override type = 'review' as const
+
+		override getInfo(action: Streaming<ReviewAction>) {
+			const label = action.complete ? 'Review' : 'Reviewing'
+			const text = action.intent?.startsWith('#') ? `\n\n${action.intent}` : action.intent
+			const description = `**${label}:** ${text ?? ''}`
+
+			return {
+				icon: 'search' as const,
+				description,
+			}
+		}
+
+		override applyAction(action: Streaming<ReviewAction>, helpers: AgentHelpers) {
+			if (!action.complete) return
+
+			const reviewBounds = helpers.removeOffsetFromBox({
+				x: action.x,
+				y: action.y,
+				w: action.w,
+				h: action.h,
+			})
+
+			const contextArea: AreaContextItem = {
+				type: 'area',
+				bounds: reviewBounds,
+				source: 'agent',
+			}
+
+			// If the review area is outside the already-scheduled bounds, expand the bounds to include it
+			const scheduledRequest = this.agent.requests.getScheduledRequest()
+			const bounds = scheduledRequest
+				? Box.From(scheduledRequest.bounds).union(reviewBounds)
+				: reviewBounds
+
+			// Schedule the review
+			this.agent.schedule({
+				bounds,
+				message: getReviewMessage(action.intent),
+				contextItems: [contextArea],
+			})
+		}
+	}
+)
+
+function getReviewMessage(intent: string) {
+	return `Examine the actions that you (the agent) took since the most recent user message, with the intent: "${intent}". What's next?
+
+- Is there still more work to do? If so, continue it.
+- Is the task supposed to be complete? If so, it's time to review the results of that. Did you do what the user asked for? Did the plan work? Think through your findings and pay close attention to the image, because that's what you can see right now. If you make any corrections, let the user know what you did and why. If no corrections are needed, there's no need to say anything.
+- Make sure to reference your last actions (denoted by [ACTION]) in order to see if you completed the task. Assume each action you see in the chat history completed successfully.`
+}

templates/agent/client/actions/RotateActionUtil.ts

@@ -0,0 +1,35 @@
+import { TLShapeId } from 'tldraw'
+import { RotateAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentHelpers } from '../AgentHelpers'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const RotateActionUtil = registerActionUtil(
+	class RotateActionUtil extends AgentActionUtil<RotateAction> {
+		static override type = 'rotate' as const
+
+		override getInfo(action: Streaming<RotateAction>) {
+			return {
+				icon: 'cursor' as const,
+				description: action.intent ?? '',
+			}
+		}
+
+		override sanitizeAction(action: Streaming<RotateAction>, helpers: AgentHelpers) {
+			action.shapeIds = helpers.ensureShapeIdsExist(action.shapeIds ?? [])
+			return action
+		}
+
+		override applyAction(action: Streaming<RotateAction>, helpers: AgentHelpers) {
+			if (!action.shapeIds || !action.degrees || !action.originX || !action.originY) {
+				return
+			}
+
+			const origin = helpers.removeOffsetFromVec({ x: action.originX, y: action.originY })
+			const shapeIds = action.shapeIds.map((shapeId) => `shape:${shapeId}` as TLShapeId)
+			const radians = (action.degrees * Math.PI) / 180
+
+			this.editor.rotateShapesBy(shapeIds, radians, { center: origin })
+		}
+	}
+)

templates/agent/client/actions/SendToBackActionUtil.ts

@@ -0,0 +1,28 @@
+import { TLShapeId } from 'tldraw'
+import { SendToBackAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentHelpers } from '../AgentHelpers'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const SendToBackActionUtil = registerActionUtil(
+	class SendToBackActionUtil extends AgentActionUtil<SendToBackAction> {
+		static override type = 'sendToBack' as const
+
+		override getInfo(action: Streaming<SendToBackAction>) {
+			return {
+				icon: 'cursor' as const,
+				description: action.intent ?? '',
+			}
+		}
+
+		override sanitizeAction(action: Streaming<SendToBackAction>, helpers: AgentHelpers) {
+			action.shapeIds = helpers.ensureShapeIdsExist(action.shapeIds ?? [])
+			return action
+		}
+
+		override applyAction(action: Streaming<SendToBackAction>) {
+			if (!action.shapeIds) return
+			this.editor.sendToBack(action.shapeIds.map((shapeId) => `shape:${shapeId}` as TLShapeId))
+		}
+	}
+)

templates/agent/client/actions/SetMyViewActionUtil.ts

@@ -0,0 +1,42 @@
+import { structuredClone } from 'tldraw'
+import { SetMyViewAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentHelpers } from '../AgentHelpers'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const SetMyViewActionUtil = registerActionUtil(
+	class SetMyViewActionUtil extends AgentActionUtil<SetMyViewAction> {
+		static override type = 'setMyView' as const
+
+		override getInfo(action: Streaming<SetMyViewAction>) {
+			const label = action.complete ? 'Move camera' : 'Moving camera'
+			const text = action.intent?.startsWith('#') ? `\n\n${action.intent}` : action.intent
+			return {
+				icon: 'eye' as const,
+				description: `**${label}**: ${text ?? ''}`,
+			}
+		}
+
+		override applyAction(action: Streaming<SetMyViewAction>, helpers: AgentHelpers) {
+			if (!action.complete) return
+
+			const roundedAction = helpers.roundBox(structuredClone(action))
+
+			const bounds = helpers.removeOffsetFromBox({
+				x: action.x,
+				y: action.y,
+				w: action.w,
+				h: action.h,
+			})
+
+			this.agent.interrupt({
+				input: {
+					bounds,
+					agentMessages: [
+						`Just navigated to new area with the intent: ${action.intent}. Can now see the new area at (${roundedAction.x}, ${roundedAction.y}) and is ${roundedAction.w}x${roundedAction.h} in size.`, // this uses the action instead of the bounds because this will go in the chat history and so must be consistent with what the agent thinks the chat origin is
+					],
+				},
+			})
+		}
+	}
+)

templates/agent/client/actions/StackActionUtil.ts

@@ -0,0 +1,36 @@
+import { TLShapeId } from 'tldraw'
+import { StackAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentHelpers } from '../AgentHelpers'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const StackActionUtil = registerActionUtil(
+	class StackActionUtil extends AgentActionUtil<StackAction> {
+		static override type = 'stack' as const
+
+		override getInfo(action: Streaming<StackAction>) {
+			return {
+				icon: 'cursor' as const,
+				description: action.intent ?? '',
+			}
+		}
+
+		override sanitizeAction(action: Streaming<StackAction>, helpers: AgentHelpers) {
+			if (!action.complete) return action
+
+			action.shapeIds = helpers.ensureShapeIdsExist(action.shapeIds)
+
+			return action
+		}
+
+		override applyAction(action: Streaming<StackAction>) {
+			if (!action.complete) return
+
+			this.editor.stackShapes(
+				action.shapeIds.map((id) => `shape:${id}` as TLShapeId),
+				action.direction,
+				Math.max(action.gap, 0)
+			)
+		}
+	}
+)

templates/agent/client/actions/ThinkActionUtil.ts

@@ -0,0 +1,22 @@
+import { ThinkAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const ThinkActionUtil = registerActionUtil(
+	class ThinkActionUtil extends AgentActionUtil<ThinkAction> {
+		static override type = 'think' as const
+
+		override getInfo(action: Streaming<ThinkAction>) {
+			const time = Math.floor(action.time / 1000)
+			let summary = `Thought for ${time} seconds`
+			if (time === 0) summary = 'Thought for less than a second'
+			if (time === 1) summary = 'Thought for 1 second'
+
+			return {
+				icon: 'brain' as const,
+				description: action.text ?? (action.complete ? null : 'Thinking...'),
+				summary,
+			}
+		}
+	}
+)

templates/agent/client/actions/UnknownActionUtil.ts

@@ -0,0 +1,18 @@
+import { UnknownAction } from '../../shared/schema/AgentActionSchemas'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+/**
+ * This event util is used when the event type is unknown.
+ * This usually happens because the event type has not finished streaming yet.
+ * Sometimes it happens because the model makes a mistake.
+ */
+export const UnknownActionUtil = registerActionUtil(
+	class UnknownActionUtil extends AgentActionUtil<UnknownAction> {
+		static override type = 'unknown' as const
+
+		override getInfo() {
+			// Don't show anything in the UI for unknown actions
+			return null
+		}
+	}
+)

templates/agent/client/actions/UpdateActionUtil.ts

@@ -0,0 +1,106 @@
+import { TLBindingId } from 'tldraw'
+import {
+	convertFocusedShapeToTldrawShape,
+	convertSimpleIdToTldrawId,
+} from '../../shared/format/convertFocusedShapeToTldrawShape'
+import { UpdateAction } from '../../shared/schema/AgentActionSchemas'
+import { toSimpleShapeId } from '../../shared/types/ids-schema'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentHelpers } from '../AgentHelpers'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const UpdateActionUtil = registerActionUtil(
+	class UpdateActionUtil extends AgentActionUtil<UpdateAction> {
+		static override type = 'update' as const
+
+		override getInfo(action: Streaming<UpdateAction>) {
+			return {
+				icon: 'cursor' as const,
+				description: action.intent ?? '',
+			}
+		}
+
+		override sanitizeAction(action: Streaming<UpdateAction>, helpers: AgentHelpers) {
+			if (!action.complete) return action
+
+			const { update } = action
+
+			// Ensure the shape ID refers to a real shape
+			const shapeId = helpers.ensureShapeIdExists(toSimpleShapeId(update.shapeId))
+			if (!shapeId) return null
+			update.shapeId = shapeId
+
+			// If it's an arrow, ensure the from and to IDs refer to real shapes
+			if (update._type === 'arrow') {
+				if (update.fromId) {
+					update.fromId = helpers.ensureShapeIdExists(update.fromId)
+				}
+				if (update.toId) {
+					update.toId = helpers.ensureShapeIdExists(update.toId)
+				}
+
+				if ('x1' in update) {
+					update.x1 = helpers.ensureValueIsNumber(update.x1) ?? 0
+				}
+				if ('y1' in update) {
+					update.y1 = helpers.ensureValueIsNumber(update.y1) ?? 0
+				}
+				if ('x2' in update) {
+					update.x2 = helpers.ensureValueIsNumber(update.x2) ?? 0
+				}
+				if ('y2' in update) {
+					update.y2 = helpers.ensureValueIsNumber(update.y2) ?? 0
+				}
+				if ('bend' in update) {
+					update.bend = helpers.ensureValueIsNumber(update.bend) ?? 0
+				}
+			}
+
+			// Unround the shape to restore the original values
+			action.update = helpers.unroundShape(action.update)
+
+			return action
+		}
+
+		override applyAction(action: Streaming<UpdateAction>, helpers: AgentHelpers) {
+			if (!action.complete) return
+			const { editor } = this
+
+			// Translate the shape back to the chat's position
+			action.update = helpers.removeOffsetFromShape(action.update)
+
+			const shapeId = convertSimpleIdToTldrawId(action.update.shapeId)
+			const existingShape = editor.getShape(shapeId)
+
+			if (!existingShape) {
+				throw new Error(`Shape ${shapeId} not found in canvas`)
+			}
+
+			const result = convertFocusedShapeToTldrawShape(editor, action.update, {
+				defaultShape: existingShape,
+			})
+
+			editor.updateShape(result.shape)
+
+			// Handle arrow bindings if they exist
+			if (result.bindings) {
+				// First, clean up existing bindings
+				const existingBindings = editor.getBindingsFromShape(shapeId, 'arrow')
+				for (const binding of existingBindings) {
+					editor.deleteBinding(binding.id as TLBindingId)
+				}
+
+				// Create new bindings
+				for (const binding of result.bindings) {
+					editor.createBinding({
+						type: binding.type,
+						fromId: binding.fromId,
+						toId: binding.toId,
+						props: binding.props,
+						meta: binding.meta,
+					})
+				}
+			}
+		}
+	}
+)

templates/agent/client/actions/UpsertTodoListItemActionUtil.ts

@@ -0,0 +1,33 @@
+import { UpsertPersonalTodoItemAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const UpsertTodoListItemActionUtil = registerActionUtil(
+	class UpsertTodoListItemActionUtil extends AgentActionUtil<UpsertPersonalTodoItemAction> {
+		static override type = 'update-todo-list' as const
+
+		override getInfo() {
+			// Don't show todo actions in the chat history because we show them in the dedicated todo list UI
+			return null
+		}
+
+		override applyAction(action: Streaming<UpsertPersonalTodoItemAction>) {
+			if (!action.complete) return
+
+			const { id, text, status } = action
+
+			const index = this.agent.todos.getTodos().findIndex((item) => item.id === id)
+			if (index === -1) {
+				if (!text) {
+					this.agent.interrupt({
+						input: 'You must provide text when creating a new todo item.',
+					})
+					return
+				}
+				this.agent.todos.push(id, text)
+			} else {
+				this.agent.todos.update({ id, status, text })
+			}
+		}
+	}
+)