snapshot of https://github.com/tldraw/tldraw.git (main, 2026-09-23; blobs > 131072B omitted)

templates/agent/client/AgentHelpers.ts

@@ -0,0 +1,532 @@
+import { BoxModel, createShapeId, Editor, TLShapeId, VecModel } from 'tldraw'
+import { FocusedFill, FocusedFillSchema } from '../shared/format/FocusedFill'
+import { FocusedShape } from '../shared/format/FocusedShape'
+import { ContextItem } from '../shared/types/ContextItem'
+import { SimpleShapeId } from '../shared/types/ids-schema'
+import { TldrawAgent } from './agent/TldrawAgent'
+
+/**
+ * This class contains handles the transformations that happen throughout a
+ * request. It contains helpers that can be used to change prompt parts
+ * before they get sent to the model, as well as helpers that can be used to
+ * change incoming actions as they get streamed back from the model.
+ *
+ * For example, `applyOffsetToShape` adjusts the position of a shape to make it
+ * relative to the current chat origin. The `removeOffsetFromShape` method
+ * reverses it. This is helpful because it helps to keep numbers low, which is
+ * easier for the model to deal with.
+ *
+ * Many transformation methods save some state. For example, the
+ * `ensureShapeIdIsUnique` method changes a shape's ID if it's not unique, and
+ * it saves a record of this change so that further actions can continue to
+ * refer to the shape by its untransformed ID.
+ */
+export class AgentHelpers {
+	/**
+	 * The agent that the this intance of AgentHelpers is for.
+	 */
+	agent: TldrawAgent
+
+	/**
+	 * The editor that the this intance of AgentHelpers is for.
+	 */
+	editor: Editor
+
+	constructor(agent: TldrawAgent) {
+		this.agent = agent
+		this.editor = agent.editor
+		const origin = agent.chatOrigin.getOrigin()
+		this.offset = {
+			x: -origin.x,
+			y: -origin.y,
+		}
+	}
+
+	/**
+	 * The offset of the current request from the chat origin.
+	 */
+	offset: VecModel = { x: 0, y: 0 }
+
+	/**
+	 * A map of shape ids that have been transformed as part of this request.
+	 * The key is the original id, and the value is the transformed id.
+	 */
+	shapeIdMap = new Map<string, string>()
+
+	/**
+	 * A map of rounding diffs, stored by key.
+	 * These are used to restore the original values of rounded numbers.
+	 */
+	roundingDiffMap = new Map<string, number>()
+
+	/**
+	 * Apply the offset of this request to a position.
+	 */
+	applyOffsetToVec(position: VecModel): VecModel {
+		return {
+			x: position.x + this.offset.x,
+			y: position.y + this.offset.y,
+		}
+	}
+
+	/**
+	 * Remove the offset of this request from a position.
+	 */
+	removeOffsetFromVec(position: VecModel): VecModel {
+		return {
+			x: position.x - this.offset.x,
+			y: position.y - this.offset.y,
+		}
+	}
+
+	/**
+	 * Apply the offset of this request to a box.
+	 */
+	applyOffsetToBox(box: BoxModel): BoxModel {
+		return {
+			x: box.x + this.offset.x,
+			y: box.y + this.offset.y,
+			w: box.w,
+			h: box.h,
+		}
+	}
+
+	/**
+	 * Remove the offset of this request from a box.
+	 */
+	removeOffsetFromBox(box: BoxModel): BoxModel {
+		return {
+			x: box.x - this.offset.x,
+			y: box.y - this.offset.y,
+			w: box.w,
+			h: box.h,
+		}
+	}
+
+	/**
+	 * Apply the offset of this request to a shape.
+	 */
+	applyOffsetToShape(shape: FocusedShape): FocusedShape {
+		if ('x1' in shape) {
+			return {
+				...shape,
+				x1: shape.x1 + this.offset.x,
+				y1: shape.y1 + this.offset.y,
+				x2: shape.x2 + this.offset.x,
+				y2: shape.y2 + this.offset.y,
+			}
+		}
+		if ('x' in shape) {
+			return {
+				...shape,
+				x: shape.x + this.offset.x,
+				y: shape.y + this.offset.y,
+			}
+		}
+		return shape
+	}
+
+	/**
+	 * Apply the offset of this request to a shape partial.
+	 */
+	applyOffsetToShapePartial(shape: Partial<FocusedShape>): Partial<FocusedShape> {
+		const result = { ...shape }
+		if ('x' in result && result.x !== undefined) {
+			;(result as any).x = result.x + this.offset.x
+		}
+		if ('y' in result && result.y !== undefined) {
+			;(result as any).y = result.y + this.offset.y
+		}
+		if ('x1' in result && result.x1 !== undefined) {
+			;(result as any).x1 = result.x1 + this.offset.x
+		}
+		if ('y1' in result && result.y1 !== undefined) {
+			;(result as any).y1 = result.y1 + this.offset.y
+		}
+		if ('x2' in result && result.x2 !== undefined) {
+			;(result as any).x2 = result.x2 + this.offset.x
+		}
+		if ('y2' in result && result.y2 !== undefined) {
+			;(result as any).y2 = result.y2 + this.offset.y
+		}
+		return result
+	}
+
+	/**
+	 * Remove the offset of this request from a shape partial.
+	 */
+	removeOffsetFromShapePartial(shape: Partial<FocusedShape>): Partial<FocusedShape> {
+		const result = { ...shape }
+		if ('x' in result && result.x !== undefined) {
+			;(result as any).x = result.x - this.offset.x
+		}
+		if ('y' in result && result.y !== undefined) {
+			;(result as any).y = result.y - this.offset.y
+		}
+		if ('x1' in result && result.x1 !== undefined) {
+			;(result as any).x1 = result.x1 - this.offset.x
+		}
+		if ('y1' in result && result.y1 !== undefined) {
+			;(result as any).y1 = result.y1 - this.offset.y
+		}
+		if ('x2' in result && result.x2 !== undefined) {
+			;(result as any).x2 = result.x2 - this.offset.x
+		}
+		if ('y2' in result && result.y2 !== undefined) {
+			;(result as any).y2 = result.y2 - this.offset.y
+		}
+		return result
+	}
+
+	/**
+	 * Remove the offset of this request from a shape.
+	 */
+	removeOffsetFromShape(shape: FocusedShape): FocusedShape {
+		if ('x1' in shape) {
+			return {
+				...shape,
+				x1: shape.x1 - this.offset.x,
+				y1: shape.y1 - this.offset.y,
+				x2: shape.x2 - this.offset.x,
+				y2: shape.y2 - this.offset.y,
+			}
+		}
+		if ('x' in shape) {
+			return {
+				...shape,
+				x: shape.x - this.offset.x,
+				y: shape.y - this.offset.y,
+			}
+		}
+		return shape
+	}
+
+	/**
+	 * Apply the offset of this request to a context item.
+	 */
+	applyOffsetToContextItem(contextItem: ContextItem) {
+		switch (contextItem.type) {
+			case 'shape': {
+				contextItem.shape = this.applyOffsetToShape(contextItem.shape)
+				return contextItem
+			}
+			case 'shapes': {
+				contextItem.shapes = contextItem.shapes.map((shape) => {
+					return this.applyOffsetToShape(shape)
+				})
+				return contextItem
+			}
+			case 'area': {
+				contextItem.bounds = this.applyOffsetToBox(contextItem.bounds)
+				return contextItem
+			}
+			case 'point': {
+				contextItem.point = this.applyOffsetToVec(contextItem.point)
+				return contextItem
+			}
+		}
+	}
+
+	/**
+	 * Round the numbers of a context item.
+	 */
+	roundContextItem(contextItem: ContextItem) {
+		switch (contextItem.type) {
+			case 'shape': {
+				contextItem.shape = this.roundShape(contextItem.shape)
+				return contextItem
+			}
+			case 'shapes': {
+				contextItem.shapes = contextItem.shapes.map((shape) => {
+					return this.roundShape(shape)
+				})
+				return contextItem
+			}
+			case 'area': {
+				contextItem.bounds = this.roundBox(contextItem.bounds)
+				return contextItem
+			}
+			case 'point': {
+				contextItem.point = this.roundVec(contextItem.point)
+				return contextItem
+			}
+		}
+	}
+
+	/**
+	 * Ensure that a shape ID is unique.
+	 * @param id - The id to check (should be a SimpleShapeId from the model, not a TLShapeId).
+	 * @returns The unique id (always without the "shape:" prefix).
+	 */
+	ensureShapeIdIsUnique(id = 'shape' as SimpleShapeId): SimpleShapeId {
+		// todo: remove default and have a better handling of cases where id is undefined
+
+		const { editor } = this.agent
+		// Defensively strip the prefix in case the model incorrectly includes it
+
+		// Ensure the id is unique by incrementing a number at the end
+		let newId = id
+		let existingShape = editor.getShape(`shape:${newId}` as TLShapeId)
+		while (existingShape) {
+			newId = /^.*(\d+)$/.exec(newId)?.[1]
+				? (newId.replace(/(\d+)$/, (m) => {
+						return (+m + 1).toString()
+					}) as SimpleShapeId)
+				: (`${newId}-1` as SimpleShapeId)
+			existingShape = editor.getShape(`shape:${newId}` as TLShapeId)
+		}
+
+		// If the id was transformed, track it so that future events can refer to it by its original id.
+		if (id !== newId) {
+			this.shapeIdMap.set(id, newId)
+		}
+
+		return newId as SimpleShapeId
+	}
+
+	/**
+	 * Ensure that a shape ID refers to a real shape.
+	 * @param id - The id to check.
+	 * @returns The real id, or null if the shape doesn't exist.
+	 */
+	ensureShapeIdExists(id: SimpleShapeId): SimpleShapeId | null {
+		const { editor } = this.agent
+
+		// If there's already a transformed ID, use that
+		const existingId = this.shapeIdMap.get(id)
+		if (existingId) {
+			return existingId as SimpleShapeId
+		}
+
+		// If there's an existing shape with this ID, use that
+		const existingShape = editor.getShape(createShapeId(id))
+		if (existingShape) {
+			return id
+		}
+
+		// Otherwise, give up
+		return null
+	}
+
+	/**
+	 * Ensure that all shape IDs refer to real shapes.
+	 * @param ids - An array of ids to check.
+	 * @returns The array of ids, with imaginary ids removed.
+	 */
+	ensureShapeIdsExist(ids: SimpleShapeId[]): SimpleShapeId[] {
+		return ids.map((id) => this.ensureShapeIdExists(id)).filter((v) => v !== null)
+	}
+
+	/**
+	 * Round the coordinates, width, and height of a focused shape.
+	 * Save the diffs so that they can be restored later.
+	 * @param shape - The shape to round.
+	 * @returns The rounded shape.
+	 */
+	roundShape(shape: FocusedShape): FocusedShape {
+		if ('x1' in shape) {
+			shape = this.roundProperty(shape, 'x1')
+			shape = this.roundProperty(shape, 'y1')
+			shape = this.roundProperty(shape, 'x2')
+			shape = this.roundProperty(shape, 'y2')
+		} else if ('x' in shape) {
+			shape = this.roundProperty(shape, 'x')
+			shape = this.roundProperty(shape, 'y')
+		}
+
+		if ('w' in shape) {
+			shape = this.roundProperty(shape, 'w')
+			shape = this.roundProperty(shape, 'h')
+		}
+
+		return shape
+	}
+
+	/**
+	 * Round the coordinates, width, and height of a focused shape partial.
+	 * Save the diffs so that they can be restored later.
+	 * @param shape - The shape partial to round.
+	 * @returns The rounded shape partial.
+	 */
+	roundShapePartial(shape: Partial<FocusedShape>): Partial<FocusedShape> {
+		for (const prop of ['x1', 'y1', 'x2', 'y2', 'x', 'y', 'w', 'h'] as const) {
+			if (prop in shape) {
+				shape = this.roundProperty(shape, prop as keyof Partial<FocusedShape>)
+			}
+		}
+		return shape
+	}
+
+	/**
+	 * Reverse the rounding of a shape that we did earlier.
+	 * This ensures that shape's don't do a tiny jitter when they are updated.
+	 * @param shape - The shape to unround.
+	 * @returns The unrounded shape.
+	 */
+	unroundShape(shape: FocusedShape): FocusedShape {
+		if ('x1' in shape) {
+			shape = this.unroundProperty(shape, 'x1')
+			shape = this.unroundProperty(shape, 'y1')
+			shape = this.unroundProperty(shape, 'x2')
+			shape = this.unroundProperty(shape, 'y2')
+		} else if ('x' in shape) {
+			shape = this.unroundProperty(shape, 'x')
+			shape = this.unroundProperty(shape, 'y')
+		}
+
+		if ('w' in shape) {
+			shape = this.unroundProperty(shape, 'w')
+			shape = this.unroundProperty(shape, 'h')
+		}
+		return shape
+	}
+
+	/**
+	 * Round a number and save its diff so that it can be restored later.
+	 * @param number - The number to round.
+	 * @param key - The key to save the diff under.
+	 * @returns The rounded number.
+	 */
+	roundAndSaveNumber(number: number, key: string): number {
+		const roundedNumber = Math.round(number)
+		const diff = roundedNumber - number
+		this.roundingDiffMap.set(key, diff)
+		return roundedNumber
+	}
+
+	/**
+	 * Reverse the rounding of a number and restore the original value.
+	 * @param number - The number to unround.
+	 * @param key - The key to restore the diff from.
+	 * @returns The unrounded number.
+	 */
+	unroundAndRestoreNumber(number: number, key: string): number {
+		const diff = this.roundingDiffMap.get(key)
+		if (diff === undefined) return number
+		return number + diff
+	}
+
+	/**
+	 * Round a number property of a shape, and save the diff so that it can be restored later.
+	 * @param shape - The shape to round.
+	 * @param property - The property to round.
+	 * @returns The rounded shape.
+	 */
+	roundProperty<T extends Partial<FocusedShape>>(shape: T, property: keyof T): T {
+		if (typeof shape[property] !== 'number') return shape
+
+		const value = shape[property]
+		const key = `${shape.shapeId}_${property as string}`
+		const roundedValue = this.roundAndSaveNumber(value, key)
+
+		;(shape[property] as number) = roundedValue
+		return shape
+	}
+
+	/**
+	 * Reverse the rounding of a number property of a shape, and restore the original value.
+	 * @param shape - The shape to unround.
+	 * @param property - The property to unround.
+	 * @returns The unrounded shape.
+	 */
+	unroundProperty<T extends FocusedShape>(shape: T, property: keyof T): T {
+		if (typeof shape[property] !== 'number') return shape
+
+		const key = `${shape.shapeId}_${property as string}`
+		const diff = this.roundingDiffMap.get(key)
+		if (diff === undefined) return shape
+		;(shape[property] as number) += diff
+		return shape
+	}
+
+	/**
+	 * Ensure that a value is a number.
+	 * Used for checking incoming data from the model.
+	 * @returns The number, or null if the value is not a number.
+	 */
+	ensureValueIsNumber(value: any): number | null {
+		if (typeof value === 'number') {
+			return value
+		}
+
+		if (typeof value === 'string') {
+			const parsedValue = parseFloat(value)
+			if (isNaN(parsedValue)) {
+				return null
+			}
+			return parsedValue
+		}
+
+		return null
+	}
+
+	/**
+	 * Ensure that a value is a vector.
+	 * Used for checking incoming data from the model.
+	 * @returns The vector, or null if the value is not a vector.
+	 */
+	ensureValueIsVec(value: any): VecModel | null {
+		if (!value) return null
+		if (typeof value !== 'object') return null
+		if (!('x' in value) || !('y' in value)) return null
+
+		const x = this.ensureValueIsNumber(value.x)
+		const y = this.ensureValueIsNumber(value.y)
+		if (x === null || y === null) return null
+		return { x, y }
+	}
+
+	/**
+	 * Ensure that a value is a boolean.
+	 * Used for checking incoming data from the model.
+	 * @returns The boolean, or null if the value is not a boolean.
+	 */
+	ensureValueIsBoolean(value: any): boolean | null {
+		if (typeof value === 'boolean') {
+			return value
+		}
+
+		if (typeof value === 'number') {
+			return value > 0
+		}
+
+		if (typeof value === 'string') {
+			return value !== 'false'
+		}
+
+		return null
+	}
+
+	/**
+	 * Ensure that a value is a focused fill.
+	 * Used for checking incoming data from the model.
+	 * @returns The focused fill, or null if the value is not a focused fill.
+	 */
+	ensureValueIsFocusedFill(value: any): FocusedFill | null {
+		const focusedFill = FocusedFillSchema.safeParse(value)
+		if (focusedFill.success) {
+			return focusedFill.data
+		}
+		return null
+	}
+
+	/**
+	 * Round the corners of a box.
+	 */
+	roundBox(boxModel: BoxModel): BoxModel {
+		boxModel.x = Math.round(boxModel.x)
+		boxModel.y = Math.round(boxModel.y)
+		boxModel.w = Math.round(boxModel.w)
+		boxModel.h = Math.round(boxModel.h)
+		return boxModel
+	}
+
+	/**
+	 * Round the numbers of a vector.
+	 */
+	roundVec(vecModel: VecModel): VecModel {
+		vecModel.x = Math.round(vecModel.x)
+		vecModel.y = Math.round(vecModel.y)
+		return vecModel
+	}
+}

templates/agent/client/App.tsx

@@ -0,0 +1,99 @@
+import { useCallback, useMemo, useState } from 'react'
+import {
+	DefaultSizeStyle,
+	ErrorBoundary,
+	TLComponents,
+	Tldraw,
+	TldrawUiToastsProvider,
+	TLUiOverrides,
+} from 'tldraw'
+import { TldrawAgentApp } from './agent/TldrawAgentApp'
+import {
+	TldrawAgentAppContextProvider,
+	TldrawAgentAppProvider,
+} from './agent/TldrawAgentAppProvider'
+import { ChatPanel } from './components/ChatPanel'
+import { ChatPanelFallback } from './components/ChatPanelFallback'
+import { CustomHelperButtons } from './components/CustomHelperButtons'
+import { AgentHighlightOverlayUtil } from './overlays/AgentHighlightOverlayUtil'
+import { TargetAreaTool } from './tools/TargetAreaTool'
+import { TargetShapeTool } from './tools/TargetShapeTool'
+
+// Customize tldraw's styles to play to the agent's strengths
+DefaultSizeStyle.setDefaultValue('s')
+
+// Custom tools for picking context items
+const tools = [TargetShapeTool, TargetAreaTool]
+const overlayUtils = [AgentHighlightOverlayUtil]
+const overrides: TLUiOverrides = {
+	tools: (editor, tools) => {
+		return {
+			...tools,
+			'target-area': {
+				id: 'target-area',
+				label: 'Pick Area',
+				kbd: 'c',
+				icon: 'tool-frame',
+				onSelect() {
+					editor.setCurrentTool('target-area')
+				},
+			},
+			'target-shape': {
+				id: 'target-shape',
+				label: 'Pick Shape',
+				kbd: 's',
+				icon: 'tool-frame',
+				onSelect() {
+					editor.setCurrentTool('target-shape')
+				},
+			},
+		}
+	},
+}
+
+function App() {
+	const [app, setApp] = useState<TldrawAgentApp | null>(null)
+
+	const handleUnmount = useCallback(() => {
+		setApp(null)
+	}, [])
+
+	// Custom components that need the agent app's React context
+	const components: TLComponents = useMemo(() => {
+		return {
+			HelperButtons: () =>
+				app && (
+					<TldrawAgentAppContextProvider app={app}>
+						<CustomHelperButtons />
+					</TldrawAgentAppContextProvider>
+				),
+		}
+	}, [app])
+
+	return (
+		<TldrawUiToastsProvider>
+			<div className="tldraw-agent-container">
+				<div className="tldraw-canvas">
+					<Tldraw
+						persistenceKey="tldraw-agent-demo"
+						tools={tools}
+						overlayUtils={overlayUtils}
+						overrides={overrides}
+						components={components}
+					>
+						<TldrawAgentAppProvider onMount={setApp} onUnmount={handleUnmount} />
+					</Tldraw>
+				</div>
+				<ErrorBoundary fallback={ChatPanelFallback}>
+					{app && (
+						<TldrawAgentAppContextProvider app={app}>
+							<ChatPanel />
+						</TldrawAgentAppContextProvider>
+					)}
+				</ErrorBoundary>
+			</div>
+		</TldrawUiToastsProvider>
+	)
+}
+
+export default App

templates/agent/client/actions/AddDetailActionUtil.ts

@@ -0,0 +1,25 @@
+import { AddDetailAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const AddDetailActionUtil = registerActionUtil(
+	class AddDetailActionUtil extends AgentActionUtil<AddDetailAction> {
+		static override type = 'add-detail' as const
+
+		override getInfo(action: Streaming<AddDetailAction>) {
+			const label = 'Adding detail'
+			const text = action.intent?.startsWith('#') ? `\n\n${action.intent}` : action.intent
+			const description = `**${label}:** ${text ?? ''}`
+
+			return {
+				icon: 'pencil' as const,
+				description,
+			}
+		}
+
+		override applyAction(action: Streaming<AddDetailAction>) {
+			if (!action.complete) return
+			this.agent.schedule('Add detail to the canvas.')
+		}
+	}
+)

templates/agent/client/actions/AgentActionUtil.ts

@@ -0,0 +1,167 @@
+import { Editor } from 'tldraw'
+import { AgentAction } from '../../shared/types/AgentAction'
+import { BaseAgentAction } from '../../shared/types/BaseAgentAction'
+import { ChatHistoryInfo } from '../../shared/types/ChatHistoryInfo'
+import { Streaming } from '../../shared/types/Streaming'
+import { TldrawAgent } from '../agent/TldrawAgent'
+import { AgentHelpers } from '../AgentHelpers'
+
+// ============================================================================
+// Registry
+// ============================================================================
+
+/**
+ * Options for registering an action util.
+ */
+export interface RegisterActionUtilOptions {
+	/**
+	 * If specified, this util will only be used when the agent is in one of these modes.
+	 * Otherwise, it will be the default util for this action type.
+	 */
+	forModes?: string[]
+}
+
+// Default registry: actionType -> util (used when no mode-specific util is registered)
+const defaultRegistry = new Map<string, AgentActionUtilConstructor<BaseAgentAction>>()
+
+// Mode registry: actionType -> (mode -> util) (mode-specific overrides)
+const modeRegistry = new Map<string, Map<string, AgentActionUtilConstructor<BaseAgentAction>>>()
+
+/**
+ * Register an agent action util class. Call this after defining each util class.
+ *
+ * @param util - The action util class to register.
+ * @param options - Optional configuration for mode-specific registration.
+ * @returns The registered util class.
+ */
+export function registerActionUtil<T extends AgentActionUtilConstructor<BaseAgentAction>>(
+	util: T,
+	options?: RegisterActionUtilOptions
+): T {
+	const { forModes } = options ?? {}
+
+	if (forModes && forModes.length > 0) {
+		// Mode-specific registration
+		if (!modeRegistry.has(util.type)) {
+			modeRegistry.set(util.type, new Map())
+		}
+		const modeMap = modeRegistry.get(util.type)!
+		for (const mode of forModes) {
+			if (modeMap.has(mode)) {
+				throw new Error(`Action util for ${util.type} already registered for mode ${mode}`)
+			}
+			modeMap.set(mode, util)
+		}
+	} else {
+		// Default registration (existing behavior)
+		if (defaultRegistry.has(util.type)) {
+			throw new Error(`Agent action util already registered: ${util.type}`)
+		}
+		defaultRegistry.set(util.type, util)
+	}
+
+	return util
+}
+
+/**
+ * Get all registered agent action util classes.
+ * Returns both default and mode-specific utils (deduplicated).
+ */
+export function getAllActionUtils(): AgentActionUtilConstructor<AgentAction>[] {
+	const allUtils = new Set<AgentActionUtilConstructor<BaseAgentAction>>()
+
+	// Add all default utils
+	for (const util of defaultRegistry.values()) {
+		allUtils.add(util)
+	}
+
+	// Add all mode-specific utils
+	for (const modeMap of modeRegistry.values()) {
+		for (const util of modeMap.values()) {
+			allUtils.add(util)
+		}
+	}
+
+	return Array.from(allUtils) as AgentActionUtilConstructor<AgentAction>[]
+}
+
+/**
+ * Get an object containing instantiated agent action utils for an agent,
+ * resolved for a specific mode. Mode-specific utils override defaults.
+ *
+ * @param agent - The agent to create utils for.
+ * @param mode - The mode to resolve utils for.
+ * @returns A record of action utils keyed by action type.
+ */
+export function getAgentActionUtilsRecordForMode(agent: TldrawAgent, mode: string) {
+	const object = {} as Record<AgentAction['_type'], AgentActionUtil<AgentAction>>
+
+	// Start with defaults
+	for (const [type, util] of defaultRegistry.entries()) {
+		object[type as AgentAction['_type']] = new util(agent) as AgentActionUtil<AgentAction>
+	}
+
+	// Override with mode-specific utils
+	for (const [type, modeMap] of modeRegistry.entries()) {
+		const modeUtil = modeMap.get(mode)
+		if (modeUtil) {
+			object[type as AgentAction['_type']] = new modeUtil(agent) as AgentActionUtil<AgentAction>
+		}
+	}
+
+	return object
+}
+
+// ============================================================================
+// Base Class
+// ============================================================================
+
+export abstract class AgentActionUtil<T extends BaseAgentAction = BaseAgentAction> {
+	static type: string
+
+	agent: TldrawAgent
+	editor: Editor
+
+	constructor(agent: TldrawAgent) {
+		this.agent = agent
+		this.editor = agent?.editor
+	}
+
+	/**
+	 * Get information about the action to display within the chat history UI.
+	 * Return null to not show anything.
+	 * Defaults to the stringified action if not set.
+	 */
+	getInfo(_action: Streaming<T>): Partial<ChatHistoryInfo> | null {
+		return {}
+	}
+
+	/**
+	 * Transforms the action before saving it to chat history.
+	 * Useful for sanitizing or correcting actions.
+	 * @returns The transformed action, or null to reject the action
+	 */
+	sanitizeAction(action: Streaming<T>, _helpers: AgentHelpers): Streaming<T> | null {
+		return action
+	}
+
+	/**
+	 * Apply the action to the editor.
+	 * Any changes that happen during this function will be displayed as a diff.
+	 */
+	applyAction(_action: Streaming<T>, _helpers: AgentHelpers): Promise<void> | void {
+		// Do nothing by default
+	}
+
+	/**
+	 * Whether the action gets saved to history.
+	 */
+	savesToHistory(): boolean {
+		return true
+	}
+}
+
+export interface AgentActionUtilConstructor<T extends BaseAgentAction = BaseAgentAction> {
+	new (agent: TldrawAgent): AgentActionUtil<T>
+	type: T['_type']
+}

templates/agent/client/actions/AlignActionUtil.ts

@@ -0,0 +1,32 @@
+import { TLShapeId } from 'tldraw'
+import { AlignAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentHelpers } from '../AgentHelpers'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const AlignActionUtil = registerActionUtil(
+	class AlignActionUtil extends AgentActionUtil<AlignAction> {
+		static override type = 'align' as const
+
+		override getInfo(action: Streaming<AlignAction>) {
+			return {
+				icon: 'cursor' as const,
+				description: action.intent ?? '',
+			}
+		}
+
+		override sanitizeAction(action: Streaming<AlignAction>, helpers: AgentHelpers) {
+			action.shapeIds = helpers.ensureShapeIdsExist(action.shapeIds ?? [])
+			return action
+		}
+
+		override applyAction(action: Streaming<AlignAction>) {
+			if (!action.complete) return
+
+			this.editor.alignShapes(
+				action.shapeIds.map((id) => `shape:${id}` as TLShapeId),
+				action.alignment
+			)
+		}
+	}
+)

templates/agent/client/actions/BringToFrontActionUtil.ts

@@ -0,0 +1,28 @@
+import { TLShapeId } from 'tldraw'
+import { BringToFrontAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentHelpers } from '../AgentHelpers'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const BringToFrontActionUtil = registerActionUtil(
+	class BringToFrontActionUtil extends AgentActionUtil<BringToFrontAction> {
+		static override type = 'bringToFront' as const
+
+		override getInfo(action: Streaming<BringToFrontAction>) {
+			return {
+				icon: 'cursor' as const,
+				description: action.intent ?? '',
+			}
+		}
+
+		override sanitizeAction(action: Streaming<BringToFrontAction>, helpers: AgentHelpers) {
+			action.shapeIds = helpers.ensureShapeIdsExist(action.shapeIds ?? [])
+			return action
+		}
+
+		override applyAction(action: Streaming<BringToFrontAction>) {
+			if (!action.shapeIds) return
+			this.editor.bringToFront(action.shapeIds.map((shapeId) => `shape:${shapeId}` as TLShapeId))
+		}
+	}
+)

templates/agent/client/actions/ClearActionUtil.ts

@@ -0,0 +1,37 @@
+import { ClearAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const ClearActionUtil = registerActionUtil(
+	class ClearActionUtil extends AgentActionUtil<ClearAction> {
+		static override type = 'clear' as const
+
+		/**
+		 * Tell the model what the action's schema is
+		 */
+
+		/**
+		 * Tell the model how to display this action in the chat history UI
+		 */
+		override getInfo() {
+			return {
+				icon: 'trash' as const,
+				description: 'Cleared the canvas',
+			}
+		}
+
+		/**
+		 * Tell the model how to apply the action
+		 */
+		override applyAction(action: Streaming<ClearAction>) {
+			// Don't do anything if the action hasn't finished streaming
+			if (!action.complete) return
+
+			// Delete all shapes on the page
+			const { editor } = this
+
+			const allShapes = editor.getCurrentPageShapes()
+			editor.deleteShapes(allShapes)
+		}
+	}
+)

templates/agent/client/actions/CountShapesActionUtil.ts

@@ -0,0 +1,28 @@
+import { CountShapesAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentHelpers } from '../AgentHelpers'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const CountShapesActionUtil = registerActionUtil(
+	class CountShapesActionUtil extends AgentActionUtil<CountShapesAction> {
+		static override type = 'count' as const
+
+		override getInfo(action: Streaming<CountShapesAction>) {
+			const description = action.complete ? 'Counted shapes' : 'Counting shapes'
+			return {
+				icon: 'search' as const,
+				description,
+			}
+		}
+
+		override async applyAction(action: Streaming<CountShapesAction>, helpers: AgentHelpers) {
+			if (!action.complete) return
+			const { agent, editor } = helpers
+
+			// Add the shape count to the next request
+			agent.schedule({
+				data: [`Number of shapes on the canvas: ${editor.getCurrentPageShapes().length}`],
+			})
+		}
+	}
+)

templates/agent/client/actions/CountryInfoActionUtil.ts

@@ -0,0 +1,41 @@
+import { JsonValue } from 'tldraw'
+import { CountryInfoAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const CountryInfoActionUtil = registerActionUtil(
+	class CountryInfoActionUtil extends AgentActionUtil<CountryInfoAction> {
+		static override type = 'countryInfo' as const
+
+		override getInfo(action: Streaming<CountryInfoAction>) {
+			const description = action.complete
+				? 'Searched for country info'
+				: 'Searching for country info'
+			return {
+				icon: 'search' as const,
+				description,
+			}
+		}
+
+		override async applyAction(action: Streaming<CountryInfoAction>) {
+			// Wait until the action has finished streaming
+			if (!action.complete) return
+			const data = await fetchCountryInfo(action.code)
+			this.agent.schedule({ data: [data] })
+		}
+	}
+)
+
+export async function fetchCountryInfo(code: string) {
+	const response = await fetch(`https://restcountries.com/v3.1/alpha/${code}`)
+
+	if (!response.ok) {
+		throw new Error(`Country API returned status ${response.status}, ${response.statusText}`)
+	}
+
+	const json = await response.json()
+	if (Array.isArray(json)) {
+		return json[0] as JsonValue
+	}
+	return json as JsonValue
+}

templates/agent/client/actions/CreateActionUtil.ts

@@ -0,0 +1,230 @@
+import { IndexKey, TLShape, TLShapeId, toRichText } from 'tldraw'
+import {
+	convertPartialFocusedShapeToTldrawShape,
+	FOCUSED_TO_GEO_TYPES,
+} from '../../shared/format/convertFocusedShapeToTldrawShape'
+import { FocusedShape } from '../../shared/format/FocusedShape'
+import { CreateAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentHelpers } from '../AgentHelpers'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const CreateActionUtil = registerActionUtil(
+	class CreateActionUtil extends AgentActionUtil<CreateAction> {
+		static override type = 'create' as const
+
+		override getInfo(action: Streaming<CreateAction>) {
+			return {
+				icon: 'pencil' as const,
+				description: action.intent ?? '',
+			}
+		}
+
+		override sanitizeAction(action: Streaming<CreateAction>, helpers: AgentHelpers) {
+			const { shape } = action
+
+			// If there's no shape yet, return action (will be filtered in applyAction)
+			if (!shape) return action
+
+			// Ensure the created shape has a unique ID (only if shapeId is present)
+			if (shape.shapeId) {
+				shape.shapeId = helpers.ensureShapeIdIsUnique(shape.shapeId)
+			}
+
+			// If the shape is an arrow and complete, ensure the from and to IDs are real shapes
+			if (action.complete && shape._type === 'arrow') {
+				if (shape.fromId) {
+					shape.fromId = helpers.ensureShapeIdExists(shape.fromId)
+				}
+				if (shape.toId) {
+					shape.toId = helpers.ensureShapeIdExists(shape.toId)
+				}
+				if ('x1' in shape) {
+					shape.x1 = helpers.ensureValueIsNumber(shape.x1) ?? 0
+				}
+				if ('y1' in shape) {
+					shape.y1 = helpers.ensureValueIsNumber(shape.y1) ?? 0
+				}
+				if ('x2' in shape) {
+					shape.x2 = helpers.ensureValueIsNumber(shape.x2) ?? 0
+				}
+				if ('y2' in shape) {
+					shape.y2 = helpers.ensureValueIsNumber(shape.y2) ?? 0
+				}
+				if ('bend' in shape) {
+					shape.bend = helpers.ensureValueIsNumber(shape.bend) ?? 0
+				}
+			}
+
+			return action
+		}
+
+		override applyAction(action: Streaming<CreateAction>, helpers: AgentHelpers) {
+			const { editor } = this
+			const { shape } = action
+
+			// If there's no shape yet, return early
+			if (!shape || !shape._type) return
+
+			// Translate the shape back to the chat's position
+			const shapePartial = helpers.removeOffsetFromShapePartial(shape)
+
+			const result = convertPartialFocusedShapeToTldrawShape(editor, shapePartial, {
+				defaultShape: getDefaultShape(shape._type, action.complete),
+				complete: action.complete,
+			})
+
+			if (!result.shape) return
+
+			editor.createShape(result.shape)
+
+			// Handle arrow bindings if they exist
+			if (result.bindings) {
+				for (const binding of result.bindings) {
+					editor.createBinding({
+						type: binding.type,
+						fromId: binding.fromId,
+						toId: binding.toId,
+						props: binding.props,
+						meta: binding.meta,
+					})
+				}
+			}
+		}
+	}
+)
+
+function getDefaultShape(shapeType: FocusedShape['_type'], complete: boolean): Partial<TLShape> {
+	const isGeo = shapeType in FOCUSED_TO_GEO_TYPES
+	const defaultShape = isGeo
+		? SHAPE_DEFAULTS.geo
+		: (SHAPE_DEFAULTS[shapeType as keyof typeof SHAPE_DEFAULTS] ?? SHAPE_DEFAULTS.unknown)
+	return complete ? defaultShape : { ...defaultShape, isLocked: true }
+}
+
+type BaseShapeDefaults = Pick<TLShape, 'id' | 'isLocked' | 'opacity' | 'rotation' | 'meta'>
+
+const SHARED_DEFAULTS: BaseShapeDefaults = {
+	isLocked: false,
+	opacity: 1,
+	rotation: 0,
+	meta: {},
+	id: 'shape:shape' as TLShapeId,
+}
+
+type ShapeDefaults<T extends TLShape = TLShape> = BaseShapeDefaults & {
+	props?: Partial<T['props']>
+}
+
+type SupportedShapeTypes = 'text' | 'line' | 'arrow' | 'geo' | 'note' | 'draw'
+
+type ShapeDefaultsMap = {
+	[K in SupportedShapeTypes]: ShapeDefaults<TLShape<K>>
+} & {
+	unknown: ShapeDefaults
+}
+
+const SHAPE_DEFAULTS: ShapeDefaultsMap = {
+	text: {
+		...SHARED_DEFAULTS,
+		props: {
+			autoSize: true,
+			color: 'black',
+			font: 'draw',
+			richText: toRichText(''),
+			scale: 1,
+			size: 's',
+			textAlign: 'start',
+			w: 100,
+		},
+	},
+	line: {
+		...SHARED_DEFAULTS,
+		props: {
+			size: 's',
+			color: 'black',
+			dash: 'draw',
+			points: {
+				a1: {
+					id: 'a1',
+					index: 'a1' as IndexKey,
+					x: 0,
+					y: 0,
+				},
+				a2: {
+					id: 'a2',
+					index: 'a2' as IndexKey,
+					x: 100,
+					y: 0,
+				},
+			},
+			scale: 1,
+			spline: 'line',
+		},
+	},
+	arrow: {
+		...SHARED_DEFAULTS,
+		props: {
+			arrowheadEnd: 'arrow',
+			arrowheadStart: 'none',
+			bend: 0,
+			color: 'black',
+			dash: 'draw',
+			elbowMidPoint: 0.5,
+			end: { x: 100, y: 0 },
+			fill: 'none',
+			font: 'draw',
+			kind: 'arc',
+			labelColor: 'black',
+			labelPosition: 0.5,
+			richText: toRichText(''),
+			scale: 1,
+			size: 's',
+			start: { x: 0, y: 0 },
+		},
+	},
+	geo: {
+		...SHARED_DEFAULTS,
+		props: {
+			align: 'middle',
+			color: 'black',
+			dash: 'draw',
+			fill: 'none',
+			font: 'draw',
+			geo: 'rectangle',
+			growY: 0,
+			h: 100,
+			labelColor: 'black',
+			richText: toRichText(''),
+			scale: 1,
+			size: 's',
+			url: '',
+			verticalAlign: 'middle',
+			w: 100,
+		},
+	},
+	note: {
+		...SHARED_DEFAULTS,
+		props: {
+			color: 'black',
+			richText: toRichText(''),
+			size: 's',
+			align: 'middle',
+			font: 'draw',
+			fontSizeAdjustment: 1,
+			growY: 0,
+			labelColor: 'black',
+			scale: 1,
+			url: '',
+			verticalAlign: 'middle',
+		},
+	},
+	draw: {
+		...SHARED_DEFAULTS,
+		props: {},
+	},
+	unknown: {
+		...SHARED_DEFAULTS,
+		props: {},
+	},
+}

templates/agent/client/actions/DeleteActionUtil.ts

@@ -0,0 +1,36 @@
+import { TLShapeId } from 'tldraw'
+import { DeleteAction } from '../../shared/schema/AgentActionSchemas'
+import { BaseAgentAction } from '../../shared/types/BaseAgentAction'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentHelpers } from '../AgentHelpers'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const DeleteActionUtil = registerActionUtil(
+	class DeleteActionUtil extends AgentActionUtil<DeleteAction> {
+		static override type = 'delete' as const
+
+		override getInfo(action: Streaming<DeleteAction>) {
+			return {
+				icon: 'trash' as const,
+				description: action.intent ?? '',
+				canGroup: (other: Streaming<BaseAgentAction>) => other._type === 'delete',
+			}
+		}
+
+		override sanitizeAction(action: Streaming<DeleteAction>, helpers: AgentHelpers) {
+			if (!action.complete) return action
+
+			const shapeId = helpers.ensureShapeIdExists(action.shapeId)
+			if (!shapeId) return null
+
+			action.shapeId = shapeId
+			return action
+		}
+
+		override applyAction(action: Streaming<DeleteAction>) {
+			if (!action.complete) return
+
+			this.editor.deleteShape(`shape:${action.shapeId}` as TLShapeId)
+		}
+	}
+)

templates/agent/client/actions/DistributeActionUtil.ts

@@ -0,0 +1,32 @@
+import { TLShapeId } from 'tldraw'
+import { DistributeAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentHelpers } from '../AgentHelpers'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const DistributeActionUtil = registerActionUtil(
+	class DistributeActionUtil extends AgentActionUtil<DistributeAction> {
+		static override type = 'distribute' as const
+
+		override getInfo(action: Streaming<DistributeAction>) {
+			return {
+				icon: 'cursor' as const,
+				description: action.intent ?? '',
+			}
+		}
+
+		override sanitizeAction(action: Streaming<DistributeAction>, helpers: AgentHelpers) {
+			action.shapeIds = helpers.ensureShapeIdsExist(action.shapeIds ?? [])
+			return action
+		}
+
+		override applyAction(action: Streaming<DistributeAction>) {
+			if (!action.complete) return
+
+			this.editor.distributeShapes(
+				action.shapeIds.map((id) => `shape:${id}` as TLShapeId),
+				action.direction
+			)
+		}
+	}
+)

templates/agent/client/actions/LabelActionUtil.ts

@@ -0,0 +1,66 @@
+import { TLRichText, TLShape, TLShapeId, toRichText } from 'tldraw'
+import { LabelAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentHelpers } from '../AgentHelpers'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+type ShapeWithRichText = Extract<TLShape, { props: { richText: TLRichText } }>
+
+function isShapeWithRichText(shape: TLShape | null | undefined): shape is ShapeWithRichText {
+	return !!(shape && 'richText' in shape.props)
+}
+
+function assertShapeWithRichText(
+	shape: TLShape | null | undefined
+): asserts shape is ShapeWithRichText {
+	if (!isShapeWithRichText(shape)) {
+		throw new Error('Shape is not a valid ShapeWithRichText')
+	}
+}
+
+export const LabelActionUtil = registerActionUtil(
+	class LabelActionUtil extends AgentActionUtil<LabelAction> {
+		static override type = 'label' as const
+
+		override getInfo(action: Streaming<LabelAction>) {
+			return {
+				icon: 'pencil' as const,
+				description: action.intent ?? '',
+			}
+		}
+
+		override sanitizeAction(action: Streaming<LabelAction>, helpers: AgentHelpers) {
+			if (!action.complete) return action
+
+			const shapeId = helpers.ensureShapeIdExists(action.shapeId)
+			if (!shapeId) {
+				return null
+			}
+			const shape = this.editor.getShape(`shape:${shapeId}` as TLShapeId)
+			if (!shape) {
+				return null
+			}
+			if (!isShapeWithRichText(shape)) {
+				console.warn(`Shape type "${shape.type}" does not support richText labels`)
+				return null
+			}
+			action.shapeId = shapeId
+			return action
+		}
+
+		override applyAction(action: Streaming<LabelAction>) {
+			if (!action.complete) return
+			const { editor } = this
+
+			const shapeId = `shape:${action.shapeId}` as TLShapeId
+			const shape = editor.getShape(shapeId)
+			assertShapeWithRichText(shape)
+
+			editor.updateShape({
+				id: shapeId,
+				type: shape.type,
+				props: { richText: toRichText(action.text ?? '') },
+			})
+		}
+	}
+)

templates/agent/client/actions/MessageActionUtil.ts

@@ -0,0 +1,16 @@
+import { MessageAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const MessageActionUtil = registerActionUtil(
+	class MessageActionUtil extends AgentActionUtil<MessageAction> {
+		static override type = 'message' as const
+
+		override getInfo(action: Streaming<MessageAction>) {
+			return {
+				description: action.text ?? '',
+				canGroup: () => false,
+			}
+		}
+	}
+)

templates/agent/client/actions/MoveActionUtil.ts

@@ -0,0 +1,128 @@
+import { TLShapeId, Vec } from 'tldraw'
+import { MoveAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentHelpers } from '../AgentHelpers'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const MoveActionUtil = registerActionUtil(
+	class MoveActionUtil extends AgentActionUtil<MoveAction> {
+		static override type = 'move' as const
+
+		override getInfo(action: Streaming<MoveAction>) {
+			return {
+				icon: 'cursor' as const,
+				description: action.intent ?? '',
+			}
+		}
+
+		override sanitizeAction(action: Streaming<MoveAction>, helpers: AgentHelpers) {
+			if (!action.complete) return action
+
+			// Make sure the shape ID refers to a real shape
+			const shapeId = helpers.ensureShapeIdExists(action.shapeId)
+			if (!shapeId) return null
+			action.shapeId = shapeId
+
+			// Make sure the x and y values are numbers
+			const floatX = helpers.ensureValueIsNumber(action.x)
+			const floatY = helpers.ensureValueIsNumber(action.y)
+			if (floatX === null || floatY === null) return null
+			action.x = floatX
+			action.y = floatY
+
+			return action
+		}
+
+		override applyAction(action: Streaming<MoveAction>, helpers: AgentHelpers) {
+			if (!action.complete) return
+			const { editor } = this
+
+			// Translate the position back to the chat's position
+			const { x, y } = helpers.removeOffsetFromVec({ x: action.x, y: action.y })
+
+			const shapeId = `shape:${action.shapeId}` as TLShapeId
+			const shape = editor.getShape(shapeId)
+			if (!shape) return
+
+			const shapeBounds = editor.getShapePageBounds(shapeId)
+			if (!shapeBounds) return
+
+			const moveTarget = new Vec(x, y)
+			const shapeOrigin = new Vec(shape.x, shape.y)
+			const shapeBoundsOrigin = new Vec(shapeBounds.minX, shapeBounds.minY)
+
+			// Calculate the offset from the shape bounds origin to the shape origin
+			const shapeOriginDelta = shapeOrigin.sub(shapeBoundsOrigin)
+
+			// Adjust the target position based on the anchor point
+			const boundsWidth = shapeBounds.w
+			const boundsHeight = shapeBounds.h
+
+			// Calculate the anchor point offset from the bounds origin
+			let anchorOffsetX = 0
+			let anchorOffsetY = 0
+
+			switch (action.anchor) {
+				case 'top-left': {
+					anchorOffsetX = 0
+					anchorOffsetY = 0
+					break
+				}
+				case 'top-center': {
+					anchorOffsetX = boundsWidth / 2
+					anchorOffsetY = 0
+					break
+				}
+				case 'top-right': {
+					anchorOffsetX = boundsWidth
+					anchorOffsetY = 0
+					break
+				}
+				case 'bottom-left': {
+					anchorOffsetX = 0
+					anchorOffsetY = boundsHeight
+					break
+				}
+				case 'bottom-center': {
+					anchorOffsetX = boundsWidth / 2
+					anchorOffsetY = boundsHeight
+					break
+				}
+				case 'bottom-right': {
+					anchorOffsetX = boundsWidth
+					anchorOffsetY = boundsHeight
+					break
+				}
+				case 'center-left': {
+					anchorOffsetX = 0
+					anchorOffsetY = boundsHeight / 2
+					break
+				}
+				case 'center-right': {
+					anchorOffsetX = boundsWidth
+					anchorOffsetY = boundsHeight / 2
+					break
+				}
+				case 'center': {
+					anchorOffsetX = boundsWidth / 2
+					anchorOffsetY = boundsHeight / 2
+					break
+				}
+			}
+
+			// Adjust the target to account for the anchor point
+			// The target x,y should be where the anchor point is positioned
+			// So we subtract the anchor offset to get the bounds origin position
+			const adjustedTarget = moveTarget.sub(new Vec(anchorOffsetX, anchorOffsetY))
+
+			const newTarget = adjustedTarget.add(shapeOriginDelta)
+
+			editor.updateShape({
+				id: shapeId,
+				type: shape.type,
+				x: newTarget.x,
+				y: newTarget.y,
+			})
+		}
+	}
+)

templates/agent/client/actions/PenActionUtil.ts

@@ -0,0 +1,115 @@
+import { b64Vecs, TLDrawShapeSegment, TLShapeId, Vec, VecModel } from 'tldraw'
+import { asColor } from '../../shared/format/FocusedColor'
+import { convertFocusedFillToTldrawFill } from '../../shared/format/FocusedFill'
+import { PenAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentHelpers } from '../AgentHelpers'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const PenActionUtil = registerActionUtil(
+	class PenActionUtil extends AgentActionUtil<PenAction> {
+		static override type = 'pen' as const
+
+		override getInfo(action: Streaming<PenAction>) {
+			return {
+				icon: 'pencil' as const,
+				description: action.intent ?? '',
+			}
+		}
+
+		override sanitizeAction(action: Streaming<PenAction>, helpers: AgentHelpers) {
+			if (!action.points) return action
+
+			// Ensure the shape has a unique ID
+			action.shapeId = helpers.ensureShapeIdIsUnique(action.shapeId)
+
+			// Don't include the final point if we're still streaming.
+			// Its numbers might be incomplete.
+			const points = action.complete ? action.points : action.points.slice(0, -1)
+
+			// This is a complex action for the model, so validate the data it gives us
+			const validPoints = points
+				.map((point) => helpers.ensureValueIsVec(point))
+				.filter((v) => v !== null)
+
+			action.points = validPoints
+			action.closed = helpers.ensureValueIsBoolean(action.closed) ?? false
+			action.fill = helpers.ensureValueIsFocusedFill(action.fill) ?? 'none'
+
+			return action
+		}
+
+		override applyAction(action: Streaming<PenAction>, helpers: AgentHelpers) {
+			if (!action.points) return
+			if (action.points.length === 0) return
+
+			if (!action.shapeId) return
+			const shapeId = `shape:${action.shapeId}` as TLShapeId
+
+			action.points = action.points.map((point) => helpers.removeOffsetFromVec(point))
+
+			if (action.closed) {
+				const firstPoint = action.points[0]
+				action.points.push(firstPoint)
+			}
+
+			const minX = Math.min(...action.points.map((p) => p.x))
+			const minY = Math.min(...action.points.map((p) => p.y))
+
+			const points: VecModel[] = []
+			const maxDistanceBetweenPoints = action.style === 'smooth' ? 10 : 2
+			for (let i = 0; i < action.points.length - 1; i++) {
+				const point = action.points[i]
+				points.push(point)
+
+				const nextPoint = action.points[i + 1]
+				if (!nextPoint) continue
+
+				const distance = Vec.Dist(point, nextPoint)
+				const numPointsToAdd = Math.floor(distance / maxDistanceBetweenPoints)
+				const pointsToAdd = Array.from({ length: numPointsToAdd }, (_, j) => {
+					const t = (j + 1) / (numPointsToAdd + 1)
+					return Vec.Lrp(point, nextPoint, t)
+				})
+				points.push(...pointsToAdd)
+			}
+
+			// Require at least 2 points to draw a line
+			if (points.length <= 1) {
+				return
+			}
+
+			const segmentPoints = points.map((point) => ({
+				x: point.x - minX,
+				y: point.y - minY,
+				z: 0.75,
+			}))
+			const base64Points = b64Vecs.encodePoints(segmentPoints)
+
+			const segments: TLDrawShapeSegment[] = [
+				{
+					type: 'free',
+					path: base64Points,
+				},
+			]
+
+			this.editor.createShape({
+				id: shapeId,
+				type: 'draw',
+				x: minX,
+				y: minY,
+				isLocked: !action.complete,
+				props: {
+					color: asColor(action.color ?? 'black'),
+					fill: convertFocusedFillToTldrawFill(action.fill ?? 'none'),
+					dash: 'draw',
+					size: 's',
+					segments,
+					isComplete: action.complete,
+					isClosed: action.closed,
+					isPen: true,
+				},
+			})
+		}
+	}
+)

templates/agent/client/actions/PlaceActionUtil.ts

@@ -0,0 +1,135 @@
+import { TLShapeId } from 'tldraw'
+import { PlaceAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentHelpers } from '../AgentHelpers'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const PlaceActionUtil = registerActionUtil(
+	class PlaceActionUtil extends AgentActionUtil<PlaceAction> {
+		static override type = 'place' as const
+
+		override getInfo(action: Streaming<PlaceAction>) {
+			return {
+				icon: 'target' as const,
+				description: action.intent ?? '',
+			}
+		}
+
+		override sanitizeAction(action: Streaming<PlaceAction>, helpers: AgentHelpers) {
+			if (!action.complete) return action
+
+			const shapeId = helpers.ensureShapeIdExists(action.shapeId)
+			if (!shapeId) return null
+			action.shapeId = shapeId
+
+			const referenceShapeId = helpers.ensureShapeIdExists(action.referenceShapeId)
+			if (!referenceShapeId) return null
+			action.referenceShapeId = referenceShapeId
+
+			return action
+		}
+
+		override applyAction(action: Streaming<PlaceAction>) {
+			if (!action.complete) return
+			const { editor } = this
+
+			const { side, sideOffset = 0, align, alignOffset = 0 } = action
+			const referenceShapeId = `shape:${action.referenceShapeId}` as TLShapeId
+			const shapeId = `shape:${action.shapeId}` as TLShapeId
+
+			const shape = editor.getShape(shapeId)
+			const referenceShape = editor.getShape(referenceShapeId)
+			if (!shape || !referenceShape) return
+
+			const bbA = editor.getShapePageBounds(shape)!
+			const bbR = editor.getShapePageBounds(referenceShape)!
+			if (side === 'top' && align === 'start') {
+				editor.updateShape({
+					id: shapeId,
+					type: shape.type,
+					x: bbR.minX + alignOffset,
+					y: bbR.minY - bbA.height - sideOffset,
+				})
+			} else if (side === 'top' && align === 'center') {
+				editor.updateShape({
+					id: shapeId,
+					type: shape.type,
+					x: bbR.midX - bbA.width / 2 + alignOffset,
+					y: bbR.minY - bbA.height - sideOffset,
+				})
+			} else if (side === 'top' && align === 'end') {
+				editor.updateShape({
+					id: shapeId,
+					type: shape.type,
+					x: bbR.maxX - bbA.width - alignOffset,
+					y: bbR.minY - bbA.height - sideOffset,
+				})
+			} else if (side === 'bottom' && align === 'start') {
+				editor.updateShape({
+					id: shapeId,
+					type: shape.type,
+					x: bbR.minX + alignOffset,
+					y: bbR.maxY + sideOffset,
+				})
+			} else if (side === 'bottom' && align === 'center') {
+				editor.updateShape({
+					id: shapeId,
+					type: shape.type,
+					x: bbR.midX - bbA.width / 2 + alignOffset,
+					y: bbR.maxY + sideOffset,
+				})
+			} else if (side === 'bottom' && align === 'end') {
+				editor.updateShape({
+					id: shapeId,
+					type: shape.type,
+					x: bbR.maxX - bbA.width - alignOffset,
+					y: bbR.maxY + sideOffset,
+				})
+				// LEFT SIDE (corrected)
+			} else if (side === 'left' && align === 'start') {
+				editor.updateShape({
+					id: shapeId,
+					type: shape.type,
+					x: bbR.minX - bbA.width - sideOffset,
+					y: bbR.minY + alignOffset,
+				})
+			} else if (side === 'left' && align === 'center') {
+				editor.updateShape({
+					id: shapeId,
+					type: shape.type,
+					x: bbR.minX - bbA.width - sideOffset,
+					y: bbR.midY - bbA.height / 2 + alignOffset,
+				})
+			} else if (side === 'left' && align === 'end') {
+				editor.updateShape({
+					id: shapeId,
+					type: shape.type,
+					x: bbR.minX - bbA.width - sideOffset,
+					y: bbR.maxY - bbA.height - alignOffset,
+				})
+				// RIGHT SIDE (corrected)
+			} else if (side === 'right' && align === 'start') {
+				editor.updateShape({
+					id: shapeId,
+					type: shape.type,
+					x: bbR.maxX + sideOffset,
+					y: bbR.minY + alignOffset,
+				})
+			} else if (side === 'right' && align === 'center') {
+				editor.updateShape({
+					id: shapeId,
+					type: shape.type,
+					x: bbR.maxX + sideOffset,
+					y: bbR.midY - bbA.height / 2 + alignOffset,
+				})
+			} else if (side === 'right' && align === 'end') {
+				editor.updateShape({
+					id: shapeId,
+					type: shape.type,
+					x: bbR.maxX + sideOffset,
+					y: bbR.maxY - bbA.height - alignOffset,
+				})
+			}
+		}
+	}
+)

templates/agent/client/actions/ResizeActionUtil.ts

@@ -0,0 +1,49 @@
+import { TLShapeId } from 'tldraw'
+import { ResizeAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentHelpers } from '../AgentHelpers'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const ResizeActionUtil = registerActionUtil(
+	class ResizeActionUtil extends AgentActionUtil<ResizeAction> {
+		static override type = 'resize' as const
+
+		override getInfo(action: Streaming<ResizeAction>) {
+			return {
+				icon: 'cursor' as const,
+				description: action.intent ?? '',
+			}
+		}
+
+		override sanitizeAction(action: Streaming<ResizeAction>, helpers: AgentHelpers) {
+			const shapeIds = helpers.ensureShapeIdsExist(action.shapeIds ?? [])
+			if (shapeIds.length === 0) return null
+
+			action.shapeIds = shapeIds
+			return action
+		}
+
+		override applyAction(action: Streaming<ResizeAction>, helpers: AgentHelpers) {
+			if (
+				!action.shapeIds ||
+				!action.scaleX ||
+				!action.scaleY ||
+				!action.originX ||
+				!action.originY
+			) {
+				return
+			}
+
+			const origin = helpers.removeOffsetFromVec({ x: action.originX, y: action.originY })
+			const shapeIds = action.shapeIds.map((shapeId) => `shape:${shapeId}` as TLShapeId)
+
+			for (const shapeId of shapeIds) {
+				this.editor.resizeShape(
+					shapeId,
+					{ x: action.scaleX, y: action.scaleY },
+					{ scaleOrigin: origin }
+				)
+			}
+		}
+	}
+)

templates/agent/client/actions/ReviewActionUtil.ts

@@ -0,0 +1,61 @@
+import { Box } from 'tldraw'
+import { ReviewAction } from '../../shared/schema/AgentActionSchemas'
+import { AreaContextItem } from '../../shared/types/ContextItem'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentHelpers } from '../AgentHelpers'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const ReviewActionUtil = registerActionUtil(
+	class ReviewActionUtil extends AgentActionUtil<ReviewAction> {
+		static override type = 'review' as const
+
+		override getInfo(action: Streaming<ReviewAction>) {
+			const label = action.complete ? 'Review' : 'Reviewing'
+			const text = action.intent?.startsWith('#') ? `\n\n${action.intent}` : action.intent
+			const description = `**${label}:** ${text ?? ''}`
+
+			return {
+				icon: 'search' as const,
+				description,
+			}
+		}
+
+		override applyAction(action: Streaming<ReviewAction>, helpers: AgentHelpers) {
+			if (!action.complete) return
+
+			const reviewBounds = helpers.removeOffsetFromBox({
+				x: action.x,
+				y: action.y,
+				w: action.w,
+				h: action.h,
+			})
+
+			const contextArea: AreaContextItem = {
+				type: 'area',
+				bounds: reviewBounds,
+				source: 'agent',
+			}
+
+			// If the review area is outside the already-scheduled bounds, expand the bounds to include it
+			const scheduledRequest = this.agent.requests.getScheduledRequest()
+			const bounds = scheduledRequest
+				? Box.From(scheduledRequest.bounds).union(reviewBounds)
+				: reviewBounds
+
+			// Schedule the review
+			this.agent.schedule({
+				bounds,
+				message: getReviewMessage(action.intent),
+				contextItems: [contextArea],
+			})
+		}
+	}
+)
+
+function getReviewMessage(intent: string) {
+	return `Examine the actions that you (the agent) took since the most recent user message, with the intent: "${intent}". What's next?
+
+- Is there still more work to do? If so, continue it.
+- Is the task supposed to be complete? If so, it's time to review the results of that. Did you do what the user asked for? Did the plan work? Think through your findings and pay close attention to the image, because that's what you can see right now. If you make any corrections, let the user know what you did and why. If no corrections are needed, there's no need to say anything.
+- Make sure to reference your last actions (denoted by [ACTION]) in order to see if you completed the task. Assume each action you see in the chat history completed successfully.`
+}

templates/agent/client/actions/RotateActionUtil.ts

@@ -0,0 +1,35 @@
+import { TLShapeId } from 'tldraw'
+import { RotateAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentHelpers } from '../AgentHelpers'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const RotateActionUtil = registerActionUtil(
+	class RotateActionUtil extends AgentActionUtil<RotateAction> {
+		static override type = 'rotate' as const
+
+		override getInfo(action: Streaming<RotateAction>) {
+			return {
+				icon: 'cursor' as const,
+				description: action.intent ?? '',
+			}
+		}
+
+		override sanitizeAction(action: Streaming<RotateAction>, helpers: AgentHelpers) {
+			action.shapeIds = helpers.ensureShapeIdsExist(action.shapeIds ?? [])
+			return action
+		}
+
+		override applyAction(action: Streaming<RotateAction>, helpers: AgentHelpers) {
+			if (!action.shapeIds || !action.degrees || !action.originX || !action.originY) {
+				return
+			}
+
+			const origin = helpers.removeOffsetFromVec({ x: action.originX, y: action.originY })
+			const shapeIds = action.shapeIds.map((shapeId) => `shape:${shapeId}` as TLShapeId)
+			const radians = (action.degrees * Math.PI) / 180
+
+			this.editor.rotateShapesBy(shapeIds, radians, { center: origin })
+		}
+	}
+)

templates/agent/client/actions/SendToBackActionUtil.ts

@@ -0,0 +1,28 @@
+import { TLShapeId } from 'tldraw'
+import { SendToBackAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentHelpers } from '../AgentHelpers'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const SendToBackActionUtil = registerActionUtil(
+	class SendToBackActionUtil extends AgentActionUtil<SendToBackAction> {
+		static override type = 'sendToBack' as const
+
+		override getInfo(action: Streaming<SendToBackAction>) {
+			return {
+				icon: 'cursor' as const,
+				description: action.intent ?? '',
+			}
+		}
+
+		override sanitizeAction(action: Streaming<SendToBackAction>, helpers: AgentHelpers) {
+			action.shapeIds = helpers.ensureShapeIdsExist(action.shapeIds ?? [])
+			return action
+		}
+
+		override applyAction(action: Streaming<SendToBackAction>) {
+			if (!action.shapeIds) return
+			this.editor.sendToBack(action.shapeIds.map((shapeId) => `shape:${shapeId}` as TLShapeId))
+		}
+	}
+)

templates/agent/client/actions/SetMyViewActionUtil.ts

@@ -0,0 +1,42 @@
+import { structuredClone } from 'tldraw'
+import { SetMyViewAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentHelpers } from '../AgentHelpers'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const SetMyViewActionUtil = registerActionUtil(
+	class SetMyViewActionUtil extends AgentActionUtil<SetMyViewAction> {
+		static override type = 'setMyView' as const
+
+		override getInfo(action: Streaming<SetMyViewAction>) {
+			const label = action.complete ? 'Move camera' : 'Moving camera'
+			const text = action.intent?.startsWith('#') ? `\n\n${action.intent}` : action.intent
+			return {
+				icon: 'eye' as const,
+				description: `**${label}**: ${text ?? ''}`,
+			}
+		}
+
+		override applyAction(action: Streaming<SetMyViewAction>, helpers: AgentHelpers) {
+			if (!action.complete) return
+
+			const roundedAction = helpers.roundBox(structuredClone(action))
+
+			const bounds = helpers.removeOffsetFromBox({
+				x: action.x,
+				y: action.y,
+				w: action.w,
+				h: action.h,
+			})
+
+			this.agent.interrupt({
+				input: {
+					bounds,
+					agentMessages: [
+						`Just navigated to new area with the intent: ${action.intent}. Can now see the new area at (${roundedAction.x}, ${roundedAction.y}) and is ${roundedAction.w}x${roundedAction.h} in size.`, // this uses the action instead of the bounds because this will go in the chat history and so must be consistent with what the agent thinks the chat origin is
+					],
+				},
+			})
+		}
+	}
+)

templates/agent/client/actions/StackActionUtil.ts

@@ -0,0 +1,36 @@
+import { TLShapeId } from 'tldraw'
+import { StackAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentHelpers } from '../AgentHelpers'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const StackActionUtil = registerActionUtil(
+	class StackActionUtil extends AgentActionUtil<StackAction> {
+		static override type = 'stack' as const
+
+		override getInfo(action: Streaming<StackAction>) {
+			return {
+				icon: 'cursor' as const,
+				description: action.intent ?? '',
+			}
+		}
+
+		override sanitizeAction(action: Streaming<StackAction>, helpers: AgentHelpers) {
+			if (!action.complete) return action
+
+			action.shapeIds = helpers.ensureShapeIdsExist(action.shapeIds)
+
+			return action
+		}
+
+		override applyAction(action: Streaming<StackAction>) {
+			if (!action.complete) return
+
+			this.editor.stackShapes(
+				action.shapeIds.map((id) => `shape:${id}` as TLShapeId),
+				action.direction,
+				Math.max(action.gap, 0)
+			)
+		}
+	}
+)

templates/agent/client/actions/ThinkActionUtil.ts

@@ -0,0 +1,22 @@
+import { ThinkAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const ThinkActionUtil = registerActionUtil(
+	class ThinkActionUtil extends AgentActionUtil<ThinkAction> {
+		static override type = 'think' as const
+
+		override getInfo(action: Streaming<ThinkAction>) {
+			const time = Math.floor(action.time / 1000)
+			let summary = `Thought for ${time} seconds`
+			if (time === 0) summary = 'Thought for less than a second'
+			if (time === 1) summary = 'Thought for 1 second'
+
+			return {
+				icon: 'brain' as const,
+				description: action.text ?? (action.complete ? null : 'Thinking...'),
+				summary,
+			}
+		}
+	}
+)

templates/agent/client/actions/UnknownActionUtil.ts

@@ -0,0 +1,18 @@
+import { UnknownAction } from '../../shared/schema/AgentActionSchemas'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+/**
+ * This event util is used when the event type is unknown.
+ * This usually happens because the event type has not finished streaming yet.
+ * Sometimes it happens because the model makes a mistake.
+ */
+export const UnknownActionUtil = registerActionUtil(
+	class UnknownActionUtil extends AgentActionUtil<UnknownAction> {
+		static override type = 'unknown' as const
+
+		override getInfo() {
+			// Don't show anything in the UI for unknown actions
+			return null
+		}
+	}
+)

templates/agent/client/actions/UpdateActionUtil.ts

@@ -0,0 +1,106 @@
+import { TLBindingId } from 'tldraw'
+import {
+	convertFocusedShapeToTldrawShape,
+	convertSimpleIdToTldrawId,
+} from '../../shared/format/convertFocusedShapeToTldrawShape'
+import { UpdateAction } from '../../shared/schema/AgentActionSchemas'
+import { toSimpleShapeId } from '../../shared/types/ids-schema'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentHelpers } from '../AgentHelpers'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const UpdateActionUtil = registerActionUtil(
+	class UpdateActionUtil extends AgentActionUtil<UpdateAction> {
+		static override type = 'update' as const
+
+		override getInfo(action: Streaming<UpdateAction>) {
+			return {
+				icon: 'cursor' as const,
+				description: action.intent ?? '',
+			}
+		}
+
+		override sanitizeAction(action: Streaming<UpdateAction>, helpers: AgentHelpers) {
+			if (!action.complete) return action
+
+			const { update } = action
+
+			// Ensure the shape ID refers to a real shape
+			const shapeId = helpers.ensureShapeIdExists(toSimpleShapeId(update.shapeId))
+			if (!shapeId) return null
+			update.shapeId = shapeId
+
+			// If it's an arrow, ensure the from and to IDs refer to real shapes
+			if (update._type === 'arrow') {
+				if (update.fromId) {
+					update.fromId = helpers.ensureShapeIdExists(update.fromId)
+				}
+				if (update.toId) {
+					update.toId = helpers.ensureShapeIdExists(update.toId)
+				}
+
+				if ('x1' in update) {
+					update.x1 = helpers.ensureValueIsNumber(update.x1) ?? 0
+				}
+				if ('y1' in update) {
+					update.y1 = helpers.ensureValueIsNumber(update.y1) ?? 0
+				}
+				if ('x2' in update) {
+					update.x2 = helpers.ensureValueIsNumber(update.x2) ?? 0
+				}
+				if ('y2' in update) {
+					update.y2 = helpers.ensureValueIsNumber(update.y2) ?? 0
+				}
+				if ('bend' in update) {
+					update.bend = helpers.ensureValueIsNumber(update.bend) ?? 0
+				}
+			}
+
+			// Unround the shape to restore the original values
+			action.update = helpers.unroundShape(action.update)
+
+			return action
+		}
+
+		override applyAction(action: Streaming<UpdateAction>, helpers: AgentHelpers) {
+			if (!action.complete) return
+			const { editor } = this
+
+			// Translate the shape back to the chat's position
+			action.update = helpers.removeOffsetFromShape(action.update)
+
+			const shapeId = convertSimpleIdToTldrawId(action.update.shapeId)
+			const existingShape = editor.getShape(shapeId)
+
+			if (!existingShape) {
+				throw new Error(`Shape ${shapeId} not found in canvas`)
+			}
+
+			const result = convertFocusedShapeToTldrawShape(editor, action.update, {
+				defaultShape: existingShape,
+			})
+
+			editor.updateShape(result.shape)
+
+			// Handle arrow bindings if they exist
+			if (result.bindings) {
+				// First, clean up existing bindings
+				const existingBindings = editor.getBindingsFromShape(shapeId, 'arrow')
+				for (const binding of existingBindings) {
+					editor.deleteBinding(binding.id as TLBindingId)
+				}
+
+				// Create new bindings
+				for (const binding of result.bindings) {
+					editor.createBinding({
+						type: binding.type,
+						fromId: binding.fromId,
+						toId: binding.toId,
+						props: binding.props,
+						meta: binding.meta,
+					})
+				}
+			}
+		}
+	}
+)

templates/agent/client/actions/UpsertTodoListItemActionUtil.ts

@@ -0,0 +1,33 @@
+import { UpsertPersonalTodoItemAction } from '../../shared/schema/AgentActionSchemas'
+import { Streaming } from '../../shared/types/Streaming'
+import { AgentActionUtil, registerActionUtil } from './AgentActionUtil'
+
+export const UpsertTodoListItemActionUtil = registerActionUtil(
+	class UpsertTodoListItemActionUtil extends AgentActionUtil<UpsertPersonalTodoItemAction> {
+		static override type = 'update-todo-list' as const
+
+		override getInfo() {
+			// Don't show todo actions in the chat history because we show them in the dedicated todo list UI
+			return null
+		}
+
+		override applyAction(action: Streaming<UpsertPersonalTodoItemAction>) {
+			if (!action.complete) return
+
+			const { id, text, status } = action
+
+			const index = this.agent.todos.getTodos().findIndex((item) => item.id === id)
+			if (index === -1) {
+				if (!text) {
+					this.agent.interrupt({
+						input: 'You must provide text when creating a new todo item.',
+					})
+					return
+				}
+				this.agent.todos.push(id, text)
+			} else {
+				this.agent.todos.update({ id, status, text })
+			}
+		}
+	}
+)

templates/agent/client/agent/TldrawAgent.ts

@@ -0,0 +1,735 @@
+import { Editor, RecordsDiff, reverseRecordsDiff, structuredClone, TLRecord } from 'tldraw'
+import { convertTldrawShapeToFocusedShape } from '../../shared/format/convertTldrawShapeToFocusedShape'
+import { AgentModelName } from '../../shared/models'
+import { AgentAction } from '../../shared/types/AgentAction'
+import { AgentInput } from '../../shared/types/AgentInput'
+import { AgentPrompt, BaseAgentPrompt } from '../../shared/types/AgentPrompt'
+import { AgentRequest } from '../../shared/types/AgentRequest'
+import { ChatHistoryItem, ChatHistoryPromptItem } from '../../shared/types/ChatHistoryItem'
+import { ContextItem } from '../../shared/types/ContextItem'
+import { PromptPart } from '../../shared/types/PromptPart'
+import { Streaming } from '../../shared/types/Streaming'
+import { TodoItem } from '../../shared/types/TodoItem'
+import { AgentHelpers } from '../AgentHelpers'
+import { getModeNode } from '../modes/AgentModeChart'
+import { AgentModeType } from '../modes/AgentModeDefinitions'
+import { getPromptPartUtilsRecord, PromptPartUtil } from '../parts/PromptPartUtil'
+import { AgentActionManager } from './managers/AgentActionManager'
+import { AgentChatManager } from './managers/AgentChatManager'
+import { AgentChatOriginManager } from './managers/AgentChatOriginManager'
+import { AgentContextManager } from './managers/AgentContextManager'
+import { AgentDebugFlags, AgentDebugManager } from './managers/AgentDebugManager'
+import { AgentLintManager } from './managers/AgentLintManager'
+import { AgentModelNameManager } from './managers/AgentModelNameManager'
+import { AgentModeManager } from './managers/AgentModeManager'
+import { AgentRequestManager } from './managers/AgentRequestManager'
+import { AgentTodoManager } from './managers/AgentTodoManager'
+import { AgentUserActionTracker } from './managers/AgentUserActionTracker'
+
+/**
+ * The persisted state of an agent.
+ * Used for saving and loading agent state.
+ */
+export interface PersistedAgentState {
+	chatHistory?: ChatHistoryItem[]
+	chatOrigin?: { x: number; y: number }
+	todoList?: TodoItem[]
+	contextItems?: ContextItem[]
+	modelName?: AgentModelName
+	debugFlags?: AgentDebugFlags
+}
+
+export interface TldrawAgentOptions {
+	/** The editor to associate the agent with. */
+	editor: Editor
+	/** A key used to differentiate the agent from other agents. */
+	id: string
+	/** A callback for when an error occurs. */
+	onError: (e: any) => void
+}
+
+/**
+ * An agent that can be prompted to edit the canvas.
+ * Access the agent via `useAgent()` hook from TldrawAgentAppProvider,
+ * or via `AgentAppAgentsManager.getAgent(editor)`.
+ *
+ * @example
+ * ```tsx
+ * const agent = useAgent()
+ * agent.prompt('Draw a snowman')
+ * ```
+ */
+export class TldrawAgent {
+	/** The editor associated with this agent. */
+	editor: Editor
+
+	/** An id to differentiate the agent from other agents. */
+	id: string
+
+	/** A callback for when an error occurs. */
+	onError: (e: any) => void
+
+	// ==================== Managers ====================
+
+	/** The action manager associated with this agent. */
+	actions: AgentActionManager
+
+	/** The chat manager associated with this agent. */
+	chat: AgentChatManager
+
+	/** The chat origin manager associated with this agent. */
+	chatOrigin: AgentChatOriginManager
+
+	/** The context manager associated with this agent. */
+	context: AgentContextManager
+
+	/** The debug manager associated with this agent. */
+	debug: AgentDebugManager
+
+	/** The lint manager associated with this agent. */
+	lints: AgentLintManager
+
+	/** The mode manager associated with this agent. */
+	mode: AgentModeManager
+
+	/** The model name manager associated with this agent. */
+	modelName: AgentModelNameManager
+
+	/** The request manager associated with this agent. */
+	requests: AgentRequestManager
+
+	/** The todo manager associated with this agent. */
+	todos: AgentTodoManager
+
+	/** The user action tracker associated with this agent. */
+	userAction: AgentUserActionTracker
+
+	// ==================== Prompt Part Utils ====================
+
+	/**
+	 * A record of the agent's prompt part util instances.
+	 * Used by the `getPromptPartUtil` method.
+	 */
+	promptPartUtils: Record<PromptPart['type'], PromptPartUtil<PromptPart>>
+
+	/**
+	 * Get a prompt part util for a specific part type.
+	 *
+	 * @param type - The type of part to get the util for.
+	 * @returns The part util.
+	 */
+	getPromptPartUtil(type: PromptPart['type']) {
+		return this.promptPartUtils[type]
+	}
+
+	/**
+	 * Create a new tldraw agent.
+	 */
+	constructor({ editor, id, onError }: TldrawAgentOptions) {
+		this.editor = editor
+		this.id = id
+		this.onError = onError
+
+		// Initialize managers
+		// Note: mode must be initialized before actions, since actions depends on mode
+		this.mode = new AgentModeManager(this)
+		this.actions = new AgentActionManager(this)
+		this.chat = new AgentChatManager(this)
+		this.chatOrigin = new AgentChatOriginManager(this)
+		this.context = new AgentContextManager(this)
+		this.debug = new AgentDebugManager(this)
+		this.lints = new AgentLintManager(this)
+		this.modelName = new AgentModelNameManager(this)
+		this.requests = new AgentRequestManager(this)
+		this.todos = new AgentTodoManager(this)
+		this.userAction = new AgentUserActionTracker(this)
+
+		// Note: Agent registration is handled by AgentAppAgentsManager.createAgent()
+
+		// Initialize prompt part utils
+		this.promptPartUtils = getPromptPartUtilsRecord(this)
+
+		// Start recording user actions
+		this.userAction.startRecording()
+	}
+
+	// ==================== State Persistence ====================
+
+	/**
+	 * Serialize the agent's state to a plain object for persistence.
+	 * This is called by the app-level persistence manager to save agent state.
+	 */
+	serializeState(): PersistedAgentState {
+		return {
+			chatHistory: this.chat.getHistory(),
+			chatOrigin: this.chatOrigin.getOrigin(),
+			todoList: this.todos.getTodos(),
+			contextItems: this.context.getItems(),
+			modelName: this.modelName.getModelName(),
+			debugFlags: this.debug.getDebugFlags(),
+		}
+	}
+
+	/**
+	 * Load previously persisted state into the agent.
+	 * This is called by the app-level persistence manager to restore agent state.
+	 *
+	 * @param state - The persisted state to load.
+	 */
+	loadState(state: PersistedAgentState) {
+		if (state.chatHistory) {
+			this.chat.setHistory(state.chatHistory)
+		}
+		if (state.chatOrigin) {
+			this.chatOrigin.setOrigin(state.chatOrigin)
+		}
+		if (state.todoList) {
+			this.todos.setTodos(state.todoList)
+		}
+		if (state.contextItems) {
+			this.context.setItems(state.contextItems)
+		}
+		if (state.modelName) {
+			this.modelName.setModelName(state.modelName)
+		}
+		if (state.debugFlags) {
+			this.debug.setDebugFlags(state.debugFlags)
+		}
+	}
+
+	/**
+	 * Dispose of the agent by cancelling requests and stopping listeners.
+	 */
+	dispose() {
+		this.cancel()
+		this.userAction.dispose()
+
+		// Dispose all managers
+		this.actions.dispose()
+		this.chat.dispose()
+		this.chatOrigin.dispose()
+		this.context.dispose()
+		this.debug.dispose()
+		this.lints.dispose()
+		this.mode.dispose()
+		this.modelName.dispose()
+		this.requests.dispose()
+		this.todos.dispose()
+
+		// Note: Agent removal from registry is handled by AgentAppAgentsManager.deleteAgent()
+	}
+
+	/**
+	 * Whether the agent is currently acting on the editor or not.
+	 * This flag is used to prevent agent actions from being recorded as user actions.
+	 *
+	 * Do not use this to check if the agent is currently working on a request. Use `isGenerating` instead.
+	 */
+	private isActingOnEditor = false
+
+	/**
+	 * Get whether the agent is currently acting on the editor.
+	 * @returns true if the agent is currently acting, false otherwise.
+	 */
+	getIsActingOnEditor(): boolean {
+		return this.isActingOnEditor
+	}
+
+	/**
+	 * Set whether the agent is currently acting on the editor.
+	 * @param value - true if the agent is acting, false otherwise.
+	 */
+	setIsActingOnEditor(value: boolean): void {
+		this.isActingOnEditor = value
+	}
+
+	// ==================== Request Handling ====================
+
+	/**
+	 * Get a full prompt based on a request.
+	 *
+	 * @param request - The request to use for the prompt.
+	 * @param helpers - The helpers to use.
+	 * @returns The fully assembled prompt.
+	 */
+	async preparePrompt(request: AgentRequest, helpers: AgentHelpers): Promise<AgentPrompt> {
+		const { promptPartUtils } = this
+		const transformedParts: PromptPart[] = []
+
+		// Get available prompt part types from the current mode
+		const modeDefinition = this.mode.getCurrentModeDefinition()
+		if (!modeDefinition.active) {
+			throw new Error(
+				`Fairy is not in an active mode so can't act right now. Current mode: ${modeDefinition.type}`
+			)
+		}
+
+		const availablePromptPartTypes = modeDefinition.parts
+
+		for (const promptPartType of availablePromptPartTypes) {
+			const util = promptPartUtils[promptPartType]
+			if (!util) throw new Error(`Prompt part util not found for part type: ${promptPartType}`)
+			const part = await util.getPart(structuredClone(request), helpers)
+			if (!part) continue
+			transformedParts.push(part)
+		}
+
+		return Object.fromEntries(transformedParts.map((part) => [part.type, part])) as AgentPrompt
+	}
+
+	/**
+	 * Prompt the agent to edit the canvas.
+	 *
+	 * @example
+	 * ```tsx
+	 * const agent = useAgent()
+	 * agent.prompt('Draw a cat')
+	 * ```
+	 *
+	 * ```tsx
+	 * agent.prompt({
+	 *   message: 'Draw a cat in this area',
+	 *   bounds: {
+	 *     x: 0,
+	 *     y: 0,
+	 *     w: 300,
+	 *     h: 400,
+	 *   },
+	 * })
+	 * ```
+	 *
+	 * @returns A promise for when the agent has finished its work.
+	 */
+	async prompt(input: AgentInput, { nested = false }: { nested?: boolean } = {}) {
+		if (this.requests.isGenerating() && !nested) {
+			throw new Error('Agent is already prompting. Please wait for the current prompt to finish.')
+		}
+
+		if (this.isActingOnEditor) {
+			throw new Error(
+				"Agent is already acting. It's illegal to prompt an agent during an action. Please use schedule instead."
+			)
+		}
+
+		this.requests.setIsPrompting(true)
+
+		const request = this.requests.getFullRequestFromInput(input)
+		const startingNode = this.mode.getCurrentModeNode()
+		startingNode.onPromptStart?.(this, request)
+
+		// Submit the request to the agent.
+		try {
+			await this.request(request)
+		} catch (e) {
+			console.error('Error data:', e)
+			this.requests.setIsPrompting(false)
+			this.requests.setCancelFn(null)
+			return
+		}
+
+		let modeChanged = true
+		while (!this.requests.getScheduledRequest() && modeChanged) {
+			modeChanged = false
+			const currentModeType = this.mode.getCurrentModeType()
+			const currentModeNode = this.mode.getCurrentModeNode()
+			currentModeNode.onPromptEnd?.(this, request) // in case onPromptEnd switches modes
+			const newModeType = this.mode.getCurrentModeType()
+			if (newModeType !== currentModeType) {
+				modeChanged = true
+			}
+		}
+
+		// If there's still no scheduled request, quit
+		const scheduledRequest = this.requests.getScheduledRequest()
+		const eventualModeType = this.mode.getCurrentModeType()
+		const eventualModeDefinition = this.mode.getCurrentModeDefinition()
+		if (!scheduledRequest) {
+			if (eventualModeDefinition.active) {
+				throw new Error(
+					`Agent is not allowed to become inactive during the active mode: ${eventualModeType}`
+				)
+			}
+			this.requests.setIsPrompting(false)
+			this.requests.setCancelFn(null)
+			return
+		}
+
+		// If there *is* a scheduled request...
+		// Add the scheduled request to chat history
+		const resolvedData = await Promise.all(scheduledRequest.data)
+		this.chat.push({
+			type: 'continuation',
+			data: resolvedData,
+		})
+
+		// Handle the scheduled request and clear it
+		this.requests.clearScheduledRequest()
+		await this.prompt(scheduledRequest, { nested: true })
+	}
+
+	/**
+	 * Send a single request to the agent and handle its response.
+	 *
+	 * Note: This method does not chain multiple requests together. For a full
+	 * agentic system, use the `prompt` method.
+	 *
+	 * Most developers will not want to use this method directly. It's mostly
+	 * used internally by the `prompt` method, but can also be useful for
+	 * carrying out evals.
+	 *
+	 * @param input - The input to form the request from.
+	 * @returns A promise for when the request is complete and a cancel function
+	 * to abort the request.
+	 */
+	async request(input: AgentInput) {
+		const request = this.requests.getFullRequestFromInput(input)
+
+		// Interrupt any currently active request
+		if (this.requests.getActiveRequest() !== null) {
+			this.cancel()
+		}
+		this.requests.setActiveRequest(request)
+
+		// Call an external helper function to request the agent
+		const { promise, cancel } = this.requestAgentActions(request)
+
+		this.requests.setCancelFn(cancel)
+
+		const results = await promise
+		this.requests.clearActiveRequest()
+
+		return results
+	}
+
+	/**
+	 * Schedule further work for the agent to do after this request has finished.
+	 * What you schedule will get merged with the currently scheduled request, if there is one.
+	 *
+	 * @example
+	 * ```tsx
+	 * // Add an instruction
+	 * agent.schedule('Add more detail.')
+	 * ```
+	 *
+	 * @example
+	 * ```tsx
+	 * // Move the viewport
+	 * agent.schedule({
+	 *  bounds: { x: 0, y: 0, w: 100, h: 100 },
+	 * })
+	 * ```
+	 *
+	 * @example
+	 * ```tsx
+	 * // Add data to the request
+	 * agent.schedule({ data: [value] })
+	 * ```
+	 */
+	schedule(input: AgentInput) {
+		const scheduledRequest = this.requests.getScheduledRequest()
+
+		// If there's no request scheduled yet, schedule one
+		if (!scheduledRequest) {
+			this._schedule(input)
+			return
+		}
+
+		const newRequest = this.requests.getPartialRequestFromInput(input)
+
+		this._schedule({
+			// Append to properties where possible
+			agentMessages: [...scheduledRequest.agentMessages, ...(newRequest.agentMessages ?? [])],
+			userMessages: [...scheduledRequest.userMessages, ...(newRequest.userMessages ?? [])],
+			data: [...scheduledRequest.data, ...(newRequest.data ?? [])],
+
+			// Override specific properties
+			bounds: newRequest.bounds ?? scheduledRequest.bounds,
+			contextItems: [...scheduledRequest.contextItems, ...(newRequest.contextItems ?? [])],
+			source: newRequest.source ?? scheduledRequest.source ?? 'self',
+		})
+	}
+
+	/**
+	 * Manually override what the agent should do next.
+	 *
+	 * @example
+	 * ```tsx
+	 * agent.setScheduledRequest('Add more detail.')
+	 * ```
+	 *
+	 * @example
+	 * ```tsx
+	 * agent.setScheduledRequest({
+	 *  message: 'Add more detail to this area.',
+	 *  bounds: { x: 0, y: 0, w: 100, h: 100 },
+	 * })
+	 * ```
+	 *
+	 * @example
+	 * ```tsx
+	 * // Cancel the scheduled request
+	 * agent.setScheduledRequest(null)
+	 * ```
+	 *
+	 * @param input - What to set the scheduled request to, or null to cancel
+	 * the scheduled request.
+	 */
+	private _schedule(input: AgentInput | null) {
+		if (input === null) {
+			this.requests.clearScheduledRequest()
+			return
+		}
+
+		const partialRequest = this.requests.getPartialRequestFromInput(input)
+		partialRequest.source = partialRequest.source ?? 'self' // when scheduling, we want the default source to be 'self' if none is provided
+		const request = this.requests.getFullRequestFromInput(partialRequest)
+
+		const isCurrentlyActive = this.requests.isGenerating()
+
+		if (isCurrentlyActive) {
+			this.requests.setScheduledRequest(request)
+		} else {
+			this.prompt(request)
+		}
+	}
+
+	/**
+	 * Interrupt the agent and set their mode.
+	 * Optionally, schedule a request.
+	 */
+	interrupt({ input, mode }: { input: AgentInput | null; mode?: AgentModeType }) {
+		this.requests.cancel()
+		if (mode) {
+			this.mode.setMode(mode)
+		}
+		if (input !== null) {
+			this.schedule(input)
+		}
+	}
+
+	// ==================== Cancel & Reset ====================
+
+	/**
+	 * Cancel the agent's current prompt, if one is active.
+	 */
+	cancel() {
+		const activeRequest = this.requests.getActiveRequest()
+
+		if (activeRequest) {
+			const modeType = this.mode.getCurrentModeType()
+			const modeNode = getModeNode(modeType)
+			modeNode.onPromptCancel?.(this, activeRequest)
+
+			const newModeDefinition = this.mode.getCurrentModeDefinition()
+			if (newModeDefinition.active) {
+				throw new Error(
+					`Agent is not allowed to become inactive during the active mode: ${this.mode.getCurrentModeType()}`
+				)
+			}
+		}
+
+		this.requests.cancel()
+	}
+
+	/**
+	 * Reset the agent's chat and memory.
+	 * Cancel the current request if there's one active.
+	 */
+	reset() {
+		this.cancel()
+
+		// Reset all managers
+		this.actions.reset()
+		this.chat.reset()
+		this.chatOrigin.reset()
+		this.context.reset()
+		this.lints.reset()
+		this.mode.reset()
+		this.requests.reset()
+		this.todos.reset()
+		this.userAction.reset()
+	}
+
+	// ==================== Request Helpers ====================
+
+	/**
+	 * Send a request to the agent and handle its response.
+	 *
+	 * This is a helper function that is used internally by the agent.
+	 */
+	private requestAgentActions(request: AgentRequest) {
+		const { editor } = this
+
+		// Add user prompt to chat history
+		const promptHistoryItem: ChatHistoryPromptItem = {
+			type: 'prompt',
+			promptSource: request.source,
+			agentFacingMessage: request.agentMessages.join('\n'),
+			userFacingMessage: request.userMessages.length > 0 ? request.userMessages.join('\n') : null,
+			contextItems: structuredClone(request.contextItems),
+			selectedShapes: this.editor
+				.getSelectedShapes()
+				.map((shape) => convertTldrawShapeToFocusedShape(this.editor, structuredClone(shape))),
+		}
+		this.chat.push(promptHistoryItem)
+
+		let cancelled = false
+		const controller = new AbortController()
+		const signal = controller.signal
+		const helpers = new AgentHelpers(this)
+
+		const modeDefinition = this.mode.getCurrentModeDefinition()
+		if (!modeDefinition.active) {
+			this.cancel()
+			throw new Error(
+				`Agent is not in an active mode so cannot take actions. Current mode: ${modeDefinition.type}`
+			)
+		}
+
+		const availableActions = modeDefinition.actions
+
+		const requestPromise = (async () => {
+			const prompt = await this.preparePrompt(request, helpers)
+			let incompleteDiff: RecordsDiff<TLRecord> | null = null
+			const actionPromises: Promise<void>[] = []
+			try {
+				for await (const action of this.streamAgentActions({ prompt, signal })) {
+					if (cancelled) break
+
+					// Set acting flag BEFORE editor.run so user action tracker ignores all changes
+					// including diff reverts that happen before act() is called
+					this.setIsActingOnEditor(true)
+					try {
+						editor.run(
+							() => {
+								const actionUtilType = this.actions.getAgentActionUtilType(action._type)
+								const actionUtil = this.actions.getAgentActionUtil(action._type)
+
+								// If the action is not in the mode's available actions, skip it
+								if (!availableActions.includes(actionUtilType)) {
+									return
+								}
+
+								// If there was a diff from an incomplete action, revert it so that we can reapply the action
+								// This must happen BEFORE sanitize so we're working with clean state
+								if (incompleteDiff) {
+									const inversePrevDiff = reverseRecordsDiff(incompleteDiff)
+									editor.store.applyDiff(inversePrevDiff)
+									// Track the inverse diff to update created shapes tracking
+									this.lints.trackShapesFromDiff(inversePrevDiff)
+									incompleteDiff = null
+								}
+
+								// Sanitize the agent's action
+								const transformedAction = actionUtil.sanitizeAction(action, helpers)
+								if (!transformedAction) {
+									return
+								}
+
+								// Apply the action to the app and editor
+								const { diff, promise } = this.actions.act(transformedAction, helpers)
+
+								if (promise) {
+									actionPromises.push(promise)
+								}
+
+								// Track shapes from diff for both complete and incomplete actions
+								this.lints.trackShapesFromDiff(diff)
+
+								// If the action is incomplete, save the diff so that we can revert it in the future
+								if (transformedAction.complete) {
+									// Log completed action if debug logging is enabled
+									this.debug.logCompletedAction(transformedAction)
+								} else {
+									incompleteDiff = diff
+								}
+							},
+							{
+								ignoreShapeLock: true,
+								history: 'ignore',
+							}
+						)
+					} finally {
+						this.setIsActingOnEditor(false)
+					}
+				}
+				await Promise.all(actionPromises)
+			} catch (e) {
+				if (e === 'Cancelled by user' || (e instanceof Error && e.name === 'AbortError')) {
+					return
+				}
+				this.onError(e)
+			}
+		})()
+
+		const cancel = () => {
+			cancelled = true
+			controller.abort('Cancelled by user')
+		}
+
+		return { promise: requestPromise, cancel }
+	}
+
+	/**
+	 * Stream a response from the model.
+	 * Act on the model's events as they come in.
+	 *
+	 * This is a helper function that is used internally by the agent.
+	 */
+	private async *streamAgentActions({
+		prompt,
+		signal,
+	}: {
+		prompt: BaseAgentPrompt
+		signal: AbortSignal
+	}): AsyncGenerator<Streaming<AgentAction>> {
+		const res = await fetch('/stream', {
+			method: 'POST',
+			body: JSON.stringify(prompt),
+			headers: {
+				'Content-Type': 'application/json',
+			},
+			signal,
+		})
+
+		if (!res.body) {
+			throw Error('No body in response')
+		}
+
+		const reader = res.body.getReader()
+		const decoder = new TextDecoder()
+		let buffer = ''
+
+		try {
+			while (true) {
+				const { value, done } = await reader.read()
+				if (done) break
+
+				buffer += decoder.decode(value, { stream: true })
+				const actions = buffer.split('\n\n')
+				buffer = actions.pop() || ''
+
+				for (const action of actions) {
+					const match = action.match(/^data: (.+)$/m)
+					if (match) {
+						try {
+							const data = JSON.parse(match[1])
+
+							// If the response contains an error, throw it
+							if ('error' in data) {
+								throw new Error(data.error)
+							}
+
+							const agentAction: Streaming<AgentAction> = data
+							yield agentAction
+						} catch (err: any) {
+							throw new Error(err.message)
+						}
+					}
+				}
+			}
+		} finally {
+			reader.releaseLock()
+		}
+	}
+}

templates/agent/client/agent/TldrawAgentApp.ts

@@ -0,0 +1,83 @@
+import { Editor } from 'tldraw'
+import { AgentAppAgentsManager } from './managers/AgentAppAgentsManager'
+import { AgentAppPersistenceManager } from './managers/AgentAppPersistenceManager'
+
+/**
+ * The TldrawAgentApp class manages the agent system for a given editor instance.
+ *
+ * This is a coordinator class that handles app-level concerns shared across agents,
+ * such as agent lifecycle management, persistence, and global settings.
+ *
+ * Individual agents (TldrawAgent) handle their own concerns like chat, context, and requests.
+ * The app manages the agents and coordinates shared state.
+ *
+ * @example
+ * ```tsx
+ * const app = new TldrawAgentApp(editor, { onError: handleError })
+ * const agent = app.agents.getAgent()
+ * agent.prompt('Draw a cat')
+ * ```
+ */
+export class TldrawAgentApp {
+	/**
+	 * Manager for agent lifecycle - creation, disposal, and tracking.
+	 */
+	agents: AgentAppAgentsManager
+
+	/**
+	 * Manager for state persistence - loading, saving, and auto-save.
+	 */
+	persistence: AgentAppPersistenceManager
+
+	/**
+	 * Handle crash and dispose events.
+	 */
+	private handleCrash = () => this.dispose()
+	private handleDispose = () => this.dispose()
+
+	private _editor: Editor | null
+
+	/**
+	 * The editor associated with this app.
+	 * @throws Error if the app has been disposed.
+	 */
+	get editor(): Editor {
+		if (!this._editor) {
+			throw new Error('TldrawAgentApp has been disposed')
+		}
+		return this._editor
+	}
+
+	constructor(
+		editor: Editor,
+		public options: {
+			onError: (e: any) => void
+		}
+	) {
+		this._editor = editor
+		this.agents = new AgentAppAgentsManager(this)
+		this.persistence = new AgentAppPersistenceManager(this)
+		editor.on('crash', this.handleCrash)
+		editor.on('dispose', this.handleDispose)
+	}
+
+	/**
+	 * Dispose of all resources. Call this during cleanup.
+	 */
+	dispose() {
+		if (!this._editor) return
+		this._editor.off('crash', this.handleCrash)
+		this._editor.off('dispose', this.handleDispose)
+		this.persistence.dispose()
+		this.agents.dispose()
+		this._editor = null
+	}
+
+	/**
+	 * Reset everything to initial state.
+	 */
+	reset() {
+		this.agents.reset()
+		this.persistence.reset()
+	}
+}

templates/agent/client/agent/TldrawAgentAppProvider.tsx

@@ -0,0 +1,213 @@
+import { createContext, memo, ReactNode, useCallback, useContext, useEffect, useState } from 'react'
+import { useEditor, useToasts, useValue } from 'tldraw'
+import { TldrawAgent } from './TldrawAgent'
+import { TldrawAgentApp } from './TldrawAgentApp'
+
+const TldrawAgentAppContext = createContext<TldrawAgentApp | null>(null)
+
+export interface TldrawAgentAppProviderProps {
+	children?: ReactNode
+	/**
+	 * Callback fired when the app is created. Use this to pass the app
+	 * to components outside the Tldraw component via TldrawAgentAppContextProvider.
+	 */
+	onMount?: (app: TldrawAgentApp) => void
+	/**
+	 * Callback fired when the app is disposed.
+	 */
+	onUnmount?: () => void
+}
+
+/**
+ * Provider component that creates and manages a TldrawAgentApp instance for the current editor.
+ *
+ * This component should be rendered inside a `<Tldraw>` component to have access to the editor context.
+ * It creates a TldrawAgentApp instance, provides it via context, and cleans up on unmount.
+ *
+ * The provider waits until the app and agent exist before rendering children,
+ * ensuring useAgent() always returns a valid agent.
+ *
+ * For components defined in Tldraw's `components` prop, use `onMount` to get the app instance
+ * and wrap them with `TldrawAgentAppContextProvider`.
+ *
+ * @example
+ * ```tsx
+ * function App() {
+ *   const [app, setApp] = useState<TldrawAgentApp | null>(null)
+ *
+ *   const components = useMemo(() => ({
+ *     InFrontOfTheCanvas: () => app && (
+ *       <TldrawAgentAppContextProvider app={app}>
+ *         <AgentHighlights />
+ *       </TldrawAgentAppContextProvider>
+ *     ),
+ *   }), [app])
+ *
+ *   return (
+ *     <Tldraw components={components}>
+ *       <TldrawAgentAppProvider onMount={setApp} onUnmount={() => setApp(null)} />
+ *     </Tldraw>
+ *   )
+ * }
+ * ```
+ */
+export const TldrawAgentAppProvider = memo(function TldrawAgentAppProvider({
+	children,
+	onMount,
+	onUnmount,
+}: TldrawAgentAppProviderProps) {
+	const editor = useEditor()
+	const toasts = useToasts()
+	const [app, setApp] = useState<TldrawAgentApp | null>(null)
+
+	// Error handler for agent errors
+	const handleError = useCallback(
+		(e: any) => {
+			const message = typeof e === 'string' ? e : e instanceof Error && e.message
+			toasts.addToast({
+				title: 'Error',
+				description: message || 'An error occurred',
+				severity: 'error',
+			})
+			console.error(e)
+		},
+		[toasts]
+	)
+
+	// Create the TldrawAgentApp instance
+	useEffect(() => {
+		const instance = new TldrawAgentApp(editor, { onError: handleError })
+
+		// Load persisted state first (this will create agents from persisted data)
+		instance.persistence.loadState()
+
+		// Ensure at least one agent exists (creates one if none were loaded)
+		const defaultAgent = instance.agents.ensureAtLeastOneAgent()
+
+		// Start auto-saving (must be after loadState to avoid saving during load)
+		instance.persistence.startAutoSave()
+
+		setApp(instance)
+
+		// Notify parent
+		onMount?.(instance)
+
+		// Expose to window for debugging
+		;(window as any).agentApp = instance
+		;(window as any).agent = defaultAgent
+		;(window as any).editor = editor
+
+		return () => {
+			instance.dispose()
+			setApp(null)
+			onUnmount?.()
+			delete (window as any).agentApp
+			delete (window as any).agent
+			delete (window as any).editor
+		}
+	}, [editor, handleError, onMount, onUnmount])
+
+	// Don't render children until app exists
+	if (!app) {
+		return null
+	}
+
+	return <TldrawAgentAppContext.Provider value={app}>{children}</TldrawAgentAppContext.Provider>
+})
+
+/**
+ * Context provider that wraps children with the TldrawAgentApp context.
+ * Use this to provide the app context to components defined in Tldraw's `components` prop.
+ *
+ * @example
+ * ```tsx
+ * const components = useMemo(() => ({
+ *   InFrontOfTheCanvas: () => app && (
+ *     <TldrawAgentAppContextProvider app={app}>
+ *       <AgentHighlights />
+ *     </TldrawAgentAppContextProvider>
+ *   ),
+ * }), [app])
+ * ```
+ */
+export function TldrawAgentAppContextProvider({
+	app,
+	children,
+}: {
+	app: TldrawAgentApp
+	children: ReactNode
+}) {
+	return <TldrawAgentAppContext.Provider value={app}>{children}</TldrawAgentAppContext.Provider>
+}
+
+/**
+ * Hook to get the TldrawAgentApp instance from context.
+ * Must be used inside a TldrawAgentAppProvider or TldrawAgentAppContextProvider.
+ *
+ * @throws Error if called outside of a provider
+ * @returns The TldrawAgentApp instance (guaranteed non-null)
+ *
+ * @example
+ * ```tsx
+ * const app = useTldrawAgentApp()
+ * // app is guaranteed to exist here
+ * const agent = app.agents.getAgent()
+ * ```
+ */
+export function useTldrawAgentApp(): TldrawAgentApp {
+	const app = useContext(TldrawAgentAppContext)
+	if (!app) {
+		throw new Error('useTldrawAgentApp must be used inside a TldrawAgentAppProvider')
+	}
+	return app
+}
+
+/**
+ * Hook to get the default TldrawAgent instance from the app context.
+ * Must be used inside a TldrawAgentAppProvider or TldrawAgentAppContextProvider.
+ *
+ * @throws Error if called outside of a provider or if agent doesn't exist
+ * @returns The default TldrawAgent instance (guaranteed non-null)
+ *
+ * @example
+ * ```tsx
+ * function ChatPanel() {
+ *   const agent = useAgent()
+ *   // agent is guaranteed to exist here
+ *   agent.prompt('Draw a cat')
+ * }
+ * ```
+ */
+export function useAgent(): TldrawAgent {
+	const app = useTldrawAgentApp()
+	const agent = useValue('agent', () => app.agents.getAgent(), [app])
+	if (!agent) {
+		throw new Error('No agent found. Make sure an agent has been created.')
+	}
+	return agent
+}
+
+/**
+ * Hook to get all TldrawAgent instances from the app context.
+ * Returns a reactive array that updates when agents are added or removed.
+ * Must be used inside a TldrawAgentAppProvider or TldrawAgentAppContextProvider.
+ *
+ * @throws Error if called outside of a provider
+ * @returns Array of all TldrawAgent instances
+ *
+ * @example
+ * ```tsx
+ * function AgentList() {
+ *   const agents = useAgents()
+ *   return (
+ *     <ul>
+ *       {agents.map(agent => <li key={agent.id}>{agent.id}</li>)}
+ *     </ul>
+ *   )
+ * }
+ * ```
+ */
+export function useAgents(): TldrawAgent[] {
+	const app = useTldrawAgentApp()
+	return useValue('agents', () => app.agents.getAgents(), [app])
+}

templates/agent/client/agent/managers/AgentActionManager.ts

@@ -0,0 +1,159 @@
+import { RecordsDiff, structuredClone, TLRecord } from 'tldraw'
+import { AgentAction } from '../../../shared/types/AgentAction'
+import { ChatHistoryItem } from '../../../shared/types/ChatHistoryItem'
+import { Streaming } from '../../../shared/types/Streaming'
+import { AgentActionUtil, getAgentActionUtilsRecordForMode } from '../../actions/AgentActionUtil'
+import { AgentHelpers } from '../../AgentHelpers'
+import type { TldrawAgent } from '../TldrawAgent'
+import { BaseAgentManager } from './BaseAgentManager'
+
+/**
+ * Manages action-related functionality for the agent.
+ * Handles action utils, action execution, and chat history updates for actions.
+ */
+export class AgentActionManager extends BaseAgentManager {
+	/**
+	 * A record of the agent's action util instances.
+	 * Used by the `getAgentActionUtil` method.
+	 */
+	private agentActionUtils: Record<AgentAction['_type'], AgentActionUtil<AgentAction>>
+
+	/**
+	 * The agent action util instance for the "unknown" action type.
+	 *
+	 * This is returned by the `getAgentActionUtil` method when the action type
+	 * isn't properly specified. This can happen if the model isn't finished
+	 * streaming yet or makes a mistake.
+	 */
+	unknownActionUtil: AgentActionUtil<AgentAction>
+
+	constructor(agent: TldrawAgent) {
+		super(agent)
+		this.agentActionUtils = getAgentActionUtilsRecordForMode(
+			this.agent,
+			agent.mode.getCurrentModeType()
+		)
+		this.unknownActionUtil = this.agentActionUtils.unknown
+	}
+
+	/**
+	 * Reset the action manager to its initial state.
+	 * Currently no state to reset as action utils are stateless.
+	 */
+	reset(): void {
+		// Reset state if needed - currently no state to reset
+	}
+
+	/**
+	 * Rebuild action utils for a specific mode.
+	 * Called when the agent's mode changes to ensure mode-specific
+	 * action utils are used.
+	 *
+	 * @param mode - The mode to rebuild utils for.
+	 */
+	rebuildUtilsForMode(mode: string): void {
+		this.agentActionUtils = getAgentActionUtilsRecordForMode(this.agent, mode)
+		this.unknownActionUtil = this.agentActionUtils.unknown
+	}
+
+	/**
+	 * Get an agent action util for a specific action type.
+	 *
+	 * @param type - The type of action to get the util for.
+	 * @returns The action util.
+	 */
+	getAgentActionUtil(type?: string) {
+		const utilType = this.getAgentActionUtilType(type)
+		return this.agentActionUtils[utilType]
+	}
+
+	/**
+	 * Get the util type for a provided action type.
+	 * If no util type is found, returns 'unknown'.
+	 * @param type - The action type to get the util type for.
+	 * @returns The action util type, or 'unknown' if not found.
+	 */
+	getAgentActionUtilType(type?: string): AgentAction['_type'] {
+		if (!type) return 'unknown'
+		const util = this.agentActionUtils[type as AgentAction['_type']]
+		if (!util) return 'unknown'
+		return type as AgentAction['_type']
+	}
+
+	/**
+	 * Make the agent perform an action.
+	 * Applies the action to the editor and tracks it in chat history.
+	 * @param action - The action to make the agent do.
+	 * @param helpers - The helpers to use for action execution.
+	 * @returns An object containing the diff of changes made and a promise that resolves when the action completes.
+	 */
+	act(
+		action: Streaming<AgentAction>,
+		helpers: AgentHelpers = new AgentHelpers(this.agent)
+	): {
+		diff: RecordsDiff<TLRecord>
+		promise: Promise<void> | null
+	} {
+		const { editor } = this.agent
+		const util = this.getAgentActionUtil(action._type)
+		this.agent.setIsActingOnEditor(true)
+
+		let promise: Promise<void> | null = null
+		let diff: RecordsDiff<TLRecord>
+		try {
+			diff = editor.store.extractingChanges(() => {
+				promise = util.applyAction(structuredClone(action), helpers) ?? null
+			})
+		} catch (error) {
+			// always toast the error
+			this.agent.onError(error)
+			promise = null
+			throw error // you may not want to throw in productions
+		} finally {
+			this.agent.setIsActingOnEditor(false)
+		}
+
+		// Add the action to chat history
+		if (util.savesToHistory()) {
+			const historyItem: ChatHistoryItem = {
+				type: 'action',
+				action,
+				diff,
+				acceptance: 'pending',
+			}
+
+			this.agent.chat.update((historyItems) => {
+				// If there are no items, start off the chat history with the first item
+				if (historyItems.length === 0) return [historyItem]
+
+				// Find the last EXTERNAL prompt index (ignore prompts from 'self' which are internal state transitions)
+				const lastPromptIndex = historyItems.findLastIndex(
+					(item) => item.type === 'prompt' && item.promptSource !== 'self'
+				)
+
+				// If the last action is still in progress AND it's after the last external prompt, replace it
+				const lastActionHistoryItemIndex = historyItems.findLastIndex(
+					(item) => item.type === 'action'
+				)
+				const lastActionHistoryItem =
+					lastActionHistoryItemIndex !== -1 ? historyItems[lastActionHistoryItemIndex] : null
+				if (
+					lastActionHistoryItem &&
+					lastActionHistoryItem.type === 'action' &&
+					!lastActionHistoryItem.action.complete &&
+					(lastPromptIndex === -1 || lastActionHistoryItemIndex > lastPromptIndex)
+				) {
+					const newHistoryItems = [...historyItems]
+					// Replace the incomplete action with the complete one (timestamp already set above)
+					newHistoryItems[lastActionHistoryItemIndex] = historyItem
+					return newHistoryItems
+				} else {
+					// Otherwise, just add the new item to the end of the list
+					return [...historyItems, historyItem]
+				}
+			})
+		}
+
+		return { diff, promise }
+	}
+}

templates/agent/client/agent/managers/AgentAppAgentsManager.ts

@@ -0,0 +1,162 @@
+import { Editor, EditorAtom, uniqueId } from 'tldraw'
+import { TldrawAgent } from '../TldrawAgent'
+import { BaseAgentAppManager } from './BaseAgentAppManager'
+
+/**
+ * Generate a unique agent ID.
+ */
+function generateAgentId(): string {
+	return uniqueId()
+}
+
+/**
+ * Manager for agent lifecycle - creation, disposal, and tracking.
+ *
+ * Manages multiple agents per editor. The agents are stored in an EditorAtom
+ * so they can be accessed from tools that only have access to the editor.
+ *
+ * Use the static methods `getAgents(editor)` and `getAgent(editor, id)` to access
+ * agents from tools. Use instance methods for agent lifecycle management.
+ */
+export class AgentAppAgentsManager extends BaseAgentAppManager {
+	/**
+	 * Static EditorAtom containing agents.
+	 * This allows tools to access agents without needing the full TldrawAgentApp.
+	 */
+	private static $agents = new EditorAtom<TldrawAgent[]>('agents', () => [])
+
+	/**
+	 * Get all agents for an editor.
+	 * Use this static method from tools that only have access to the editor.
+	 */
+	static getAgents(editor: Editor): TldrawAgent[] {
+		return AgentAppAgentsManager.$agents.get(editor)
+	}
+
+	/**
+	 * Get an agent by ID for an editor.
+	 * If no ID is provided, returns the first agent.
+	 * Use this static method from tools that only have access to the editor.
+	 */
+	static getAgent(editor: Editor, id?: string): TldrawAgent | undefined {
+		const agents = AgentAppAgentsManager.$agents.get(editor)
+		if (id) {
+			return agents.find((agent) => agent.id === id)
+		}
+		return agents[0]
+	}
+
+	/**
+	 * Get all agents.
+	 */
+	getAgents(): TldrawAgent[] {
+		return AgentAppAgentsManager.$agents.get(this.app.editor)
+	}
+
+	/**
+	 * Get an agent by ID.
+	 * If no ID is provided, returns the first agent.
+	 */
+	getAgent(id?: string): TldrawAgent | undefined {
+		const agents = AgentAppAgentsManager.$agents.get(this.app.editor)
+		if (id) {
+			return agents.find((agent) => agent.id === id)
+		}
+		return agents[0]
+	}
+
+	/**
+	 * Create an agent with the given ID.
+	 * If an agent with the ID already exists, returns the existing agent.
+	 *
+	 * @param id - The ID for the new agent
+	 * @returns The created or existing agent
+	 */
+	createAgent(id: string): TldrawAgent {
+		const existingAgent = this.getAgent(id)
+		if (existingAgent) {
+			return existingAgent
+		}
+
+		const agent = new TldrawAgent({
+			editor: this.app.editor,
+			id,
+			onError: this.app.options.onError,
+		})
+
+		// Register the agent in the static atom
+		AgentAppAgentsManager.$agents.update(this.app.editor, (agents) => [...agents, agent])
+
+		return agent
+	}
+
+	/**
+	 * Ensure at least one agent exists.
+	 * Returns the first existing agent, or creates a new one with a generated ID.
+	 * Call this after the app is initialized.
+	 */
+	ensureAtLeastOneAgent(): TldrawAgent {
+		const existingAgent = this.getAgent()
+		if (existingAgent) {
+			return existingAgent
+		}
+		return this.createAgent(generateAgentId())
+	}
+
+	/**
+	 * Delete an agent by ID.
+	 * Disposes the agent and removes it from the registry.
+	 *
+	 * @param id - The ID of the agent to delete
+	 * @returns true if the agent was found and deleted, false otherwise
+	 */
+	deleteAgent(id: string): boolean {
+		const agent = this.getAgent(id)
+		if (!agent) {
+			return false
+		}
+
+		// Dispose the agent first
+		agent.dispose()
+
+		// Remove from the static atom
+		AgentAppAgentsManager.$agents.update(this.app.editor, (agents) =>
+			agents.filter((a) => a.id !== id)
+		)
+
+		return true
+	}
+
+	/**
+	 * Reset the state of all agents without disposing them.
+	 * Clears chats, todos, context, and returns agents to initial mode.
+	 */
+	resetAllAgents() {
+		const agents = AgentAppAgentsManager.$agents.get(this.app.editor)
+		agents.forEach((agent) => agent.reset())
+	}
+
+	/**
+	 * Dispose all agents. Call this during cleanup.
+	 */
+	disposeAllAgents() {
+		const agents = AgentAppAgentsManager.$agents.get(this.app.editor)
+		agents.forEach((agent) => agent.dispose())
+		AgentAppAgentsManager.$agents.set(this.app.editor, [])
+	}
+
+	/**
+	 * Reset the manager to its initial state.
+	 */
+	reset() {
+		this.resetAllAgents()
+	}
+
+	/**
+	 * Dispose of the manager and all agents.
+	 */
+	override dispose() {
+		this.disposeAllAgents()
+		super.dispose()
+	}
+}

templates/agent/client/agent/managers/AgentAppPersistenceManager.ts

@@ -0,0 +1,236 @@
+import { react } from 'tldraw'
+import { PersistedAgentState, TldrawAgent } from '../TldrawAgent'
+import { BaseAgentAppManager } from './BaseAgentAppManager'
+
+/**
+ * The key prefix used for localStorage persistence.
+ */
+const STORAGE_PREFIX = 'tldraw-agent-app'
+
+/**
+ * The persisted state for the entire app.
+ * Contains state for all agents.
+ */
+export interface PersistedAppState {
+	agents: Record<string, PersistedAgentState>
+}
+
+/**
+ * Manager for app-level state persistence.
+ *
+ * Coordinates loading and saving agent state to localStorage.
+ * Calls agent-level serializeState() and loadState() methods
+ * to handle the actual state serialization/deserialization.
+ */
+export class AgentAppPersistenceManager extends BaseAgentAppManager {
+	/**
+	 * Whether we're currently loading state to prevent premature saves.
+	 */
+	private isLoadingState = false
+
+	/**
+	 * Cleanup function for the agents list watcher.
+	 */
+	private agentsListCleanup: (() => void) | null = null
+
+	/**
+	 * Cleanup functions for per-agent state watchers, keyed by agent ID.
+	 */
+	private agentWatcherCleanupFns = new Map<string, () => void>()
+
+	/**
+	 * Check if state is currently being loaded.
+	 */
+	getIsLoadingState(): boolean {
+		return this.isLoadingState
+	}
+
+	/**
+	 * Serialize the current app state for persistence.
+	 */
+	serializeState(): PersistedAppState {
+		const agents = this.app.agents.getAgents()
+
+		return {
+			agents: agents.reduce(
+				(acc, agent) => {
+					acc[agent.id] = agent.serializeState()
+					return acc
+				},
+				{} as Record<string, PersistedAgentState>
+			),
+		}
+	}
+
+	/**
+	 * Load app state from localStorage.
+	 * Call this after the app is initialized.
+	 * Creates agents for all persisted agent IDs that don't already exist.
+	 */
+	loadState() {
+		this.isLoadingState = true
+
+		try {
+			const appState = this.loadValue<PersistedAppState>('state')
+			if (!appState) {
+				this.isLoadingState = false
+				return
+			}
+
+			// Create agents for all persisted IDs (createAgent returns existing if already exists)
+			for (const agentId of Object.keys(appState.agents)) {
+				this.app.agents.createAgent(agentId)
+			}
+
+			// Load state for each agent
+			const agents = this.app.agents.getAgents()
+			agents.forEach((agent) => {
+				const agentState = appState.agents[agent.id]
+				if (agentState) {
+					agent.loadState(agentState)
+				}
+			})
+		} catch (e) {
+			console.error('Failed to load app state:', e)
+		} finally {
+			this.isLoadingState = false
+		}
+	}
+
+	/**
+	 * Start auto-saving app state changes.
+	 * Call this after loadState() to avoid saving during load.
+	 * Reactively watches the agents list and all agent state.
+	 */
+	startAutoSave() {
+		// Watch for changes to the agents list and set up per-agent watchers
+		this.agentsListCleanup = react('agents list', () => {
+			const agents = this.app.agents.getAgents()
+			const currentAgentIds = new Set(agents.map((a) => a.id))
+
+			// Set up watchers for new agents
+			for (const agent of agents) {
+				if (!this.agentWatcherCleanupFns.has(agent.id)) {
+					const cleanup = this.createAgentStateWatcher(agent)
+					this.agentWatcherCleanupFns.set(agent.id, cleanup)
+				}
+			}
+
+			// Clean up watchers for removed agents
+			for (const id of this.agentWatcherCleanupFns.keys()) {
+				if (!currentAgentIds.has(id)) {
+					const cleanup = this.agentWatcherCleanupFns.get(id)
+					if (cleanup) {
+						cleanup()
+					}
+					this.agentWatcherCleanupFns.delete(id)
+				}
+			}
+
+			// Save when agent list changes (if not loading)
+			if (!this.isLoadingState) {
+				this.saveState()
+			}
+		})
+	}
+
+	/**
+	 * Create a reactive watcher for a single agent's state.
+	 */
+	private createAgentStateWatcher(agent: TldrawAgent): () => void {
+		return react(`${agent.id} state`, () => {
+			// Access reactive state to trigger on changes
+			agent.chat.getHistory()
+			agent.chatOrigin.getOrigin()
+			agent.todos.getTodos()
+			agent.context.getItems()
+			agent.modelName.getModelName()
+			agent.debug.getDebugFlags()
+
+			// Save if not currently loading
+			if (!this.isLoadingState) {
+				this.saveState()
+			}
+		})
+	}
+
+	/**
+	 * Save the current app state to localStorage.
+	 */
+	private saveState() {
+		const agents = this.app.agents.getAgents()
+		// Don't save if no agents exist (e.g., during dispose)
+		if (agents.length === 0) {
+			return
+		}
+		const appState = this.serializeState()
+		this.saveValue('state', appState)
+	}
+
+	/**
+	 * Stop auto-saving and clean up watchers.
+	 */
+	stopAutoSave() {
+		if (this.agentsListCleanup) {
+			this.agentsListCleanup()
+			this.agentsListCleanup = null
+		}
+		for (const cleanup of this.agentWatcherCleanupFns.values()) {
+			cleanup()
+		}
+		this.agentWatcherCleanupFns.clear()
+	}
+
+	/**
+	 * Reset the manager to its initial state.
+	 */
+	reset() {
+		this.stopAutoSave()
+		this.isLoadingState = false
+	}
+
+	/**
+	 * Dispose of the persistence manager.
+	 */
+	override dispose() {
+		this.stopAutoSave()
+		super.dispose()
+	}
+
+	// --- Helper methods ---
+
+	/**
+	 * Load a value from localStorage.
+	 */
+	private loadValue<T>(key: string): T | null {
+		const localStorage = globalThis.localStorage
+		if (!localStorage) return null
+
+		try {
+			const fullKey = `${STORAGE_PREFIX}:${key}`
+			const stored = localStorage.getItem(fullKey)
+			if (stored) {
+				return JSON.parse(stored) as T
+			}
+		} catch {
+			console.warn(`Couldn't load ${key} from localStorage`)
+		}
+
+		return null
+	}
+
+	/**
+	 * Save a value to localStorage.
+	 */
+	private saveValue<T>(key: string, value: T): void {
+		const localStorage = globalThis.localStorage
+		if (!localStorage) return
+
+		try {
+			const fullKey = `${STORAGE_PREFIX}:${key}`
+			localStorage.setItem(fullKey, JSON.stringify(value))
+		} catch {
+			console.warn(`Couldn't save ${key} to localStorage`)
+		}
+	}
+}

templates/agent/client/agent/managers/AgentChatManager.ts

@@ -0,0 +1,86 @@
+import { Atom, atom } from 'tldraw'
+import { ChatHistoryItem } from '../../../shared/types/ChatHistoryItem'
+import type { TldrawAgent } from '../TldrawAgent'
+import { BaseAgentManager } from './BaseAgentManager'
+
+/**
+ * Manages chat history for an agent.
+ * The chat history stores all interactions between the user and the agent,
+ * including prompts, actions, and continuations.
+ */
+export class AgentChatManager extends BaseAgentManager {
+	/**
+	 * An atom containing the agent's chat history.
+	 * Stores all chat interactions including prompts, actions, and continuations.
+	 */
+	private $chatHistory: Atom<ChatHistoryItem[]>
+
+	/**
+	 * Creates a new AgentChatManager instance.
+	 * Initializes chat history atom with an empty array.
+	 */
+	constructor(agent: TldrawAgent) {
+		super(agent)
+		this.$chatHistory = atom('chatHistory', [])
+	}
+
+	/**
+	 * Get the current chat history.
+	 * @returns The array of chat history items.
+	 */
+	getHistory() {
+		return this.$chatHistory.get()
+	}
+
+	/**
+	 * Set the chat history directly.
+	 * Primarily used for loading persisted state.
+	 * @param history - The chat history items to set.
+	 */
+	setHistory(history: ChatHistoryItem[]) {
+		this.$chatHistory.set(history)
+	}
+
+	/**
+	 * Reset the chat manager to its initial state.
+	 * Clears all chat history.
+	 */
+	reset(): void {
+		this.$chatHistory.set([])
+	}
+
+	/**
+	 * Push one or more items to the chat history.
+	 * Items are appended to the end of the history array.
+	 * If no items are provided, this method does nothing.
+	 * @param items - The chat history item(s) to add.
+	 * @example
+	 * ```ts
+	 * chatManager.push(promptItem)
+	 * chatManager.push(actionItem1, actionItem2)
+	 * ```
+	 */
+	push(...items: ChatHistoryItem[]) {
+		if (items.length === 0) return
+		this.$chatHistory.update((prev) => [...prev, ...items])
+	}
+
+	/**
+	 * Update chat history items in place using an updater function.
+	 * This is used for operations like marking actions as accepted/rejected,
+	 * modifying existing items, or filtering the history.
+	 * @param updater - A function that receives the current history and returns the updated history.
+	 * @example
+	 * ```ts
+	 * // Mark all actions as accepted
+	 * chatManager.update((history) =>
+	 *   history.map((item) =>
+	 *     item.type === 'action' ? { ...item, acceptance: 'accepted' } : item
+	 *   )
+	 * )
+	 * ```
+	 */
+	update(updater: (history: ChatHistoryItem[]) => ChatHistoryItem[]) {
+		this.$chatHistory.update(updater)
+	}
+}

templates/agent/client/agent/managers/AgentChatOriginManager.ts

@@ -0,0 +1,51 @@
+import { Atom, atom, VecModel } from 'tldraw'
+import type { TldrawAgent } from '../TldrawAgent'
+import { BaseAgentManager } from './BaseAgentManager'
+
+/**
+ * Manages the chat origin position for an agent.
+ * The chat origin represents where on the canvas a conversation started,
+ * and is used to track the spatial context of the chat session.
+ */
+export class AgentChatOriginManager extends BaseAgentManager {
+	/**
+	 * An atom containing the position on the page where the current chat started.
+	 * Used to track the origin point of the conversation on the canvas.
+	 */
+	private $chatOrigin: Atom<VecModel>
+
+	/**
+	 * Creates a new chat origin manager for the given agent.
+	 * Initializes the origin to (0, 0).
+	 */
+	constructor(agent: TldrawAgent) {
+		super(agent)
+		this.$chatOrigin = atom('chatOrigin', { x: 0, y: 0 })
+	}
+
+	/**
+	 * Set the chat origin position on the canvas.
+	 * The origin represents where the conversation started spatially.
+	 * @param origin - The position where the chat started, with x and y coordinates.
+	 */
+	setOrigin(origin: VecModel) {
+		this.$chatOrigin.set(origin)
+	}
+
+	/**
+	 * Get the current chat origin position.
+	 * @returns The position where the chat started on the canvas.
+	 */
+	getOrigin() {
+		return this.$chatOrigin.get()
+	}
+
+	/**
+	 * Reset the chat origin manager to its initial state.
+	 * Sets the origin to the current viewport position.
+	 */
+	reset(): void {
+		const viewport = this.agent.editor.getViewportPageBounds()
+		this.$chatOrigin.set({ x: viewport.x, y: viewport.y })
+	}
+}

templates/agent/client/agent/managers/AgentContextManager.ts

@@ -0,0 +1,219 @@
+import { Atom, atom, Box, structuredClone, Vec } from 'tldraw'
+import { FocusedShape } from '../../../shared/format/FocusedShape'
+import {
+	AreaContextItem,
+	ContextItem,
+	PointContextItem,
+	ShapeContextItem,
+	ShapesContextItem,
+} from '../../../shared/types/ContextItem'
+import type { TldrawAgent } from '../TldrawAgent'
+import { BaseAgentManager } from './BaseAgentManager'
+
+/**
+ * Manages context items for an agent.
+ * Context items are pieces of information that can be sent to the model
+ * to provide additional context for the conversation.
+ */
+export class AgentContextManager extends BaseAgentManager {
+	/**
+	 * An atom containing currently selected context items.
+	 *
+	 * To send context items to the model, include them in the `contextItems`
+	 * field of a request.
+	 */
+	private $contextItems: Atom<ContextItem[]>
+
+	/**
+	 * Creates a new context manager for the given agent.
+	 * Initializes with an empty context items array.
+	 */
+	constructor(agent: TldrawAgent) {
+		super(agent)
+		this.$contextItems = atom('contextItems', [])
+	}
+
+	/**
+	 * Reset the context manager to its initial state.
+	 * Clears all context items.
+	 */
+	reset(): void {
+		this.$contextItems.set([])
+	}
+
+	/**
+	 * Get the current context items.
+	 * @returns An array of context items.
+	 */
+	getItems() {
+		return this.$contextItems.get()
+	}
+
+	/**
+	 * Set the context items directly.
+	 * Primarily used for loading persisted state.
+	 * @param items - The context items to set.
+	 */
+	setItems(items: ContextItem[]) {
+		this.$contextItems.set(items)
+	}
+
+	/**
+	 * Add a context item to the agent's context, ensuring that duplicates are
+	 * not included.
+	 *
+	 * @param item The context item to add.
+	 */
+	add(item: ContextItem) {
+		this.$contextItems.update((items) => {
+			// Don't add shapes that are already within context
+			if (item.type === 'shapes') {
+				const newItems = dedupeShapesContextItem(item, items)
+				return [...items, ...newItems]
+			}
+
+			// Don't add items that are already in context
+			if (this.has(item)) {
+				return items
+			}
+
+			return [...items, structuredClone(item)]
+		})
+	}
+
+	/**
+	 * Remove a context item from the agent's context.
+	 * @param item The context item to remove.
+	 */
+	remove(item: ContextItem) {
+		this.$contextItems.update((items) => items.filter((v) => item !== v))
+	}
+
+	/**
+	 * Clear all context items.
+	 */
+	clear() {
+		this.$contextItems.set([])
+	}
+
+	/**
+	 * Check if the agent's context contains a specific context item. This could
+	 * mean as an individual item, or as part of a group of items.
+	 *
+	 * @param item The context item to check for.
+	 * @returns True if the agent's context contains the item, false otherwise.
+	 */
+	has(item: ContextItem) {
+		const items = this.$contextItems.get()
+		if (items.some((v) => areContextItemsEqual(v, item))) {
+			return true
+		}
+
+		if (item.type === 'shape') {
+			for (const existingItem of items) {
+				if (existingItem.type === 'shapes') {
+					if (existingItem.shapes.some((shape) => shape.shapeId === item.shape.shapeId)) {
+						return true
+					}
+				}
+			}
+		}
+
+		return false
+	}
+}
+
+/**
+ * Check if two context items are equal.
+ *
+ * This is a helper function that is used internally by the manager.
+ */
+function areContextItemsEqual(a: ContextItem, b: ContextItem): boolean {
+	if (a.type !== b.type) return false
+
+	switch (a.type) {
+		case 'shape': {
+			const _b = b as ShapeContextItem
+			return a.shape.shapeId === _b.shape.shapeId
+		}
+		case 'shapes': {
+			const _b = b as ShapesContextItem
+			if (a.shapes.length !== _b.shapes.length) return false
+			return a.shapes.every((shape) => _b.shapes.find((s) => s.shapeId === shape.shapeId))
+		}
+		case 'area': {
+			const _b = b as AreaContextItem
+			return Box.Equals(a.bounds, _b.bounds)
+		}
+		case 'point': {
+			const _b = b as PointContextItem
+			return Vec.Equals(a.point, _b.point)
+		}
+		default: {
+			exhaustiveSwitchError(a)
+		}
+	}
+}
+
+/**
+ * Remove duplicate shapes from a shapes context item.
+ * If there's only one shape left, return it as a shape item instead.
+ *
+ * This is a helper function that is used internally by the manager.
+ */
+function dedupeShapesContextItem(
+	item: ShapesContextItem,
+	existingItems: ContextItem[]
+): ContextItem[] {
+	// Get all shape IDs that are already in the context
+	const existingShapeIds = new Set<string>()
+
+	// Check individual shapes
+	existingItems.forEach((contextItem) => {
+		if (contextItem.type === 'shape') {
+			existingShapeIds.add(contextItem.shape.shapeId)
+		} else if (contextItem.type === 'shapes') {
+			contextItem.shapes.forEach((shape: FocusedShape) => {
+				existingShapeIds.add(shape.shapeId)
+			})
+		}
+	})
+
+	// Filter out shapes that are already in the context
+	const newShapes = item.shapes.filter((shape) => !existingShapeIds.has(shape.shapeId))
+
+	// Only add if there are remaining shapes
+	if (newShapes.length > 0) {
+		// If only one shape remains, add it as a single shape item
+		if (newShapes.length === 1) {
+			const newItem: ContextItem = {
+				type: 'shape',
+				shape: newShapes[0],
+				source: item.source,
+			}
+			return [structuredClone(newItem)]
+		}
+
+		// Otherwise add as a shapes group
+		const newItem: ContextItem = {
+			type: 'shapes',
+			shapes: newShapes,
+			source: item.source,
+		}
+		return [structuredClone(newItem)]
+	}
+
+	// No new shapes to add
+	return []
+}
+
+/**
+ * Throw an error if a switch case is not exhaustive.
+ *
+ * This is a helper function that is used internally by the manager.
+ */
+function exhaustiveSwitchError(value: never, property?: string): never {
+	const debugValue =
+		property && value && typeof value === 'object' && property in value ? value[property] : value
+	throw new Error(`Unknown switch case ${debugValue}`)
+}

templates/agent/client/agent/managers/AgentDebugManager.ts

@@ -0,0 +1,81 @@
+import { Atom, atom } from 'tldraw'
+import type { AgentAction } from '../../../shared/types/AgentAction'
+import type { Streaming } from '../../../shared/types/Streaming'
+import type { TldrawAgent } from '../TldrawAgent'
+
+/**
+ * Debug flags for controlling logging behavior.
+ */
+export interface AgentDebugFlags {
+	/** Log the system prompt to the console (worker-side). */
+	logSystemPrompt: boolean
+	/** Log messages sent to the model (worker-side). */
+	logMessages: boolean
+	/** Log completed actions to the console (client-side). */
+	logCompletedActions: boolean
+}
+
+const DEFAULT_DEBUG_FLAGS: AgentDebugFlags = {
+	logSystemPrompt: false,
+	logMessages: false,
+	logCompletedActions: false,
+}
+
+/**
+ * Manages debug functionality for an agent.
+ * Provides flags for controlling logging of system prompts, messages, and actions.
+ */
+export class AgentDebugManager {
+	/**
+	 * Debug flags for controlling logging behavior.
+	 */
+	private $debugFlags: Atom<AgentDebugFlags>
+
+	constructor(public agent: TldrawAgent) {
+		this.$debugFlags = atom<AgentDebugFlags>('debugFlags', DEFAULT_DEBUG_FLAGS)
+	}
+
+	/**
+	 * Get the current debug flags.
+	 */
+	getDebugFlags(): AgentDebugFlags {
+		return this.$debugFlags.get()
+	}
+
+	/**
+	 * Set debug flags. Merges with existing flags.
+	 */
+	setDebugFlags(flags: Partial<AgentDebugFlags>): void {
+		this.$debugFlags.update((current) => ({ ...current, ...flags }))
+	}
+
+	/**
+	 * Toggle a specific debug flag.
+	 * @returns The updated debug flags.
+	 */
+	toggleFlag(flag: keyof AgentDebugFlags): AgentDebugFlags {
+		this.$debugFlags.update((current) => ({
+			...current,
+			[flag]: !current[flag],
+		}))
+		return this.$debugFlags.get()
+	}
+
+	/**
+	 * Log a completed action if logging is enabled.
+	 */
+	logCompletedAction(action: Streaming<AgentAction>): void {
+		const flags = this.$debugFlags.get()
+		if (flags.logCompletedActions && action.complete) {
+			console.log('[DEBUG] Completed Action:', action)
+		}
+	}
+
+	reset(): void {
+		this.$debugFlags.set(DEFAULT_DEBUG_FLAGS)
+	}
+
+	dispose(): void {
+		// No cleanup needed
+	}
+}

templates/agent/client/agent/managers/AgentLintManager.ts

@@ -0,0 +1,329 @@
+import {
+	Box,
+	getArrowBindings,
+	intersectPolygonPolygon,
+	RecordsDiff,
+	TLArrowShape,
+	TLRecord,
+	TLShape,
+	TLShapeId,
+} from 'tldraw'
+import { convertTldrawIdToSimpleId } from '../../../shared/format/convertTldrawShapeToFocusedShape'
+import { AgentCanvasLint } from '../../../shared/types/AgentCanvasLint'
+import { BaseAgentManager } from './BaseAgentManager'
+
+/**
+ * Generate a unique key for a lint based on its type and shape IDs.
+ */
+function getLintKey(lint: AgentCanvasLint): string {
+	const sortedIds = [...lint.shapeIds].sort().join(',')
+	return `${lint.type}:${sortedIds}`
+}
+
+/**
+ * Tracks shapes created by the agent during a prompt chain and computes lints on them.
+ * This is cleared when starting a new top-level prompt (not nested prompts).
+ */
+export class AgentLintManager extends BaseAgentManager {
+	/**
+	 * Set of shape IDs created during the current prompt chain.
+	 * Persists across nested prompts but clears when a new top-level prompt starts.
+	 */
+	private createdShapeIds: Set<TLShapeId> = new Set()
+
+	/**
+	 * Set of lint keys that have already been surfaced to the agent.
+	 * These won't be shown again in subsequent prompts.
+	 */
+	private surfacedLintKeys: Set<string> = new Set()
+
+	/**
+	 * Reset the lint manager to its initial state.
+	 */
+	reset(): void {
+		this.createdShapeIds.clear()
+		this.surfacedLintKeys.clear()
+	}
+
+	/**
+	 * Clear tracking of created shapes (called at start of new prompt chain).
+	 * Also clears surfaced lints since we're starting fresh.
+	 */
+	clearCreatedShapes(): void {
+		this.createdShapeIds.clear()
+		this.surfacedLintKeys.clear()
+	}
+
+	/**
+	 * Track shapes created from a diff.
+	 * Should be called after each action is applied.
+	 */
+	trackShapesFromDiff(diff: RecordsDiff<TLRecord>): void {
+		for (const [id, record] of Object.entries(diff.added)) {
+			if (record.typeName === 'shape') {
+				this.createdShapeIds.add(id as TLShapeId)
+			}
+		}
+
+		// Track removed shapes and remove from created set
+		for (const id of Object.keys(diff.removed)) {
+			this.createdShapeIds.delete(id as TLShapeId)
+		}
+	}
+
+	/**
+	 * Get all shape IDs created during the current prompt chain.
+	 */
+	getCreatedShapeIds(): TLShapeId[] {
+		return Array.from(this.createdShapeIds)
+	}
+
+	/**
+	 * Get shapes created during the prompt chain that still exist.
+	 */
+	getCreatedShapes() {
+		const { editor } = this.agent
+		return this.getCreatedShapeIds()
+			.map((id) => editor.getShape(id))
+			.filter((shape) => shape !== undefined)
+	}
+
+	/**
+	 * Unlock all shapes created during the current prompt chain.
+	 * Called when the prompt completes to make shapes editable again.
+	 */
+	unlockCreatedShapes(): void {
+		const { editor } = this.agent
+		const createdShapes = this.getCreatedShapes()
+		const lockedShapes = createdShapes.filter((shape) => shape.isLocked)
+		if (lockedShapes.length === 0) return
+
+		editor.run(
+			() => {
+				for (const shape of lockedShapes) {
+					editor.updateShape({ id: shape.id, type: shape.type, isLocked: false })
+				}
+			},
+			{ ignoreShapeLock: true, history: 'ignore' }
+		)
+	}
+
+	/**
+	 * Check if there are any unsurfaced lints for the given shapes.
+	 */
+	hasUnsurfacedLints(shapes: TLShape[]): boolean {
+		return this.getUnsurfacedLintsForShapes(shapes).length > 0
+	}
+
+	/**
+	 * Get unsurfaced lints for a specific set of shapes.
+	 */
+	getUnsurfacedLintsForShapes(shapes: TLShape[]): AgentCanvasLint[] {
+		const lints = this.detectCanvasLints(shapes)
+		return lints.filter((lint) => !this.surfacedLintKeys.has(getLintKey(lint)))
+	}
+
+	/**
+	 * Mark the given lints as surfaced.
+	 * Call this when scheduling a lint-fixing prompt.
+	 */
+	markLintsAsSurfaced(lints: AgentCanvasLint[]): void {
+		for (const lint of lints) {
+			this.surfacedLintKeys.add(getLintKey(lint))
+		}
+	}
+
+	// ============================================================================
+	// Lint Detection Functions
+	// ============================================================================
+
+	/**
+	 * Detect all canvas lints on a set of shapes.
+	 */
+	detectCanvasLints(shapes: TLShape[]): AgentCanvasLint[] {
+		const lints: AgentCanvasLint[] = []
+
+		// Collect shapes for each lint type
+		const growYShapes = this.getShapesWithGrowY(shapes)
+		const overlappingTextGroups = this.getOverlappingTextGroups(shapes)
+		const friendlessArrows = this.getFriendlessArrows(shapes)
+
+		// Convert shapes to lints (converting shape IDs to strings)
+		for (const shape of growYShapes) {
+			lints.push({
+				type: 'growY-on-shape',
+				shapeIds: [convertTldrawIdToSimpleId(shape.id)],
+			})
+		}
+
+		for (const group of overlappingTextGroups) {
+			lints.push({
+				type: 'overlapping-text',
+				shapeIds: group.map((shape) => convertTldrawIdToSimpleId(shape.id)),
+			})
+		}
+
+		for (const arrow of friendlessArrows) {
+			lints.push({
+				type: 'friendless-arrow',
+				shapeIds: [convertTldrawIdToSimpleId(arrow.id)],
+			})
+		}
+
+		return lints
+	}
+
+	/**
+	 * Get shapes where text exceeds the shape bounds (growY > 0) and overlaps with geo shapes.
+	 */
+	private getShapesWithGrowY(shapes: TLShape[]): TLShape[] {
+		const shapesWithGrowY = shapes.filter((shape) => {
+			if ('growY' in shape.props) {
+				return shape.props.growY > 5 // use 5 because 0 flags shapes that don't need to be changed
+			}
+			return false
+		})
+
+		// Get all geo shapes to check for overlaps
+		const geoShapes = shapes.filter((shape) => shape.type === 'geo')
+
+		// Only return shapes that overlap with any geo shape
+		const result: TLShape[] = []
+		for (const shape of shapesWithGrowY) {
+			const overlapsWithGeo = geoShapes.some((geoShape) => this.shapesOverlap(shape, geoShape))
+			if (overlapsWithGeo) {
+				result.push(shape)
+			}
+		}
+		return result
+	}
+
+	/**
+	 * Get groups of shapes with overlapping text.
+	 */
+	private getOverlappingTextGroups(shapes: TLShape[]): TLShape[][] {
+		const { editor } = this.agent
+		const groups: TLShape[][] = []
+		const shapesWithText = shapes.filter((shape) => {
+			// Exclude arrows from overlapping text detection
+			if (shape.type === 'arrow') return false
+			const util = editor.getShapeUtil(shape)
+			const text = util.getText(shape)
+			return text !== undefined && text.length > 0
+		})
+
+		if (shapesWithText.length < 2) {
+			return groups
+		}
+
+		// Use union-find to group overlapping shapes
+		const parent = new Map<TLShape, TLShape>()
+
+		const find = (shape: TLShape): TLShape => {
+			if (!parent.has(shape)) {
+				parent.set(shape, shape)
+			}
+			const p = parent.get(shape)!
+			if (p !== shape) {
+				parent.set(shape, find(p))
+			}
+			return parent.get(shape)!
+		}
+
+		const union = (shapeA: TLShape, shapeB: TLShape) => {
+			const rootA = find(shapeA)
+			const rootB = find(shapeB)
+			if (rootA !== rootB) {
+				parent.set(rootB, rootA)
+			}
+		}
+
+		// Check all pairs for overlaps using geometry-based detection
+		for (let i = 0; i < shapesWithText.length; i++) {
+			const shapeA = shapesWithText[i]
+			for (let j = i + 1; j < shapesWithText.length; j++) {
+				const shapeB = shapesWithText[j]
+				if (this.shapesOverlap(shapeA, shapeB)) {
+					union(shapeA, shapeB)
+				}
+			}
+		}
+
+		// Group shapes by their root
+		const rootGroups = new Map<TLShape, TLShape[]>()
+		for (const shape of shapesWithText) {
+			const root = find(shape)
+			if (!rootGroups.has(root)) {
+				rootGroups.set(root, [])
+			}
+			rootGroups.get(root)!.push(shape)
+		}
+
+		// Collect groups with 2+ shapes (overlapping)
+		for (const [, group] of rootGroups) {
+			if (group.length >= 2) {
+				groups.push(group)
+			}
+		}
+
+		return groups
+	}
+
+	/**
+	 * Get arrows that are not connected to anything.
+	 */
+	private getFriendlessArrows(shapes: TLShape[]): TLArrowShape[] {
+		const { editor } = this.agent
+		const arrowShapes = shapes.filter((shape) => shape.type === 'arrow') as TLArrowShape[]
+
+		const friendlessArrows = arrowShapes.filter((arrow) => {
+			const bindings = getArrowBindings(editor, arrow)
+			// An arrow is "friendless" if it has no start or end binding
+			return !bindings.start || !bindings.end
+		})
+
+		return friendlessArrows
+	}
+
+	// ============================================================================
+	// Helper Functions
+	// ============================================================================
+
+	/**
+	 * Check if two shapes overlap using geometry-based detection.
+	 */
+	private shapesOverlap(shapeA: TLShape, shapeB: TLShape): boolean {
+		const { editor } = this.agent
+
+		// Quick bounds check first for early exit
+		const boundsA = editor.getShapePageBounds(shapeA)
+		const boundsB = editor.getShapePageBounds(shapeB)
+		if (!boundsA || !boundsB || !Box.Collides(boundsA, boundsB)) {
+			return false
+		}
+
+		// Get geometry and transform for shape A
+		const geometryA = editor.getShapeGeometry(shapeA)
+		const pageTransformA = editor.getShapePageTransform(shapeA)
+		const verticesA = pageTransformA.applyToPoints(geometryA.vertices)
+
+		// Get clip path if it exists
+		const shapeUtilA = editor.getShapeUtil(shapeA.type)
+		const clipPathA = shapeUtilA.getClipPath?.(shapeA)
+		const polygonA = clipPathA
+			? intersectPolygonPolygon(pageTransformA.applyToPoints(clipPathA), verticesA)
+			: verticesA
+
+		if (!polygonA || polygonA.length === 0) {
+			return false
+		}
+
+		// Transform polygon A into shape B's local space
+		const pageTransformB = editor.getShapePageTransform(shapeB)
+		const polygonAInShapeBSpace = pageTransformB.clone().invert().applyToPoints(polygonA)
+
+		// Check if shape B's geometry overlaps with the transformed polygon
+		const geometryB = editor.getShapeGeometry(shapeB)
+		return geometryB.overlapsPolygon(polygonAInShapeBSpace)
+	}
+}

templates/agent/client/agent/managers/AgentModeManager.ts

@@ -0,0 +1,83 @@
+import { Atom, atom } from 'tldraw'
+import { getModeNode } from '../../modes/AgentModeChart'
+import { AgentModeType, getAgentModeDefinition } from '../../modes/AgentModeDefinitions'
+import type { TldrawAgent } from '../TldrawAgent'
+import { BaseAgentManager } from './BaseAgentManager'
+
+/**
+ * Manages the mode/state of an agent.
+ * The mode determines what prompt parts and actions are available.
+ */
+export class AgentModeManager extends BaseAgentManager {
+	/**
+	 * An atom containing the current agent mode.
+	 */
+	private $mode: Atom<AgentModeType>
+
+	/**
+	 * Creates a new mode manager for the given agent.
+	 * Initializes the mode to 'idling'.
+	 */
+	constructor(agent: TldrawAgent) {
+		super(agent)
+		this.$mode = atom('mode', 'idling')
+	}
+
+	/**
+	 * Resets the mode manager to its initial state.
+	 * Sets the mode to 'idling'.
+	 */
+	reset(): void {
+		this.$mode.set('idling')
+	}
+
+	/**
+	 * Get the current mode of the agent.
+	 * @returns The current mode type.
+	 */
+	getCurrentModeType(): AgentModeType {
+		return this.$mode.get()
+	}
+
+	/**
+	 * Set the mode of the agent.
+	 * Calls onExit for the current mode and onEnter for the new mode.
+	 * Also rebuilds action utils to use mode-specific implementations.
+	 * @param newMode - The mode to set.
+	 */
+	setMode(newMode: AgentModeType) {
+		const fromMode = this.getCurrentModeType()
+
+		// TODO see if this is needed, or if it should just be a return, or if we can remove it entirely
+		if (fromMode === newMode) {
+			throw new Error(`Agent is already in mode: ${newMode}`)
+		}
+
+		const fromModeNode = this.getCurrentModeNode()
+		const newModeNode = getModeNode(newMode)
+		fromModeNode.onExit?.(this.agent, newMode)
+		newModeNode.onEnter?.(this.agent, fromMode)
+
+		// Update the mode
+		this.$mode.set(newMode)
+
+		// Rebuild action utils for the new mode
+		this.agent.actions.rebuildUtilsForMode(newMode)
+	}
+
+	/**
+	 * Get the mode definition for the current mode.
+	 * @returns The mode definition containing parts and actions.
+	 */
+	getCurrentModeDefinition() {
+		return getAgentModeDefinition(this.getCurrentModeType())
+	}
+
+	/**
+	 * Get current mode node.
+	 * @returns The current mode node.
+	 */
+	getCurrentModeNode() {
+		return getModeNode(this.getCurrentModeType())
+	}
+}

templates/agent/client/agent/managers/AgentModelNameManager.ts

@@ -0,0 +1,55 @@
+import { Atom, atom } from 'tldraw'
+import { AgentModelName, DEFAULT_MODEL_NAME, isValidModelName } from '../../../shared/models'
+import type { TldrawAgent } from '../TldrawAgent'
+import { BaseAgentManager } from './BaseAgentManager'
+
+/**
+ * Manages the model name selection for an agent.
+ * The model name determines which AI model is used to generate responses.
+ */
+export class AgentModelNameManager extends BaseAgentManager {
+	/**
+	 * An atom containing the model name that the user has selected.
+	 * This gets passed through to prompts unless manually overridden.
+	 *
+	 * Note: Prompt part utils may ignore or override this value. See the
+	 * ModelNamePartUtil for an example.
+	 */
+	private $modelName: Atom<AgentModelName>
+
+	/**
+	 * Creates a new model name manager for the given agent.
+	 * Initializes with the default model name.
+	 */
+	constructor(agent: TldrawAgent) {
+		super(agent)
+		this.$modelName = atom<AgentModelName>('modelName', DEFAULT_MODEL_NAME)
+	}
+
+	/**
+	 * Get the current model name.
+	 * @returns The currently selected model name.
+	 */
+	getModelName(): AgentModelName {
+		return this.$modelName.get()
+	}
+
+	/**
+	 * Set the model name.
+	 * @param modelName - The model name to set.
+	 */
+	setModelName(modelName: AgentModelName): void {
+		// A persisted selection from an older version may name a model that no
+		// longer exists. Fall back to the default rather than holding an unknown
+		// model name that would later fail when sent to the worker.
+		this.$modelName.set(isValidModelName(modelName) ? modelName : DEFAULT_MODEL_NAME)
+	}
+
+	/**
+	 * Reset the model name manager to its initial state.
+	 * Sets the model name back to the default.
+	 */
+	reset(): void {
+		this.$modelName.set(DEFAULT_MODEL_NAME)
+	}
+}

templates/agent/client/agent/managers/AgentRequestManager.ts

@@ -0,0 +1,188 @@
+import { atom, Atom } from 'tldraw'
+import { AgentInput } from '../../../shared/types/AgentInput'
+import { AgentRequest } from '../../../shared/types/AgentRequest'
+import type { TldrawAgent } from '../TldrawAgent'
+import { BaseAgentManager } from './BaseAgentManager'
+
+/**
+ * Manages request lifecycle for an agent.
+ * Handles active requests, scheduled requests, and request state management.
+ */
+export class AgentRequestManager extends BaseAgentManager {
+	/**
+	 * An atom containing the currently active request.
+	 * This is mainly used to render highlights and other UI elements.
+	 */
+	private $activeRequest: Atom<AgentRequest | null>
+
+	/**
+	 * An atom containing the next request that the agent has scheduled for itself.
+	 * Null if there is no scheduled request.
+	 */
+	private $scheduledRequest: Atom<AgentRequest | null>
+
+	/**
+	 * Whether the agent is currently prompting (working on a request).
+	 */
+	private $isPrompting: Atom<boolean>
+
+	/**
+	 * A function that cancels the agent's current prompt, if one is active.
+	 */
+	private cancelFn: (() => void) | null = null
+
+	/**
+	 * Creates a new request manager for the given agent.
+	 * Initializes all request-related atoms with default values.
+	 */
+	constructor(agent: TldrawAgent) {
+		super(agent)
+		this.$activeRequest = atom('activeRequest', null)
+		this.$scheduledRequest = atom('scheduledRequest', null)
+		this.$isPrompting = atom('isPrompting', false)
+	}
+
+	/**
+	 * Reset the request manager to its initial state.
+	 * Clears all active and scheduled requests.
+	 */
+	reset(): void {
+		this.$activeRequest.set(null)
+		this.$scheduledRequest.set(null)
+		this.$isPrompting.set(false)
+		this.cancelFn = null
+	}
+
+	/**
+	 * Check if the agent is currently generating a response.
+	 * @returns True if the agent is prompting, false otherwise.
+	 */
+	isGenerating() {
+		return this.$isPrompting.get()
+	}
+
+	/**
+	 * Set the prompting state.
+	 * @param value - True if the agent is prompting, false otherwise.
+	 */
+	setIsPrompting(value: boolean) {
+		this.$isPrompting.set(value)
+	}
+
+	/**
+	 * Clear the scheduled request without affecting the active request.
+	 */
+	clearScheduledRequest() {
+		this.$scheduledRequest.set(null)
+	}
+
+	/**
+	 * Clear the active request without affecting the scheduled request.
+	 */
+	clearActiveRequest() {
+		this.$activeRequest.set(null)
+	}
+
+	/**
+	 * Get a full agent request from a user input by filling out any missing values with defaults.
+	 * @param input - A partial agent request or a string message.
+	 */
+	getFullRequestFromInput(input: AgentInput): AgentRequest {
+		const request = this.getPartialRequestFromInput(input)
+		const activeRequest = this.getActiveRequest()
+
+		return {
+			source: request.source ?? 'user',
+			agentMessages: request.agentMessages ?? [],
+			userMessages: request.userMessages ?? [],
+			data: request.data ?? [],
+			bounds: request.bounds ?? activeRequest?.bounds ?? this.agent.editor.getViewportPageBounds(),
+			contextItems: request.contextItems ?? [],
+		}
+	}
+
+	/**
+	 * Convert an input into a partial request.
+	 * This involves handling the various ways that the input can be provided.
+	 * @param input - The input to convert (string, array, or object).
+	 * @returns A partial request object.
+	 */
+	getPartialRequestFromInput(input: AgentInput): Partial<AgentRequest> {
+		// eg: agent.prompt('Draw a cat')
+		if (typeof input === 'string') {
+			return { agentMessages: [input] }
+		}
+
+		// eg: agent.prompt(['Draw a cat', 'Draw a dog'])
+		if (Array.isArray(input)) {
+			return { agentMessages: input }
+		}
+
+		// eg: agent.prompt({ message: 'Draw a cat' })
+		if ('message' in input && typeof input.message === 'string') {
+			const { message, ...rest } = input
+			return { ...rest, agentMessages: [message], userMessages: [message] }
+		}
+
+		return input
+	}
+
+	/**
+	 * Set the active request.
+	 * @param request - The request to set as active, or null to clear.
+	 */
+	setActiveRequest(request: AgentRequest | null) {
+		this.$activeRequest.set(request)
+	}
+
+	/**
+	 * Get the active request.
+	 * @returns The currently active request, or null if none.
+	 */
+	getActiveRequest() {
+		return this.$activeRequest.get()
+	}
+
+	/**
+	 * Set the scheduled request.
+	 * @param request - The request to schedule, or null to clear.
+	 */
+	setScheduledRequest(request: AgentRequest | null) {
+		this.$scheduledRequest.set(request)
+	}
+
+	/**
+	 * Get the scheduled request.
+	 * @returns The scheduled request, or null if none.
+	 */
+	getScheduledRequest() {
+		return this.$scheduledRequest.get()
+	}
+
+	/**
+	 * Set the cancel function for the current request.
+	 * @param fn - A function that cancels the current request, or null to clear.
+	 */
+	setCancelFn(fn: (() => void) | null) {
+		this.cancelFn = fn
+	}
+
+	/**
+	 * Get the cancel function for the current request.
+	 * @returns The cancel function, or null if none.
+	 */
+	getCancelFn() {
+		return this.cancelFn
+	}
+
+	/**
+	 * Cancel the current request if one is active.
+	 * Clears both active and scheduled requests.
+	 */
+	cancel() {
+		this.cancelFn?.()
+		this.$activeRequest.set(null)
+		this.$scheduledRequest.set(null)
+		this.cancelFn = null
+	}
+}

templates/agent/client/agent/managers/AgentTodoManager.ts

@@ -0,0 +1,109 @@
+import { Atom, atom } from 'tldraw'
+import { TodoId } from '../../../shared/types/ids-schema'
+import { TodoItem } from '../../../shared/types/TodoItem'
+import type { TldrawAgent } from '../TldrawAgent'
+import { BaseAgentManager } from './BaseAgentManager'
+
+/**
+ * Manages the todo list for an agent.
+ * Each agent maintains its own todo list for tracking tasks.
+ */
+export class AgentTodoManager extends BaseAgentManager {
+	/**
+	 * An atom containing the agent's todo list.
+	 */
+	private $todoList: Atom<TodoItem[]>
+
+	/**
+	 * Creates a new todo manager for the given agent.
+	 * Initializes with an empty todo list.
+	 */
+	constructor(agent: TldrawAgent) {
+		super(agent)
+		this.$todoList = atom('todoList', [])
+	}
+
+	/**
+	 * Reset the todo manager to its initial state.
+	 * Clears all todo items from the list.
+	 */
+	reset(): void {
+		this.$todoList.set([])
+	}
+
+	/**
+	 * Get the current todo list.
+	 * @returns An array of todo items.
+	 */
+	getTodos() {
+		return this.$todoList.get()
+	}
+
+	/**
+	 * Set the todo list directly.
+	 * Primarily used for loading persisted state.
+	 * @param todos - The todo items to set.
+	 */
+	setTodos(todos: TodoItem[]) {
+		this.$todoList.set(todos)
+	}
+
+	/**
+	 * Add a todo item to the agent's todo list.
+	 * @param id - The id of the todo item.
+	 * @param text - The text of the todo item.
+	 * @returns The id of the todo item.
+	 */
+	push(id: TodoId, text: string) {
+		this.$todoList.update((todoItems) => {
+			return [
+				...todoItems,
+				{
+					id,
+					status: 'todo' as const,
+					text,
+				},
+			]
+		})
+		return id
+	}
+
+	/**
+	 * Update a todo item's status and optionally its text.
+	 * @param params - The update parameters
+	 */
+	update(params: { id: number; status: TodoItem['status']; text?: string }) {
+		const { id, status, text } = params
+		this.$todoList.update((todoItems) => {
+			const index = todoItems.findIndex((item) => item.id === id)
+			if (index !== -1) {
+				return [
+					...todoItems.slice(0, index),
+					{ ...todoItems[index], status, ...(text !== undefined && { text }) },
+					...todoItems.slice(index + 1),
+				]
+			}
+			return todoItems
+		})
+	}
+
+	/**
+	 * Delete specific todo items by their ids.
+	 * @param ids - The ids of the todos to delete
+	 */
+	delete(ids: number[]) {
+		const idsSet = new Set(ids)
+		this.$todoList.update((todoItems) => {
+			return todoItems.filter((item) => !idsSet.has(item.id))
+		})
+	}
+
+	/**
+	 * Remove all completed todo items from the todo list.
+	 */
+	flush() {
+		this.$todoList.update((todoItems) => {
+			return todoItems.filter((item) => item.status !== 'done')
+		})
+	}
+}

templates/agent/client/agent/managers/AgentUserActionTracker.ts

@@ -0,0 +1,132 @@
+import { Atom, atom, RecordsDiff, TLRecord } from 'tldraw'
+import type { TldrawAgent } from '../TldrawAgent'
+import { BaseAgentManager } from './BaseAgentManager'
+
+/**
+ * Tracks user actions on the canvas for an agent.
+ * This allows the agent to be aware of changes the user makes between prompts.
+ */
+export class AgentUserActionTracker extends BaseAgentManager {
+	/**
+	 * An atom that stores document changes made by the user since the previous request.
+	 */
+	private $userActionHistory: Atom<RecordsDiff<TLRecord>[]>
+
+	/**
+	 * A function that stops recording user actions.
+	 */
+	private stopRecordingFn: (() => void) | null = null
+
+	/**
+	 * Creates a new user action tracker for the given agent.
+	 * Initializes with an empty action history.
+	 */
+	constructor(agent: TldrawAgent) {
+		super(agent)
+		this.$userActionHistory = atom('userActionHistory', [])
+	}
+
+	/**
+	 * Reset the user action tracker to its initial state.
+	 * Clears the action history but does not stop recording.
+	 */
+	reset(): void {
+		this.$userActionHistory.set([])
+	}
+
+	/**
+	 * Start recording user actions.
+	 * Sets up listeners for create, delete, and change events.
+	 * @returns A cleanup function to stop recording user actions.
+	 */
+	startRecording() {
+		const { editor } = this.agent
+		const cleanUpCreate = editor.sideEffects.registerAfterCreateHandler(
+			'shape',
+			(shape, source) => {
+				if (source !== 'user') return
+				if (this.agent.getIsActingOnEditor()) return
+				const change = {
+					added: { [shape.id]: shape },
+					updated: {},
+					removed: {},
+				}
+				this.$userActionHistory.update((prev) => [...prev, change])
+				return
+			}
+		)
+
+		const cleanUpDelete = editor.sideEffects.registerAfterDeleteHandler(
+			'shape',
+			(shape, source) => {
+				if (source !== 'user') return
+				if (this.agent.getIsActingOnEditor()) return
+				const change = {
+					added: {},
+					updated: {},
+					removed: { [shape.id]: shape },
+				}
+				this.$userActionHistory.update((prev) => [...prev, change])
+				return
+			}
+		)
+
+		const cleanUpChange = editor.sideEffects.registerAfterChangeHandler(
+			'shape',
+			(prev, next, source) => {
+				if (source !== 'user') return
+				if (this.agent.getIsActingOnEditor()) return
+				const change: RecordsDiff<TLRecord> = {
+					added: {},
+					updated: { [prev.id]: [prev, next] },
+					removed: {},
+				}
+				this.$userActionHistory.update((prev) => [...prev, change])
+				return
+			}
+		)
+
+		const cleanUp = () => {
+			cleanUpCreate()
+			cleanUpDelete()
+			cleanUpChange()
+		}
+
+		this.stopRecordingFn = cleanUp
+		return cleanUp
+	}
+
+	/**
+	 * Stop recording user actions.
+	 * Cleans up event listeners but preserves the action history.
+	 */
+	stopRecording() {
+		this.stopRecordingFn?.()
+		this.stopRecordingFn = null
+	}
+
+	/**
+	 * Clear the user action history.
+	 * Does not affect recording status.
+	 */
+	clearHistory() {
+		this.$userActionHistory.set([])
+	}
+
+	/**
+	 * Get the current user action history.
+	 * @returns An array of record diffs representing user changes.
+	 */
+	getHistory() {
+		return this.$userActionHistory.get()
+	}
+
+	/**
+	 * Dispose of the tracker by stopping recording.
+	 * Called automatically during manager cleanup.
+	 */
+	override dispose() {
+		this.stopRecording()
+		super.dispose()
+	}
+}

templates/agent/client/agent/managers/BaseAgentAppManager.ts

@@ -0,0 +1,37 @@
+import type { TldrawAgentApp } from '../TldrawAgentApp'
+
+/**
+ * Base class for all agent app managers.
+ * These managers handle app-level concerns (shared across all agents),
+ * as opposed to BaseAgentManager which handles per-agent concerns.
+ *
+ * Provides common lifecycle methods and resource cleanup functionality.
+ */
+export abstract class BaseAgentAppManager {
+	constructor(public app: TldrawAgentApp) {}
+
+	/**
+	 * Set of cleanup functions to be called when the manager is disposed.
+	 * Subclasses can add disposables during initialization or operation.
+	 * @protected
+	 */
+	protected disposables = new Set<() => void>()
+
+	/**
+	 * Reset the manager to its initial state.
+	 * Must be implemented by all subclasses to handle their specific state.
+	 * @abstract
+	 */
+	abstract reset(): void
+
+	/**
+	 * Clean up all resources held by this manager.
+	 * Calls all registered dispose functions and clears the disposables set.
+	 */
+	dispose(): void {
+		for (const fn of this.disposables) {
+			fn()
+		}
+		this.disposables.clear()
+	}
+}

templates/agent/client/agent/managers/BaseAgentManager.ts

@@ -0,0 +1,34 @@
+import type { TldrawAgent } from '../TldrawAgent'
+
+/**
+ * Base class for all agent managers.
+ * Provides common lifecycle methods and resource cleanup functionality.
+ */
+export abstract class BaseAgentManager {
+	constructor(public agent: TldrawAgent) {}
+
+	/**
+	 * Set of cleanup functions to be called when the manager is disposed.
+	 * Subclasses can add disposables during initialization or operation.
+	 * @protected
+	 */
+	protected disposables = new Set<() => void>()
+
+	/**
+	 * Reset the manager to its initial state.
+	 * Must be implemented by all subclasses to handle their specific state.
+	 * @abstract
+	 */
+	abstract reset(): void
+
+	/**
+	 * Clean up all resources held by this manager.
+	 * Calls all registered dispose functions and clears the disposables set.
+	 */
+	dispose(): void {
+		for (const fn of this.disposables) {
+			fn()
+		}
+		this.disposables.clear()
+	}
+}

templates/agent/client/components/ChatInput.tsx

@@ -0,0 +1,149 @@
+import { FormEventHandler, useState } from 'react'
+import { Editor, useValue } from 'tldraw'
+import { AtIcon } from '../../shared/icons/AtIcon'
+import { BrainIcon } from '../../shared/icons/BrainIcon'
+import { ChevronDownIcon } from '../../shared/icons/ChevronDownIcon'
+import { AGENT_MODEL_DEFINITIONS, AgentModelName } from '../../shared/models'
+import { useAgent } from '../agent/TldrawAgentAppProvider'
+import { ContextItemTag } from './ContextItemTag'
+import { SelectionTag } from './SelectionTag'
+
+export function ChatInput({
+	handleSubmit,
+	inputRef,
+}: {
+	handleSubmit: FormEventHandler<HTMLFormElement>
+	inputRef: React.RefObject<HTMLTextAreaElement | null>
+}) {
+	const agent = useAgent()
+	const { editor } = agent
+	const [inputValue, setInputValue] = useState('')
+	const isGenerating = useValue('isGenerating', () => agent.requests.isGenerating(), [agent])
+
+	const isContextToolActive = useValue(
+		'isContextToolActive',
+		() => {
+			const tool = editor.getCurrentTool()
+			return tool.id === 'target-shape' || tool.id === 'target-area'
+		},
+		[editor]
+	)
+
+	const selectedShapes = useValue('selectedShapes', () => editor.getSelectedShapes(), [editor])
+	const contextItems = useValue('contextItems', () => agent.context.getItems(), [agent])
+	const modelName = useValue('modelName', () => agent.modelName.getModelName(), [agent])
+
+	return (
+		<div className="chat-input">
+			<form
+				onSubmit={(e) => {
+					e.preventDefault()
+					setInputValue('')
+					handleSubmit(e)
+				}}
+			>
+				<div className="prompt-tags">
+					<div className={'chat-context-select ' + (isContextToolActive ? 'active' : '')}>
+						<div className="chat-context-select-label">
+							<AtIcon /> Add Context
+						</div>
+						<select
+							id="chat-context-select"
+							value=" "
+							onChange={(e) => {
+								const action = ADD_CONTEXT_ACTIONS.find((action) => action.name === e.target.value)
+								if (action) action.onSelect(editor)
+							}}
+						>
+							{ADD_CONTEXT_ACTIONS.map((action) => {
+								return (
+									<option key={action.name} value={action.name}>
+										{action.name}
+									</option>
+								)
+							})}
+						</select>
+					</div>
+					{selectedShapes.length > 0 && <SelectionTag onClick={() => editor.selectNone()} />}
+					{contextItems.map((item, i) => (
+						<ContextItemTag
+							editor={editor}
+							onClick={() => agent.context.remove(item)}
+							key={'context-item-' + i}
+							item={item}
+						/>
+					))}
+				</div>
+
+				<textarea
+					ref={inputRef}
+					name="input"
+					autoComplete="off"
+					placeholder="Ask, learn, brainstorm, draw"
+					value={inputValue}
+					onInput={(e) => setInputValue(e.currentTarget.value)}
+					onKeyDown={(e) => {
+						if (e.key === 'Enter' && !e.shiftKey) {
+							e.preventDefault()
+							//idk about this but it works oops -max
+							const form = e.currentTarget.closest('form')
+							if (form) {
+								const submitEvent = new Event('submit', { bubbles: true, cancelable: true })
+								form.dispatchEvent(submitEvent)
+							}
+						}
+					}}
+				/>
+				<span className="chat-actions">
+					<div className="chat-actions-left">
+						<div className="chat-model-select">
+							<div className="chat-model-select-label">
+								<BrainIcon /> {modelName}
+							</div>
+							<select
+								value={modelName}
+								onChange={(e) => agent.modelName.setModelName(e.target.value as AgentModelName)}
+							>
+								{Object.values(AGENT_MODEL_DEFINITIONS).map((model) => (
+									<option key={model.name} value={model.name}>
+										{model.name}
+									</option>
+								))}
+							</select>
+							<ChevronDownIcon />
+						</div>
+					</div>
+					<button className="chat-input-submit" disabled={inputValue === '' && !isGenerating}>
+						{isGenerating && inputValue === '' ? '◼' : '⬆'}
+					</button>
+				</span>
+			</form>
+		</div>
+	)
+}
+
+const ADD_CONTEXT_ACTIONS = [
+	{
+		name: 'Pick Shapes',
+		onSelect: (editor: Editor) => {
+			editor.setCurrentTool('target-shape')
+			editor.focus()
+		},
+	},
+	{
+		name: 'Pick Area',
+		onSelect: (editor: Editor) => {
+			editor.setCurrentTool('target-area')
+			editor.focus()
+		},
+	},
+	{
+		name: ' ',
+		onSelect: (editor: Editor) => {
+			const currentTool = editor.getCurrentTool()
+			if (currentTool.id === 'target-area' || currentTool.id === 'target-shape') {
+				editor.setCurrentTool('select')
+			}
+		},
+	},
+]

templates/agent/client/components/ChatPanel.tsx

@@ -0,0 +1,58 @@
+import { FormEventHandler, useCallback, useRef } from 'react'
+import { useAgent } from '../agent/TldrawAgentAppProvider'
+import { ChatHistory } from './chat-history/ChatHistory'
+import { ChatInput } from './ChatInput'
+import { TodoList } from './TodoList'
+
+export function ChatPanel() {
+	const agent = useAgent()
+	const inputRef = useRef<HTMLTextAreaElement>(null)
+
+	const handleSubmit = useCallback<FormEventHandler<HTMLFormElement>>(
+		async (e) => {
+			e.preventDefault()
+			if (!inputRef.current) return
+			const formData = new FormData(e.currentTarget)
+			const value = formData.get('input') as string
+
+			// If the user's message is empty, just cancel the current request (if there is one)
+			if (value === '') {
+				agent.cancel()
+				return
+			}
+
+			// Clear the chat input (context is cleared after it's captured in requestAgentActions)
+			inputRef.current.value = ''
+
+			// Sending a new message to the agent should interrupt the current request
+			agent.interrupt({
+				input: {
+					agentMessages: [value],
+					bounds: agent.editor.getViewportPageBounds(),
+					source: 'user',
+					contextItems: agent.context.getItems(),
+				},
+			})
+		},
+		[agent]
+	)
+
+	const handleNewChat = useCallback(() => {
+		agent.reset()
+	}, [agent])
+
+	return (
+		<div className="chat-panel tl-theme__dark">
+			<div className="chat-header">
+				<button className="new-chat-button" onClick={handleNewChat}>
+					+
+				</button>
+			</div>
+			<ChatHistory agent={agent} />
+			<div className="chat-input-container">
+				<TodoList agent={agent} />
+				<ChatInput handleSubmit={handleSubmit} inputRef={inputRef} />
+			</div>
+		</div>
+	)
+}

templates/agent/client/components/ChatPanelFallback.tsx

@@ -0,0 +1,15 @@
+export function ChatPanelFallback() {
+	return (
+		<div className="chat-fallback">
+			<p>Error loading chat history</p>
+			<button
+				onClick={() => {
+					localStorage.clear()
+					window.location.reload()
+				}}
+			>
+				Clear chat history
+			</button>
+		</div>
+	)
+}

templates/agent/client/components/ContextItemTag.tsx

@@ -0,0 +1,19 @@
+import { Editor } from 'tldraw'
+import { CONTEXT_TYPE_DEFINITIONS, ContextItem } from '../../shared/types/ContextItem'
+import { PromptTag } from './PromptTag'
+
+export function ContextItemTag({
+	item,
+	editor,
+	onClick,
+}: {
+	item: ContextItem
+	editor: Editor
+	onClick?: () => void
+}) {
+	const definition = CONTEXT_TYPE_DEFINITIONS[item.type]
+	const name = definition.name(item, editor)
+	const icon = definition.icon
+
+	return <PromptTag text={name} icon={icon} onClick={onClick} />
+}

templates/agent/client/components/CustomHelperButtons.tsx

@@ -0,0 +1,17 @@
+import {
+	DefaultHelperButtons,
+	DefaultHelperButtonsContent,
+	TldrawUiMenuContextProvider,
+} from 'tldraw'
+import { GoToAgentButtons } from './GoToAgentButton'
+
+export function CustomHelperButtons() {
+	return (
+		<DefaultHelperButtons>
+			<TldrawUiMenuContextProvider type="helper-buttons" sourceId="helper-buttons">
+				<DefaultHelperButtonsContent />
+				<GoToAgentButtons />
+			</TldrawUiMenuContextProvider>
+		</DefaultHelperButtons>
+	)
+}

templates/agent/client/components/GoToAgentButton.tsx

@@ -0,0 +1,108 @@
+import { useCallback } from 'react'
+import {
+	Box,
+	TldrawUiButton,
+	TldrawUiButtonIcon,
+	TldrawUiButtonLabel,
+	useEditor,
+	useValue,
+	Vec,
+} from 'tldraw'
+import { TldrawAgent } from '../agent/TldrawAgent'
+import { useAgents } from '../agent/TldrawAgentAppProvider'
+
+/**
+ * Renders GoToAgentButton for all agents.
+ */
+export function GoToAgentButtons() {
+	const agents = useAgents()
+
+	return (
+		<>
+			{agents.map((agent) => (
+				<GoToAgentButton key={agent.id} agent={agent} />
+			))}
+		</>
+	)
+}
+
+/**
+ * A button that zooms the viewport to show where an agent is working.
+ */
+export function GoToAgentButton({ agent }: { agent: TldrawAgent }) {
+	const editor = useEditor()
+	const currentRequest = useValue('activeRequest', () => agent.requests.getActiveRequest(), [agent])
+	const agentViewport = currentRequest?.bounds
+
+	// We only show the button if the agent is offscreen
+	const agentIsOffscreen = useValue(
+		'agentIsOffscreen',
+		() => {
+			if (!agentViewport) return false
+			const screenBounds = editor.getViewportScreenBounds()
+			const agentViewportScreenCorners = Box.From(agentViewport).corners.map((corner) =>
+				editor.pageToViewport(corner)
+			)
+
+			return !Box.FromPoints(agentViewportScreenCorners).collides(screenBounds)
+		},
+		[agentViewport]
+	)
+
+	// The button's arrow points towards the agent
+	const angleToAgent = useValue(
+		'angleToAgent',
+		() => {
+			if (!agentViewport) return
+			if (agentIsOffscreen) return
+
+			const agentCenter = Box.From(agentViewport).center
+			const agentScreenCenter = editor.pageToViewport(agentCenter)
+			const screenBounds = editor.getViewportScreenBounds()
+			const screenCenter = Box.From(screenBounds).center
+			const displacement = Vec.From(agentScreenCenter).sub(screenCenter)
+			return Math.atan2(displacement.y, displacement.x) * (180 / Math.PI)
+		},
+		[agentViewport]
+	)
+
+	const handleClick = useCallback(() => {
+		if (agentViewport) {
+			const bounds = Box.From(agentViewport).expandBy(50)
+			editor.zoomToBounds(bounds, { animation: { duration: 220 } })
+		}
+	}, [agentViewport, editor])
+
+	if (!agentViewport) return null
+	if (!agentIsOffscreen) return null
+
+	return (
+		<TldrawUiButton type="low" onClick={handleClick}>
+			<div style={{ display: 'flex', flexDirection: 'row', alignItems: 'center', gap: 8 }}>
+				<TldrawUiButtonIcon
+					icon={
+						<svg
+							width="16"
+							height="16"
+							viewBox="0 0 24 24"
+							style={{
+								transform: `rotate(${angleToAgent}deg)`,
+								transition: 'transform 0.2s ease',
+							}}
+						>
+							<path
+								d="M8 6l6 6-6 6"
+								stroke="currentColor"
+								strokeWidth="2"
+								strokeLinecap="round"
+								strokeLinejoin="round"
+								fill="none"
+							/>
+						</svg>
+					}
+				/>
+				<TldrawUiButtonLabel>Go to {agent.id}</TldrawUiButtonLabel>
+			</div>
+		</TldrawUiButton>
+	)
+}

templates/agent/client/components/PromptTag.tsx

@@ -0,0 +1,21 @@
+import { AgentIcon, AgentIconType } from '../../shared/icons/AgentIcon'
+
+export function PromptTag({
+	text,
+	icon,
+	onClick,
+}: {
+	text: string
+	icon: AgentIconType
+	onClick?: () => void
+}) {
+	return onClick ? (
+		<button type="button" className="prompt-tag" onClick={onClick}>
+			<AgentIcon type={icon} /> <span>{text}</span>
+		</button>
+	) : (
+		<div className="prompt-tag">
+			<AgentIcon type={icon} /> <span>{text}</span>
+		</div>
+	)
+}

templates/agent/client/components/SelectionTag.tsx

@@ -0,0 +1,5 @@
+import { PromptTag } from './PromptTag'
+
+export function SelectionTag({ onClick }: { onClick?: () => void }) {
+	return <PromptTag text="Selection" icon="cursor" onClick={onClick} />
+}

templates/agent/client/components/TodoList.tsx

@@ -0,0 +1,51 @@
+import { useCallback } from 'react'
+import { useValue } from 'tldraw'
+import { TodoItem } from '../../shared/types/TodoItem'
+import { TldrawAgent } from '../agent/TldrawAgent'
+
+export function TodoList({ agent }: { agent: TldrawAgent }) {
+	const todoItems = useValue('todoList', () => agent.todos.getTodos(), [agent])
+
+	if (todoItems.length === 0) {
+		return null
+	}
+
+	return (
+		<div className="todo-list">
+			<div className="todo-list-items">
+				{todoItems.map((item) => (
+					<TodoListItem key={item.id} item={item} agent={agent} />
+				))}
+			</div>
+		</div>
+	)
+}
+
+function TodoListItem({ item, agent }: { item: TodoItem; agent: TldrawAgent }) {
+	const deleteTodo = useCallback(() => {
+		agent.todos.delete([item.id])
+	}, [item.id, agent])
+
+	const getStatusIcon = (status: TodoItem['status']) => {
+		switch (status) {
+			case 'todo':
+				return '○'
+			case 'in-progress':
+				return '➤'
+			case 'done':
+				return '●'
+		}
+	}
+
+	const icon = getStatusIcon(item.status)
+
+	return (
+		<div className={`todo-item todo-item-${item.status}`}>
+			<span className="todo-item-icon">{icon}</span>
+			<span className="todo-item-text">{item.text}</span>
+			<button className="todo-item-delete" onClick={deleteTodo}>
+				×
+			</button>
+		</div>
+	)
+}

templates/agent/client/components/chat-history/ChatHistory.tsx

@@ -0,0 +1,90 @@
+import { useEffect, useRef } from 'react'
+import { useValue } from 'tldraw'
+import { TldrawAgent } from '../../agent/TldrawAgent'
+import { ChatHistorySection, getAgentHistorySections } from './ChatHistorySection'
+
+/*
+Chat history is stored as a list of history items.
+
+Within the UI, we split this list of items into sections.
+Each section contains a user prompt item, and all the agent's actions that follow it.
+
+Each section is rendered as a separate component.
+The model's actions are grouped together into collapsible groups if appropriate.
+
+
+Here's an example of how the UI might look:
+
+- Chat history
+	- Section
+		- Prompt
+		- Action group
+			- Action
+			- Action
+		- Action group
+			- Action
+	- Section
+		- Prompt
+		- Action group
+			- Action
+
+*/
+
+export function ChatHistory({ agent }: { agent: TldrawAgent }) {
+	const historyItems = useValue('chatHistory', () => agent.chat.getHistory(), [agent])
+	const sections = getAgentHistorySections(historyItems)
+	const historyRef = useRef<HTMLDivElement>(null)
+	const previousScrollDistanceFromBottomRef = useRef(0)
+
+	useEffect(() => {
+		if (!historyRef.current) return
+
+		// If a new prompt is submitted by the user, scroll to the bottom
+		if (historyItems.at(-1)?.type === 'prompt') {
+			if (previousScrollDistanceFromBottomRef.current <= 0) {
+				historyRef.current.scrollTo(0, historyRef.current.scrollHeight)
+				previousScrollDistanceFromBottomRef.current = 0
+			}
+			return
+		}
+
+		// If the user is scrolled to the bottom, keep them there while new actions appear
+		if (previousScrollDistanceFromBottomRef.current <= 0) {
+			const scrollDistanceFromBottom =
+				historyRef.current.scrollHeight -
+				historyRef.current.scrollTop -
+				historyRef.current.clientHeight
+
+			if (scrollDistanceFromBottom > 0) {
+				historyRef.current.scrollTo(0, historyRef.current.scrollHeight)
+			}
+		}
+	}, [historyRef, historyItems])
+
+	// Keep track of the user's scroll position
+	const handleScroll = () => {
+		if (!historyRef.current) return
+		const scrollDistanceFromBottom =
+			historyRef.current.scrollHeight -
+			historyRef.current.scrollTop -
+			historyRef.current.clientHeight
+
+		previousScrollDistanceFromBottomRef.current = scrollDistanceFromBottom
+	}
+
+	const isGenerating = useValue('isGenerating', () => agent.requests.isGenerating(), [agent])
+
+	return (
+		<div className="chat-history" ref={historyRef} onScroll={handleScroll}>
+			{sections.map((section, i) => {
+				return (
+					<ChatHistorySection
+						key={'history-section-' + i}
+						section={section}
+						loading={i === sections.length - 1 && isGenerating}
+					/>
+				)
+			})}
+		</div>
+	)
+}

templates/agent/client/components/chat-history/ChatHistoryGroup.tsx

@@ -0,0 +1,82 @@
+import { isRecordsDiffEmpty } from 'tldraw'
+import { ChatHistoryActionItem } from '../../../shared/types/ChatHistoryItem'
+import { TldrawAgent } from '../../agent/TldrawAgent'
+import { ChatHistoryGroupWithDiff } from './ChatHistoryGroupWithDiff'
+import { ChatHistoryGroupWithoutDiff } from './ChatHistoryGroupWithoutDiff'
+import { getActionInfo } from './getActionInfo'
+
+export interface ChatHistoryGroup {
+	items: ChatHistoryActionItem[]
+	withDiff: boolean
+}
+
+export function ChatHistoryGroup({ group }: { group: ChatHistoryGroup }) {
+	if (group.withDiff) {
+		return <ChatHistoryGroupWithDiff group={group} />
+	}
+
+	return <ChatHistoryGroupWithoutDiff group={group} />
+}
+
+/**
+ * Merge adjacent actions into groups where possible.
+ */
+export function getActionHistoryGroups(
+	items: ChatHistoryActionItem[],
+	agent: TldrawAgent
+): ChatHistoryGroup[] {
+	const groups: ChatHistoryGroup[] = []
+
+	for (const item of items) {
+		const { description } = getActionInfo(item.action, agent)
+		if (description === null) {
+			continue
+		}
+
+		const group = groups[groups.length - 1]
+		if (group && canActionBeGrouped({ item, group, agent })) {
+			group.items.push(item)
+		} else {
+			groups.push({
+				items: [item],
+				withDiff: !isRecordsDiffEmpty(item.diff) && item.action.complete,
+			})
+		}
+	}
+
+	return groups
+}
+
+/**
+ * Check if an action can be merged with a group.
+ */
+export function canActionBeGrouped({
+	item,
+	group,
+	agent,
+}: {
+	item: ChatHistoryActionItem
+	group: ChatHistoryGroup
+	agent: TldrawAgent
+}) {
+	if (!item.action.complete) return false
+	if (!group) return false
+
+	const showDiff = !isRecordsDiffEmpty(item.diff)
+	if (showDiff !== group.withDiff) return false
+
+	const groupAcceptance = group.items[0]?.acceptance
+	if (groupAcceptance !== item.acceptance) return false
+
+	const prevAction = group.items.at(-1)?.action
+	if (!prevAction) return false
+
+	const actionInfo = getActionInfo(item.action, agent)
+	const prevActionInfo = getActionInfo(prevAction, agent)
+
+	if (actionInfo.canGroup(prevAction) && prevActionInfo.canGroup(item.action)) {
+		return true
+	}
+
+	return false
+}

templates/agent/client/components/chat-history/ChatHistoryGroupWithDiff.tsx

@@ -0,0 +1,122 @@
+import { useCallback, useMemo } from 'react'
+import { reverseRecordsDiff, squashRecordDiffs } from 'tldraw'
+import { AgentIcon, AgentIconType } from '../../../shared/icons/AgentIcon'
+import { ChatHistoryActionItem } from '../../../shared/types/ChatHistoryItem'
+import { useAgent } from '../../agent/TldrawAgentAppProvider'
+import { ChatHistoryGroup } from './ChatHistoryGroup'
+import { getActionInfo } from './getActionInfo'
+import { TldrawDiffViewer } from './TldrawDiffViewer'
+
+export function ChatHistoryGroupWithDiff({ group }: { group: ChatHistoryGroup }) {
+	const agent = useAgent()
+	const { items } = group
+	const { editor } = agent
+	const diff = useMemo(() => squashRecordDiffs(items.map((item) => item.diff)), [items])
+
+	// Accept all changes from this group
+	const handleAccept = useCallback(() => {
+		agent.chat.update((currentChatHistoryItems) => {
+			const newItems = [...currentChatHistoryItems]
+			for (const item of items) {
+				const index = newItems.findIndex((v) => v === item)
+
+				// Mark the item as accepted
+				if (index !== -1) {
+					newItems[index] = { ...item, acceptance: 'accepted' }
+				}
+
+				// Apply the diff if needed
+				if (item.acceptance === 'rejected') {
+					editor.store.applyDiff(item.diff)
+				}
+			}
+			return newItems
+		})
+	}, [items, editor, agent.chat])
+
+	// Reject all changes from this group
+	const handleReject = useCallback(() => {
+		agent.chat.update((currentChatHistoryItems) => {
+			const newItems = [...currentChatHistoryItems]
+			for (const item of items) {
+				const index = newItems.findIndex((v) => v === item)
+
+				// Mark the item as rejected
+				if (index !== -1) {
+					newItems[index] = { ...item, acceptance: 'rejected' }
+				}
+
+				// Reverse the diff if needed
+				if (item.acceptance !== 'rejected') {
+					const reverseDiff = reverseRecordsDiff(item.diff)
+					editor.store.applyDiff(reverseDiff)
+				}
+			}
+			return newItems
+		})
+	}, [items, editor, agent.chat])
+
+	// Get the acceptance status of the group
+	// If all items are accepted, the group is accepted
+	// If all items are rejected, the group is rejected
+	// Otherwise, the group is pending
+	const acceptance = useMemo<ChatHistoryActionItem['acceptance']>(() => {
+		if (items.length === 0) return 'pending'
+		const acceptance = items[0].acceptance
+		for (let i = 1; i < items.length; i++) {
+			if (items[i].acceptance !== acceptance) {
+				return 'pending'
+			}
+		}
+		return acceptance
+	}, [items])
+
+	const steps = useMemo(
+		() => items.map((item) => getActionInfo(item.action, agent)),
+		[items, agent]
+	)
+
+	return (
+		<div className="chat-history-change">
+			<div className="chat-history-change-acceptance">
+				<button onClick={handleReject} disabled={acceptance === 'rejected'}>
+					{acceptance === 'rejected' ? 'Rejected' : 'Reject'}
+				</button>
+				<button onClick={handleAccept} disabled={acceptance === 'accepted'}>
+					{acceptance === 'accepted' ? 'Accepted' : 'Accept'}
+				</button>
+			</div>
+			<DiffSteps steps={steps} />
+			<TldrawDiffViewer diff={diff} />
+		</div>
+	)
+}
+
+interface DiffStep {
+	icon: AgentIconType | null
+	description: string | null
+}
+
+function DiffSteps({ steps }: { steps: DiffStep[] }) {
+	let previousDescription = ''
+	return (
+		<div className="agent-changes">
+			{steps.map((step, i) => {
+				if (!step.description) return null
+
+				if (step.description === previousDescription) return null
+				previousDescription = step.description
+				return (
+					<div className="agent-change" key={'intent-' + i}>
+						{step.icon && (
+							<span className="agent-change-icon">
+								<AgentIcon type={step.icon} />
+							</span>
+						)}
+						{step.description}
+					</div>
+				)
+			})}
+		</div>
+	)
+}

templates/agent/client/components/chat-history/ChatHistoryGroupWithoutDiff.tsx

@@ -0,0 +1,111 @@
+import { useMemo, useState } from 'react'
+import Markdown from 'react-markdown'
+import { AgentIcon } from '../../../shared/icons/AgentIcon'
+import { ChevronDownIcon } from '../../../shared/icons/ChevronDownIcon'
+import { ChevronRightIcon } from '../../../shared/icons/ChevronRightIcon'
+import { AgentAction } from '../../../shared/types/AgentAction'
+import { ChatHistoryActionItem } from '../../../shared/types/ChatHistoryItem'
+import { Streaming } from '../../../shared/types/Streaming'
+import { useAgent } from '../../agent/TldrawAgentAppProvider'
+import { ChatHistoryGroup } from './ChatHistoryGroup'
+import { getActionInfo } from './getActionInfo'
+
+export function ChatHistoryGroupWithoutDiff({ group }: { group: ChatHistoryGroup }) {
+	const agent = useAgent()
+	const { items } = group
+
+	const nonEmptyItems = useMemo(() => {
+		return items.filter((item) => {
+			const { description } = getActionInfo(item.action, agent)
+			return description !== null
+		})
+	}, [items, agent])
+
+	const [collapsed, setCollapsed] = useState(true)
+
+	const complete = useMemo(() => {
+		return items.every((item) => item.action.complete)
+	}, [items])
+
+	const summary = useMemo(() => {
+		const time = Math.floor(items.reduce((acc, item) => acc + item.action.time, 0) / 1000)
+		if (time === 0) return 'Thought for less than a second'
+		if (time === 1) return 'Thought for 1 second'
+		return `Thought for ${time} seconds`
+	}, [items])
+
+	if (nonEmptyItems.length === 0) {
+		return null
+	}
+
+	if (nonEmptyItems.length < 2) {
+		return (
+			<div className="chat-history-group">
+				{nonEmptyItems.map((item, i) => {
+					return <ChatHistoryItem item={item} key={'action-' + i} />
+				})}
+			</div>
+		)
+	}
+
+	const showContent = !collapsed || !complete
+
+	return (
+		<div className="chat-history-group">
+			{complete && (
+				<button onClick={() => setCollapsed((v) => !v)}>
+					<span>{showContent ? <ChevronDownIcon /> : <ChevronRightIcon />}</span>
+					{summary}
+				</button>
+			)}
+			{showContent && (
+				<div className="agent-actions-container">
+					{nonEmptyItems.map((item, i) => {
+						return <ChatHistoryItemExpanded action={item.action} key={'action-' + i} />
+					})}
+				</div>
+			)}
+		</div>
+	)
+}
+
+function ChatHistoryItem({ item }: { item: ChatHistoryActionItem }) {
+	const agent = useAgent()
+	const { action } = item
+	const { description, summary } = getActionInfo(action, agent)
+	const collapsible = summary !== null
+	const [collapsed, setCollapsed] = useState(collapsible)
+
+	if (!description) return null
+
+	return (
+		<div className="agent-actions-container">
+			{action.complete && collapsible && (
+				<button onClick={() => setCollapsed((v) => !v)}>
+					<span>{collapsed ? <ChevronRightIcon /> : <ChevronDownIcon />}</span>
+					{summary}
+				</button>
+			)}
+
+			{(!collapsed || !action.complete) && <ChatHistoryItemExpanded action={action} />}
+		</div>
+	)
+}
+
+function ChatHistoryItemExpanded({ action }: { action: Streaming<AgentAction> }) {
+	const agent = useAgent()
+	const { icon, description } = getActionInfo(action, agent)
+
+	return (
+		<div className={`agent-action agent-action-type-${action._type}`}>
+			{icon && (
+				<span>
+					<AgentIcon type={icon} />
+				</span>
+			)}
+			<span className="agent-action-description">
+				<Markdown>{description}</Markdown>
+			</span>
+		</div>
+	)
+}

templates/agent/client/components/chat-history/ChatHistoryPrompt.tsx

@@ -0,0 +1,38 @@
+import { Editor } from 'tldraw'
+import { ChatHistoryPromptItem } from '../../../shared/types/ChatHistoryItem'
+import { ContextItemTag } from '../ContextItemTag'
+import { SelectionTag } from '../SelectionTag'
+
+export function ChatHistoryPrompt({
+	item,
+	editor,
+}: {
+	item: ChatHistoryPromptItem
+	editor: Editor
+}) {
+	const { contextItems, agentFacingMessage, userFacingMessage, selectedShapes, promptSource } = item
+
+	const showTags = selectedShapes.length > 0 || contextItems.length > 0
+
+	// Display the user-facing message if available, otherwise fall back to the agent-facing message
+	const displayMessage = userFacingMessage ?? agentFacingMessage
+
+	// Get the CSS class modifier based on the prompt source
+	const sourceClass = `chat-history-prompt-${promptSource}`
+
+	return (
+		<div className="chat-history-prompt-container">
+			<div className={`chat-history-prompt ${sourceClass}`}>
+				{showTags && (
+					<div className="prompt-tags">
+						{selectedShapes.length > 0 && <SelectionTag />}
+						{contextItems.map((contextItem, i) => (
+							<ContextItemTag editor={editor} key={'context-item-' + i} item={contextItem} />
+						))}
+					</div>
+				)}
+				<span className="chat-history-prompt-content">{displayMessage}</span>
+			</div>
+		</div>
+	)
+}

templates/agent/client/components/chat-history/ChatHistorySection.tsx

@@ -0,0 +1,56 @@
+import { SmallSpinner } from '../../../shared/icons/SmallSpinner'
+import {
+	ChatHistoryActionItem,
+	ChatHistoryContinuationItem,
+	ChatHistoryItem,
+	ChatHistoryPromptItem,
+} from '../../../shared/types/ChatHistoryItem'
+import { useAgent } from '../../agent/TldrawAgentAppProvider'
+import { ChatHistoryGroup, getActionHistoryGroups } from './ChatHistoryGroup'
+import { ChatHistoryPrompt } from './ChatHistoryPrompt'
+
+export interface ChatHistorySection {
+	prompt: ChatHistoryPromptItem
+	items: (ChatHistoryActionItem | ChatHistoryContinuationItem)[]
+}
+
+export function ChatHistorySection({
+	section,
+	loading,
+}: {
+	section: ChatHistorySection
+	loading: boolean
+}) {
+	const agent = useAgent()
+	const actions = section.items.filter((item) => item.type === 'action')
+	const groups = getActionHistoryGroups(actions, agent)
+	return (
+		<div className="chat-history-section">
+			<ChatHistoryPrompt item={section.prompt} editor={agent.editor} />
+			{groups.map((group, i) => {
+				return <ChatHistoryGroup key={'chat-history-group-' + i} group={group} />
+			})}
+			{loading && <SmallSpinner />}
+		</div>
+	)
+}
+
+export function getAgentHistorySections(items: ChatHistoryItem[]): ChatHistorySection[] {
+	const sections: ChatHistorySection[] = []
+
+	for (const item of items) {
+		if (item.type === 'prompt') {
+			// Filter out 'self' prompts from the UI
+			if (item.promptSource === 'self') continue
+			sections.push({ prompt: item, items: [] })
+			continue
+		}
+
+		// Only add to the last section if one exists
+		if (sections.length > 0) {
+			sections[sections.length - 1].items.push(item)
+		}
+	}
+
+	return sections
+}

templates/agent/client/components/chat-history/TldrawDiffViewer.tsx

@@ -0,0 +1,135 @@
+import { forwardRef, useMemo } from 'react'
+import {
+	DefaultShapeWrapper,
+	RecordsDiff,
+	TLRecord,
+	TLShape,
+	TLShapeId,
+	TLShapeWrapperProps,
+} from 'tldraw'
+import { TldrawViewer } from './TldrawViewer'
+
+export function TldrawDiffViewer({ diff }: { diff: RecordsDiff<TLRecord> }) {
+	const diffShapes = useMemo(() => getDiffShapesFromDiff(diff), [diff])
+	return <TldrawViewer shapes={diffShapes} components={{ ShapeWrapper: DiffShapeWrapper }} />
+}
+
+function getDiffShapesFromDiff(diff: RecordsDiff<TLRecord>): TLShape[] {
+	const diffShapes: TLShape[] = []
+
+	const numberOfShapes =
+		Object.keys(diff.added).length +
+		Object.keys(diff.updated).length +
+		Object.keys(diff.removed).length
+
+	// If there are many shapes in the diff, don't show shadows (for performance reasons)
+	const showShadows = numberOfShapes < 20
+
+	for (const key in diff.removed) {
+		const id = key as TLShapeId
+		const prevShape = diff.removed[id]
+		if (prevShape.typeName !== 'shape') continue
+		const shape = {
+			...prevShape,
+			opacity: showShadows ? prevShape.opacity : prevShape.opacity / 2,
+			meta: { ...prevShape.meta, changeType: showShadows ? 'delete-shadow' : 'delete' },
+		}
+
+		if ('dash' in shape.props) {
+			shape.props = {
+				...shape.props,
+				dash: 'solid',
+			}
+		}
+
+		diffShapes.push(shape)
+	}
+
+	for (const key in diff.updated) {
+		const id = key as TLShapeId
+
+		const prevBefore = diff.updated[id][0]
+		const prevAfter = diff.updated[id][1]
+		if (prevBefore.typeName !== 'shape' || prevAfter.typeName !== 'shape') continue
+
+		const before = {
+			...prevBefore,
+			id: (id + '-before') as TLShapeId,
+			opacity: prevAfter.opacity / 2,
+			meta: {
+				...prevBefore.meta,
+				changeType: showShadows ? 'update-before-shadow' : 'update-before',
+			},
+		}
+
+		const after = {
+			...prevAfter,
+			meta: {
+				...prevAfter.meta,
+				changeType: showShadows ? 'update-after-shadow' : 'update-after',
+			},
+		}
+
+		if ('dash' in before.props) {
+			before.props = {
+				...before.props,
+				dash: 'dashed',
+			}
+		}
+		if ('fill' in before.props) {
+			before.props = {
+				...before.props,
+				fill: 'none',
+			}
+		}
+		if ('dash' in after.props) {
+			after.props = {
+				...after.props,
+				dash: 'solid',
+			}
+		}
+
+		diffShapes.push(before)
+		diffShapes.push(after)
+	}
+
+	for (const key in diff.added) {
+		const id = key as TLShapeId
+		const prevShape = diff.added[id]
+		if (prevShape.typeName !== 'shape') continue
+		const shape = {
+			...prevShape,
+			meta: {
+				...prevShape.meta,
+				changeType: showShadows ? 'create-shadow' : 'create',
+			},
+		}
+		if ('dash' in shape.props) {
+			shape.props = {
+				...shape.props,
+				dash: 'solid',
+			}
+		}
+		diffShapes.push(shape)
+	}
+
+	return diffShapes
+}
+
+const DiffShapeWrapper = forwardRef(function DiffShapeWrapper(
+	{ children, shape, isBackground }: TLShapeWrapperProps,
+	ref: React.Ref<HTMLDivElement>
+) {
+	const changeType = shape.meta.changeType
+
+	return (
+		<DefaultShapeWrapper
+			ref={ref}
+			shape={shape}
+			isBackground={isBackground}
+			className={changeType ? 'diff-shape-' + changeType : undefined}
+		>
+			{children}
+		</DefaultShapeWrapper>
+	)
+})

templates/agent/client/components/chat-history/TldrawViewer.tsx

@@ -0,0 +1,96 @@
+import { useEffect, useRef, useState } from 'react'
+import {
+	defaultAddFontsFromNode,
+	defaultBindingUtils,
+	defaultEditorAssetUrls,
+	defaultShapeUtils,
+	Editor,
+	StateNode,
+	tipTapDefaultExtensions,
+	TLComponents,
+	TldrawEditor,
+	TldrawUiContextProvider,
+	TLShape,
+} from 'tldraw'
+
+export function TldrawViewer({
+	shapes,
+	components = {},
+}: {
+	shapes: TLShape[]
+	components?: TLComponents
+}) {
+	const [editor, setEditor] = useState<Editor | null>(null)
+	const [isVisible, setIsVisible] = useState(false)
+	const containerRef = useRef<HTMLDivElement>(null)
+
+	// Hide the component if it's outside of the current scroll area
+	useEffect(() => {
+		const observer = new IntersectionObserver(
+			([entry]) => {
+				setIsVisible(entry.isIntersecting)
+			},
+			{ rootMargin: '200px', threshold: 0 }
+		)
+
+		const currentElement = containerRef.current
+		if (currentElement) observer.observe(currentElement)
+
+		return () => {
+			if (currentElement) observer.unobserve(currentElement)
+		}
+	}, [])
+
+	useEffect(() => {
+		if (!editor || !isVisible) return
+		editor.updateInstanceState({ isReadonly: false })
+		editor.setCameraOptions({ isLocked: false })
+		editor.deleteShapes(editor.getCurrentPageShapes())
+		editor.createShapes(shapes)
+		editor.updateInstanceState({ isReadonly: true })
+		editor.selectAll()
+		const bounds = editor.getSelectionPageBounds()
+		if (bounds) {
+			editor.zoomToBounds(bounds, { inset: 30 })
+		}
+		editor.selectNone()
+		editor.setCameraOptions({ isLocked: true })
+	}, [shapes, editor, isVisible])
+
+	if (!isVisible) {
+		return <div ref={containerRef} className="tldraw-viewer" />
+	}
+
+	return (
+		<div ref={containerRef} className="tldraw-viewer">
+			<TldrawUiContextProvider>
+				<TldrawEditor
+					autoFocus={false}
+					components={components ?? {}}
+					onMount={setEditor}
+					shapeUtils={defaultShapeUtils}
+					bindingUtils={defaultBindingUtils}
+					tools={tools}
+					options={defaultOptions}
+					assetUrls={defaultEditorAssetUrls}
+					initialState="inspect"
+				/>
+			</TldrawUiContextProvider>
+		</div>
+	)
+}
+
+class InspectTool extends StateNode {
+	static override id = 'inspect'
+}
+
+const tools = [InspectTool]
+
+const defaultOptions = {
+	text: {
+		tipTapConfig: {
+			extensions: tipTapDefaultExtensions,
+		},
+		addFontsFromNode: defaultAddFontsFromNode,
+	},
+}

templates/agent/client/components/chat-history/getActionInfo.ts

@@ -0,0 +1,27 @@
+import { AgentAction } from '../../../shared/types/AgentAction'
+import { ChatHistoryInfo } from '../../../shared/types/ChatHistoryInfo'
+import { Streaming } from '../../../shared/types/Streaming'
+import { TldrawAgent } from '../../agent/TldrawAgent'
+
+/**
+ * Get the full info for an action to display in chat history UI.
+ * This function adds default values for any unset properties.
+ * If the action's util returns null, the action will not be shown in chat history.
+ */
+export function getActionInfo(action: Streaming<AgentAction>, agent: TldrawAgent): ChatHistoryInfo {
+	const util = agent.actions.getAgentActionUtil(action._type)
+	const info = util.getInfo(action) ?? { description: null }
+	const {
+		icon = null,
+		description = JSON.stringify(action, null, 2),
+		summary = null,
+		canGroup = () => true,
+	} = info
+
+	return {
+		icon,
+		description,
+		summary,
+		canGroup,
+	}
+}

templates/agent/client/index.css

@@ -0,0 +1,772 @@
+@import url('https://fonts.googleapis.com/css2?family=Inter:wght@500;700&display=swap');
+@import url('tldraw/tldraw.css');
+
+* {
+	box-sizing: border-box;
+	overscroll-behavior: none;
+}
+
+body {
+	font-family: 'Inter', sans-serif;
+}
+
+.tldraw-agent-container {
+	position: fixed;
+	inset: 0;
+	display: grid;
+	grid-template-columns: 1fr 350px;
+}
+
+@media (max-width: 768px) {
+	.tldraw-agent-container {
+		grid-template-columns: 1fr;
+		grid-template-rows: 1fr auto;
+	}
+}
+
+.tldraw-canvas {
+	width: 100%;
+	height: 100%;
+}
+
+/* Chat Panel */
+
+.chat-panel.tl-theme__light {
+	--tl-color-text-2: hsl(0, 0%, 55%);
+	--tl-color-primary-hover: hsl(214, 84%, 58%);
+}
+
+.chat-panel.tl-theme__dark {
+	--tl-color-text-2: hsl(0, 0%, 55%);
+	--tl-color-primary-hover: hsl(214, 84%, 50%);
+}
+
+.chat-panel {
+	display: flex;
+	flex-direction: column;
+	height: 100vh;
+	font-size: 14px;
+	background-color: var(--tl-color-panel);
+	border-left: 1px solid var(--tl-color-low);
+	color: var(--tl-color-text-1);
+	font-family: 'Inter', sans-serif;
+	font-weight: 400;
+	font-size: 12px;
+	line-height: 1.5;
+}
+
+@media (max-width: 768px) {
+	.chat-panel {
+		height: 50vh;
+		width: 100vw;
+	}
+}
+
+.chat-panel {
+	position: relative;
+}
+
+.chat-panel * {
+	user-select: text;
+}
+
+.chat-panel button {
+	user-select: none;
+}
+
+.chat-header {
+	display: flex;
+	justify-content: flex-end;
+}
+
+.new-chat-button {
+	width: 32px;
+	height: 32px;
+	font-size: 16px;
+	align-self: flex-end;
+	appearance: none;
+	border: 0;
+	border-radius: 4px;
+	cursor: pointer;
+	background: none;
+	margin: 2px;
+	color: var(--tl-color-text);
+}
+
+.new-chat-button:hover {
+	background-color: var(--tl-color-muted-2);
+}
+
+/* Chat Input */
+.chat-input-container {
+	display: flex;
+	flex-direction: column;
+	position: relative;
+}
+
+.chat-input {
+	padding: 0px 8px 8px 8px;
+	flex-shrink: 0;
+	position: sticky;
+	bottom: 0;
+	border-radius: 12px;
+	margin: 0px 8px 8px 8px;
+	padding: 0px;
+	background-color: var(--tl-color-panel);
+	z-index: 999999;
+}
+
+.chat-input form {
+	display: flex;
+	flex-direction: column;
+	gap: 8px;
+	padding: 8px;
+	border-radius: 12px;
+	color: var(--tl-color-text-2);
+	background-color: var(--tl-color-muted-1);
+	box-shadow: var(--tl-shadow-1);
+	font-size: 11px;
+}
+
+.chat-input input[type='text'] {
+	flex: 1;
+	padding: 8px 0px;
+	margin-bottom: 4px;
+	border: none;
+	background-color: transparent;
+	color: var(--tl-color-text);
+	border-radius: 4px;
+	font-size: 12px;
+	outline: none;
+	width: 100%;
+}
+
+.chat-input input[type='text']::placeholder {
+	color: var(--tl-color-text-3);
+}
+
+.chat-input input[type='text']:focus {
+	border-color: var(--tl-color-primary);
+}
+
+.chat-input textarea {
+	flex: 1;
+	padding: 8px 0px;
+	margin-bottom: 4px;
+	border: none;
+	background-color: transparent;
+	color: var(--tl-color-text);
+	font-family: sans-serif;
+	border-radius: 4px;
+	font-size: 12px;
+	outline: none;
+	width: 100%;
+	resize: none;
+	scrollbar-color: var(--tl-color-text-3) transparent;
+}
+
+.chat-input textarea::placeholder {
+	color: var(--tl-color-text-3);
+}
+
+.chat-input textarea:focus {
+	border-color: var(--tl-color-primary);
+}
+
+.chat-input textarea::-webkit-scrollbar {
+	width: 8px;
+	background: transparent;
+}
+
+.chat-input textarea::-webkit-scrollbar-track {
+	background: transparent;
+}
+
+.chat-input textarea::-webkit-scrollbar-thumb {
+	background: var(--tl-color-text-3);
+	border-radius: 4px;
+}
+
+.chat-input-submit {
+	width: 16px;
+	height: 16px;
+	align-self: flex-end;
+	background: var(--tl-color-primary);
+	box-shadow: var(--tl-shadow-1);
+	color: var(--tl-color-panel-contrast);
+	appearance: none;
+	border: 0;
+	border-radius: 4px;
+	display: flex;
+	align-items: center;
+	justify-content: center;
+	cursor: pointer;
+	font-size: 10px;
+	text-align: center;
+	font-weight: 500;
+	border-radius: 100%;
+}
+
+.chat-input-submit:hover {
+	background: var(--tl-color-primary-hover);
+}
+
+.chat-input-submit:disabled {
+	background: var(--tl-color-grid);
+	cursor: not-allowed;
+}
+
+.chat-actions {
+	display: flex;
+	gap: 8px;
+	width: 100%;
+	justify-content: space-between;
+}
+
+.chat-actions-left {
+	display: flex;
+	gap: 0px;
+}
+
+.chat-icon {
+	width: 11px;
+	height: 11px;
+	margin-right: 4px;
+}
+
+.chat-mode-select {
+	display: flex;
+	align-items: center;
+	gap: 0px;
+}
+
+.chat-mode-select span {
+	margin-right: 4px;
+}
+
+.chat-context-select-label {
+	display: flex;
+	align-items: center;
+	justify-content: center;
+	font-size: 10px;
+}
+
+.chat-model-select,
+.chat-context-select {
+	position: relative;
+	display: flex;
+	align-items: center;
+	gap: 4px;
+	padding: 0px 8px;
+}
+
+.chat-model-select-label {
+	display: flex;
+	align-items: center;
+	gap: 2px;
+	margin-left: -6px;
+}
+
+.chat-context-select {
+	border: 1px solid var(--tl-color-text-2);
+	padding: 0px 4px 0px 3px;
+	border-radius: 6px;
+	height: 18px;
+}
+
+.chat-model-select:hover,
+.chat-context-select:hover {
+	color: var(--tl-color-text);
+	border-color: var(--tl-color-text);
+}
+
+.chat-model-select select,
+.chat-context-select select {
+	opacity: 0;
+	position: absolute;
+	left: -1px;
+	right: -1px;
+	background-color: transparent;
+	border-radius: 6px;
+	padding: 2px 16px 2px 0px;
+	font-size: 11px;
+	border: none;
+	-webkit-appearance: none;
+	-moz-appearance: none;
+	appearance: none;
+	font-family: 'Inter', sans-serif;
+}
+
+.chat-context-select select {
+	cursor: pointer;
+	font-size: 10px;
+	font-weight: 500;
+	line-height: 1.5;
+}
+
+.chat-context-select.active {
+	background-color: var(--tl-color-primary);
+	color: black;
+	border-color: black;
+}
+
+.chat-context-select.active:hover {
+	background-color: var(--tl-color-primary);
+	color: var(--tl-color-text);
+	border-color: var(--tl-color-text);
+}
+
+/* Chat History */
+
+.chat-history {
+	padding: 0px 16px 32px 16px;
+	overflow-y: auto;
+	display: flex;
+	flex-direction: column;
+	gap: 10px;
+	height: 100%;
+	position: relative;
+	width: 100%;
+	scrollbar-width: 8px;
+	scrollbar-color: var(--tl-color-muted-1) transparent;
+}
+
+.chat-history::-webkit-scrollbar {
+	width: 8px;
+}
+
+.chat-history::-webkit-scrollbar-track {
+	background: transparent;
+}
+
+.chat-history::-webkit-scrollbar-thumb {
+	background: var(--tl-color-muted-1);
+	border-radius: 4px;
+}
+
+.chat-history-section {
+	display: flex;
+	flex-direction: column;
+	gap: 14px;
+	flex-shrink: 0;
+}
+
+.chat-history-section:last-child {
+	min-height: calc(100% - 1px);
+}
+
+.chat-history-prompt-container {
+	position: sticky;
+	top: -6px;
+	padding: 8px;
+	margin: -8px;
+	z-index: 9999;
+	background-color: var(--tl-color-panel);
+}
+
+.chat-history-prompt {
+	background-color: var(--tl-color-muted-1);
+	box-shadow: var(--tl-shadow-1);
+	color: var(--tl-color-text-1);
+	width: 100%;
+	font-family: 'Inter', sans-serif;
+	font-weight: 400;
+	font-size: 12px;
+	padding: 8px;
+	border-radius: 8px;
+	display: flex;
+	flex-direction: column;
+	gap: 4px;
+	white-space: pre-wrap;
+	max-height: 25vh;
+	overflow-y: auto;
+	scrollbar-color: var(--tl-color-text-3) transparent;
+}
+
+.chat-history-prompt::-webkit-scrollbar {
+	background: transparent;
+}
+
+.chat-history-prompt::-webkit-scrollbar-thumb {
+	background: var(--tl-color-text-3);
+}
+
+.chat-history-prompt-content {
+	white-space: pre-wrap;
+}
+
+/* Messages from other agents are italicized and lighter */
+.chat-history-prompt-other-agent {
+	color: var(--tl-color-text-3);
+	font-style: italic;
+}
+
+.chat-history-group {
+	display: flex;
+	flex-direction: column;
+	gap: 14px;
+}
+
+.chat-history-group button {
+	display: flex;
+	align-items: center;
+	gap: 4px;
+	padding: 12px 0px;
+	margin: -12px 0px;
+	font-size: 12px;
+	font-family: 'Inter', sans-serif;
+	font-weight: 400;
+	color: var(--tl-color-text-2);
+	background-color: transparent;
+	border: none;
+	cursor: pointer;
+}
+
+.chat-history-group button:hover {
+	color: var(--tl-color-text);
+}
+
+.chat-history-group button:disabled {
+	color: var(--tl-color-text-1);
+	cursor: default;
+}
+
+.chat-history-change-acceptance {
+	display: flex;
+	justify-content: flex-end;
+	width: 100%;
+	gap: 2px;
+}
+
+.chat-history-change-acceptance button {
+	border: 0;
+	cursor: pointer;
+	background-color: var(--tl-color-muted-1);
+	color: var(--tl-color-text-2);
+	padding: 4px 8px;
+	font-size: 10px;
+	font-weight: 500;
+	cursor: pointer;
+	height: 20px;
+	user-select: none;
+	display: flex;
+	align-items: center;
+	gap: 4px;
+	flex-direction: row;
+	user-select: none;
+}
+
+.chat-history-change-acceptance button:hover {
+	color: var(--tl-color-text);
+}
+
+.chat-history-change-acceptance button:disabled {
+	color: var(--tl-color-text-1);
+	cursor: default;
+}
+
+.chat-history-change-acceptance button:first-child {
+	border-radius: 4px 0px 0px 0px;
+}
+
+.chat-history-change-acceptance button:last-child {
+	border-radius: 0px 4px 0px 0px;
+}
+
+/* Agent Actions */
+
+.agent-action {
+	display: flex;
+	align-items: flex-start;
+	gap: 4px;
+	font-size: 12px;
+	font-family: 'Inter', sans-serif;
+	font-weight: 400;
+	color: var(--tl-color-text-2);
+}
+
+.agent-action-description {
+	margin: 0;
+	width: 100%;
+}
+
+.agent-action-description > p:first-child {
+	margin-top: 0;
+}
+
+.agent-action-description > p:last-child {
+	margin-bottom: 0;
+}
+
+.agent-action-description p {
+	white-space: pre-wrap;
+}
+
+.agent-action-description ul,
+.agent-action-description ol {
+	padding-left: 20px;
+}
+
+.agent-action-description pre {
+	width: 100%;
+	overflow-x: scroll;
+}
+
+.agent-action-description blockquote {
+	padding-left: 14px;
+	border-left: 2px solid var(--tl-color-text-2);
+	margin: 0;
+}
+
+.agent-actions-container {
+	display: flex;
+	flex-direction: column;
+	gap: 10px;
+}
+
+.agent-changes {
+	background-color: var(--tl-color-muted-1);
+	padding: 6px 8px;
+	margin: 0px 0px;
+	border-radius: 4px 0px 0px 0px;
+	color: var(--tl-color-text-2);
+	max-height: 70px;
+	overflow-y: auto;
+	scrollbar-width: none;
+	-ms-overflow-style: none;
+}
+
+.agent-changes::-webkit-scrollbar {
+	display: none;
+}
+
+.agent-changes:last-child {
+	border-radius: 4px 0px 4px 4px;
+	max-height: 100%;
+}
+
+.agent-change {
+	display: flex;
+	align-items: flex-start;
+	gap: 4px;
+	font-size: 12px;
+	font-family: 'Inter', sans-serif;
+	font-weight: 400;
+}
+
+/* Prompt Tags */
+
+.prompt-tags {
+	display: flex;
+	gap: 4px;
+	align-items: center;
+	flex-wrap: wrap;
+}
+
+.prompt-tag {
+	display: flex;
+	align-items: center;
+	border-radius: 6px;
+	background-color: transparent;
+	font-size: 10px;
+	border: 1px solid var(--tl-color-text-2);
+	padding: 8px 4px 8px 3px;
+	font-weight: 500;
+	height: 16px;
+	color: var(--tl-color-text-2);
+	white-space: nowrap;
+	font-family: 'Inter', sans-serif;
+	max-width: 100%;
+}
+
+button.prompt-tag {
+	cursor: pointer;
+}
+
+button.prompt-tag:hover {
+	color: var(--tl-color-text);
+	border-color: var(--tl-color-text);
+}
+
+button.prompt-tag span {
+	user-select: none;
+}
+
+.prompt-tag span {
+	text-overflow: ellipsis;
+	overflow: hidden;
+}
+
+.prompt-tag svg {
+	flex-shrink: 0;
+}
+
+/* Diff viewer */
+
+.tldraw-viewer {
+	height: 100px;
+	position: relative;
+}
+
+.tldraw-viewer .tlui-tooltip {
+	display: none;
+}
+
+.diff-shape-create-shadow {
+	filter: drop-shadow(
+			calc(var(--tl-scale) * 1px) calc(var(--tl-scale) * 1px) 0px rgba(0, 200, 0, 50%)
+		)
+		drop-shadow(calc(var(--tl-scale) * -1px) calc(var(--tl-scale) * -1px) 0px rgba(0, 200, 0, 50%))
+		drop-shadow(calc(var(--tl-scale) * 1px) calc(var(--tl-scale) * -1px) 0px rgba(0, 200, 0, 50%))
+		drop-shadow(calc(var(--tl-scale) * -1px) calc(var(--tl-scale) * 1px) 0px rgba(0, 200, 0, 50%));
+}
+
+.diff-shape-delete-shadow {
+	filter: drop-shadow(
+			calc(var(--tl-scale) * 1px) calc(var(--tl-scale) * 1px) 0px rgba(200, 0, 0, 50%)
+		)
+		drop-shadow(calc(var(--tl-scale) * -1px) calc(var(--tl-scale) * -1px) 0px rgba(200, 0, 0, 50%))
+		drop-shadow(calc(var(--tl-scale) * 1px) calc(var(--tl-scale) * -1px) 0px rgba(200, 0, 0, 50%))
+		drop-shadow(calc(var(--tl-scale) * -1px) calc(var(--tl-scale) * 1px) 0px rgba(200, 0, 0, 50%));
+}
+
+.diff-shape-update-after-shadow {
+	filter: drop-shadow(
+			calc(var(--tl-scale) * 1px) calc(var(--tl-scale) * 1px) 0px rgba(0, 100, 255, 50%)
+		)
+		drop-shadow(
+			calc(var(--tl-scale) * -1px) calc(var(--tl-scale) * -1px) 0px rgba(0, 100, 255, 50%)
+		)
+		drop-shadow(calc(var(--tl-scale) * 1px) calc(var(--tl-scale) * -1px) 0px rgba(0, 100, 255, 50%))
+		drop-shadow(calc(var(--tl-scale) * -1px) calc(var(--tl-scale) * 1px) 0px rgba(0, 100, 255, 50%));
+}
+
+/* Chat History Fallback */
+
+.chat-fallback {
+	padding: 10px;
+	display: flex;
+	align-items: center;
+	justify-content: center;
+	flex-direction: column;
+	color: var(--tl-color-text-2);
+	user-select: none;
+	font-size: 14px;
+}
+
+.chat-fallback button {
+	background-color: hsl(214, 84%, 50%);
+	color: white;
+	border: none;
+	border-radius: 4px;
+	padding: 6px 12px;
+	font-weight: 500;
+	cursor: pointer;
+}
+
+.chat-fallback button:hover {
+	background-color: hsl(214, 84%, 40%);
+}
+
+/* Todo list */
+
+.todo-list {
+	margin: 0px 16px -18px 16px;
+	background-color: var(--tl-color-panel);
+	border-radius: 12px 12px 0px 0px;
+	padding: 8px 4px 0 4px;
+	box-shadow: var(--tl-shadow-1);
+	border: 1px solid var(--tl-color-muted-2);
+	max-height: 120px;
+	overflow-y: auto;
+	flex-shrink: 0;
+	scrollbar-width: none;
+	-ms-overflow-style: none;
+	position: relative;
+	z-index: 99999;
+	transform: translateY(-16px);
+}
+
+.todo-list::-webkit-scrollbar {
+	display: none;
+}
+
+.todo-list-items {
+	display: flex;
+	flex-direction: column;
+}
+
+.todo-item {
+	display: flex;
+	align-items: flex-start;
+	gap: 8px;
+	padding-left: 4px;
+	border-radius: 6px;
+	font-size: 12px;
+	line-height: 1.3;
+}
+
+.todo-item-icon {
+	font-size: 10px;
+	width: 12px;
+	text-align: center;
+	flex-shrink: 0;
+	user-select: none;
+}
+
+.todo-item-text {
+	flex: 1;
+	color: var(--tl-color-text-3);
+	user-select: none;
+	padding-bottom: 8px;
+}
+
+.todo-item-todo .todo-item-icon {
+	color: var(--tl-color-text-3);
+}
+
+.todo-item-in-progress .todo-item-icon {
+	color: var(--tl-color-primary);
+}
+
+.todo-item-in-progress .todo-item-text {
+	font-weight: 500;
+	color: var(--tl-color-text);
+}
+
+.todo-item-done .todo-item-icon {
+	color: rgb(76, 200, 94);
+}
+
+.todo-item-done .todo-item-text {
+	color: var(--tl-color-text-2);
+	text-decoration: line-through;
+}
+
+.todo-item-delete {
+	appearance: none;
+	background: none;
+	border: none;
+	width: 16px;
+	height: 16px;
+	flex-shrink: 0;
+	display: flex;
+	padding: 12px;
+	padding-right: 14px;
+	margin-right: -3px;
+	margin-top: -3px;
+	align-items: center;
+	justify-content: center;
+	cursor: pointer;
+	color: var(--tl-color-text-2);
+}
+
+.todo-item-delete:hover {
+	color: var(--tl-color-text);
+}
+
+/* Custom appearance for specific agent actions */
+
+.agent-action-type-message {
+	color: var(--tl-color-text-1);
+}

templates/agent/client/main.tsx

@@ -0,0 +1,10 @@
+import React from 'react'
+import ReactDOM from 'react-dom/client'
+import App from './App'
+import './index.css'
+
+ReactDOM.createRoot(document.getElementById('root') as HTMLElement).render(
+	<React.StrictMode>
+		<App />
+	</React.StrictMode>
+)

templates/agent/client/modes/AgentModeChart.ts

@@ -0,0 +1,107 @@
+import type { AgentRequest } from '../../shared/types/AgentRequest'
+import type { TldrawAgent } from '../agent/TldrawAgent'
+import type { AgentModeDefinition, AgentModeType } from './AgentModeDefinitions'
+
+/**
+ * Lifecycle hooks for an agent mode.
+ * Each mode can optionally implement these hooks to respond to state changes.
+ */
+export interface AgentModeNode {
+	onEnter?(agent: TldrawAgent, fromMode: AgentModeType): void
+	onExit?(agent: TldrawAgent, toMode: AgentModeType): void
+	onPromptStart?(agent: TldrawAgent, request: AgentRequest): void
+	onPromptEnd?(agent: TldrawAgent, request: AgentRequest): void
+	onPromptCancel?(agent: TldrawAgent, request: AgentRequest): void
+}
+
+/**
+ * Lifecycle implementations for each agent mode.
+ *
+ * This chart maps mode types to their lifecycle hooks.
+ * Modes can implement any subset of hooks (all are optional).
+ * Not all modes need an entry - modes without entries simply have no lifecycle behavior.
+ *
+ * To add lifecycle behavior for a new mode:
+ * 1. Add the mode to AGENT_MODE_DEFINITIONS in AgentModeDefinitions.ts
+ * 2. Add an entry here with the lifecycle hooks you need
+ */
+const _AGENT_MODE_CHART: Record<AgentModeDefinition['type'], AgentModeNode> = {
+	idling: {
+		onPromptStart(agent) {
+			agent.mode.setMode('working')
+		},
+		onEnter(agent, _fromMode) {
+			agent.todos.reset()
+			agent.userAction.clearHistory()
+		},
+	},
+	working: {
+		onEnter(agent, fromMode) {
+			// Reset state when entering working mode
+			agent.todos.reset()
+			// agent.userAction.clearHistory()
+			agent.context.clear()
+
+			// When entering working mode from idling, clear created shapes tracking
+			// This handles the case where a user prompt starts while in idling mode,
+			// which transitions to working before working.onPromptStart is called
+			if (fromMode === 'idling') {
+				agent.lints.clearCreatedShapes()
+			}
+		},
+
+		onExit(agent, _toMode) {
+			// Unlock all shapes created during the prompt when exiting working mode
+			agent.lints.unlockCreatedShapes()
+		},
+
+		onPromptStart(agent, request) {
+			// Clear created shapes tracking and flush todos when a new user prompt starts
+			// This handles cases where a prompt starts while already in working mode (e.g., continuation, interrupt)
+			if (request.source === 'user') {
+				agent.todos.flush()
+				agent.lints.clearCreatedShapes()
+			}
+		},
+
+		onPromptEnd(agent, _request) {
+			// Check if there are incomplete todos
+			const todoList = agent.todos.getTodos()
+			const incompleteTodos = todoList.filter((item) => item.status !== 'done')
+
+			if (incompleteTodos.length > 0) {
+				// Schedule continuation to complete remaining work
+				agent.schedule(
+					"Continue until all your todo items are marked as done. If you've completed the work, mark them as done, otherwise keep going."
+				)
+				return
+			}
+
+			// Check if there are unsurfaced lints on created shapes
+			if (agent.lints.hasUnsurfacedLints(agent.lints.getCreatedShapes())) {
+				agent.schedule({
+					agentMessages: [
+						'The automated linter has detected potential visual problems in the canvas. Decide if they need to be addressed.',
+					],
+				})
+				return
+			}
+
+			// All work complete - return to idling
+			agent.mode.setMode('idling')
+		},
+
+		onPromptCancel(agent, _request) {
+			// Return to idling on cancel
+			agent.mode.setMode('idling')
+		},
+	},
+}
+
+/**
+ * Get the lifecycle node for a mode, if one exists.
+ * This function helps TypeScript resolve types correctly with circular imports.
+ */
+export function getModeNode(mode: AgentModeType): AgentModeNode {
+	return _AGENT_MODE_CHART[mode]
+}

templates/agent/client/modes/AgentModeDefinitions.ts

@@ -0,0 +1,194 @@
+import type { AgentAction } from '../../shared/types/AgentAction'
+import type { PromptPart } from '../../shared/types/PromptPart'
+// Import action utils to ensure they register themselves
+import { AddDetailActionUtil } from '../actions/AddDetailActionUtil'
+import { AlignActionUtil } from '../actions/AlignActionUtil'
+import { BringToFrontActionUtil } from '../actions/BringToFrontActionUtil'
+import { ClearActionUtil } from '../actions/ClearActionUtil'
+import { CountryInfoActionUtil } from '../actions/CountryInfoActionUtil'
+import { CountShapesActionUtil } from '../actions/CountShapesActionUtil'
+import { CreateActionUtil } from '../actions/CreateActionUtil'
+import { DeleteActionUtil } from '../actions/DeleteActionUtil'
+import { DistributeActionUtil } from '../actions/DistributeActionUtil'
+import { LabelActionUtil } from '../actions/LabelActionUtil'
+import { MessageActionUtil } from '../actions/MessageActionUtil'
+import { MoveActionUtil } from '../actions/MoveActionUtil'
+import { PenActionUtil } from '../actions/PenActionUtil'
+import { PlaceActionUtil } from '../actions/PlaceActionUtil'
+import { ResizeActionUtil } from '../actions/ResizeActionUtil'
+import { ReviewActionUtil } from '../actions/ReviewActionUtil'
+import { RotateActionUtil } from '../actions/RotateActionUtil'
+import { SendToBackActionUtil } from '../actions/SendToBackActionUtil'
+import { SetMyViewActionUtil } from '../actions/SetMyViewActionUtil'
+import { StackActionUtil } from '../actions/StackActionUtil'
+import { ThinkActionUtil } from '../actions/ThinkActionUtil'
+import { UnknownActionUtil } from '../actions/UnknownActionUtil'
+import { UpdateActionUtil } from '../actions/UpdateActionUtil'
+import { UpsertTodoListItemActionUtil } from '../actions/UpsertTodoListItemActionUtil'
+// Import prompt part utils to ensure they register themselves
+import { AgentViewportBoundsPartUtil } from '../parts/AgentViewportBoundsPartUtil'
+import { BlurryShapesPartUtil } from '../parts/BlurryShapesPartUtil'
+import { CanvasLintsPartUtil } from '../parts/CanvasLintsPartUtil'
+import { ChatHistoryPartUtil } from '../parts/ChatHistoryPartUtil'
+import { ContextItemsPartUtil } from '../parts/ContextItemsPartUtil'
+import { DataPartUtil } from '../parts/DataPartUtil'
+import { DebugPartUtil } from '../parts/DebugPartUtil'
+import { MessagesPartUtil } from '../parts/MessagesPartUtil'
+import { ModelNamePartUtil } from '../parts/ModelNamePartUtil'
+import { ModePartUtil } from '../parts/ModePartUtil'
+import { PeripheralShapesPartUtil } from '../parts/PeripheralShapesPartUtil'
+import { ScreenshotPartUtil } from '../parts/ScreenshotPartUtil'
+import { SelectedShapesPartUtil } from '../parts/SelectedShapesPartUtil'
+import { TimePartUtil } from '../parts/TimePartUtil'
+import { TodoListPartUtil } from '../parts/TodoListPartUtil'
+import { UserActionHistoryPartUtil } from '../parts/UserActionHistoryPartUtil'
+import { UserViewportBoundsPartUtil } from '../parts/UserViewportBoundsPartUtil'
+
+/**
+ * What an agent can see and do when in a given mode.
+ *
+ * This is a discriminated union based on the `active` property:
+ * - Active modes can take actions and receive prompt parts
+ * - Inactive modes cannot take actions (agent is idle/waiting)
+ */
+export type AgentModeDefinition = {
+	/** A unique identifier for the agent mode. */
+	type: string
+} & (
+	| {
+			/** Whether the agent is active in this mode and can take actions. */
+			active: true
+			/** The prompt parts that determine what information will be sent to the model. */
+			parts: PromptPart['type'][]
+			/** The actions that the agent can take. */
+			actions: AgentAction['_type'][]
+	  }
+	| {
+			/** Whether the agent is active in this mode and can take actions. */
+			active: false
+	  }
+)
+
+/**
+ * All agent mode definitions.
+ *
+ * To add a new mode, add a new object to this array.
+ * To change what the agent can see, edit the `parts` array.
+ * To change what the agent can do, edit the `actions` array.
+ *
+ * By importing utils directly, we ensure they are registered when this module is loaded.
+ */
+export const AGENT_MODE_DEFINITIONS = [
+	{
+		type: 'idling',
+		active: false,
+	},
+	{
+		type: 'working',
+		active: true,
+
+		/**
+		 * Prompt parts determine what information will be sent to the model.
+		 */
+		parts: [
+			// Mode (sends metadata to worker)
+			ModePartUtil.type,
+
+			// Debug (sends debug flags to worker)
+			DebugPartUtil.type,
+
+			// Model
+			ModelNamePartUtil.type,
+
+			// Request
+			MessagesPartUtil.type,
+			DataPartUtil.type,
+			ContextItemsPartUtil.type,
+
+			// Viewport
+			ScreenshotPartUtil.type,
+			UserViewportBoundsPartUtil.type,
+			AgentViewportBoundsPartUtil.type,
+
+			// Shapes
+			BlurryShapesPartUtil.type,
+			PeripheralShapesPartUtil.type,
+			SelectedShapesPartUtil.type,
+
+			// History
+			ChatHistoryPartUtil.type,
+			UserActionHistoryPartUtil.type,
+			TodoListPartUtil.type,
+
+			// Lints
+			CanvasLintsPartUtil.type,
+
+			// Metadata
+			TimePartUtil.type,
+		],
+
+		/**
+		 * Agent actions determine what actions the agent can take.
+		 */
+		actions: [
+			// Communication
+			MessageActionUtil.type,
+
+			// Planning
+			ThinkActionUtil.type,
+			ReviewActionUtil.type,
+			AddDetailActionUtil.type,
+			UpsertTodoListItemActionUtil.type,
+			SetMyViewActionUtil.type,
+
+			// Individual shapes
+			CreateActionUtil.type,
+			DeleteActionUtil.type,
+			UpdateActionUtil.type,
+			LabelActionUtil.type,
+			MoveActionUtil.type,
+
+			// Groups of shapes
+			PlaceActionUtil.type,
+			BringToFrontActionUtil.type,
+			SendToBackActionUtil.type,
+			RotateActionUtil.type,
+			ResizeActionUtil.type,
+			AlignActionUtil.type,
+			DistributeActionUtil.type,
+			StackActionUtil.type,
+			ClearActionUtil.type,
+
+			// Drawing
+			PenActionUtil.type,
+
+			// External APIs
+			CountryInfoActionUtil.type,
+			CountShapesActionUtil.type,
+
+			// Internal (required)
+			UnknownActionUtil.type,
+		],
+	},
+] as const satisfies AgentModeDefinition[]
+
+/**
+ * An agent mode definition from the AGENT_MODE_DEFINITIONS array.
+ */
+export type AgentModeDefinitionType = (typeof AGENT_MODE_DEFINITIONS)[number]
+
+/**
+ * The type of an agent mode.
+ */
+export type AgentModeType = AgentModeDefinitionType['type']
+
+/**
+ * Get the mode definition for a given mode type.
+ * @param type - The mode type to look up.
+ * @returns The mode definition.
+ */
+export function getAgentModeDefinition(type: AgentModeType): AgentModeDefinitionType {
+	const mode = AGENT_MODE_DEFINITIONS.find((m) => m.type === type)
+	if (!mode) throw new Error(`Unknown agent mode: ${type}`)
+	return mode
+}

templates/agent/client/overlays/AgentHighlightOverlayUtil.ts

@@ -0,0 +1,383 @@
+import {
+	atom,
+	Box,
+	BoxModel,
+	Editor,
+	OverlayUtil,
+	PI2,
+	TLOverlay,
+	TLShapeId,
+	VecModel,
+} from 'tldraw'
+import { ContextItem } from '../../shared/types/ContextItem'
+import { AgentAppAgentsManager } from '../agent/managers/AgentAppAgentsManager'
+
+type AgentHighlightColor = 'selected' | 'tooltip'
+const ANIMATION_PERIOD = 100_000
+const DASH_SPEED = 20
+const MINIMAP_FILL_ALPHA = 0.12
+const MINIMAP_PULSE_PERIOD = 2400
+
+interface AgentAreaHighlightOverlay extends TLOverlay {
+	type: 'agent-highlight'
+	props: {
+		kind: 'area'
+		bounds: BoxModel
+		color: AgentHighlightColor
+		generating: boolean
+		label?: string
+	}
+}
+
+interface AgentPointHighlightOverlay extends TLOverlay {
+	type: 'agent-highlight'
+	props: {
+		kind: 'point'
+		point: VecModel
+		color: AgentHighlightColor
+		generating: boolean
+	}
+}
+
+type AgentHighlightOverlay = AgentAreaHighlightOverlay | AgentPointHighlightOverlay
+
+export class AgentHighlightOverlayUtil extends OverlayUtil<AgentHighlightOverlay> {
+	static override type = 'agent-highlight' as const
+	override options = { zIndex: 700 }
+
+	private readonly $animationElapsed = atom('agent highlight animation elapsed', 0)
+	private hasAnimatedOverlays = false
+
+	constructor(editor: Editor) {
+		super(editor)
+		editor.on('tick', this.handleTick)
+	}
+
+	override dispose() {
+		this.editor.off('tick', this.handleTick)
+	}
+
+	override isActive(): boolean {
+		return AgentAppAgentsManager.getAgents(this.editor).length > 0
+	}
+
+	override getOverlays(): AgentHighlightOverlay[] {
+		const overlays: AgentHighlightOverlay[] = []
+		const agents = AgentAppAgentsManager.getAgents(this.editor)
+
+		for (const agent of agents) {
+			const activeRequest = agent.requests.getActiveRequest()
+			const activeContextItems = activeRequest?.contextItems ?? []
+
+			if (
+				activeRequest?.bounds &&
+				!this.isEquivalentToPendingContextArea(activeRequest.bounds, activeContextItems)
+			) {
+				overlays.push({
+					id: `agent-highlight:${agent.id}:viewport`,
+					type: 'agent-highlight',
+					props: {
+						kind: 'area',
+						bounds: toBoxModel(activeRequest.bounds),
+						color: 'tooltip',
+						generating: true,
+						label: `Agent ${agent.id.slice(0, 6)}'s view`,
+					},
+				})
+			}
+
+			const selectedContextItems = agent.requests.isGenerating() ? [] : agent.context.getItems()
+			this.addContextItemOverlays(overlays, agent.id, 'selected', selectedContextItems, false)
+			this.addContextItemOverlays(overlays, agent.id, 'active', activeContextItems, true)
+		}
+
+		return overlays
+	}
+
+	override render(ctx: CanvasRenderingContext2D, overlays: AgentHighlightOverlay[]): void {
+		// Track whether any overlay is animating so the tick handler knows when to
+		// advance the animation. This is a side effect, so it belongs in render
+		// (an effect) rather than getOverlays (a reactive computation).
+		this.hasAnimatedOverlays = overlays.some((overlay) => overlay.props.generating)
+
+		const zoom = this.editor.getZoomLevel()
+		const colors = this.getColors()
+		const dashOffset = getDashOffset(this.$animationElapsed.get(), zoom)
+
+		for (const overlay of overlays) {
+			const color = colors[overlay.props.color]
+
+			if (overlay.props.kind === 'area') {
+				this.renderArea(ctx, overlay.props.bounds, {
+					color,
+					generating: overlay.props.generating,
+					dashOffset,
+					label: overlay.props.label,
+					labelText: colors.labelText,
+					zoom,
+				})
+			} else {
+				this.renderPoint(ctx, overlay.props.point, {
+					color,
+					generating: overlay.props.generating,
+					dashOffset,
+					zoom,
+				})
+			}
+		}
+	}
+
+	override renderMinimap(
+		ctx: CanvasRenderingContext2D,
+		overlays: AgentHighlightOverlay[],
+		zoom: number
+	): void {
+		const colors = this.getColors()
+		const animationElapsed = this.$animationElapsed.get()
+
+		for (const overlay of overlays) {
+			const color = colors[overlay.props.color]
+
+			if (overlay.props.kind === 'area') {
+				const bounds = Box.From(overlay.props.bounds).expandBy(4)
+				const radius = Math.min(bounds.w, bounds.h, 3 / zoom)
+				ctx.globalAlpha = overlay.props.generating
+					? getMinimapPulseAlpha(animationElapsed)
+					: MINIMAP_FILL_ALPHA
+				ctx.fillStyle = color
+				ctx.beginPath()
+				ctx.roundRect(bounds.x, bounds.y, bounds.w, bounds.h, radius)
+				ctx.fill()
+				ctx.globalAlpha = 0.85
+				ctx.lineWidth = 1 / zoom
+				ctx.strokeStyle = color
+				ctx.stroke()
+				ctx.globalAlpha = 1
+			} else {
+				const radius = 3 / zoom
+				ctx.beginPath()
+				ctx.arc(overlay.props.point.x, overlay.props.point.y, radius, 0, PI2)
+				ctx.fillStyle = color
+				ctx.fill()
+			}
+		}
+	}
+
+	private addContextItemOverlays(
+		overlays: AgentHighlightOverlay[],
+		agentId: string,
+		group: 'selected' | 'active',
+		contextItems: ContextItem[],
+		generating: boolean
+	): void {
+		const areaHighlights: AgentAreaHighlightOverlay[] = []
+		const pointHighlights: AgentPointHighlightOverlay[] = []
+
+		for (let i = 0; i < contextItems.length; i++) {
+			const item = contextItems[i]
+			const id = `agent-highlight:${agentId}:${group}:${i}`
+
+			switch (item.type) {
+				case 'area': {
+					areaHighlights.push({
+						id,
+						type: 'agent-highlight',
+						props: {
+							kind: 'area',
+							bounds: toBoxModel(item.bounds),
+							color: 'selected',
+							generating,
+							label: generating && item.source === 'agent' ? 'Reviewing' : undefined,
+						},
+					})
+					break
+				}
+				case 'shapes': {
+					const bounds = this.editor.getShapesPageBounds(
+						item.shapes.map((shape) => `shape:${shape.shapeId}` as TLShapeId)
+					)
+					if (!bounds) break
+					areaHighlights.push({
+						id,
+						type: 'agent-highlight',
+						props: {
+							kind: 'area',
+							bounds: toBoxModel(bounds),
+							color: 'selected',
+							generating,
+						},
+					})
+					break
+				}
+				case 'shape': {
+					const bounds = this.editor.getShapePageBounds(`shape:${item.shape.shapeId}` as TLShapeId)
+					if (!bounds) break
+					areaHighlights.push({
+						id,
+						type: 'agent-highlight',
+						props: {
+							kind: 'area',
+							bounds: toBoxModel(bounds),
+							color: 'selected',
+							generating,
+						},
+					})
+					break
+				}
+				case 'point': {
+					pointHighlights.push({
+						id,
+						type: 'agent-highlight',
+						props: {
+							kind: 'point',
+							point: item.point,
+							color: 'selected',
+							generating,
+						},
+					})
+					break
+				}
+			}
+		}
+
+		overlays.push(...areaHighlights, ...pointHighlights)
+	}
+
+	private isEquivalentToPendingContextArea(
+		agentViewportBounds: BoxModel,
+		contextItems: ContextItem[]
+	): boolean {
+		return contextItems.some(
+			(item) =>
+				item.type === 'area' &&
+				item.source === 'agent' &&
+				Box.Equals(item.bounds, agentViewportBounds)
+		)
+	}
+
+	private renderArea(
+		ctx: CanvasRenderingContext2D,
+		pageBounds: BoxModel,
+		{
+			color,
+			generating,
+			dashOffset,
+			label,
+			labelText,
+			zoom,
+		}: {
+			color: string
+			generating: boolean
+			dashOffset: number
+			label?: string
+			labelText: string
+			zoom: number
+		}
+	): void {
+		const bounds = Box.From(pageBounds).expandBy(4)
+		ctx.save()
+		ctx.strokeStyle = color
+		ctx.lineWidth = 1 / zoom
+		ctx.lineCap = 'round'
+		if (generating) {
+			ctx.setLineDash([4 / zoom, 4 / zoom])
+			ctx.lineDashOffset = dashOffset
+		}
+		ctx.strokeRect(bounds.x, bounds.y, bounds.w, bounds.h)
+		ctx.restore()
+
+		if (label) {
+			this.renderLabel(ctx, bounds, label, color, labelText, zoom)
+		}
+	}
+
+	private renderPoint(
+		ctx: CanvasRenderingContext2D,
+		pagePoint: VecModel,
+		{
+			color,
+			generating,
+			dashOffset,
+			zoom,
+		}: { color: string; generating: boolean; dashOffset: number; zoom: number }
+	): void {
+		const radius = 3 / zoom
+		ctx.save()
+		ctx.beginPath()
+		ctx.arc(pagePoint.x, pagePoint.y, radius, 0, PI2)
+		ctx.strokeStyle = color
+		ctx.fillStyle = color
+		ctx.lineCap = 'round'
+		ctx.lineJoin = 'round'
+		if (generating) {
+			ctx.setLineDash([3 / zoom, 3 / zoom])
+			ctx.lineDashOffset = dashOffset
+			ctx.lineWidth = 2 / zoom
+			ctx.stroke()
+		} else {
+			ctx.lineWidth = 1 / zoom
+			ctx.fill()
+			ctx.stroke()
+		}
+		ctx.restore()
+	}
+
+	private renderLabel(
+		ctx: CanvasRenderingContext2D,
+		bounds: Box,
+		label: string,
+		color: string,
+		labelText: string,
+		zoom: number
+	): void {
+		const scale = 1 / zoom
+		const padding = 4
+		const fontSize = 12
+		const fontFamily = this.editor.getCurrentTheme().fonts.sans.fontFamily
+
+		ctx.save()
+		ctx.translate(bounds.x, bounds.y)
+		ctx.scale(scale, scale)
+		ctx.font = `500 ${fontSize}px ${fontFamily}`
+		ctx.textBaseline = 'top'
+
+		const width = Math.ceil(ctx.measureText(label).width) + padding * 2
+		const height = fontSize + padding * 2
+
+		ctx.fillStyle = color
+		ctx.beginPath()
+		ctx.roundRect(0, 0, width, height, 4)
+		ctx.fill()
+
+		ctx.fillStyle = labelText
+		ctx.fillText(label, padding, padding)
+		ctx.restore()
+	}
+
+	private getColors() {
+		const themeColors = this.editor.getCurrentTheme().colors[this.editor.getColorMode()]
+		return {
+			selected: themeColors.selectionStroke,
+			tooltip: themeColors.text,
+			labelText: themeColors.background,
+		}
+	}
+
+	private handleTick = (elapsed: number) => {
+		if (!this.hasAnimatedOverlays) return
+		this.$animationElapsed.update((phase) => (phase + elapsed) % ANIMATION_PERIOD)
+	}
+}
+
+function toBoxModel(box: BoxModel): BoxModel {
+	return { x: box.x, y: box.y, w: box.w, h: box.h }
+}
+
+function getMinimapPulseAlpha(phase: number): number {
+	const progress = (phase % MINIMAP_PULSE_PERIOD) / MINIMAP_PULSE_PERIOD
+	return MINIMAP_FILL_ALPHA + 0.06 * (0.5 + Math.sin(progress * PI2 - Math.PI / 2) * 0.5)
+}
+
+function getDashOffset(elapsed: number, zoom: number): number {
+	return -((elapsed / 1000) * DASH_SPEED) / zoom
+}

templates/agent/client/parts/AgentViewportBoundsPartUtil.ts

@@ -0,0 +1,19 @@
+import { AgentViewportBoundsPart } from '../../shared/schema/PromptPartDefinitions'
+import { AgentRequest } from '../../shared/types/AgentRequest'
+import { AgentHelpers } from '../AgentHelpers'
+import { PromptPartUtil, registerPromptPartUtil } from './PromptPartUtil'
+
+export const AgentViewportBoundsPartUtil = registerPromptPartUtil(
+	class AgentViewportBoundsPartUtil extends PromptPartUtil<AgentViewportBoundsPart> {
+		static override type = 'agentViewportBounds' as const
+
+		override getPart(request: AgentRequest, helpers: AgentHelpers): AgentViewportBoundsPart {
+			const offsetAgentBounds = helpers.applyOffsetToBox(request.bounds)
+
+			return {
+				type: 'agentViewportBounds',
+				agentBounds: helpers.roundBox(offsetAgentBounds),
+			}
+		}
+	}
+)

templates/agent/client/parts/BlurryShapesPartUtil.ts

@@ -0,0 +1,49 @@
+import { Box } from 'tldraw'
+import { convertTldrawShapeToBlurryShape } from '../../shared/format/convertTldrawShapeToBlurryShape'
+import { BlurryShapesPart } from '../../shared/schema/PromptPartDefinitions'
+import { AgentRequest } from '../../shared/types/AgentRequest'
+import { AgentHelpers } from '../AgentHelpers'
+import { PromptPartUtil, registerPromptPartUtil } from './PromptPartUtil'
+
+export const BlurryShapesPartUtil = registerPromptPartUtil(
+	class BlurryShapesPartUtil extends PromptPartUtil<BlurryShapesPart> {
+		static override type = 'blurryShapes' as const
+
+		override getPart(request: AgentRequest, helpers: AgentHelpers): BlurryShapesPart {
+			const { editor } = this
+
+			const shapes = editor.getCurrentPageShapesSorted()
+			const contextBoundsBox = Box.From(request.bounds)
+
+			// Get all shapes within the agent's viewport
+			const shapesInBounds = shapes.filter((shape) => {
+				const bounds = editor.getShapeMaskedPageBounds(shape)
+				if (!bounds) return false
+				return contextBoundsBox.includes(bounds)
+			})
+
+			// Convert the shapes to the blurry shape format
+			const blurryShapes = shapesInBounds
+				.map((shape) => convertTldrawShapeToBlurryShape(editor, shape))
+				.filter((s) => s !== null)
+
+			// Apply the offset and round the blurry shapes
+			const normalizedBlurryShapes = blurryShapes.map((shape) => {
+				const bounds = helpers.roundBox(
+					helpers.applyOffsetToBox({
+						x: shape.x,
+						y: shape.y,
+						w: shape.w,
+						h: shape.h,
+					})
+				)
+				return { ...shape, x: bounds.x, y: bounds.y, w: bounds.w, h: bounds.h }
+			})
+
+			return {
+				type: 'blurryShapes',
+				shapes: normalizedBlurryShapes,
+			}
+		}
+	}
+)

templates/agent/client/parts/CanvasLintsPartUtil.ts

@@ -0,0 +1,39 @@
+import { Box } from 'tldraw'
+import { CanvasLintsPart } from '../../shared/schema/PromptPartDefinitions'
+import { AgentRequest } from '../../shared/types/AgentRequest'
+import { AgentHelpers } from '../AgentHelpers'
+import { PromptPartUtil, registerPromptPartUtil } from './PromptPartUtil'
+
+export const CanvasLintsPartUtil = registerPromptPartUtil(
+	class CanvasLintsPartUtil extends PromptPartUtil<CanvasLintsPart> {
+		static override type = 'canvasLints' as const
+
+		override getPart(request: AgentRequest, _helpers: AgentHelpers): CanvasLintsPart {
+			const { editor, agent } = this
+			if (!editor) return { type: 'canvasLints', lints: [] }
+
+			const shapes = editor.getCurrentPageShapesSorted()
+			const contextBoundsBox = Box.From(request.bounds)
+			const shapesInBounds = shapes.filter((shape) => {
+				const bounds = editor.getShapeMaskedPageBounds(shape)
+				if (!bounds) return false
+				return contextBoundsBox.includes(bounds)
+			})
+
+			// Use created shapes when in working mode, otherwise use shapes in request bounds
+			const shapesToCheck =
+				agent.mode.getCurrentModeType() === 'working'
+					? agent.lints.getCreatedShapes()
+					: shapesInBounds
+
+			// Get unsurfaced lints and mark them as surfaced
+			const lints = agent.lints.getUnsurfacedLintsForShapes(shapesToCheck)
+			agent.lints.markLintsAsSurfaced(lints)
+
+			return {
+				type: 'canvasLints',
+				lints,
+			}
+		}
+	}
+)

templates/agent/client/parts/ChatHistoryPartUtil.ts

@@ -0,0 +1,32 @@
+import { structuredClone } from 'tldraw'
+import { ChatHistoryPart } from '../../shared/schema/PromptPartDefinitions'
+import { AgentRequest } from '../../shared/types/AgentRequest'
+import { AgentHelpers } from '../AgentHelpers'
+import { PromptPartUtil, registerPromptPartUtil } from './PromptPartUtil'
+
+export const ChatHistoryPartUtil = registerPromptPartUtil(
+	class ChatHistoryPartUtil extends PromptPartUtil<ChatHistoryPart> {
+		static override type = 'chatHistory' as const
+
+		override async getPart(_request: AgentRequest, helpers: AgentHelpers) {
+			const history = structuredClone(this.agent.chat.getHistory())
+
+			for (const historyItem of history) {
+				if (historyItem.type !== 'prompt') continue
+
+				// Offset and round the context items of each history item
+				const contextItems = historyItem.contextItems.map((contextItem) => {
+					const offsetContextItem = helpers.applyOffsetToContextItem(contextItem)
+					return helpers.roundContextItem(offsetContextItem)
+				})
+
+				historyItem.contextItems = contextItems
+			}
+
+			return {
+				type: 'chatHistory' as const,
+				history,
+			}
+		}
+	}
+)

templates/agent/client/parts/ContextItemsPartUtil.ts

@@ -0,0 +1,23 @@
+import { ContextItemsPart } from '../../shared/schema/PromptPartDefinitions'
+import { AgentRequest } from '../../shared/types/AgentRequest'
+import { AgentHelpers } from '../AgentHelpers'
+import { PromptPartUtil, registerPromptPartUtil } from './PromptPartUtil'
+
+export const ContextItemsPartUtil = registerPromptPartUtil(
+	class ContextItemsPartUtil extends PromptPartUtil<ContextItemsPart> {
+		static override type = 'contextItems' as const
+
+		override getPart(request: AgentRequest, helpers: AgentHelpers): ContextItemsPart {
+			const items = request.contextItems.map((contextItem) => {
+				const offsetContextItem = helpers.applyOffsetToContextItem(contextItem)
+				return helpers.roundContextItem(offsetContextItem)
+			})
+
+			return {
+				type: 'contextItems',
+				items,
+				requestSource: request.source,
+			}
+		}
+	}
+)

templates/agent/client/parts/DataPartUtil.ts

@@ -0,0 +1,33 @@
+import { DataPart } from '../../shared/schema/PromptPartDefinitions'
+import { AgentRequest } from '../../shared/types/AgentRequest'
+import { PromptPartUtil, registerPromptPartUtil } from './PromptPartUtil'
+
+/**
+ * This prompt part collects up data retrieved by agent actions in the previous request.
+ */
+export const DataPartUtil = registerPromptPartUtil(
+	class DataPartUtil extends PromptPartUtil<DataPart> {
+		static override type = 'data' as const
+
+		override async getPart(request: AgentRequest): Promise<DataPart> {
+			const { data } = request
+
+			const values = await Promise.all(
+				data.map(async (item) => {
+					try {
+						return await item
+					} catch (error) {
+						console.error('Error retrieving data:', error)
+						// Tell the agent that something went wrong
+						return 'An error occurred while retrieving some data.'
+					}
+				})
+			)
+
+			return {
+				type: 'data',
+				data: values,
+			}
+		}
+	}
+)

templates/agent/client/parts/DebugPartUtil.ts

@@ -0,0 +1,18 @@
+import { DebugPart } from '../../shared/schema/PromptPartDefinitions'
+import { PromptPartUtil, registerPromptPartUtil } from './PromptPartUtil'
+
+export const DebugPartUtil = registerPromptPartUtil(
+	class DebugPartUtil extends PromptPartUtil<DebugPart> {
+		static override type = 'debug' as const
+
+		override getPart(): DebugPart {
+			const debugFlags = this.agent.debug.getDebugFlags()
+
+			return {
+				type: 'debug',
+				logSystemPrompt: debugFlags.logSystemPrompt,
+				logMessages: debugFlags.logMessages,
+			}
+		}
+	}
+)

templates/agent/client/parts/MessagesPartUtil.ts

@@ -0,0 +1,18 @@
+import { MessagesPart } from '../../shared/schema/PromptPartDefinitions'
+import { AgentRequest } from '../../shared/types/AgentRequest'
+import { PromptPartUtil, registerPromptPartUtil } from './PromptPartUtil'
+
+export const MessagesPartUtil = registerPromptPartUtil(
+	class MessagesPartUtil extends PromptPartUtil<MessagesPart> {
+		static override type = 'messages' as const
+
+		override getPart(request: AgentRequest): MessagesPart {
+			const { agentMessages, source } = request
+			return {
+				type: 'messages',
+				agentMessages,
+				requestSource: source,
+			}
+		}
+	}
+)

templates/agent/client/parts/ModePartUtil.ts

@@ -0,0 +1,24 @@
+import { ModePart } from '../../shared/schema/PromptPartDefinitions'
+import { AgentRequest } from '../../shared/types/AgentRequest'
+import { PromptPartUtil, registerPromptPartUtil } from './PromptPartUtil'
+
+export const ModePartUtil = registerPromptPartUtil(
+	class ModePartUtil extends PromptPartUtil<ModePart> {
+		static override type = 'mode' as const
+
+		override getPart(_request: AgentRequest): ModePart {
+			const modeDefinition = this.agent.mode.getCurrentModeDefinition()
+
+			if (!modeDefinition.active) {
+				throw new Error(`Cannot get mode part for inactive mode: ${modeDefinition.type}`)
+			}
+
+			return {
+				type: 'mode',
+				modeType: modeDefinition.type,
+				partTypes: modeDefinition.parts,
+				actionTypes: modeDefinition.actions,
+			}
+		}
+	}
+)

templates/agent/client/parts/ModelNamePartUtil.ts

@@ -0,0 +1,17 @@
+import { DEFAULT_MODEL_NAME } from '../../shared/models'
+import { ModelNamePart } from '../../shared/schema/PromptPartDefinitions'
+import { AgentRequest } from '../../shared/types/AgentRequest'
+import { PromptPartUtil, registerPromptPartUtil } from './PromptPartUtil'
+
+export const ModelNamePartUtil = registerPromptPartUtil(
+	class ModelNamePartUtil extends PromptPartUtil<ModelNamePart> {
+		static override type = 'modelName' as const
+
+		override getPart(_request: AgentRequest): ModelNamePart {
+			return {
+				type: 'modelName',
+				modelName: this.agent.modelName.getModelName() ?? DEFAULT_MODEL_NAME,
+			}
+		}
+	}
+)

templates/agent/client/parts/PeripheralShapesPartUtil.ts

@@ -0,0 +1,48 @@
+import { Box } from 'tldraw'
+import { convertTldrawShapesToPeripheralShapes } from '../../shared/format/convertTldrawShapesToPeripheralShapes'
+import { PeripheralShapesPart } from '../../shared/schema/PromptPartDefinitions'
+import { AgentRequest } from '../../shared/types/AgentRequest'
+import { AgentHelpers } from '../AgentHelpers'
+import { PromptPartUtil, registerPromptPartUtil } from './PromptPartUtil'
+
+export const PeripheralShapesPartUtil = registerPromptPartUtil(
+	class PeripheralShapesPartUtil extends PromptPartUtil<PeripheralShapesPart> {
+		static override type = 'peripheralShapes' as const
+
+		override getPart(request: AgentRequest, helpers: AgentHelpers): PeripheralShapesPart {
+			const { editor } = this
+
+			const shapes = editor.getCurrentPageShapesSorted()
+			const contextBounds = request.bounds
+
+			const contextBoundsBox = Box.From(contextBounds)
+
+			// Get all shapes that are outside the context bounds (these are what we want to peripheralize)
+			const shapesOutsideViewport = shapes.filter((shape) => {
+				const bounds = editor.getShapeMaskedPageBounds(shape)
+				if (!bounds) return
+				if (contextBoundsBox.includes(bounds)) return
+				return true
+			})
+
+			// Convert the shapes to peripheral shape cluster format
+			const clusters = convertTldrawShapesToPeripheralShapes(editor, shapesOutsideViewport, {
+				padding: 75,
+			})
+
+			// Apply the offset and round the clusters
+			const normalizedClusters = clusters.map((cluster) => {
+				const offsetBounds = helpers.applyOffsetToBox(cluster.bounds)
+				return {
+					numberOfShapes: cluster.numberOfShapes,
+					bounds: helpers.roundBox(offsetBounds),
+				}
+			})
+
+			return {
+				type: 'peripheralShapes',
+				clusters: normalizedClusters,
+			}
+		}
+	}
+)

templates/agent/client/parts/PromptPartUtil.ts

@@ -0,0 +1,70 @@
+import { Editor } from 'tldraw'
+import { AgentRequest } from '../../shared/types/AgentRequest'
+import { BasePromptPart } from '../../shared/types/BasePromptPart'
+import { PromptPart } from '../../shared/types/PromptPart'
+import { TldrawAgent } from '../agent/TldrawAgent'
+import { AgentHelpers } from '../AgentHelpers'
+
+// ============================================================================
+// Registry
+// ============================================================================
+
+const registry = new Map<string, PromptPartUtilConstructor<BasePromptPart>>()
+
+/**
+ * Register a prompt part util class. Call this after defining each util class.
+ */
+export function registerPromptPartUtil<T extends PromptPartUtilConstructor<BasePromptPart>>(
+	util: T
+): T {
+	if (registry.has(util.type)) {
+		throw new Error(`Prompt part util already registered: ${util.type}`)
+	}
+	registry.set(util.type, util)
+	return util
+}
+
+/**
+ * Get all registered prompt part util classes.
+ */
+export function getAllPromptPartUtils(): PromptPartUtilConstructor<PromptPart>[] {
+	return Array.from(registry.values()) as PromptPartUtilConstructor<PromptPart>[]
+}
+
+/**
+ * Get an object containing instantiated prompt part utils for an agent.
+ */
+export function getPromptPartUtilsRecord(agent: TldrawAgent) {
+	const object = {} as Record<PromptPart['type'], PromptPartUtil<PromptPart>>
+	for (const util of registry.values()) {
+		object[util.type as PromptPart['type']] = new util(agent) as PromptPartUtil<PromptPart>
+	}
+	return object
+}
+
+// ============================================================================
+// Base Class
+// ============================================================================
+
+export abstract class PromptPartUtil<T extends BasePromptPart = BasePromptPart> {
+	static type: string
+
+	agent: TldrawAgent
+	editor: Editor
+
+	constructor(agent: TldrawAgent) {
+		this.agent = agent
+		this.editor = agent?.editor
+	}
+
+	/**
+	 * Get some data to add to the prompt.
+	 * @returns The prompt part.
+	 */
+	abstract getPart(request: AgentRequest, helpers: AgentHelpers): Promise<T> | T
+}
+
+export interface PromptPartUtilConstructor<T extends BasePromptPart = BasePromptPart> {
+	new (agent: TldrawAgent): PromptPartUtil<T>
+	type: T['type']
+}

templates/agent/client/parts/ScreenshotPartUtil.ts

@@ -0,0 +1,45 @@
+import { Box, FileHelpers } from 'tldraw'
+import { ScreenshotPart } from '../../shared/schema/PromptPartDefinitions'
+import { AgentRequest } from '../../shared/types/AgentRequest'
+import { PromptPartUtil, registerPromptPartUtil } from './PromptPartUtil'
+
+export const ScreenshotPartUtil = registerPromptPartUtil(
+	class ScreenshotPartUtil extends PromptPartUtil<ScreenshotPart> {
+		static override type = 'screenshot' as const
+
+		override async getPart(request: AgentRequest): Promise<ScreenshotPart> {
+			const { editor } = this
+
+			const contextBounds = request.bounds
+
+			const contextBoundsBox = Box.From(contextBounds)
+
+			const shapes = editor.getCurrentPageShapesSorted().filter((shape) => {
+				const bounds = editor.getShapeMaskedPageBounds(shape)
+				if (!bounds) return false
+				return contextBoundsBox.includes(bounds)
+			})
+
+			if (shapes.length === 0) {
+				return { type: 'screenshot', screenshot: '' }
+			}
+
+			const largestDimension = Math.max(request.bounds.w, request.bounds.h)
+			const scale = largestDimension > 8000 ? 8000 / largestDimension : 1
+
+			const result = await editor.toImage(shapes, {
+				format: 'jpeg',
+				background: true,
+				bounds: Box.From(request.bounds),
+				padding: 0,
+				pixelRatio: 1,
+				scale,
+			})
+
+			return {
+				type: 'screenshot',
+				screenshot: await FileHelpers.blobToDataUrl(result.blob),
+			}
+		}
+	}
+)

templates/agent/client/parts/SelectedShapesPartUtil.ts

@@ -0,0 +1,22 @@
+import { structuredClone } from 'tldraw'
+import { convertTldrawIdToSimpleId } from '../../shared/format/convertTldrawShapeToFocusedShape'
+import { SelectedShapesPart } from '../../shared/schema/PromptPartDefinitions'
+import { AgentRequest } from '../../shared/types/AgentRequest'
+import { PromptPartUtil, registerPromptPartUtil } from './PromptPartUtil'
+
+export const SelectedShapesPartUtil = registerPromptPartUtil(
+	class SelectedShapesPartUtil extends PromptPartUtil<SelectedShapesPart> {
+		static override type = 'selectedShapes' as const
+
+		override getPart(_request: AgentRequest): SelectedShapesPart {
+			const { editor } = this
+
+			const userSelectedShapes = editor.getSelectedShapes().map((v) => structuredClone(v)) ?? []
+
+			return {
+				type: 'selectedShapes',
+				shapeIds: userSelectedShapes.map((shape) => convertTldrawIdToSimpleId(shape.id)),
+			}
+		}
+	}
+)

templates/agent/client/parts/TimePartUtil.ts

@@ -0,0 +1,15 @@
+import { TimePart } from '../../shared/schema/PromptPartDefinitions'
+import { PromptPartUtil, registerPromptPartUtil } from './PromptPartUtil'
+
+export const TimePartUtil = registerPromptPartUtil(
+	class TimePartUtil extends PromptPartUtil<TimePart> {
+		static override type = 'time' as const
+
+		override getPart(): TimePart {
+			return {
+				type: 'time',
+				time: new Date().toLocaleTimeString(),
+			}
+		}
+	}
+)

templates/agent/client/parts/TodoListPartUtil.ts

@@ -0,0 +1,17 @@
+import { TodoListPart } from '../../shared/schema/PromptPartDefinitions'
+import { AgentRequest } from '../../shared/types/AgentRequest'
+import { AgentHelpers } from '../AgentHelpers'
+import { PromptPartUtil, registerPromptPartUtil } from './PromptPartUtil'
+
+export const TodoListPartUtil = registerPromptPartUtil(
+	class TodoListPartUtil extends PromptPartUtil<TodoListPart> {
+		static override type = 'todoList' as const
+
+		override getPart(_request: AgentRequest, _helpers: AgentHelpers): TodoListPart {
+			return {
+				type: 'todoList',
+				items: this.agent.todos.getTodos(),
+			}
+		}
+	}
+)

templates/agent/client/parts/UserActionHistoryPartUtil.ts

@@ -0,0 +1,110 @@
+import { squashRecordDiffs } from 'tldraw'
+import {
+	convertTldrawIdToSimpleId,
+	convertTldrawShapeToFocusedShape,
+	convertTldrawShapeToFocusedType,
+} from '../../shared/format/convertTldrawShapeToFocusedShape'
+import { FocusedShape } from '../../shared/format/FocusedShape'
+import { UserActionHistoryPart } from '../../shared/schema/PromptPartDefinitions'
+import { AgentRequest } from '../../shared/types/AgentRequest'
+import { AgentHelpers } from '../AgentHelpers'
+import { PromptPartUtil, registerPromptPartUtil } from './PromptPartUtil'
+
+export const UserActionHistoryPartUtil = registerPromptPartUtil(
+	class UserActionHistoryPartUtil extends PromptPartUtil<UserActionHistoryPart> {
+		static override type = 'userActionHistory' as const
+
+		override getPart(_request: AgentRequest, helpers: AgentHelpers): UserActionHistoryPart {
+			const { editor, agent } = helpers
+
+			// Get the action history and clear it so that we can start tracking changes for the next request
+			const diffs = agent.userAction.getHistory()
+			agent.userAction.clearHistory()
+
+			const part: UserActionHistoryPart = {
+				type: 'userActionHistory',
+				added: [],
+				removed: [],
+				updated: [],
+			}
+
+			const squashedDiff = squashRecordDiffs(diffs)
+			const { added, updated, removed } = squashedDiff
+
+			// Collect user-added shapes
+			for (const shape of Object.values(added)) {
+				if (shape.typeName !== 'shape') continue
+				part.added.push({
+					shapeId: convertTldrawIdToSimpleId(shape.id),
+					type: convertTldrawShapeToFocusedType(shape),
+				})
+			}
+
+			// Collect user-removed shapes
+			for (const shape of Object.values(removed)) {
+				if (shape.typeName !== 'shape') continue
+				const focusedShape = convertTldrawShapeToFocusedShape(editor, shape)
+				part.removed.push({
+					shapeId: focusedShape.shapeId,
+					type: focusedShape._type,
+				})
+			}
+
+			// Collect user-updated shapes
+			for (const [from, to] of Object.values(updated)) {
+				if (from.typeName !== 'shape' || to.typeName !== 'shape') continue
+				const fromFocusedShape = convertTldrawShapeToFocusedShape(editor, from)
+				const toFocusedShape = convertTldrawShapeToFocusedShape(editor, to)
+
+				const changeFocusedShape = getFocusedShapeChange(fromFocusedShape, toFocusedShape)
+				if (!changeFocusedShape) continue
+
+				const before = helpers.applyOffsetToShapePartial(changeFocusedShape.from)
+				const after = helpers.applyOffsetToShapePartial(changeFocusedShape.to)
+
+				part.updated.push({
+					shapeId: toFocusedShape.shapeId,
+					type: toFocusedShape._type,
+					before: helpers.roundShapePartial(before),
+					after: helpers.roundShapePartial(after),
+				})
+			}
+
+			return part
+		}
+	}
+)
+
+/**
+ * Get any changed properties between two focused shapes.
+ * @param from - The original shape.
+ * @param to - The new shape.
+ * @returns The changed properties.
+ */
+function getFocusedShapeChange<T extends FocusedShape['_type']>(
+	from: FocusedShape & { _type: T },
+	to: FocusedShape & { _type: T }
+) {
+	if (from._type !== to._type) {
+		return null
+	}
+
+	const change: {
+		from: Partial<FocusedShape>
+		to: Partial<FocusedShape>
+	} = {
+		from: {},
+		to: {},
+	}
+
+	for (const key in to) {
+		const fromValue = from[key]
+		const toValue = to[key]
+		if (fromValue === toValue) {
+			continue
+		}
+		;(change.from as any)[key] = fromValue
+		;(change.to as any)[key] = toValue
+	}
+	return change
+}

templates/agent/client/parts/UserViewportBoundsPartUtil.ts

@@ -0,0 +1,19 @@
+import { UserViewportBoundsPart } from '../../shared/schema/PromptPartDefinitions'
+import { AgentHelpers } from '../AgentHelpers'
+import { PromptPartUtil, registerPromptPartUtil } from './PromptPartUtil'
+
+export const UserViewportBoundsPartUtil = registerPromptPartUtil(
+	class UserViewportBoundsPartUtil extends PromptPartUtil<UserViewportBoundsPart> {
+		static override type = 'userViewportBounds' as const
+
+		override getPart(_request: unknown, helpers: AgentHelpers): UserViewportBoundsPart {
+			const userBounds = this.editor.getViewportPageBounds()
+			const offsetUserBounds = helpers.applyOffsetToBox(userBounds)
+
+			return {
+				type: 'userViewportBounds',
+				userBounds: helpers.roundBox(offsetUserBounds),
+			}
+		}
+	}
+)

templates/agent/client/tools/TargetAreaTool.tsx

@@ -0,0 +1,119 @@
+import { BoxModel, StateNode, VecModel } from 'tldraw'
+import { AgentAppAgentsManager } from '../agent/managers/AgentAppAgentsManager'
+
+export class TargetAreaTool extends StateNode {
+	static override id = 'target-area'
+	static override initial = 'idle'
+	static override children() {
+		return [TargetAreaIdle, TargetAreaPointing, TargetAreaDragging]
+	}
+
+	override isLockable = false
+
+	override onEnter() {
+		this.editor.setCursor({ type: 'cross', rotation: 0 })
+	}
+
+	override onExit() {
+		this.editor.setCursor({ type: 'default', rotation: 0 })
+	}
+
+	override onInterrupt() {
+		this.complete()
+	}
+
+	override onCancel() {
+		this.complete()
+	}
+
+	private complete() {
+		this.parent.transition('select', {})
+	}
+}
+
+class TargetAreaIdle extends StateNode {
+	static override id = 'idle'
+
+	override onPointerDown() {
+		this.parent.transition('pointing')
+	}
+}
+
+class TargetAreaPointing extends StateNode {
+	static override id = 'pointing'
+
+	private initialScreenPoint: VecModel | undefined = undefined
+	private initialPagePoint: VecModel | undefined = undefined
+
+	override onEnter() {
+		this.initialScreenPoint = this.editor.inputs.getCurrentScreenPoint().clone()
+		this.initialPagePoint = this.editor.inputs.getCurrentPagePoint().clone()
+	}
+
+	override onPointerMove() {
+		if (!this.initialScreenPoint) return
+		if (this.editor.inputs.getIsDragging()) {
+			this.parent.transition('dragging', { initialPagePoint: this.initialPagePoint })
+		}
+	}
+
+	override onPointerUp() {
+		const agents = AgentAppAgentsManager.getAgents(this.editor)
+		for (const agent of agents) {
+			agent.context.add({
+				type: 'point',
+				point: this.editor.inputs.getCurrentPagePoint().clone(),
+				source: 'user',
+			})
+		}
+		this.editor.setCurrentTool('select')
+	}
+}
+
+class TargetAreaDragging extends StateNode {
+	static override id = 'dragging'
+
+	private initialPagePoint: VecModel | undefined = undefined
+	private bounds: BoxModel | undefined = undefined
+
+	override onEnter(props: { initialPagePoint: VecModel }) {
+		this.initialPagePoint = props.initialPagePoint
+		this.updateBounds()
+	}
+
+	override onPointerMove() {
+		this.updateBounds()
+	}
+
+	override onPointerUp() {
+		this.editor.updateInstanceState({
+			brush: null,
+		})
+
+		if (!this.bounds) throw new Error('Bounds not set')
+		const agents = AgentAppAgentsManager.getAgents(this.editor)
+		for (const agent of agents) {
+			agent.context.add({
+				type: 'area',
+				bounds: this.bounds,
+				source: 'user',
+			})
+		}
+		this.editor.setCurrentTool('select')
+	}
+
+	updateBounds() {
+		if (!this.initialPagePoint) return
+		const currentPagePoint = this.editor.inputs.getCurrentPagePoint()
+		const x = Math.min(this.initialPagePoint.x, currentPagePoint.x)
+		const y = Math.min(this.initialPagePoint.y, currentPagePoint.y)
+		const w = Math.abs(currentPagePoint.x - this.initialPagePoint.x)
+		const h = Math.abs(currentPagePoint.y - this.initialPagePoint.y)
+
+		this.editor.updateInstanceState({
+			brush: { x, y, w, h },
+		})
+
+		this.bounds = { x, y, w, h }
+	}
+}

templates/agent/client/tools/TargetShapeTool.tsx

@@ -0,0 +1,165 @@
+import { Box, BoxModel, StateNode, TLShape, VecModel } from 'tldraw'
+import { convertTldrawShapeToFocusedShape } from '../../shared/format/convertTldrawShapeToFocusedShape'
+import { AgentAppAgentsManager } from '../agent/managers/AgentAppAgentsManager'
+
+export class TargetShapeTool extends StateNode {
+	static override id = 'target-shape'
+	static override initial = 'idle'
+	static override children() {
+		return [TargetShapeIdle, TargetShapePointing, TargetShapeDragging]
+	}
+
+	override isLockable = false
+
+	override onEnter() {
+		this.editor.setCursor({ type: 'cross', rotation: 0 })
+	}
+
+	override onExit() {
+		this.editor.setCursor({ type: 'default', rotation: 0 })
+	}
+
+	override onInterrupt() {
+		this.complete()
+	}
+
+	override onCancel() {
+		this.complete()
+	}
+
+	private complete() {
+		this.parent.transition('select', {})
+	}
+}
+
+class TargetShapeIdle extends StateNode {
+	static override id = 'idle'
+
+	override onPointerMove() {
+		const currentPagePoint = this.editor.inputs.getCurrentPagePoint()
+		const shape = this.editor.getShapeAtPoint(currentPagePoint, { hitInside: true })
+		if (shape) {
+			this.editor.setHintingShapes([shape])
+		} else {
+			this.editor.setHintingShapes([])
+		}
+	}
+
+	override onPointerDown() {
+		const shape = this.editor.getShapeAtPoint(this.editor.inputs.getCurrentPagePoint(), {
+			hitInside: true,
+		})
+		this.parent.transition('pointing', { shape })
+	}
+}
+
+class TargetShapePointing extends StateNode {
+	static override id = 'pointing'
+
+	private shape: TLShape | undefined = undefined
+	private initialScreenPoint: VecModel | undefined = undefined
+	private initialPagePoint: VecModel | undefined = undefined
+
+	override onEnter({ shape }: { shape: TLShape }) {
+		this.initialScreenPoint = this.editor.inputs.getCurrentScreenPoint().clone()
+		this.initialPagePoint = this.editor.inputs.getCurrentPagePoint().clone()
+		this.shape = shape
+	}
+
+	override onPointerMove() {
+		if (!this.initialScreenPoint) return
+		if (this.editor.inputs.getIsDragging()) {
+			this.parent.transition('dragging', { initialPagePoint: this.initialPagePoint })
+		}
+	}
+
+	override onPointerUp() {
+		this.editor.setHintingShapes([])
+		if (this.shape) {
+			const agents = AgentAppAgentsManager.getAgents(this.editor)
+			for (const agent of agents) {
+				agent.context.add({
+					type: 'shape',
+					shape: convertTldrawShapeToFocusedShape(this.editor, this.shape),
+					source: 'user',
+				})
+			}
+		}
+		this.editor.setCurrentTool('select')
+	}
+}
+
+class TargetShapeDragging extends StateNode {
+	static override id = 'dragging'
+
+	private shapes: TLShape[] = []
+	private initialPagePoint: VecModel | undefined = undefined
+	private bounds: BoxModel | undefined = undefined
+
+	override onEnter(props: { initialPagePoint: VecModel }) {
+		this.initialPagePoint = props.initialPagePoint
+		this.editor.setHintingShapes([])
+		this.updateBounds()
+	}
+
+	override onPointerMove() {
+		this.updateBounds()
+	}
+
+	override onPointerUp() {
+		this.editor.setHintingShapes([])
+		this.editor.updateInstanceState({
+			brush: null,
+		})
+
+		if (!this.bounds) throw new Error('Bounds not set')
+		const agents = AgentAppAgentsManager.getAgents(this.editor)
+		if (this.shapes.length <= 3) {
+			for (const shape of this.shapes) {
+				for (const agent of agents) {
+					agent.context.add({
+						type: 'shape',
+						shape: convertTldrawShapeToFocusedShape(this.editor, shape),
+						source: 'user',
+					})
+				}
+			}
+		} else {
+			for (const agent of agents) {
+				agent.context.add({
+					type: 'shapes',
+					shapes: this.shapes.map((shape) => convertTldrawShapeToFocusedShape(this.editor, shape)),
+					source: 'user',
+				})
+			}
+		}
+		this.editor.setCurrentTool('select')
+	}
+
+	updateBounds() {
+		if (!this.initialPagePoint) return
+		const currentPagePoint = this.editor.inputs.getCurrentPagePoint()
+		const x = Math.min(this.initialPagePoint.x, currentPagePoint.x)
+		const y = Math.min(this.initialPagePoint.y, currentPagePoint.y)
+		const w = Math.abs(currentPagePoint.x - this.initialPagePoint.x)
+		const h = Math.abs(currentPagePoint.y - this.initialPagePoint.y)
+
+		this.editor.updateInstanceState({
+			brush: { x, y, w, h },
+		})
+
+		this.bounds = { x, y, w, h }
+
+		const bounds = new Box(x, y, w, h)
+		const shapesInBounds = this.editor.getCurrentPageShapesSorted().filter((shape) => {
+			const geometry = this.editor.getShapeGeometry(shape)
+			const pageTransform = this.editor.getShapePageTransform(shape)
+			const shapeTransform = pageTransform.clone().invert()
+			const boundsInShapeSpace = shapeTransform.applyToPoints(bounds.corners)
+			return geometry.overlapsPolygon(boundsInShapeSpace)
+		})
+
+		this.shapes = shapesInBounds
+		this.editor.setHintingShapes(shapesInBounds)
+	}
+}

templates/agent/client/vite-env.d.ts

@@ -0,0 +1 @@
+/// <reference types="vite/client" />