snapshot of https://github.com/tldraw/tldraw.git (main, 2026-09-23; blobs > 131072B omitted)

templates/agent/client/parts/AgentViewportBoundsPartUtil.ts

@@ -0,0 +1,19 @@
+import { AgentViewportBoundsPart } from '../../shared/schema/PromptPartDefinitions'
+import { AgentRequest } from '../../shared/types/AgentRequest'
+import { AgentHelpers } from '../AgentHelpers'
+import { PromptPartUtil, registerPromptPartUtil } from './PromptPartUtil'
+
+export const AgentViewportBoundsPartUtil = registerPromptPartUtil(
+	class AgentViewportBoundsPartUtil extends PromptPartUtil<AgentViewportBoundsPart> {
+		static override type = 'agentViewportBounds' as const
+
+		override getPart(request: AgentRequest, helpers: AgentHelpers): AgentViewportBoundsPart {
+			const offsetAgentBounds = helpers.applyOffsetToBox(request.bounds)
+
+			return {
+				type: 'agentViewportBounds',
+				agentBounds: helpers.roundBox(offsetAgentBounds),
+			}
+		}
+	}
+)

templates/agent/client/parts/BlurryShapesPartUtil.ts

@@ -0,0 +1,49 @@
+import { Box } from 'tldraw'
+import { convertTldrawShapeToBlurryShape } from '../../shared/format/convertTldrawShapeToBlurryShape'
+import { BlurryShapesPart } from '../../shared/schema/PromptPartDefinitions'
+import { AgentRequest } from '../../shared/types/AgentRequest'
+import { AgentHelpers } from '../AgentHelpers'
+import { PromptPartUtil, registerPromptPartUtil } from './PromptPartUtil'
+
+export const BlurryShapesPartUtil = registerPromptPartUtil(
+	class BlurryShapesPartUtil extends PromptPartUtil<BlurryShapesPart> {
+		static override type = 'blurryShapes' as const
+
+		override getPart(request: AgentRequest, helpers: AgentHelpers): BlurryShapesPart {
+			const { editor } = this
+
+			const shapes = editor.getCurrentPageShapesSorted()
+			const contextBoundsBox = Box.From(request.bounds)
+
+			// Get all shapes within the agent's viewport
+			const shapesInBounds = shapes.filter((shape) => {
+				const bounds = editor.getShapeMaskedPageBounds(shape)
+				if (!bounds) return false
+				return contextBoundsBox.includes(bounds)
+			})
+
+			// Convert the shapes to the blurry shape format
+			const blurryShapes = shapesInBounds
+				.map((shape) => convertTldrawShapeToBlurryShape(editor, shape))
+				.filter((s) => s !== null)
+
+			// Apply the offset and round the blurry shapes
+			const normalizedBlurryShapes = blurryShapes.map((shape) => {
+				const bounds = helpers.roundBox(
+					helpers.applyOffsetToBox({
+						x: shape.x,
+						y: shape.y,
+						w: shape.w,
+						h: shape.h,
+					})
+				)
+				return { ...shape, x: bounds.x, y: bounds.y, w: bounds.w, h: bounds.h }
+			})
+
+			return {
+				type: 'blurryShapes',
+				shapes: normalizedBlurryShapes,
+			}
+		}
+	}
+)

templates/agent/client/parts/CanvasLintsPartUtil.ts

@@ -0,0 +1,39 @@
+import { Box } from 'tldraw'
+import { CanvasLintsPart } from '../../shared/schema/PromptPartDefinitions'
+import { AgentRequest } from '../../shared/types/AgentRequest'
+import { AgentHelpers } from '../AgentHelpers'
+import { PromptPartUtil, registerPromptPartUtil } from './PromptPartUtil'
+
+export const CanvasLintsPartUtil = registerPromptPartUtil(
+	class CanvasLintsPartUtil extends PromptPartUtil<CanvasLintsPart> {
+		static override type = 'canvasLints' as const
+
+		override getPart(request: AgentRequest, _helpers: AgentHelpers): CanvasLintsPart {
+			const { editor, agent } = this
+			if (!editor) return { type: 'canvasLints', lints: [] }
+
+			const shapes = editor.getCurrentPageShapesSorted()
+			const contextBoundsBox = Box.From(request.bounds)
+			const shapesInBounds = shapes.filter((shape) => {
+				const bounds = editor.getShapeMaskedPageBounds(shape)
+				if (!bounds) return false
+				return contextBoundsBox.includes(bounds)
+			})
+
+			// Use created shapes when in working mode, otherwise use shapes in request bounds
+			const shapesToCheck =
+				agent.mode.getCurrentModeType() === 'working'
+					? agent.lints.getCreatedShapes()
+					: shapesInBounds
+
+			// Get unsurfaced lints and mark them as surfaced
+			const lints = agent.lints.getUnsurfacedLintsForShapes(shapesToCheck)
+			agent.lints.markLintsAsSurfaced(lints)
+
+			return {
+				type: 'canvasLints',
+				lints,
+			}
+		}
+	}
+)

templates/agent/client/parts/ChatHistoryPartUtil.ts

@@ -0,0 +1,32 @@
+import { structuredClone } from 'tldraw'
+import { ChatHistoryPart } from '../../shared/schema/PromptPartDefinitions'
+import { AgentRequest } from '../../shared/types/AgentRequest'
+import { AgentHelpers } from '../AgentHelpers'
+import { PromptPartUtil, registerPromptPartUtil } from './PromptPartUtil'
+
+export const ChatHistoryPartUtil = registerPromptPartUtil(
+	class ChatHistoryPartUtil extends PromptPartUtil<ChatHistoryPart> {
+		static override type = 'chatHistory' as const
+
+		override async getPart(_request: AgentRequest, helpers: AgentHelpers) {
+			const history = structuredClone(this.agent.chat.getHistory())
+
+			for (const historyItem of history) {
+				if (historyItem.type !== 'prompt') continue
+
+				// Offset and round the context items of each history item
+				const contextItems = historyItem.contextItems.map((contextItem) => {
+					const offsetContextItem = helpers.applyOffsetToContextItem(contextItem)
+					return helpers.roundContextItem(offsetContextItem)
+				})
+
+				historyItem.contextItems = contextItems
+			}
+
+			return {
+				type: 'chatHistory' as const,
+				history,
+			}
+		}
+	}
+)

templates/agent/client/parts/ContextItemsPartUtil.ts

@@ -0,0 +1,23 @@
+import { ContextItemsPart } from '../../shared/schema/PromptPartDefinitions'
+import { AgentRequest } from '../../shared/types/AgentRequest'
+import { AgentHelpers } from '../AgentHelpers'
+import { PromptPartUtil, registerPromptPartUtil } from './PromptPartUtil'
+
+export const ContextItemsPartUtil = registerPromptPartUtil(
+	class ContextItemsPartUtil extends PromptPartUtil<ContextItemsPart> {
+		static override type = 'contextItems' as const
+
+		override getPart(request: AgentRequest, helpers: AgentHelpers): ContextItemsPart {
+			const items = request.contextItems.map((contextItem) => {
+				const offsetContextItem = helpers.applyOffsetToContextItem(contextItem)
+				return helpers.roundContextItem(offsetContextItem)
+			})
+
+			return {
+				type: 'contextItems',
+				items,
+				requestSource: request.source,
+			}
+		}
+	}
+)

templates/agent/client/parts/DataPartUtil.ts

@@ -0,0 +1,33 @@
+import { DataPart } from '../../shared/schema/PromptPartDefinitions'
+import { AgentRequest } from '../../shared/types/AgentRequest'
+import { PromptPartUtil, registerPromptPartUtil } from './PromptPartUtil'
+
+/**
+ * This prompt part collects up data retrieved by agent actions in the previous request.
+ */
+export const DataPartUtil = registerPromptPartUtil(
+	class DataPartUtil extends PromptPartUtil<DataPart> {
+		static override type = 'data' as const
+
+		override async getPart(request: AgentRequest): Promise<DataPart> {
+			const { data } = request
+
+			const values = await Promise.all(
+				data.map(async (item) => {
+					try {
+						return await item
+					} catch (error) {
+						console.error('Error retrieving data:', error)
+						// Tell the agent that something went wrong
+						return 'An error occurred while retrieving some data.'
+					}
+				})
+			)
+
+			return {
+				type: 'data',
+				data: values,
+			}
+		}
+	}
+)

templates/agent/client/parts/DebugPartUtil.ts

@@ -0,0 +1,18 @@
+import { DebugPart } from '../../shared/schema/PromptPartDefinitions'
+import { PromptPartUtil, registerPromptPartUtil } from './PromptPartUtil'
+
+export const DebugPartUtil = registerPromptPartUtil(
+	class DebugPartUtil extends PromptPartUtil<DebugPart> {
+		static override type = 'debug' as const
+
+		override getPart(): DebugPart {
+			const debugFlags = this.agent.debug.getDebugFlags()
+
+			return {
+				type: 'debug',
+				logSystemPrompt: debugFlags.logSystemPrompt,
+				logMessages: debugFlags.logMessages,
+			}
+		}
+	}
+)

templates/agent/client/parts/MessagesPartUtil.ts

@@ -0,0 +1,18 @@
+import { MessagesPart } from '../../shared/schema/PromptPartDefinitions'
+import { AgentRequest } from '../../shared/types/AgentRequest'
+import { PromptPartUtil, registerPromptPartUtil } from './PromptPartUtil'
+
+export const MessagesPartUtil = registerPromptPartUtil(
+	class MessagesPartUtil extends PromptPartUtil<MessagesPart> {
+		static override type = 'messages' as const
+
+		override getPart(request: AgentRequest): MessagesPart {
+			const { agentMessages, source } = request
+			return {
+				type: 'messages',
+				agentMessages,
+				requestSource: source,
+			}
+		}
+	}
+)

templates/agent/client/parts/ModePartUtil.ts

@@ -0,0 +1,24 @@
+import { ModePart } from '../../shared/schema/PromptPartDefinitions'
+import { AgentRequest } from '../../shared/types/AgentRequest'
+import { PromptPartUtil, registerPromptPartUtil } from './PromptPartUtil'
+
+export const ModePartUtil = registerPromptPartUtil(
+	class ModePartUtil extends PromptPartUtil<ModePart> {
+		static override type = 'mode' as const
+
+		override getPart(_request: AgentRequest): ModePart {
+			const modeDefinition = this.agent.mode.getCurrentModeDefinition()
+
+			if (!modeDefinition.active) {
+				throw new Error(`Cannot get mode part for inactive mode: ${modeDefinition.type}`)
+			}
+
+			return {
+				type: 'mode',
+				modeType: modeDefinition.type,
+				partTypes: modeDefinition.parts,
+				actionTypes: modeDefinition.actions,
+			}
+		}
+	}
+)

templates/agent/client/parts/ModelNamePartUtil.ts

@@ -0,0 +1,17 @@
+import { DEFAULT_MODEL_NAME } from '../../shared/models'
+import { ModelNamePart } from '../../shared/schema/PromptPartDefinitions'
+import { AgentRequest } from '../../shared/types/AgentRequest'
+import { PromptPartUtil, registerPromptPartUtil } from './PromptPartUtil'
+
+export const ModelNamePartUtil = registerPromptPartUtil(
+	class ModelNamePartUtil extends PromptPartUtil<ModelNamePart> {
+		static override type = 'modelName' as const
+
+		override getPart(_request: AgentRequest): ModelNamePart {
+			return {
+				type: 'modelName',
+				modelName: this.agent.modelName.getModelName() ?? DEFAULT_MODEL_NAME,
+			}
+		}
+	}
+)

templates/agent/client/parts/PeripheralShapesPartUtil.ts

@@ -0,0 +1,48 @@
+import { Box } from 'tldraw'
+import { convertTldrawShapesToPeripheralShapes } from '../../shared/format/convertTldrawShapesToPeripheralShapes'
+import { PeripheralShapesPart } from '../../shared/schema/PromptPartDefinitions'
+import { AgentRequest } from '../../shared/types/AgentRequest'
+import { AgentHelpers } from '../AgentHelpers'
+import { PromptPartUtil, registerPromptPartUtil } from './PromptPartUtil'
+
+export const PeripheralShapesPartUtil = registerPromptPartUtil(
+	class PeripheralShapesPartUtil extends PromptPartUtil<PeripheralShapesPart> {
+		static override type = 'peripheralShapes' as const
+
+		override getPart(request: AgentRequest, helpers: AgentHelpers): PeripheralShapesPart {
+			const { editor } = this
+
+			const shapes = editor.getCurrentPageShapesSorted()
+			const contextBounds = request.bounds
+
+			const contextBoundsBox = Box.From(contextBounds)
+
+			// Get all shapes that are outside the context bounds (these are what we want to peripheralize)
+			const shapesOutsideViewport = shapes.filter((shape) => {
+				const bounds = editor.getShapeMaskedPageBounds(shape)
+				if (!bounds) return
+				if (contextBoundsBox.includes(bounds)) return
+				return true
+			})
+
+			// Convert the shapes to peripheral shape cluster format
+			const clusters = convertTldrawShapesToPeripheralShapes(editor, shapesOutsideViewport, {
+				padding: 75,
+			})
+
+			// Apply the offset and round the clusters
+			const normalizedClusters = clusters.map((cluster) => {
+				const offsetBounds = helpers.applyOffsetToBox(cluster.bounds)
+				return {
+					numberOfShapes: cluster.numberOfShapes,
+					bounds: helpers.roundBox(offsetBounds),
+				}
+			})
+
+			return {
+				type: 'peripheralShapes',
+				clusters: normalizedClusters,
+			}
+		}
+	}
+)

templates/agent/client/parts/PromptPartUtil.ts

@@ -0,0 +1,70 @@
+import { Editor } from 'tldraw'
+import { AgentRequest } from '../../shared/types/AgentRequest'
+import { BasePromptPart } from '../../shared/types/BasePromptPart'
+import { PromptPart } from '../../shared/types/PromptPart'
+import { TldrawAgent } from '../agent/TldrawAgent'
+import { AgentHelpers } from '../AgentHelpers'
+
+// ============================================================================
+// Registry
+// ============================================================================
+
+const registry = new Map<string, PromptPartUtilConstructor<BasePromptPart>>()
+
+/**
+ * Register a prompt part util class. Call this after defining each util class.
+ */
+export function registerPromptPartUtil<T extends PromptPartUtilConstructor<BasePromptPart>>(
+	util: T
+): T {
+	if (registry.has(util.type)) {
+		throw new Error(`Prompt part util already registered: ${util.type}`)
+	}
+	registry.set(util.type, util)
+	return util
+}
+
+/**
+ * Get all registered prompt part util classes.
+ */
+export function getAllPromptPartUtils(): PromptPartUtilConstructor<PromptPart>[] {
+	return Array.from(registry.values()) as PromptPartUtilConstructor<PromptPart>[]
+}
+
+/**
+ * Get an object containing instantiated prompt part utils for an agent.
+ */
+export function getPromptPartUtilsRecord(agent: TldrawAgent) {
+	const object = {} as Record<PromptPart['type'], PromptPartUtil<PromptPart>>
+	for (const util of registry.values()) {
+		object[util.type as PromptPart['type']] = new util(agent) as PromptPartUtil<PromptPart>
+	}
+	return object
+}
+
+// ============================================================================
+// Base Class
+// ============================================================================
+
+export abstract class PromptPartUtil<T extends BasePromptPart = BasePromptPart> {
+	static type: string
+
+	agent: TldrawAgent
+	editor: Editor
+
+	constructor(agent: TldrawAgent) {
+		this.agent = agent
+		this.editor = agent?.editor
+	}
+
+	/**
+	 * Get some data to add to the prompt.
+	 * @returns The prompt part.
+	 */
+	abstract getPart(request: AgentRequest, helpers: AgentHelpers): Promise<T> | T
+}
+
+export interface PromptPartUtilConstructor<T extends BasePromptPart = BasePromptPart> {
+	new (agent: TldrawAgent): PromptPartUtil<T>
+	type: T['type']
+}

templates/agent/client/parts/ScreenshotPartUtil.ts

@@ -0,0 +1,45 @@
+import { Box, FileHelpers } from 'tldraw'
+import { ScreenshotPart } from '../../shared/schema/PromptPartDefinitions'
+import { AgentRequest } from '../../shared/types/AgentRequest'
+import { PromptPartUtil, registerPromptPartUtil } from './PromptPartUtil'
+
+export const ScreenshotPartUtil = registerPromptPartUtil(
+	class ScreenshotPartUtil extends PromptPartUtil<ScreenshotPart> {
+		static override type = 'screenshot' as const
+
+		override async getPart(request: AgentRequest): Promise<ScreenshotPart> {
+			const { editor } = this
+
+			const contextBounds = request.bounds
+
+			const contextBoundsBox = Box.From(contextBounds)
+
+			const shapes = editor.getCurrentPageShapesSorted().filter((shape) => {
+				const bounds = editor.getShapeMaskedPageBounds(shape)
+				if (!bounds) return false
+				return contextBoundsBox.includes(bounds)
+			})
+
+			if (shapes.length === 0) {
+				return { type: 'screenshot', screenshot: '' }
+			}
+
+			const largestDimension = Math.max(request.bounds.w, request.bounds.h)
+			const scale = largestDimension > 8000 ? 8000 / largestDimension : 1
+
+			const result = await editor.toImage(shapes, {
+				format: 'jpeg',
+				background: true,
+				bounds: Box.From(request.bounds),
+				padding: 0,
+				pixelRatio: 1,
+				scale,
+			})
+
+			return {
+				type: 'screenshot',
+				screenshot: await FileHelpers.blobToDataUrl(result.blob),
+			}
+		}
+	}
+)

templates/agent/client/parts/SelectedShapesPartUtil.ts

@@ -0,0 +1,22 @@
+import { structuredClone } from 'tldraw'
+import { convertTldrawIdToSimpleId } from '../../shared/format/convertTldrawShapeToFocusedShape'
+import { SelectedShapesPart } from '../../shared/schema/PromptPartDefinitions'
+import { AgentRequest } from '../../shared/types/AgentRequest'
+import { PromptPartUtil, registerPromptPartUtil } from './PromptPartUtil'
+
+export const SelectedShapesPartUtil = registerPromptPartUtil(
+	class SelectedShapesPartUtil extends PromptPartUtil<SelectedShapesPart> {
+		static override type = 'selectedShapes' as const
+
+		override getPart(_request: AgentRequest): SelectedShapesPart {
+			const { editor } = this
+
+			const userSelectedShapes = editor.getSelectedShapes().map((v) => structuredClone(v)) ?? []
+
+			return {
+				type: 'selectedShapes',
+				shapeIds: userSelectedShapes.map((shape) => convertTldrawIdToSimpleId(shape.id)),
+			}
+		}
+	}
+)

templates/agent/client/parts/TimePartUtil.ts

@@ -0,0 +1,15 @@
+import { TimePart } from '../../shared/schema/PromptPartDefinitions'
+import { PromptPartUtil, registerPromptPartUtil } from './PromptPartUtil'
+
+export const TimePartUtil = registerPromptPartUtil(
+	class TimePartUtil extends PromptPartUtil<TimePart> {
+		static override type = 'time' as const
+
+		override getPart(): TimePart {
+			return {
+				type: 'time',
+				time: new Date().toLocaleTimeString(),
+			}
+		}
+	}
+)

templates/agent/client/parts/TodoListPartUtil.ts

@@ -0,0 +1,17 @@
+import { TodoListPart } from '../../shared/schema/PromptPartDefinitions'
+import { AgentRequest } from '../../shared/types/AgentRequest'
+import { AgentHelpers } from '../AgentHelpers'
+import { PromptPartUtil, registerPromptPartUtil } from './PromptPartUtil'
+
+export const TodoListPartUtil = registerPromptPartUtil(
+	class TodoListPartUtil extends PromptPartUtil<TodoListPart> {
+		static override type = 'todoList' as const
+
+		override getPart(_request: AgentRequest, _helpers: AgentHelpers): TodoListPart {
+			return {
+				type: 'todoList',
+				items: this.agent.todos.getTodos(),
+			}
+		}
+	}
+)

templates/agent/client/parts/UserActionHistoryPartUtil.ts

@@ -0,0 +1,110 @@
+import { squashRecordDiffs } from 'tldraw'
+import {
+	convertTldrawIdToSimpleId,
+	convertTldrawShapeToFocusedShape,
+	convertTldrawShapeToFocusedType,
+} from '../../shared/format/convertTldrawShapeToFocusedShape'
+import { FocusedShape } from '../../shared/format/FocusedShape'
+import { UserActionHistoryPart } from '../../shared/schema/PromptPartDefinitions'
+import { AgentRequest } from '../../shared/types/AgentRequest'
+import { AgentHelpers } from '../AgentHelpers'
+import { PromptPartUtil, registerPromptPartUtil } from './PromptPartUtil'
+
+export const UserActionHistoryPartUtil = registerPromptPartUtil(
+	class UserActionHistoryPartUtil extends PromptPartUtil<UserActionHistoryPart> {
+		static override type = 'userActionHistory' as const
+
+		override getPart(_request: AgentRequest, helpers: AgentHelpers): UserActionHistoryPart {
+			const { editor, agent } = helpers
+
+			// Get the action history and clear it so that we can start tracking changes for the next request
+			const diffs = agent.userAction.getHistory()
+			agent.userAction.clearHistory()
+
+			const part: UserActionHistoryPart = {
+				type: 'userActionHistory',
+				added: [],
+				removed: [],
+				updated: [],
+			}
+
+			const squashedDiff = squashRecordDiffs(diffs)
+			const { added, updated, removed } = squashedDiff
+
+			// Collect user-added shapes
+			for (const shape of Object.values(added)) {
+				if (shape.typeName !== 'shape') continue
+				part.added.push({
+					shapeId: convertTldrawIdToSimpleId(shape.id),
+					type: convertTldrawShapeToFocusedType(shape),
+				})
+			}
+
+			// Collect user-removed shapes
+			for (const shape of Object.values(removed)) {
+				if (shape.typeName !== 'shape') continue
+				const focusedShape = convertTldrawShapeToFocusedShape(editor, shape)
+				part.removed.push({
+					shapeId: focusedShape.shapeId,
+					type: focusedShape._type,
+				})
+			}
+
+			// Collect user-updated shapes
+			for (const [from, to] of Object.values(updated)) {
+				if (from.typeName !== 'shape' || to.typeName !== 'shape') continue
+				const fromFocusedShape = convertTldrawShapeToFocusedShape(editor, from)
+				const toFocusedShape = convertTldrawShapeToFocusedShape(editor, to)
+
+				const changeFocusedShape = getFocusedShapeChange(fromFocusedShape, toFocusedShape)
+				if (!changeFocusedShape) continue
+
+				const before = helpers.applyOffsetToShapePartial(changeFocusedShape.from)
+				const after = helpers.applyOffsetToShapePartial(changeFocusedShape.to)
+
+				part.updated.push({
+					shapeId: toFocusedShape.shapeId,
+					type: toFocusedShape._type,
+					before: helpers.roundShapePartial(before),
+					after: helpers.roundShapePartial(after),
+				})
+			}
+
+			return part
+		}
+	}
+)
+
+/**
+ * Get any changed properties between two focused shapes.
+ * @param from - The original shape.
+ * @param to - The new shape.
+ * @returns The changed properties.
+ */
+function getFocusedShapeChange<T extends FocusedShape['_type']>(
+	from: FocusedShape & { _type: T },
+	to: FocusedShape & { _type: T }
+) {
+	if (from._type !== to._type) {
+		return null
+	}
+
+	const change: {
+		from: Partial<FocusedShape>
+		to: Partial<FocusedShape>
+	} = {
+		from: {},
+		to: {},
+	}
+
+	for (const key in to) {
+		const fromValue = from[key]
+		const toValue = to[key]
+		if (fromValue === toValue) {
+			continue
+		}
+		;(change.from as any)[key] = fromValue
+		;(change.to as any)[key] = toValue
+	}
+	return change
+}

templates/agent/client/parts/UserViewportBoundsPartUtil.ts

@@ -0,0 +1,19 @@
+import { UserViewportBoundsPart } from '../../shared/schema/PromptPartDefinitions'
+import { AgentHelpers } from '../AgentHelpers'
+import { PromptPartUtil, registerPromptPartUtil } from './PromptPartUtil'
+
+export const UserViewportBoundsPartUtil = registerPromptPartUtil(
+	class UserViewportBoundsPartUtil extends PromptPartUtil<UserViewportBoundsPart> {
+		static override type = 'userViewportBounds' as const
+
+		override getPart(_request: unknown, helpers: AgentHelpers): UserViewportBoundsPart {
+			const userBounds = this.editor.getViewportPageBounds()
+			const offsetUserBounds = helpers.applyOffsetToBox(userBounds)
+
+			return {
+				type: 'userViewportBounds',
+				userBounds: helpers.roundBox(offsetUserBounds),
+			}
+		}
+	}
+)