snapshot of https://github.com/tldraw/tldraw.git (main, 2026-09-23; blobs > 131072B omitted)

templates/agent/worker/do/AgentDurableObject.ts

@@ -0,0 +1,80 @@
+import { DurableObject } from 'cloudflare:workers'
+import { AutoRouter, error } from 'itty-router'
+import { AgentAction } from '../../shared/types/AgentAction'
+import { AgentPrompt } from '../../shared/types/AgentPrompt'
+import { Streaming } from '../../shared/types/Streaming'
+import { Environment } from '../environment'
+import { AgentService } from './AgentService'
+
+export class AgentDurableObject extends DurableObject<Environment> {
+	service: AgentService
+
+	constructor(ctx: DurableObjectState, env: Environment) {
+		super(ctx, env)
+		this.service = new AgentService(this.env) // swap this with your own service
+	}
+
+	private readonly router = AutoRouter({
+		catch: (e) => {
+			console.error(e)
+			return error(e)
+		},
+	}).post('/stream', (request) => this.stream(request))
+
+	// `fetch` is the entry point for all requests to the Durable Object
+	override fetch(request: Request): Response | Promise<Response> {
+		return this.router.fetch(request)
+	}
+
+	/**
+	 * Stream changes from the model.
+	 *
+	 * @param request - The request object containing the prompt.
+	 * @returns A Promise that resolves to a Response object containing the streamed changes.
+	 */
+	private async stream(request: Request): Promise<Response> {
+		const encoder = new TextEncoder()
+		const { readable, writable } = new TransformStream()
+		const writer = writable.getWriter()
+
+		const response: { changes: Streaming<AgentAction>[] } = { changes: [] }
+
+		;(async () => {
+			try {
+				const prompt = (await request.json()) as AgentPrompt
+
+				for await (const change of this.service.stream(prompt)) {
+					response.changes.push(change)
+					const data = `data: ${JSON.stringify(change)}\n\n`
+					await writer.write(encoder.encode(data))
+					await writer.ready
+				}
+				await writer.close()
+			} catch (error: any) {
+				console.error('Stream error:', error)
+
+				// Send error through the stream
+				const errorData = `data: ${JSON.stringify({ error: error.message })}\n\n`
+				try {
+					await writer.write(encoder.encode(errorData))
+					await writer.close()
+				} catch (writeError) {
+					await writer.abort(writeError)
+				}
+			}
+		})()
+
+		return new Response(readable, {
+			headers: {
+				'Content-Type': 'text/event-stream',
+				'Cache-Control': 'no-cache, no-transform',
+				Connection: 'keep-alive',
+				'X-Accel-Buffering': 'no',
+				'Transfer-Encoding': 'chunked',
+				'Access-Control-Allow-Origin': '*',
+				'Access-Control-Allow-Methods': 'POST, OPTIONS',
+				'Access-Control-Allow-Headers': 'Content-Type',
+			},
+		})
+	}
+}

templates/agent/worker/do/AgentService.ts

@@ -0,0 +1,228 @@
+import { AnthropicProvider, AnthropicProviderOptions, createAnthropic } from '@ai-sdk/anthropic'
+import {
+	createGoogleGenerativeAI,
+	GoogleGenerativeAIProvider,
+	GoogleGenerativeAIProviderOptions,
+} from '@ai-sdk/google'
+import { createOpenAI, OpenAIProvider, OpenAIResponsesProviderOptions } from '@ai-sdk/openai'
+import { LanguageModel, ModelMessage, streamText } from 'ai'
+import {
+	AgentModelDefinition,
+	AgentModelName,
+	getAgentModelDefinition,
+	isValidModelName,
+} from '../../shared/models'
+import { DebugPart } from '../../shared/schema/PromptPartDefinitions'
+import { AgentAction } from '../../shared/types/AgentAction'
+import { AgentPrompt } from '../../shared/types/AgentPrompt'
+import { Streaming } from '../../shared/types/Streaming'
+import { Environment } from '../environment'
+import { buildMessages } from '../prompt/buildMessages'
+import { buildSystemPrompt } from '../prompt/buildSystemPrompt'
+import { getModelName } from '../prompt/getModelName'
+import { closeAndParseJson } from './closeAndParseJson'
+
+export class AgentService {
+	openai: OpenAIProvider
+	anthropic: AnthropicProvider
+	google: GoogleGenerativeAIProvider
+
+	constructor(env: Environment) {
+		this.openai = createOpenAI({ apiKey: env.OPENAI_API_KEY })
+		this.anthropic = createAnthropic({ apiKey: env.ANTHROPIC_API_KEY })
+		this.google = createGoogleGenerativeAI({ apiKey: env.GOOGLE_API_KEY })
+	}
+
+	getModel(modelName: AgentModelName): LanguageModel {
+		const modelDefinition = getAgentModelDefinition(modelName)
+		const provider = modelDefinition.provider
+		return this[provider](modelDefinition.id)
+	}
+
+	async *stream(prompt: AgentPrompt): AsyncGenerator<Streaming<AgentAction>> {
+		try {
+			for await (const event of this.streamActions(prompt)) {
+				yield event
+			}
+		} catch (error: any) {
+			console.error('Stream error:', error)
+			throw error
+		}
+	}
+
+	private async *streamActions(prompt: AgentPrompt): AsyncGenerator<Streaming<AgentAction>> {
+		const modelName = getModelName(prompt)
+		const model = this.getModel(modelName)
+
+		if (typeof model === 'string') {
+			throw new Error('Model is a string, not a LanguageModel')
+		}
+
+		const { modelId, provider } = model
+		if (!isValidModelName(modelId)) {
+			throw new Error(`Model ${modelId} is not in AGENT_MODEL_DEFINITIONS`)
+		}
+
+		const modelDefinition = getAgentModelDefinition(modelId)
+		const systemPrompt = buildSystemPrompt(prompt)
+
+		// Build messages with provider-specific options
+		const messages: ModelMessage[] = []
+
+		// Add system prompt with Anthropic caching if applicable
+		if (provider === 'anthropic.messages') {
+			// Anthropic requires explicit cache breakpoints. We set one at the end of the
+			// system prompt to cache all system content (which generally changes together).
+			messages.push({
+				role: 'system',
+				content: systemPrompt,
+				providerOptions: {
+					anthropic: { cacheControl: { type: 'ephemeral' } },
+				},
+			})
+		} else {
+			messages.push({
+				role: 'system',
+				content: systemPrompt,
+			})
+		}
+
+		// Add prompt messages
+		const promptMessages = buildMessages(prompt)
+		messages.push(...promptMessages)
+
+		// Check for debug flags and log if enabled
+		const debugPart = prompt.debug as DebugPart | undefined
+		if (debugPart) {
+			if (debugPart.logSystemPrompt) {
+				const promptWithoutSchema = buildSystemPrompt(prompt, { withSchema: false })
+				console.log('[DEBUG] System Prompt (without schema):\n', promptWithoutSchema)
+			}
+			if (debugPart.logMessages) {
+				console.log('[DEBUG] Messages:\n', JSON.stringify(promptMessages, null, 2))
+			}
+		}
+
+		// Prefill the assistant turn to force the JSON start, where the model allows it.
+		// Opus 4.7+ and Sonnet 4.6 reject last-assistant-turn prefills (400), so skip it there.
+		if (modelDefinition.supportsPrefill) {
+			messages.push({
+				role: 'assistant',
+				content: '{"actions": [{"_type":',
+			})
+		}
+
+		try {
+			const { textStream } = streamText({
+				model,
+				messages,
+				maxOutputTokens: 8192,
+				// Opus 4.7+ removed `temperature` (and top_p/top_k); sending it returns a 400.
+				...(modelDefinition.supportsTemperature ? { temperature: 0 } : {}),
+				providerOptions: getProviderOptions(modelDefinition),
+				onAbort() {
+					console.warn('Stream actions aborted')
+				},
+				onError: (e) => {
+					console.error('Stream text error:', e)
+					throw e
+				},
+			})
+
+			const canForceResponseStart =
+				(provider === 'anthropic.messages' || provider === 'google.generative-ai') &&
+				modelDefinition.supportsPrefill
+			let buffer = canForceResponseStart ? '{"actions": [{"_type":' : ''
+			let cursor = 0
+			let maybeIncompleteAction: AgentAction | null = null
+
+			let startTime = Date.now()
+			for await (const text of textStream) {
+				buffer += text
+				const partialObject = closeAndParseJson(buffer)
+				if (!partialObject) continue
+
+				const actions = partialObject.actions
+				if (!Array.isArray(actions)) continue
+				if (actions.length === 0) continue
+
+				// If the events list is ahead of the cursor, we know we've completed the current event
+				// We can complete the event and move the cursor forward
+				if (actions.length > cursor) {
+					const action = actions[cursor - 1] as AgentAction
+					if (action) {
+						yield {
+							...action,
+							complete: true,
+							time: Date.now() - startTime,
+						}
+						maybeIncompleteAction = null
+					}
+					cursor++
+				}
+
+				// Now let's check the (potentially new) current event
+				// And let's yield it in its (potentially incomplete) state
+				const action = actions[cursor - 1] as AgentAction
+				if (action) {
+					// If we don't have an incomplete event yet, this is the start of a new one
+					if (!maybeIncompleteAction) {
+						startTime = Date.now()
+					}
+
+					maybeIncompleteAction = action
+
+					// Yield the potentially incomplete event
+					yield {
+						...action,
+						complete: false,
+						time: Date.now() - startTime,
+					}
+				}
+			}
+
+			// If we've finished receiving events, but there's still an incomplete event, we need to complete it
+			if (maybeIncompleteAction) {
+				yield {
+					...maybeIncompleteAction,
+					complete: true,
+					time: Date.now() - startTime,
+				}
+			}
+		} catch (error: any) {
+			console.error('streamActions error:', error)
+			throw error
+		}
+	}
+}
+
+type StreamTextProviderOptions = NonNullable<Parameters<typeof streamText>[0]['providerOptions']>
+
+/**
+ * Map a model definition's reasoning preferences to AI SDK provider options.
+ * Only the matching provider's options are set; the SDK ignores the rest.
+ */
+function getProviderOptions(definition: AgentModelDefinition): StreamTextProviderOptions {
+	switch (definition.provider) {
+		case 'anthropic':
+			return {
+				anthropic: {
+					thinking:
+						definition.thinking === 'adaptive' ? { type: 'adaptive' } : { type: 'disabled' },
+					...(definition.effort ? { effort: definition.effort } : {}),
+				} satisfies AnthropicProviderOptions,
+			}
+		case 'google':
+			return {
+				google: {
+					thinkingConfig: { thinkingLevel: definition.thinkingLevel },
+				} satisfies GoogleGenerativeAIProviderOptions,
+			}
+		case 'openai':
+			return {
+				openai: {
+					reasoningEffort: definition.reasoningEffort,
+				} satisfies OpenAIResponsesProviderOptions,
+			}
+	}
+}

templates/agent/worker/do/closeAndParseJson.ts

@@ -0,0 +1,72 @@
+/**
+ * JSON helper. Given a potentially incomplete JSON string, return the parsed object.
+ * The string might be missing closing braces, brackets, or other characters like quotation marks.
+ * @param string - The string to parse.
+ * @returns The parsed object.
+ */
+export function closeAndParseJson(string: string) {
+	const stackOfOpenings = []
+
+	// Track openings and closings
+	let i = 0
+	while (i < string.length) {
+		const char = string[i]
+		const lastOpening = stackOfOpenings.at(-1)
+
+		if (char === '"') {
+			// Check if this quote is escaped
+			if (i > 0 && string[i - 1] === '\\') {
+				// This is an escaped quote, skip it
+				i++
+				continue
+			}
+
+			if (lastOpening === '"') {
+				stackOfOpenings.pop()
+			} else {
+				stackOfOpenings.push('"')
+			}
+		}
+
+		if (lastOpening === '"') {
+			i++
+			continue
+		}
+
+		if (char === '{' || char === '[') {
+			stackOfOpenings.push(char)
+		}
+
+		if (char === '}' && lastOpening === '{') {
+			stackOfOpenings.pop()
+		}
+
+		if (char === ']' && lastOpening === '[') {
+			stackOfOpenings.pop()
+		}
+
+		i++
+	}
+
+	// Now close all unclosed openings
+	for (let i = stackOfOpenings.length - 1; i >= 0; i--) {
+		const opening = stackOfOpenings[i]
+		if (opening === '{') {
+			string += '}'
+		}
+
+		if (opening === '[') {
+			string += ']'
+		}
+
+		if (opening === '"') {
+			string += '"'
+		}
+	}
+
+	try {
+		return JSON.parse(string)
+	} catch (_e) {
+		return null
+	}
+}

templates/agent/worker/environment.ts

@@ -0,0 +1,6 @@
+export interface Environment {
+	AGENT_DURABLE_OBJECT: DurableObjectNamespace
+	OPENAI_API_KEY: string
+	ANTHROPIC_API_KEY: string
+	GOOGLE_API_KEY: string
+}

templates/agent/worker/prompt/buildMessages.ts

@@ -0,0 +1,98 @@
+import { ModelMessage, UserContent } from 'ai'
+import { AgentMessage, AgentMessageContent } from '../../shared/types/AgentMessage'
+import { AgentPrompt } from '../../shared/types/AgentPrompt'
+import { getPromptPartDefinition, PromptPart } from '../../shared/types/PromptPart'
+
+export function buildMessages(prompt: AgentPrompt): ModelMessage[] {
+	const allMessages: AgentMessage[] = []
+
+	// Build messages from each prompt part using shared definitions
+	for (const part of Object.values(prompt)) {
+		const messages = buildMessagesFromPart(part)
+		allMessages.push(...messages)
+	}
+
+	// Sort by priority (higher priority = later in prompt)
+	allMessages.sort((a, b) => a.priority - b.priority)
+
+	return toModelMessages(allMessages)
+}
+
+/**
+ * Build messages from a prompt part using its definition.
+ * This is used by the worker to convert prompt parts into messages for the model.
+ */
+function buildMessagesFromPart(part: PromptPart): AgentMessage[] {
+	const definition = getPromptPartDefinition(part.type)
+
+	// If the definition has a custom buildMessages function, use it
+	if (definition.buildMessages) {
+		return definition.buildMessages(part)
+	}
+
+	// Otherwise, use the default logic with buildContent
+	return defaultBuildMessagesFromPart(part)
+}
+
+/**
+ * Default message building logic that uses buildContent.
+ */
+function defaultBuildMessagesFromPart(part: PromptPart): AgentMessage[] {
+	const definition = getPromptPartDefinition(part.type)
+
+	// Get content strings from the definition
+	const content = definition.buildContent ? definition.buildContent(part) : []
+
+	if (!content || content.length === 0) {
+		return []
+	}
+
+	// Convert content strings to message content (handling images)
+	const messageContent: AgentMessageContent[] = []
+	for (const item of content) {
+		if (typeof item === 'string' && item.startsWith('data:image/')) {
+			messageContent.push({
+				type: 'image',
+				image: item,
+			})
+		} else {
+			messageContent.push({
+				type: 'text',
+				text: item,
+			})
+		}
+	}
+
+	// Get priority from definition (default to 0)
+	const priority = definition.priority ?? 0
+
+	return [{ role: 'user', content: messageContent, priority }]
+}
+
+/**
+ * Convert AgentMessage[] to ModelMessage[] for the AI SDK
+ */
+function toModelMessages(agentMessages: AgentMessage[]): ModelMessage[] {
+	return agentMessages.map((tlMessage) => {
+		const content: UserContent = []
+
+		for (const contentItem of tlMessage.content) {
+			if (contentItem.type === 'image') {
+				content.push({
+					type: 'image',
+					image: contentItem.image!,
+				})
+			} else {
+				content.push({
+					type: 'text',
+					text: contentItem.text!,
+				})
+			}
+		}
+
+		return {
+			role: tlMessage.role,
+			content,
+		} as ModelMessage
+	})
+}

templates/agent/worker/prompt/buildSystemPrompt.ts

@@ -0,0 +1,58 @@
+import { buildResponseSchema } from '../../shared/schema/buildResponseSchema'
+import type { ModePart } from '../../shared/schema/PromptPartDefinitions'
+import { AgentPrompt } from '../../shared/types/AgentPrompt'
+import { getSystemPromptFlags } from './getSystemPromptFlags'
+import { buildIntroPromptSection } from './sections/intro-section'
+import { buildRulesPromptSection } from './sections/rules-section'
+
+/**
+ * Build the system prompt for the agent.
+ *
+ * This is the main instruction set that tells the AI how to behave.
+ * The prompt is constructed from modular sections that adapt based on
+ * what actions and parts are available.
+ *
+ * @param prompt - The prompt containing all parts including the mode part.
+ * @param opts - Options for building the system prompt.
+ * @param opts.withSchema - Whether to include the JSON schema in the system prompt. Defaults to true.
+ * @returns The system prompt string.
+ */
+export function buildSystemPrompt(
+	prompt: AgentPrompt,
+	opts: { withSchema: boolean } = { withSchema: true }
+): string {
+	const { withSchema = true } = opts
+
+	const modePart = prompt.mode
+	if (!modePart) {
+		throw new Error('A mode part is always required.')
+	}
+
+	const { actionTypes, partTypes } = modePart
+	const flags = getSystemPromptFlags(actionTypes, partTypes)
+
+	const lines = [buildIntroPromptSection(flags), buildRulesPromptSection(flags)]
+
+	if (withSchema) {
+		lines.push(buildSchemaPromptSection(modePart))
+	}
+
+	const result = normalizeNewlines(lines.join('\n'))
+
+	return result
+}
+
+function buildSchemaPromptSection(modePart: ModePart) {
+	const schema = buildResponseSchema(modePart.actionTypes, modePart.modeType)
+
+	return `## JSON schema
+
+This is the JSON schema for the events you can return. You must conform to this schema. You must only return things in this format, otherwise your response will error. Output ONLY the JSON itself — do not add any preamble, explanation, or commentary before or after it, and do NOT wrap your response in a markdown code block (e.g. \`\`\`json). Just the raw JSON object.
+
+${JSON.stringify(schema, null, 2)}
+`
+}
+
+function normalizeNewlines(text: string): string {
+	return text.replace(/\n{3,}/g, '\n\n')
+}

templates/agent/worker/prompt/getModelName.ts

@@ -0,0 +1,23 @@
+import { AgentModelName, DEFAULT_MODEL_NAME, isValidModelName } from '../../shared/models'
+import { AgentPrompt } from '../../shared/types/AgentPrompt'
+import { getPromptPartDefinition } from '../../shared/types/PromptPart'
+
+/**
+ * Get the selected model name from a prompt using shared definitions.
+ */
+export function getModelName(prompt: AgentPrompt): AgentModelName {
+	for (const part of Object.values(prompt)) {
+		const definition = getPromptPartDefinition(part.type)
+
+		// Check if this definition provides a model name
+		if (definition.getModelName) {
+			const modelName = definition.getModelName(part)
+			// Ignore unknown model names (e.g. from an older client whose stored
+			// selection no longer exists) and fall through to the default so the
+			// request doesn't throw.
+			if (modelName && isValidModelName(modelName)) return modelName
+		}
+	}
+
+	return DEFAULT_MODEL_NAME
+}

templates/agent/worker/prompt/getSystemPromptFlags.ts

@@ -0,0 +1,81 @@
+import { getActionMeta } from '../../shared/types/AgentAction'
+import type { AgentAction } from '../../shared/types/AgentAction'
+import type { PromptPart } from '../../shared/types/PromptPart'
+import type { SystemPromptCategory } from '../../shared/types/SystemPromptCategory'
+
+export function getSystemPromptFlags(actions: AgentAction['_type'][], parts: PromptPart['type'][]) {
+	return {
+		// Communication
+		hasMessage: actions.includes('message'),
+
+		// Planning
+		hasThink: actions.includes('think'),
+		hasReview: actions.includes('review'),
+		hasSetMyView: actions.includes('setMyView'),
+		hasTodoList: actions.includes('update-todo-list') && parts.includes('todoList'),
+		hasAddDetail: actions.includes('add-detail'),
+
+		// Individual shapes
+		hasCreate: actions.includes('create'),
+		hasDelete: actions.includes('delete'),
+		hasUpdate: actions.includes('update'),
+		hasLabel: actions.includes('label'),
+		hasMove: actions.includes('move'),
+
+		// Groups of shapes
+		hasPlace: actions.includes('place'),
+		hasBringToFront: actions.includes('bringToFront'),
+		hasSendToBack: actions.includes('sendToBack'),
+		hasRotate: actions.includes('rotate'),
+		hasResize: actions.includes('resize'),
+		hasAlign: actions.includes('align'),
+		hasDistribute: actions.includes('distribute'),
+		hasStack: actions.includes('stack'),
+		hasClear: actions.includes('clear'),
+
+		// Drawing
+		hasPen: actions.includes('pen'),
+
+		// Request
+		hasMessagesPart: parts.includes('messages'),
+		hasDataPart: parts.includes('data'),
+		hasContextItemsPart: parts.includes('contextItems'),
+
+		// Viewport
+		hasScreenshotPart: parts.includes('screenshot'),
+		hasUserViewportBoundsPart: parts.includes('userViewportBounds'),
+		hasAgentViewportBoundsPart: parts.includes('agentViewportBounds'),
+
+		// Shapes
+		hasBlurryShapesPart: parts.includes('blurryShapes'),
+		hasPeripheralShapesPart: parts.includes('peripheralShapes'),
+		hasSelectedShapesPart: parts.includes('selectedShapes'),
+
+		// History
+		hasChatHistoryPart: parts.includes('chatHistory'),
+		hasUserActionHistoryPart: parts.includes('userActionHistory'),
+		hasTodoListPart: parts.includes('todoList'),
+
+		// Lints
+		hasCanvasLintsPart: parts.includes('canvasLints'),
+
+		// Metadata
+		hasTimePart: parts.includes('time'),
+
+		// Derived flags for convenience
+		canEdit: actions.some(isEditAction),
+	}
+}
+
+export type SystemPromptFlags = ReturnType<typeof getSystemPromptFlags>
+
+function isEditAction(type: AgentAction['_type']): boolean {
+	return isActionCategory(type, 'edit')
+}
+
+function isActionCategory(type: AgentAction['_type'], category: SystemPromptCategory): boolean {
+	const meta = getActionMeta(type)
+	if (!meta) return false
+	if (!meta._systemPromptCategory) return false
+	return meta._systemPromptCategory === category
+}

templates/agent/worker/prompt/sections/flagged.ts

@@ -0,0 +1,3 @@
+export function flagged(flag: boolean, text: string) {
+	return flag ? text : ''
+}

templates/agent/worker/prompt/sections/intro-section.ts

@@ -0,0 +1,17 @@
+import { SystemPromptFlags } from '../getSystemPromptFlags'
+import { flagged } from './flagged'
+
+export function buildIntroPromptSection(flags: SystemPromptFlags) {
+	return `You are an AI agent that helps the user use a drawing / diagramming / whiteboarding program. You and the user are both located within an infinite canvas, a 2D space that can be demarcated using x,y coordinates. You will be provided with a set of helpful information that includes a description of what the user would like you to do, along with the user's intent and the current state of the canvas${flagged(flags.hasScreenshotPart, ', including an image, which is your view of the part of the canvas contained within your viewport')}${flagged(flags.hasChatHistoryPart, ". You'll also be provided with the chat history of your conversation with the user, including the user's previous requests and your actions")}. Your goal is to generate a response that includes a list of structured events that represent the actions you would take to satisfy the user's request.
+
+You respond with structured JSON data based on a predefined schema.
+
+## Schema overview
+
+You are interacting with a system that models shapes (rectangles, ellipses, triangles, text, and many more) and carries out actions defined by events (creating, moving, labeling, deleting, thinking, and many more). Your response should include:
+
+- **A list of structured events** (\`actions\`): Each action should correspond to an action that follows the schema.
+
+For the full list of events, refer to the JSON schema.
+`
+}

templates/agent/worker/prompt/sections/rules-section.ts

@@ -0,0 +1,244 @@
+import { getFocusedShapeSchemaNames } from '../../../shared/format/FocusedShape'
+import { SystemPromptFlags } from '../getSystemPromptFlags'
+import { flagged } from './flagged'
+
+export function buildRulesPromptSection(flags: SystemPromptFlags) {
+	const shapeTypeNames = getFocusedShapeSchemaNames()
+
+	return `## Shapes
+
+${
+	flags.canEdit
+		? `Shapes can be:
+
+${shapeTypeNames.map((type) => `- **${type.charAt(0).toUpperCase() + type.slice(1)} (\`${type}\`)**`).join('\n')}
+
+Each shape has:
+
+- \`_type\` (one of ${shapeTypeNames.map((type) => `\`${type}\``).join(', ')})
+- \`x\`, \`y\` (numbers, coordinates, typically the top left corner of the shape, but text shapes use anchor-based positioning) (except for arrows and lines, which have \`x1\`, \`y1\`, \`x2\`, \`y2\`)
+- \`note\` (a description of the shape's purpose or intent) (invisible to the user)
+
+Shapes may also have different properties depending on their type:
+
+- \`w\` and \`h\` (for shapes)
+- \`color\` (optional, chosen from predefined colors)
+- \`fill\` (optional, for shapes)
+- \`text\` (optional, for text elements) (visible to the user)
+- ...and others
+
+### Arrow properties
+
+Arrows are different from shapes, in that they are lines that connect two shapes. They are different from the arrowshapes (arrow-up, arrow-down, arrow-left, arrow-right), which are two dimensional.
+
+Arrows have:
+- \`fromId\` (optional, the id of the shape that the arrow starts from)
+- \`toId\` (optional, the id of the shape that the arrow points to)
+
+### Arrow and line properties
+
+Arrows and lines are different from shapes, in that they are lines that they have two positions, not just one.
+
+Arrows and lines have:
+- \`x1\` (the x coordinate of the first point of the line)
+- \`y1\` (the y coordinate of the first point of the line)
+- \`x2\` (the x coordinate of the second point of the line)
+- \`y2\` (the y coordinate of the second point of the line)
+
+`
+		: `What you need to know about shapes is that they exist on the canvas and have x,y coordinates, as well as many different types, colors, and fills. There are also arrows that can connect two shapes. You can't create or edit them.
+`
+}
+
+## Event schema
+
+Refer to the JSON schema for the full list of available events, their properties, and their descriptions. You can only use events listed in the JSON schema, even if they are referred to within this system prompt. Use the schema as the source of truth on what is available. Make wise choices about which action types to use, but only use action types that are listed in the JSON schema.
+
+## Rules
+
+1. **Always return a valid JSON object conforming to the schema.**
+2. **Do not generate extra fields or omit required fields.**
+3. **Use meaningful \`intent\` descriptions for all actions.**
+${flagged(flags.canEdit, '4. **Ensure each `shapeId` is unique and consistent across related events.**')}
+
+## Useful notes
+
+### General tips about the canvas
+
+- The coordinate space is the same as on a website: 0,0 is the top left corner. The x-axis increases as you scroll to the right. The y-axis increases as you scroll down the canvas.
+- For most shapes, the x and y define the top left corner of the shape. However, text shapes use anchor-based positioning where x and y refer to the point specified by the anchor property.
+
+${flagged(
+	flags.canEdit,
+	`### Tips for creating and updating shapes
+
+${flagged(
+	flags.hasMove,
+	`- When moving shapes:
+	- Always use the \`move\` action to move a shape${flagged(flags.hasUpdate, ', never the `update` action')}.`
+)}
+${flagged(
+	flags.hasUpdate,
+	`- When updating shapes:
+	- Only output a single shape for each shape being updated. We know what it should update from its shapeId.`
+)}
+${flagged(
+	flags.hasCreate,
+	`- When creating shapes:
+	- Often the user will ask you to 'draw' something. If you want, you can compose the drawing using simple shapes; otherwise, use the pen to draw a custom shape.
+	- If the shape you need is not available in the schema, use the pen to draw a custom shape. The pen can be helpful when you need more control over a shape's exact shape. This can be especially helpful when you need to create shapes that need to fit together precisely.
+	- Use the \`note\` field to provide context for each shape. This will help you in the future to understand the purpose of each shape.
+	- Never create "unknown" type shapes, though you can move unknown shapes if you need to.
+	- When creating shapes that are meant to be contained within other shapes, always ensure the shapes properly fit inside of the containing or background shape. If there are overlaps, decide between making the inside shapes smaller or the outside shape bigger.`
+)}
+${flagged(
+	flags.hasCreate,
+	`- When drawing arrows between shapes:
+	- Be sure to include the shapes' ids as fromId and toId.
+	- Always ensure they are properly connected with bindings.
+	- You can make the arrow curved by using the 'bend' property. The bend value (in pixels) determines how far the arrow's midpoint is displaced perpendicular to the straight line between its endpoints. To determine the correct sign:
+		- Calculate the arrow's direction vector: (dx = x2 - x1, dy = y2 - y1)
+		- There are two vectors perpendicular to the arrow's direction vector: (-dy, dx) and (dy, -dx)
+		- A positive bend value displaces the midpoint of the arrow perpendicularly to the left of the arrow's direction, in the direction of (dy, -dx)
+		- A negative bend value displaces the midpoint of the arrow perpendicularly to the right of the arrow's direction, in the direction: (-dy, dx)
+		- Examples:
+			- Arrow going RIGHT (relatively to the canvas) (dx > 0, dy = 0): positive bend curves UP (relatively to the canvas), negative bend curves DOWN (relatively to the canvas)
+			- Arrow going LEFT (dx < 0, dy = 0): positive bend curves DOWN, negative bend curves UP
+			- Arrow going DOWN (dx = 0, dy > 0): positive bend curves RIGHT, negative bend curves LEFT
+			- Arrow going UP (dx = 0, dy < 0): positive bend curves LEFT, negative bend curves RIGHT
+		- And one diagonal example:
+			- Arrow going DOWN and RIGHT (dx > 0, dy > 0): positive bend curves UP and RIGHT, negative bend curves DOWN and LEFT
+		- Or simply: if you think of your arrow as going righty tighty, or clockwise around a circle, a positive bend with make it bend away from the center of that circle, and a negative bend will make it bend towards the center of that circle.
+		- When looking at the canvas, you might notice arrows that are bending the wrong way. To fix this, update that arrow shape's bend property to the inverse of the current bend property.
+	- Be sure not to create arrows twice—check for existing arrows that already connect the same shapes for the same purpose.
+	- Make sure your arrows are long enough to contain any labels you may add to them.
+- Text shapes
+	- When creating a text shape, you must take into account how much space the text will take up on the canvas.
+	- By default, the width of text shapes will grow to fit the text content${flagged(flags.hasScreenshotPart, '. Refer to your view of the canvas to see how much space is actually taken up by the text')}.
+	- The font size of a text shape is the height of the text.
+	- When creating a text shape, you can specify the font size of the text shape if you like. The default size is 26 pixels tall, with each character being about 18 pixels wide.
+	- The easiest way to make sure text fits within an area is to set the \`maxWidth\` property of the text shape. The text will automatically wrap to fit within that width. This works with text of any alignment.
+	- Text shapes use an \`anchor\` property to control both positioning and text alignment. The anchor determines which point of the text shape the \`x\` and \`y\` coordinates refer to. 
+		- Available anchors are: \`top-left\`, \`top-center\`, \`top-right\`, \`center-left\`, \`center\`, \`center-right\`, \`bottom-left\`, \`bottom-center\`, \`bottom-right\`.
+		- For example, if the anchor is \`top-left\`, the \`x\` and \`y\` coordinates refer to the top-left corner of the text (and text is left-aligned).
+		- If the anchor is \`top-center\`, the \`x\` and \`y\` coordinates refer to the top-center of the text (and text is center-aligned).
+		- If the anchor is \`bottom-right\`, the \`x\` and \`y\` coordinates refer to the bottom-right corner of the text (and text is right-aligned).
+		- This makes it easy to position text relative to other shapes. For example, to place text to the left of a shape, use anchor \`center-right\` with an \`x\` value just less than the shape's left edge.
+		- This behavior is unique to text shapes. No other shape uses anchor-based positioning, so be careful.
+- Labels
+	- Be careful with labels. Did the user ask for labels on their shapes? Did the user ask for a format where labels would be appropriate? If yes, add labels to shapes. If not, do not add labels to shapes. For example, a 'drawing of a cat' should not have the parts of the cat labelled; but a 'diagram of a cat' might have shapes labelled.
+	- When drawing a shape with a label, be sure that the text will fit inside of the label. Label text is generally 26 points tall and each character is about 18 pixels wide. There are 32 pixels of padding around the text on each side. You need to leave room for the padding. Factor this padding into your calculations when determining if the text will fit as you wouldn't want a word to get cut off. When a shape has a text label, it has a minimum height of 100, even if you try to set it to something smaller.
+	- You may also specify the alignment of the label text within the shape.
+	- If geometry shapes or note shapes have text, the shapes will become taller to accommodate the text. If you're adding lots of text, be sure that the shape is wide enough to fit it.
+	- Note shapes are 200x200. They're sticky notes and are only suitable for tiny sentences. Use a geometric shape or text shape if you need to write more.
+	- When drawing flow charts or other geometric shapes with labels, they should be at least 200 pixels on any side unless you have a good reason not to.
+- Colors
+	- When specifying a fill, you can use \`background\` to make the shape the same color as the background${flagged(flags.hasScreenshotPart, ", which you'll see in your viewport")}. It will either be white or black, depending on the theme of the canvas.
+		- When making shapes that are white (or black when the user is in dark mode), instead of making the color \`white\`, use \`background\` as the fill and \`grey\` as the color. This makes sure there is a border around the shape, making it easier to distinguish from the background.`
+)}`
+)}
+
+### Communicating with the user
+
+${flagged(flags.hasMessage, '- If you want to communicate with the user, use the `message` action.')}
+${flagged(
+	flags.hasReview,
+	`- Use the \`review\` action to check your work.
+- When using the \`review\` action, pass in \`x\`, \`y\`, \`w\`, and \`h\` values to define the area of the canvas where you want to focus on for your review. The more specific the better, but make sure to leave some padding around the area.
+- Do not use the \`review\` action to check your work for simple tasks like creating, updating or moving a single shape. Assume you got it right.
+- If you use the \`review\` action and find you need to make changes, carry out the changes. You are allowed to call follow-up \`review\` events after that too, but there is no need to review if the changes are simple or if there were no changes.`
+)}
+${flagged(
+	flags.hasThink && flags.hasMessage,
+	'- Your `think` events are not visible to the user, so your responses should never include only `think` events. Use a `message` action to communicate with the user.'
+)}
+
+### Starting your work
+
+${flagged(
+	flags.hasTodoList,
+	`- Use \`update-todo-list\` events liberally to keep an up to date list of your progress on the task at hand. When you are assigned a new task, use the action multiple times to sketch out your plan${flagged(flags.hasReview, '. You can then use the `review` action to check the todo list')}.
+	- Remember to always get started on the task after fleshing out a todo list.
+	- NEVER make a todo for waiting for the user to do something. If you need to wait for the user to do something, you can use the \`message\` action to communicate with the user.`
+)}
+${flagged(flags.hasThink, '- Use `think` events liberally to work through each step of your strategy.')}
+${flagged(
+	flags.hasScreenshotPart &&
+		(flags.hasBlurryShapesPart || flags.hasPeripheralShapesPart || flags.hasSelectedShapesPart),
+	'- To "see" the canvas, combine the information you have from your view of the canvas with the description of the canvas shapes on the viewport.'
+)}
+${flagged(
+	(flags.hasDistribute || flags.hasStack || flags.hasAlign || flags.hasPlace) &&
+		(flags.hasCreate || flags.hasUpdate || flags.hasMove),
+	`- Carefully plan which action types to use. For example, the higher level events like ${[flags.hasDistribute && '`distribute`', flags.hasStack && '`stack`', flags.hasAlign && '`align`', flags.hasPlace && '`place`'].filter(Boolean).join(', ')} can at times be better than the lower level events like ${[flags.hasCreate && '`create`', flags.hasUpdate && '`update`', flags.hasMove && '`move`'].filter(Boolean).join(', ')} because they're more efficient and more accurate. If lower level control is needed, the lower level events are better because they give more precise and customizable control.`
+)}
+${flagged(
+	flags.hasSelectedShapesPart,
+	"- If the user has selected shape(s) and they refer to 'this', or 'these' in their request, they are probably referring to their selected shapes."
+)}
+
+${flagged(
+	flags.hasUserViewportBoundsPart || flags.hasAgentViewportBoundsPart || flags.hasSetMyView,
+	`### Navigating the canvas
+
+${flagged(flags.hasUserViewportBoundsPart, "- Don't go out of your way to work inside the user's view unless you need to.")}
+${flagged(flags.hasPeripheralShapesPart, '- You will be provided with list of shapes that are outside of your viewport.')}
+${flagged(
+	flags.hasSetMyView,
+	`- You can use the \`setMyView\` action to change your viewport to navigate to other areas of the canvas if needed. This will provide you with an updated view of the canvas. You can also use this to functionally zoom in or out.
+- The \`setMyView\` \`x\` and \`y\` values are the top-left corner of the area you want to see, not its center. To center your view on a point, subtract half of \`w\` from \`x\` and half of \`h\` from \`y\`.`
+)}
+`
+)}
+
+${flagged(
+	flags.hasReview,
+	`## Reviewing your work
+
+- Using the \`review\` action will always give you an up to date view of the state of the canvas. You'll see the results of any actions you just completed.
+- Remember to review your work when making multiple changes so that you can see the results of your work. Otherwise, you're flying blind.
+${flagged(flags.hasSetMyView, '- If you navigate somewhere using the `setMyView` action, you get the same updated information about the canvas as if you had used the `review` action, so no need to review right after navigating.')}
+${flagged(flags.hasScreenshotPart, '- When reviewing your work, you should rely **most** on the image provided to find overlaps, assess quality, and ensure completeness.')}
+- Some important things to check for while reviewing:
+	- Are arrows properly connected to the shapes they are pointing to?
+	- Are labels properly contained within their containing shapes?
+	- Are labels properly positioned?
+	- Are any shapes overlapping? If so, decide whether to move the shapes, labels, or both.
+	- Are shapes floating in the air that were intended to be touching other shapes?
+- In a finished drawing or diagram:
+	- There should be no overlaps between shapes or labels.
+	- Arrows should be connected to the shapes they are pointing to, unless they are intended to be disconnected.
+	- Arrows should not overlap with other shapes.
+	- The overall composition should be balanced, like a good photo or directed graph.
+- It's important to review text closely. Make sure:
+	- Words are not cut off due to text wrapping. If this is the case, consider making the shape wider so that it can contain the full text, and rearranging other shapes to make room for this if necessary. Alternatively, consider shortening the text so that it can fit, or removing a text label and replacing it with a floating text shape. Important: Changing the height of a shape does not help this issue, as the text will still wrap. It's the mismatched *width* of the shape and the text that causes this issue, so adjust one of them.${flagged(
+		flags.hasMove,
+		`
+	- If text looks misaligned, it's best to manually adjust its position with the \`move\` action to put it in the right place.`
+	)}
+	- If text overflows out of a container that it's supposed to be inside, consider making the container wider, or shortening or wrapping the text so that it can fit.
+	- Spacing is important. If there is supposed to be a gap between shapes, make sure there is a gap. It's very common for text shapes to have spacing issues, so review them strictly.
+- REMEMBER: To be a good reviewer, come up with actionable steps to fix any issues you find, and carry those steps out.
+- IMPORTANT: If you made changes as part of a review, or if there is still work to do, schedule a follow-up review for tracking purposes.`
+)}
+
+### Finishing your work
+
+- Complete the task to the best of your ability. Schedule further work as many times as you need to complete the task, but be realistic about what is possible with the shapes you have available.
+${flagged(
+	flags.hasReview && flags.hasMessage,
+	"- If the task is finished to a reasonable degree, it's better to give the user a final message than to pointlessly re-review what is already reviewed."
+)}
+${flagged(flags.hasReview, "- If there's still more work to do, you must `review` it. Otherwise it won't happen.")}
+${flagged(flags.hasMessage, "- It's nice to speak to the user (with a `message` action) to let them know what you've done.")}
+
+${flagged(
+	flags.hasDataPart,
+	`### API data
+
+- When you call an API, you must end your actions in order to get response. Don't worry, you will be able to continue working after that.
+- If you want to call multiple APIs and the results of the API calls don't depend on each other, you can call them all at once before ending your response. This will help you get the results of the API calls faster.
+- If an API call fails, you should let the user know that it failed instead of trying again.`
+)}
+`
+}

templates/agent/worker/routes/stream.ts

@@ -0,0 +1,25 @@
+import { IRequest } from 'itty-router'
+import { Environment } from '../environment'
+
+export async function stream(request: IRequest, env: Environment) {
+	// eventually... use some kind of per-user id, so that each user has their own worker
+	const id = env.AGENT_DURABLE_OBJECT.idFromName('anonymous')
+	const DO = env.AGENT_DURABLE_OBJECT.get(id)
+	const response = await DO.fetch(request.url, {
+		method: 'POST',
+		body: request.body as any,
+	})
+
+	return new Response(response.body as BodyInit, {
+		headers: {
+			'Content-Type': 'text/event-stream',
+			'Cache-Control': 'no-cache, no-transform',
+			Connection: 'keep-alive',
+			'X-Accel-Buffering': 'no',
+			'Transfer-Encoding': 'chunked',
+			'Access-Control-Allow-Origin': '*',
+			'Access-Control-Allow-Methods': 'POST, OPTIONS',
+			'Access-Control-Allow-Headers': 'Content-Type',
+		},
+	})
+}

templates/agent/worker/worker.ts

@@ -0,0 +1,25 @@
+import { ExecutionContext } from '@cloudflare/workers-types'
+import { WorkerEntrypoint } from 'cloudflare:workers'
+import { AutoRouter, cors, error, IRequest } from 'itty-router'
+import { Environment } from './environment'
+import { stream } from './routes/stream'
+
+const { preflight, corsify } = cors({ origin: '*' })
+
+const router = AutoRouter<IRequest, [env: Environment, ctx: ExecutionContext]>({
+	before: [preflight],
+	finally: [corsify],
+	catch: (e) => {
+		console.error(e)
+		return error(e)
+	},
+}).post('/stream', stream)
+
+export default class extends WorkerEntrypoint<Environment> {
+	override fetch(request: Request): Promise<Response> {
+		return router.fetch(request, this.env, this.ctx)
+	}
+}
+
+// Make the durable object available to the cloudflare worker
+export { AgentDurableObject } from './do/AgentDurableObject'