snapshot of https://github.com/tldraw/tldraw.git (main, 2026-09-23; blobs > 131072B omitted)

templates/agent/client/components/ChatInput.tsx

@@ -0,0 +1,149 @@
+import { FormEventHandler, useState } from 'react'
+import { Editor, useValue } from 'tldraw'
+import { AtIcon } from '../../shared/icons/AtIcon'
+import { BrainIcon } from '../../shared/icons/BrainIcon'
+import { ChevronDownIcon } from '../../shared/icons/ChevronDownIcon'
+import { AGENT_MODEL_DEFINITIONS, AgentModelName } from '../../shared/models'
+import { useAgent } from '../agent/TldrawAgentAppProvider'
+import { ContextItemTag } from './ContextItemTag'
+import { SelectionTag } from './SelectionTag'
+
+export function ChatInput({
+	handleSubmit,
+	inputRef,
+}: {
+	handleSubmit: FormEventHandler<HTMLFormElement>
+	inputRef: React.RefObject<HTMLTextAreaElement | null>
+}) {
+	const agent = useAgent()
+	const { editor } = agent
+	const [inputValue, setInputValue] = useState('')
+	const isGenerating = useValue('isGenerating', () => agent.requests.isGenerating(), [agent])
+
+	const isContextToolActive = useValue(
+		'isContextToolActive',
+		() => {
+			const tool = editor.getCurrentTool()
+			return tool.id === 'target-shape' || tool.id === 'target-area'
+		},
+		[editor]
+	)
+
+	const selectedShapes = useValue('selectedShapes', () => editor.getSelectedShapes(), [editor])
+	const contextItems = useValue('contextItems', () => agent.context.getItems(), [agent])
+	const modelName = useValue('modelName', () => agent.modelName.getModelName(), [agent])
+
+	return (
+		<div className="chat-input">
+			<form
+				onSubmit={(e) => {
+					e.preventDefault()
+					setInputValue('')
+					handleSubmit(e)
+				}}
+			>
+				<div className="prompt-tags">
+					<div className={'chat-context-select ' + (isContextToolActive ? 'active' : '')}>
+						<div className="chat-context-select-label">
+							<AtIcon /> Add Context
+						</div>
+						<select
+							id="chat-context-select"
+							value=" "
+							onChange={(e) => {
+								const action = ADD_CONTEXT_ACTIONS.find((action) => action.name === e.target.value)
+								if (action) action.onSelect(editor)
+							}}
+						>
+							{ADD_CONTEXT_ACTIONS.map((action) => {
+								return (
+									<option key={action.name} value={action.name}>
+										{action.name}
+									</option>
+								)
+							})}
+						</select>
+					</div>
+					{selectedShapes.length > 0 && <SelectionTag onClick={() => editor.selectNone()} />}
+					{contextItems.map((item, i) => (
+						<ContextItemTag
+							editor={editor}
+							onClick={() => agent.context.remove(item)}
+							key={'context-item-' + i}
+							item={item}
+						/>
+					))}
+				</div>
+
+				<textarea
+					ref={inputRef}
+					name="input"
+					autoComplete="off"
+					placeholder="Ask, learn, brainstorm, draw"
+					value={inputValue}
+					onInput={(e) => setInputValue(e.currentTarget.value)}
+					onKeyDown={(e) => {
+						if (e.key === 'Enter' && !e.shiftKey) {
+							e.preventDefault()
+							//idk about this but it works oops -max
+							const form = e.currentTarget.closest('form')
+							if (form) {
+								const submitEvent = new Event('submit', { bubbles: true, cancelable: true })
+								form.dispatchEvent(submitEvent)
+							}
+						}
+					}}
+				/>
+				<span className="chat-actions">
+					<div className="chat-actions-left">
+						<div className="chat-model-select">
+							<div className="chat-model-select-label">
+								<BrainIcon /> {modelName}
+							</div>
+							<select
+								value={modelName}
+								onChange={(e) => agent.modelName.setModelName(e.target.value as AgentModelName)}
+							>
+								{Object.values(AGENT_MODEL_DEFINITIONS).map((model) => (
+									<option key={model.name} value={model.name}>
+										{model.name}
+									</option>
+								))}
+							</select>
+							<ChevronDownIcon />
+						</div>
+					</div>
+					<button className="chat-input-submit" disabled={inputValue === '' && !isGenerating}>
+						{isGenerating && inputValue === '' ? '◼' : '⬆'}
+					</button>
+				</span>
+			</form>
+		</div>
+	)
+}
+
+const ADD_CONTEXT_ACTIONS = [
+	{
+		name: 'Pick Shapes',
+		onSelect: (editor: Editor) => {
+			editor.setCurrentTool('target-shape')
+			editor.focus()
+		},
+	},
+	{
+		name: 'Pick Area',
+		onSelect: (editor: Editor) => {
+			editor.setCurrentTool('target-area')
+			editor.focus()
+		},
+	},
+	{
+		name: ' ',
+		onSelect: (editor: Editor) => {
+			const currentTool = editor.getCurrentTool()
+			if (currentTool.id === 'target-area' || currentTool.id === 'target-shape') {
+				editor.setCurrentTool('select')
+			}
+		},
+	},
+]

templates/agent/client/components/ChatPanel.tsx

@@ -0,0 +1,58 @@
+import { FormEventHandler, useCallback, useRef } from 'react'
+import { useAgent } from '../agent/TldrawAgentAppProvider'
+import { ChatHistory } from './chat-history/ChatHistory'
+import { ChatInput } from './ChatInput'
+import { TodoList } from './TodoList'
+
+export function ChatPanel() {
+	const agent = useAgent()
+	const inputRef = useRef<HTMLTextAreaElement>(null)
+
+	const handleSubmit = useCallback<FormEventHandler<HTMLFormElement>>(
+		async (e) => {
+			e.preventDefault()
+			if (!inputRef.current) return
+			const formData = new FormData(e.currentTarget)
+			const value = formData.get('input') as string
+
+			// If the user's message is empty, just cancel the current request (if there is one)
+			if (value === '') {
+				agent.cancel()
+				return
+			}
+
+			// Clear the chat input (context is cleared after it's captured in requestAgentActions)
+			inputRef.current.value = ''
+
+			// Sending a new message to the agent should interrupt the current request
+			agent.interrupt({
+				input: {
+					agentMessages: [value],
+					bounds: agent.editor.getViewportPageBounds(),
+					source: 'user',
+					contextItems: agent.context.getItems(),
+				},
+			})
+		},
+		[agent]
+	)
+
+	const handleNewChat = useCallback(() => {
+		agent.reset()
+	}, [agent])
+
+	return (
+		<div className="chat-panel tl-theme__dark">
+			<div className="chat-header">
+				<button className="new-chat-button" onClick={handleNewChat}>
+					+
+				</button>
+			</div>
+			<ChatHistory agent={agent} />
+			<div className="chat-input-container">
+				<TodoList agent={agent} />
+				<ChatInput handleSubmit={handleSubmit} inputRef={inputRef} />
+			</div>
+		</div>
+	)
+}

templates/agent/client/components/ChatPanelFallback.tsx

@@ -0,0 +1,15 @@
+export function ChatPanelFallback() {
+	return (
+		<div className="chat-fallback">
+			<p>Error loading chat history</p>
+			<button
+				onClick={() => {
+					localStorage.clear()
+					window.location.reload()
+				}}
+			>
+				Clear chat history
+			</button>
+		</div>
+	)
+}

templates/agent/client/components/ContextItemTag.tsx

@@ -0,0 +1,19 @@
+import { Editor } from 'tldraw'
+import { CONTEXT_TYPE_DEFINITIONS, ContextItem } from '../../shared/types/ContextItem'
+import { PromptTag } from './PromptTag'
+
+export function ContextItemTag({
+	item,
+	editor,
+	onClick,
+}: {
+	item: ContextItem
+	editor: Editor
+	onClick?: () => void
+}) {
+	const definition = CONTEXT_TYPE_DEFINITIONS[item.type]
+	const name = definition.name(item, editor)
+	const icon = definition.icon
+
+	return <PromptTag text={name} icon={icon} onClick={onClick} />
+}

templates/agent/client/components/CustomHelperButtons.tsx

@@ -0,0 +1,17 @@
+import {
+	DefaultHelperButtons,
+	DefaultHelperButtonsContent,
+	TldrawUiMenuContextProvider,
+} from 'tldraw'
+import { GoToAgentButtons } from './GoToAgentButton'
+
+export function CustomHelperButtons() {
+	return (
+		<DefaultHelperButtons>
+			<TldrawUiMenuContextProvider type="helper-buttons" sourceId="helper-buttons">
+				<DefaultHelperButtonsContent />
+				<GoToAgentButtons />
+			</TldrawUiMenuContextProvider>
+		</DefaultHelperButtons>
+	)
+}

templates/agent/client/components/GoToAgentButton.tsx

@@ -0,0 +1,108 @@
+import { useCallback } from 'react'
+import {
+	Box,
+	TldrawUiButton,
+	TldrawUiButtonIcon,
+	TldrawUiButtonLabel,
+	useEditor,
+	useValue,
+	Vec,
+} from 'tldraw'
+import { TldrawAgent } from '../agent/TldrawAgent'
+import { useAgents } from '../agent/TldrawAgentAppProvider'
+
+/**
+ * Renders GoToAgentButton for all agents.
+ */
+export function GoToAgentButtons() {
+	const agents = useAgents()
+
+	return (
+		<>
+			{agents.map((agent) => (
+				<GoToAgentButton key={agent.id} agent={agent} />
+			))}
+		</>
+	)
+}
+
+/**
+ * A button that zooms the viewport to show where an agent is working.
+ */
+export function GoToAgentButton({ agent }: { agent: TldrawAgent }) {
+	const editor = useEditor()
+	const currentRequest = useValue('activeRequest', () => agent.requests.getActiveRequest(), [agent])
+	const agentViewport = currentRequest?.bounds
+
+	// We only show the button if the agent is offscreen
+	const agentIsOffscreen = useValue(
+		'agentIsOffscreen',
+		() => {
+			if (!agentViewport) return false
+			const screenBounds = editor.getViewportScreenBounds()
+			const agentViewportScreenCorners = Box.From(agentViewport).corners.map((corner) =>
+				editor.pageToViewport(corner)
+			)
+
+			return !Box.FromPoints(agentViewportScreenCorners).collides(screenBounds)
+		},
+		[agentViewport]
+	)
+
+	// The button's arrow points towards the agent
+	const angleToAgent = useValue(
+		'angleToAgent',
+		() => {
+			if (!agentViewport) return
+			if (agentIsOffscreen) return
+
+			const agentCenter = Box.From(agentViewport).center
+			const agentScreenCenter = editor.pageToViewport(agentCenter)
+			const screenBounds = editor.getViewportScreenBounds()
+			const screenCenter = Box.From(screenBounds).center
+			const displacement = Vec.From(agentScreenCenter).sub(screenCenter)
+			return Math.atan2(displacement.y, displacement.x) * (180 / Math.PI)
+		},
+		[agentViewport]
+	)
+
+	const handleClick = useCallback(() => {
+		if (agentViewport) {
+			const bounds = Box.From(agentViewport).expandBy(50)
+			editor.zoomToBounds(bounds, { animation: { duration: 220 } })
+		}
+	}, [agentViewport, editor])
+
+	if (!agentViewport) return null
+	if (!agentIsOffscreen) return null
+
+	return (
+		<TldrawUiButton type="low" onClick={handleClick}>
+			<div style={{ display: 'flex', flexDirection: 'row', alignItems: 'center', gap: 8 }}>
+				<TldrawUiButtonIcon
+					icon={
+						<svg
+							width="16"
+							height="16"
+							viewBox="0 0 24 24"
+							style={{
+								transform: `rotate(${angleToAgent}deg)`,
+								transition: 'transform 0.2s ease',
+							}}
+						>
+							<path
+								d="M8 6l6 6-6 6"
+								stroke="currentColor"
+								strokeWidth="2"
+								strokeLinecap="round"
+								strokeLinejoin="round"
+								fill="none"
+							/>
+						</svg>
+					}
+				/>
+				<TldrawUiButtonLabel>Go to {agent.id}</TldrawUiButtonLabel>
+			</div>
+		</TldrawUiButton>
+	)
+}

templates/agent/client/components/PromptTag.tsx

@@ -0,0 +1,21 @@
+import { AgentIcon, AgentIconType } from '../../shared/icons/AgentIcon'
+
+export function PromptTag({
+	text,
+	icon,
+	onClick,
+}: {
+	text: string
+	icon: AgentIconType
+	onClick?: () => void
+}) {
+	return onClick ? (
+		<button type="button" className="prompt-tag" onClick={onClick}>
+			<AgentIcon type={icon} /> <span>{text}</span>
+		</button>
+	) : (
+		<div className="prompt-tag">
+			<AgentIcon type={icon} /> <span>{text}</span>
+		</div>
+	)
+}

templates/agent/client/components/SelectionTag.tsx

@@ -0,0 +1,5 @@
+import { PromptTag } from './PromptTag'
+
+export function SelectionTag({ onClick }: { onClick?: () => void }) {
+	return <PromptTag text="Selection" icon="cursor" onClick={onClick} />
+}

templates/agent/client/components/TodoList.tsx

@@ -0,0 +1,51 @@
+import { useCallback } from 'react'
+import { useValue } from 'tldraw'
+import { TodoItem } from '../../shared/types/TodoItem'
+import { TldrawAgent } from '../agent/TldrawAgent'
+
+export function TodoList({ agent }: { agent: TldrawAgent }) {
+	const todoItems = useValue('todoList', () => agent.todos.getTodos(), [agent])
+
+	if (todoItems.length === 0) {
+		return null
+	}
+
+	return (
+		<div className="todo-list">
+			<div className="todo-list-items">
+				{todoItems.map((item) => (
+					<TodoListItem key={item.id} item={item} agent={agent} />
+				))}
+			</div>
+		</div>
+	)
+}
+
+function TodoListItem({ item, agent }: { item: TodoItem; agent: TldrawAgent }) {
+	const deleteTodo = useCallback(() => {
+		agent.todos.delete([item.id])
+	}, [item.id, agent])
+
+	const getStatusIcon = (status: TodoItem['status']) => {
+		switch (status) {
+			case 'todo':
+				return '○'
+			case 'in-progress':
+				return '➤'
+			case 'done':
+				return '●'
+		}
+	}
+
+	const icon = getStatusIcon(item.status)
+
+	return (
+		<div className={`todo-item todo-item-${item.status}`}>
+			<span className="todo-item-icon">{icon}</span>
+			<span className="todo-item-text">{item.text}</span>
+			<button className="todo-item-delete" onClick={deleteTodo}>
+				×
+			</button>
+		</div>
+	)
+}

templates/agent/client/components/chat-history/ChatHistory.tsx

@@ -0,0 +1,90 @@
+import { useEffect, useRef } from 'react'
+import { useValue } from 'tldraw'
+import { TldrawAgent } from '../../agent/TldrawAgent'
+import { ChatHistorySection, getAgentHistorySections } from './ChatHistorySection'
+
+/*
+Chat history is stored as a list of history items.
+
+Within the UI, we split this list of items into sections.
+Each section contains a user prompt item, and all the agent's actions that follow it.
+
+Each section is rendered as a separate component.
+The model's actions are grouped together into collapsible groups if appropriate.
+
+
+Here's an example of how the UI might look:
+
+- Chat history
+	- Section
+		- Prompt
+		- Action group
+			- Action
+			- Action
+		- Action group
+			- Action
+	- Section
+		- Prompt
+		- Action group
+			- Action
+
+*/
+
+export function ChatHistory({ agent }: { agent: TldrawAgent }) {
+	const historyItems = useValue('chatHistory', () => agent.chat.getHistory(), [agent])
+	const sections = getAgentHistorySections(historyItems)
+	const historyRef = useRef<HTMLDivElement>(null)
+	const previousScrollDistanceFromBottomRef = useRef(0)
+
+	useEffect(() => {
+		if (!historyRef.current) return
+
+		// If a new prompt is submitted by the user, scroll to the bottom
+		if (historyItems.at(-1)?.type === 'prompt') {
+			if (previousScrollDistanceFromBottomRef.current <= 0) {
+				historyRef.current.scrollTo(0, historyRef.current.scrollHeight)
+				previousScrollDistanceFromBottomRef.current = 0
+			}
+			return
+		}
+
+		// If the user is scrolled to the bottom, keep them there while new actions appear
+		if (previousScrollDistanceFromBottomRef.current <= 0) {
+			const scrollDistanceFromBottom =
+				historyRef.current.scrollHeight -
+				historyRef.current.scrollTop -
+				historyRef.current.clientHeight
+
+			if (scrollDistanceFromBottom > 0) {
+				historyRef.current.scrollTo(0, historyRef.current.scrollHeight)
+			}
+		}
+	}, [historyRef, historyItems])
+
+	// Keep track of the user's scroll position
+	const handleScroll = () => {
+		if (!historyRef.current) return
+		const scrollDistanceFromBottom =
+			historyRef.current.scrollHeight -
+			historyRef.current.scrollTop -
+			historyRef.current.clientHeight
+
+		previousScrollDistanceFromBottomRef.current = scrollDistanceFromBottom
+	}
+
+	const isGenerating = useValue('isGenerating', () => agent.requests.isGenerating(), [agent])
+
+	return (
+		<div className="chat-history" ref={historyRef} onScroll={handleScroll}>
+			{sections.map((section, i) => {
+				return (
+					<ChatHistorySection
+						key={'history-section-' + i}
+						section={section}
+						loading={i === sections.length - 1 && isGenerating}
+					/>
+				)
+			})}
+		</div>
+	)
+}

templates/agent/client/components/chat-history/ChatHistoryGroup.tsx

@@ -0,0 +1,82 @@
+import { isRecordsDiffEmpty } from 'tldraw'
+import { ChatHistoryActionItem } from '../../../shared/types/ChatHistoryItem'
+import { TldrawAgent } from '../../agent/TldrawAgent'
+import { ChatHistoryGroupWithDiff } from './ChatHistoryGroupWithDiff'
+import { ChatHistoryGroupWithoutDiff } from './ChatHistoryGroupWithoutDiff'
+import { getActionInfo } from './getActionInfo'
+
+export interface ChatHistoryGroup {
+	items: ChatHistoryActionItem[]
+	withDiff: boolean
+}
+
+export function ChatHistoryGroup({ group }: { group: ChatHistoryGroup }) {
+	if (group.withDiff) {
+		return <ChatHistoryGroupWithDiff group={group} />
+	}
+
+	return <ChatHistoryGroupWithoutDiff group={group} />
+}
+
+/**
+ * Merge adjacent actions into groups where possible.
+ */
+export function getActionHistoryGroups(
+	items: ChatHistoryActionItem[],
+	agent: TldrawAgent
+): ChatHistoryGroup[] {
+	const groups: ChatHistoryGroup[] = []
+
+	for (const item of items) {
+		const { description } = getActionInfo(item.action, agent)
+		if (description === null) {
+			continue
+		}
+
+		const group = groups[groups.length - 1]
+		if (group && canActionBeGrouped({ item, group, agent })) {
+			group.items.push(item)
+		} else {
+			groups.push({
+				items: [item],
+				withDiff: !isRecordsDiffEmpty(item.diff) && item.action.complete,
+			})
+		}
+	}
+
+	return groups
+}
+
+/**
+ * Check if an action can be merged with a group.
+ */
+export function canActionBeGrouped({
+	item,
+	group,
+	agent,
+}: {
+	item: ChatHistoryActionItem
+	group: ChatHistoryGroup
+	agent: TldrawAgent
+}) {
+	if (!item.action.complete) return false
+	if (!group) return false
+
+	const showDiff = !isRecordsDiffEmpty(item.diff)
+	if (showDiff !== group.withDiff) return false
+
+	const groupAcceptance = group.items[0]?.acceptance
+	if (groupAcceptance !== item.acceptance) return false
+
+	const prevAction = group.items.at(-1)?.action
+	if (!prevAction) return false
+
+	const actionInfo = getActionInfo(item.action, agent)
+	const prevActionInfo = getActionInfo(prevAction, agent)
+
+	if (actionInfo.canGroup(prevAction) && prevActionInfo.canGroup(item.action)) {
+		return true
+	}
+
+	return false
+}

templates/agent/client/components/chat-history/ChatHistoryGroupWithDiff.tsx

@@ -0,0 +1,122 @@
+import { useCallback, useMemo } from 'react'
+import { reverseRecordsDiff, squashRecordDiffs } from 'tldraw'
+import { AgentIcon, AgentIconType } from '../../../shared/icons/AgentIcon'
+import { ChatHistoryActionItem } from '../../../shared/types/ChatHistoryItem'
+import { useAgent } from '../../agent/TldrawAgentAppProvider'
+import { ChatHistoryGroup } from './ChatHistoryGroup'
+import { getActionInfo } from './getActionInfo'
+import { TldrawDiffViewer } from './TldrawDiffViewer'
+
+export function ChatHistoryGroupWithDiff({ group }: { group: ChatHistoryGroup }) {
+	const agent = useAgent()
+	const { items } = group
+	const { editor } = agent
+	const diff = useMemo(() => squashRecordDiffs(items.map((item) => item.diff)), [items])
+
+	// Accept all changes from this group
+	const handleAccept = useCallback(() => {
+		agent.chat.update((currentChatHistoryItems) => {
+			const newItems = [...currentChatHistoryItems]
+			for (const item of items) {
+				const index = newItems.findIndex((v) => v === item)
+
+				// Mark the item as accepted
+				if (index !== -1) {
+					newItems[index] = { ...item, acceptance: 'accepted' }
+				}
+
+				// Apply the diff if needed
+				if (item.acceptance === 'rejected') {
+					editor.store.applyDiff(item.diff)
+				}
+			}
+			return newItems
+		})
+	}, [items, editor, agent.chat])
+
+	// Reject all changes from this group
+	const handleReject = useCallback(() => {
+		agent.chat.update((currentChatHistoryItems) => {
+			const newItems = [...currentChatHistoryItems]
+			for (const item of items) {
+				const index = newItems.findIndex((v) => v === item)
+
+				// Mark the item as rejected
+				if (index !== -1) {
+					newItems[index] = { ...item, acceptance: 'rejected' }
+				}
+
+				// Reverse the diff if needed
+				if (item.acceptance !== 'rejected') {
+					const reverseDiff = reverseRecordsDiff(item.diff)
+					editor.store.applyDiff(reverseDiff)
+				}
+			}
+			return newItems
+		})
+	}, [items, editor, agent.chat])
+
+	// Get the acceptance status of the group
+	// If all items are accepted, the group is accepted
+	// If all items are rejected, the group is rejected
+	// Otherwise, the group is pending
+	const acceptance = useMemo<ChatHistoryActionItem['acceptance']>(() => {
+		if (items.length === 0) return 'pending'
+		const acceptance = items[0].acceptance
+		for (let i = 1; i < items.length; i++) {
+			if (items[i].acceptance !== acceptance) {
+				return 'pending'
+			}
+		}
+		return acceptance
+	}, [items])
+
+	const steps = useMemo(
+		() => items.map((item) => getActionInfo(item.action, agent)),
+		[items, agent]
+	)
+
+	return (
+		<div className="chat-history-change">
+			<div className="chat-history-change-acceptance">
+				<button onClick={handleReject} disabled={acceptance === 'rejected'}>
+					{acceptance === 'rejected' ? 'Rejected' : 'Reject'}
+				</button>
+				<button onClick={handleAccept} disabled={acceptance === 'accepted'}>
+					{acceptance === 'accepted' ? 'Accepted' : 'Accept'}
+				</button>
+			</div>
+			<DiffSteps steps={steps} />
+			<TldrawDiffViewer diff={diff} />
+		</div>
+	)
+}
+
+interface DiffStep {
+	icon: AgentIconType | null
+	description: string | null
+}
+
+function DiffSteps({ steps }: { steps: DiffStep[] }) {
+	let previousDescription = ''
+	return (
+		<div className="agent-changes">
+			{steps.map((step, i) => {
+				if (!step.description) return null
+
+				if (step.description === previousDescription) return null
+				previousDescription = step.description
+				return (
+					<div className="agent-change" key={'intent-' + i}>
+						{step.icon && (
+							<span className="agent-change-icon">
+								<AgentIcon type={step.icon} />
+							</span>
+						)}
+						{step.description}
+					</div>
+				)
+			})}
+		</div>
+	)
+}

templates/agent/client/components/chat-history/ChatHistoryGroupWithoutDiff.tsx

@@ -0,0 +1,111 @@
+import { useMemo, useState } from 'react'
+import Markdown from 'react-markdown'
+import { AgentIcon } from '../../../shared/icons/AgentIcon'
+import { ChevronDownIcon } from '../../../shared/icons/ChevronDownIcon'
+import { ChevronRightIcon } from '../../../shared/icons/ChevronRightIcon'
+import { AgentAction } from '../../../shared/types/AgentAction'
+import { ChatHistoryActionItem } from '../../../shared/types/ChatHistoryItem'
+import { Streaming } from '../../../shared/types/Streaming'
+import { useAgent } from '../../agent/TldrawAgentAppProvider'
+import { ChatHistoryGroup } from './ChatHistoryGroup'
+import { getActionInfo } from './getActionInfo'
+
+export function ChatHistoryGroupWithoutDiff({ group }: { group: ChatHistoryGroup }) {
+	const agent = useAgent()
+	const { items } = group
+
+	const nonEmptyItems = useMemo(() => {
+		return items.filter((item) => {
+			const { description } = getActionInfo(item.action, agent)
+			return description !== null
+		})
+	}, [items, agent])
+
+	const [collapsed, setCollapsed] = useState(true)
+
+	const complete = useMemo(() => {
+		return items.every((item) => item.action.complete)
+	}, [items])
+
+	const summary = useMemo(() => {
+		const time = Math.floor(items.reduce((acc, item) => acc + item.action.time, 0) / 1000)
+		if (time === 0) return 'Thought for less than a second'
+		if (time === 1) return 'Thought for 1 second'
+		return `Thought for ${time} seconds`
+	}, [items])
+
+	if (nonEmptyItems.length === 0) {
+		return null
+	}
+
+	if (nonEmptyItems.length < 2) {
+		return (
+			<div className="chat-history-group">
+				{nonEmptyItems.map((item, i) => {
+					return <ChatHistoryItem item={item} key={'action-' + i} />
+				})}
+			</div>
+		)
+	}
+
+	const showContent = !collapsed || !complete
+
+	return (
+		<div className="chat-history-group">
+			{complete && (
+				<button onClick={() => setCollapsed((v) => !v)}>
+					<span>{showContent ? <ChevronDownIcon /> : <ChevronRightIcon />}</span>
+					{summary}
+				</button>
+			)}
+			{showContent && (
+				<div className="agent-actions-container">
+					{nonEmptyItems.map((item, i) => {
+						return <ChatHistoryItemExpanded action={item.action} key={'action-' + i} />
+					})}
+				</div>
+			)}
+		</div>
+	)
+}
+
+function ChatHistoryItem({ item }: { item: ChatHistoryActionItem }) {
+	const agent = useAgent()
+	const { action } = item
+	const { description, summary } = getActionInfo(action, agent)
+	const collapsible = summary !== null
+	const [collapsed, setCollapsed] = useState(collapsible)
+
+	if (!description) return null
+
+	return (
+		<div className="agent-actions-container">
+			{action.complete && collapsible && (
+				<button onClick={() => setCollapsed((v) => !v)}>
+					<span>{collapsed ? <ChevronRightIcon /> : <ChevronDownIcon />}</span>
+					{summary}
+				</button>
+			)}
+
+			{(!collapsed || !action.complete) && <ChatHistoryItemExpanded action={action} />}
+		</div>
+	)
+}
+
+function ChatHistoryItemExpanded({ action }: { action: Streaming<AgentAction> }) {
+	const agent = useAgent()
+	const { icon, description } = getActionInfo(action, agent)
+
+	return (
+		<div className={`agent-action agent-action-type-${action._type}`}>
+			{icon && (
+				<span>
+					<AgentIcon type={icon} />
+				</span>
+			)}
+			<span className="agent-action-description">
+				<Markdown>{description}</Markdown>
+			</span>
+		</div>
+	)
+}

templates/agent/client/components/chat-history/ChatHistoryPrompt.tsx

@@ -0,0 +1,38 @@
+import { Editor } from 'tldraw'
+import { ChatHistoryPromptItem } from '../../../shared/types/ChatHistoryItem'
+import { ContextItemTag } from '../ContextItemTag'
+import { SelectionTag } from '../SelectionTag'
+
+export function ChatHistoryPrompt({
+	item,
+	editor,
+}: {
+	item: ChatHistoryPromptItem
+	editor: Editor
+}) {
+	const { contextItems, agentFacingMessage, userFacingMessage, selectedShapes, promptSource } = item
+
+	const showTags = selectedShapes.length > 0 || contextItems.length > 0
+
+	// Display the user-facing message if available, otherwise fall back to the agent-facing message
+	const displayMessage = userFacingMessage ?? agentFacingMessage
+
+	// Get the CSS class modifier based on the prompt source
+	const sourceClass = `chat-history-prompt-${promptSource}`
+
+	return (
+		<div className="chat-history-prompt-container">
+			<div className={`chat-history-prompt ${sourceClass}`}>
+				{showTags && (
+					<div className="prompt-tags">
+						{selectedShapes.length > 0 && <SelectionTag />}
+						{contextItems.map((contextItem, i) => (
+							<ContextItemTag editor={editor} key={'context-item-' + i} item={contextItem} />
+						))}
+					</div>
+				)}
+				<span className="chat-history-prompt-content">{displayMessage}</span>
+			</div>
+		</div>
+	)
+}

templates/agent/client/components/chat-history/ChatHistorySection.tsx

@@ -0,0 +1,56 @@
+import { SmallSpinner } from '../../../shared/icons/SmallSpinner'
+import {
+	ChatHistoryActionItem,
+	ChatHistoryContinuationItem,
+	ChatHistoryItem,
+	ChatHistoryPromptItem,
+} from '../../../shared/types/ChatHistoryItem'
+import { useAgent } from '../../agent/TldrawAgentAppProvider'
+import { ChatHistoryGroup, getActionHistoryGroups } from './ChatHistoryGroup'
+import { ChatHistoryPrompt } from './ChatHistoryPrompt'
+
+export interface ChatHistorySection {
+	prompt: ChatHistoryPromptItem
+	items: (ChatHistoryActionItem | ChatHistoryContinuationItem)[]
+}
+
+export function ChatHistorySection({
+	section,
+	loading,
+}: {
+	section: ChatHistorySection
+	loading: boolean
+}) {
+	const agent = useAgent()
+	const actions = section.items.filter((item) => item.type === 'action')
+	const groups = getActionHistoryGroups(actions, agent)
+	return (
+		<div className="chat-history-section">
+			<ChatHistoryPrompt item={section.prompt} editor={agent.editor} />
+			{groups.map((group, i) => {
+				return <ChatHistoryGroup key={'chat-history-group-' + i} group={group} />
+			})}
+			{loading && <SmallSpinner />}
+		</div>
+	)
+}
+
+export function getAgentHistorySections(items: ChatHistoryItem[]): ChatHistorySection[] {
+	const sections: ChatHistorySection[] = []
+
+	for (const item of items) {
+		if (item.type === 'prompt') {
+			// Filter out 'self' prompts from the UI
+			if (item.promptSource === 'self') continue
+			sections.push({ prompt: item, items: [] })
+			continue
+		}
+
+		// Only add to the last section if one exists
+		if (sections.length > 0) {
+			sections[sections.length - 1].items.push(item)
+		}
+	}
+
+	return sections
+}

templates/agent/client/components/chat-history/TldrawDiffViewer.tsx

@@ -0,0 +1,135 @@
+import { forwardRef, useMemo } from 'react'
+import {
+	DefaultShapeWrapper,
+	RecordsDiff,
+	TLRecord,
+	TLShape,
+	TLShapeId,
+	TLShapeWrapperProps,
+} from 'tldraw'
+import { TldrawViewer } from './TldrawViewer'
+
+export function TldrawDiffViewer({ diff }: { diff: RecordsDiff<TLRecord> }) {
+	const diffShapes = useMemo(() => getDiffShapesFromDiff(diff), [diff])
+	return <TldrawViewer shapes={diffShapes} components={{ ShapeWrapper: DiffShapeWrapper }} />
+}
+
+function getDiffShapesFromDiff(diff: RecordsDiff<TLRecord>): TLShape[] {
+	const diffShapes: TLShape[] = []
+
+	const numberOfShapes =
+		Object.keys(diff.added).length +
+		Object.keys(diff.updated).length +
+		Object.keys(diff.removed).length
+
+	// If there are many shapes in the diff, don't show shadows (for performance reasons)
+	const showShadows = numberOfShapes < 20
+
+	for (const key in diff.removed) {
+		const id = key as TLShapeId
+		const prevShape = diff.removed[id]
+		if (prevShape.typeName !== 'shape') continue
+		const shape = {
+			...prevShape,
+			opacity: showShadows ? prevShape.opacity : prevShape.opacity / 2,
+			meta: { ...prevShape.meta, changeType: showShadows ? 'delete-shadow' : 'delete' },
+		}
+
+		if ('dash' in shape.props) {
+			shape.props = {
+				...shape.props,
+				dash: 'solid',
+			}
+		}
+
+		diffShapes.push(shape)
+	}
+
+	for (const key in diff.updated) {
+		const id = key as TLShapeId
+
+		const prevBefore = diff.updated[id][0]
+		const prevAfter = diff.updated[id][1]
+		if (prevBefore.typeName !== 'shape' || prevAfter.typeName !== 'shape') continue
+
+		const before = {
+			...prevBefore,
+			id: (id + '-before') as TLShapeId,
+			opacity: prevAfter.opacity / 2,
+			meta: {
+				...prevBefore.meta,
+				changeType: showShadows ? 'update-before-shadow' : 'update-before',
+			},
+		}
+
+		const after = {
+			...prevAfter,
+			meta: {
+				...prevAfter.meta,
+				changeType: showShadows ? 'update-after-shadow' : 'update-after',
+			},
+		}
+
+		if ('dash' in before.props) {
+			before.props = {
+				...before.props,
+				dash: 'dashed',
+			}
+		}
+		if ('fill' in before.props) {
+			before.props = {
+				...before.props,
+				fill: 'none',
+			}
+		}
+		if ('dash' in after.props) {
+			after.props = {
+				...after.props,
+				dash: 'solid',
+			}
+		}
+
+		diffShapes.push(before)
+		diffShapes.push(after)
+	}
+
+	for (const key in diff.added) {
+		const id = key as TLShapeId
+		const prevShape = diff.added[id]
+		if (prevShape.typeName !== 'shape') continue
+		const shape = {
+			...prevShape,
+			meta: {
+				...prevShape.meta,
+				changeType: showShadows ? 'create-shadow' : 'create',
+			},
+		}
+		if ('dash' in shape.props) {
+			shape.props = {
+				...shape.props,
+				dash: 'solid',
+			}
+		}
+		diffShapes.push(shape)
+	}
+
+	return diffShapes
+}
+
+const DiffShapeWrapper = forwardRef(function DiffShapeWrapper(
+	{ children, shape, isBackground }: TLShapeWrapperProps,
+	ref: React.Ref<HTMLDivElement>
+) {
+	const changeType = shape.meta.changeType
+
+	return (
+		<DefaultShapeWrapper
+			ref={ref}
+			shape={shape}
+			isBackground={isBackground}
+			className={changeType ? 'diff-shape-' + changeType : undefined}
+		>
+			{children}
+		</DefaultShapeWrapper>
+	)
+})

templates/agent/client/components/chat-history/TldrawViewer.tsx

@@ -0,0 +1,96 @@
+import { useEffect, useRef, useState } from 'react'
+import {
+	defaultAddFontsFromNode,
+	defaultBindingUtils,
+	defaultEditorAssetUrls,
+	defaultShapeUtils,
+	Editor,
+	StateNode,
+	tipTapDefaultExtensions,
+	TLComponents,
+	TldrawEditor,
+	TldrawUiContextProvider,
+	TLShape,
+} from 'tldraw'
+
+export function TldrawViewer({
+	shapes,
+	components = {},
+}: {
+	shapes: TLShape[]
+	components?: TLComponents
+}) {
+	const [editor, setEditor] = useState<Editor | null>(null)
+	const [isVisible, setIsVisible] = useState(false)
+	const containerRef = useRef<HTMLDivElement>(null)
+
+	// Hide the component if it's outside of the current scroll area
+	useEffect(() => {
+		const observer = new IntersectionObserver(
+			([entry]) => {
+				setIsVisible(entry.isIntersecting)
+			},
+			{ rootMargin: '200px', threshold: 0 }
+		)
+
+		const currentElement = containerRef.current
+		if (currentElement) observer.observe(currentElement)
+
+		return () => {
+			if (currentElement) observer.unobserve(currentElement)
+		}
+	}, [])
+
+	useEffect(() => {
+		if (!editor || !isVisible) return
+		editor.updateInstanceState({ isReadonly: false })
+		editor.setCameraOptions({ isLocked: false })
+		editor.deleteShapes(editor.getCurrentPageShapes())
+		editor.createShapes(shapes)
+		editor.updateInstanceState({ isReadonly: true })
+		editor.selectAll()
+		const bounds = editor.getSelectionPageBounds()
+		if (bounds) {
+			editor.zoomToBounds(bounds, { inset: 30 })
+		}
+		editor.selectNone()
+		editor.setCameraOptions({ isLocked: true })
+	}, [shapes, editor, isVisible])
+
+	if (!isVisible) {
+		return <div ref={containerRef} className="tldraw-viewer" />
+	}
+
+	return (
+		<div ref={containerRef} className="tldraw-viewer">
+			<TldrawUiContextProvider>
+				<TldrawEditor
+					autoFocus={false}
+					components={components ?? {}}
+					onMount={setEditor}
+					shapeUtils={defaultShapeUtils}
+					bindingUtils={defaultBindingUtils}
+					tools={tools}
+					options={defaultOptions}
+					assetUrls={defaultEditorAssetUrls}
+					initialState="inspect"
+				/>
+			</TldrawUiContextProvider>
+		</div>
+	)
+}
+
+class InspectTool extends StateNode {
+	static override id = 'inspect'
+}
+
+const tools = [InspectTool]
+
+const defaultOptions = {
+	text: {
+		tipTapConfig: {
+			extensions: tipTapDefaultExtensions,
+		},
+		addFontsFromNode: defaultAddFontsFromNode,
+	},
+}

templates/agent/client/components/chat-history/getActionInfo.ts

@@ -0,0 +1,27 @@
+import { AgentAction } from '../../../shared/types/AgentAction'
+import { ChatHistoryInfo } from '../../../shared/types/ChatHistoryInfo'
+import { Streaming } from '../../../shared/types/Streaming'
+import { TldrawAgent } from '../../agent/TldrawAgent'
+
+/**
+ * Get the full info for an action to display in chat history UI.
+ * This function adds default values for any unset properties.
+ * If the action's util returns null, the action will not be shown in chat history.
+ */
+export function getActionInfo(action: Streaming<AgentAction>, agent: TldrawAgent): ChatHistoryInfo {
+	const util = agent.actions.getAgentActionUtil(action._type)
+	const info = util.getInfo(action) ?? { description: null }
+	const {
+		icon = null,
+		description = JSON.stringify(action, null, 2),
+		summary = null,
+		canGroup = () => true,
+	} = info
+
+	return {
+		icon,
+		description,
+		summary,
+		canGroup,
+	}
+}