snapshot of https://github.com/tldraw/tldraw.git (main, 2026-09-23; blobs > 131072B omitted)

apps/docs/scripts/check-broken-links.ts

@@ -0,0 +1,16 @@
+import { nicelog } from '@/utils/nicelog'
+import { checkBrokenLinks } from './lib/checkBrokenLinks'
+;(async function () {
+	const strict = process.argv.includes('--fail')
+	nicelog('• Checking broken links...')
+	const brokenCount = await checkBrokenLinks()
+	if (brokenCount > 0) {
+		if (strict) {
+			process.exit(1)
+		}
+		nicelog('⚠ Run with --fail to treat broken links as errors.')
+	} else {
+		nicelog('✔ No broken links found!')
+	}
+	process.exit()
+})()

apps/docs/scripts/check-external-links.ts

@@ -0,0 +1,10 @@
+import { nicelog } from '@/utils/nicelog'
+import { checkExternalLinks } from './lib/checkExternalLinks'
+;(async function () {
+	nicelog('• Checking external links...')
+	const brokenCount = await checkExternalLinks()
+	if (brokenCount > 0) {
+		nicelog(`⚠ ${brokenCount} broken external link${brokenCount === 1 ? '' : 's'} found.`)
+	}
+	process.exit(brokenCount > 0 ? 1 : 0)
+})()

apps/docs/scripts/create-api-markdown.ts

@@ -0,0 +1,8 @@
+import { nicelog } from '@/utils/nicelog'
+import { createApiMarkdown } from './lib/createApiMarkdown'
+;(async function () {
+	nicelog('• Generating markdown files for API docs...')
+	await createApiMarkdown()
+	nicelog('✔ Complete!')
+	process.exit()
+})()

apps/docs/scripts/create-llms-txt.ts

@@ -0,0 +1,11 @@
+import { nicelog } from '@/utils/nicelog'
+import { connect } from './lib/connect'
+import { generateLlmsTxt } from './lib/generateLllmsTxt'
+;(async function () {
+	const db = await connect({ mode: 'readonly' })
+
+	nicelog('• Generating llms.txt...')
+	await generateLlmsTxt(db)
+	nicelog('✔ Complete!')
+	process.exit()
+})()

apps/docs/scripts/fetch-api-source.ts

@@ -0,0 +1,8 @@
+import { nicelog } from '@/utils/nicelog'
+import { fetchApiSource } from './lib/fetchApiSource'
+;(async function () {
+	nicelog('• Fetching API source files...')
+	await fetchApiSource()
+	nicelog('✔ Complete!')
+	process.exit()
+})()

apps/docs/scripts/lib/checkBrokenLinks.ts

@@ -0,0 +1,312 @@
+import { nicelog } from '@/utils/nicelog'
+import { connect } from './connect'
+
+export interface BrokenLink {
+	articlePath: string
+	line: number
+	url: string
+	reason: string
+}
+
+// Internal rewrites: source → destination (both are valid paths)
+const INTERNAL_REWRITES: Record<string, string> = {
+	'/releases': '/getting-started/releases',
+	'/quick-start': '/getting-started/quick-start',
+	'/installation': '/getting-started/installation',
+}
+
+// Section and category paths are not pages: only article paths are generated
+// (dynamicParams = false) and page.tsx 404s for anything else. These section roots are
+// reachable only because next.config.js redirects them to an article.
+const REDIRECTED_SECTION_PATHS = ['/docs', '/examples', '/reference', '/starter-kits']
+
+// Path prefixes that are served by another system (e.g. the marketing site)
+// rather than this Next.js app, so they will never appear in the docs DB.
+// Matches the prefix as an exact path OR as a parent of a subpath:
+// e.g. '/legal' matches '/legal' and '/legal/tldraw-license' but not '/legal-foo'.
+const EXTERNAL_ROUTE_PREFIXES = [
+	'/accessibility',
+	'/blog',
+	'/careers',
+	'/company',
+	'/events',
+	'/faq',
+	'/features',
+	'/get-a-license',
+	'/legal',
+	'/partner',
+	'/pricing',
+	'/showcase',
+	'/thanks',
+]
+
+function isExternalRoute(urlPath: string): boolean {
+	return EXTERNAL_ROUTE_PREFIXES.some(
+		(prefix) => urlPath === prefix || urlPath.startsWith(prefix + '/')
+	)
+}
+
+/**
+ * Strip fenced code blocks from markdown, replacing them with blank lines
+ * to preserve line numbering for error reporting.
+ */
+export function stripCodeBlocks(content: string): string {
+	return content.replace(/^(`{3,})[^\n]*\n[\s\S]*?^\1\s*$/gm, (match) => {
+		// Replace with same number of newlines to preserve line count
+		return '\n'.repeat((match.match(/\n/g) || []).length)
+	})
+}
+
+// File extensions that indicate static assets (not page routes)
+const STATIC_ASSET_EXTENSIONS =
+	/\.(png|jpe?g|gif|svg|webp|ico|pdf|mp4|webm|mp3|woff2?|ttf|eot|css|js|json|xml|txt|zip|tar|gz)$/i
+
+/**
+ * Extract all links from markdown/MDX content.
+ * Returns array of { url, line } for every link found (internal and external).
+ */
+export function extractLinks(content: string): Array<{ url: string; line: number }> {
+	const stripped = stripCodeBlocks(content)
+	const lines = stripped.split('\n')
+	const links: Array<{ url: string; line: number }> = []
+
+	for (let i = 0; i < lines.length; i++) {
+		const line = lines[i]
+		const lineNum = i + 1
+
+		// Standard markdown links: [text](url)
+		const mdLinkRegex = /\[[^\]]*\]\(([^)]*)\)/g
+		let match
+		while ((match = mdLinkRegex.exec(line)) !== null) {
+			const url = match[1].trim()
+			if (url) links.push({ url, line: lineNum })
+		}
+
+		// JSX href attributes: href="url"
+		const hrefRegex = /href="([^"]*)"/g
+		while ((match = hrefRegex.exec(line)) !== null) {
+			const url = match[1].trim()
+			if (url) links.push({ url, line: lineNum })
+		}
+
+		// CodeLinks JSON values: <CodeLinks links={{"Name":"/path"}}>
+		const codeLinksRegex = /<CodeLinks\s+links=\{\{([^}]+)\}\}/g
+		while ((match = codeLinksRegex.exec(line)) !== null) {
+			try {
+				const json = JSON.parse(`{${match[1]}}`)
+				for (const value of Object.values(json)) {
+					if (typeof value === 'string' && value) {
+						links.push({ url: value, line: lineNum })
+					}
+				}
+			} catch {
+				// Skip malformed JSON
+			}
+		}
+	}
+
+	return links
+}
+
+/**
+ * Extract internal links from markdown/MDX content.
+ * Returns array of { url, line } for each internal link found.
+ */
+function extractInternalLinks(content: string): Array<{ url: string; line: number }> {
+	return extractLinks(content).filter(({ url }) => url.startsWith('/'))
+}
+
+/**
+ * Try to resolve a path via redirect patterns from next.config.js.
+ * Returns the resolved destination path, or null if no redirect matches.
+ */
+function tryRedirect(urlPath: string): string | null {
+	// /examples/:cat/:id → /examples/:id
+	const exTwoParts = urlPath.match(/^\/examples\/[^/]+\/([^/]+)$/)
+	if (exTwoParts) return `/examples/${exTwoParts[1]}`
+
+	// /examples/:cat1/:cat2/:id → /examples/:id
+	const exThreeParts = urlPath.match(/^\/examples\/[^/]+\/[^/]+\/([^/]+)$/)
+	if (exThreeParts) return `/examples/${exThreeParts[1]}`
+
+	// /getting-started/:child → /:child
+	const gsMatch = urlPath.match(/^\/getting-started\/(.+)$/)
+	if (gsMatch) return `/${gsMatch[1]}`
+
+	return null
+}
+
+function anchorNotFoundReason(fragment: string, path: string, slugs: Set<string> | undefined) {
+	const reason = `anchor #${fragment} not found in ${path}`
+	const caseMismatch = slugs && [...slugs].find((s) => s.toLowerCase() === fragment.toLowerCase())
+	return caseMismatch ? `${reason} (heading ids are case-sensitive, use #${caseMismatch})` : reason
+}
+
+export async function checkBrokenLinks(): Promise<number> {
+	const db = await connect({ mode: 'readonly' })
+
+	// Build set of all valid paths
+	const validPaths = new Set<string>()
+
+	const articles = await db.all<{ path: string | null; content: string }[]>(
+		'SELECT path, content FROM articles'
+	)
+
+	for (const row of articles) {
+		if (row.path) validPaths.add(row.path)
+	}
+	for (const path of REDIRECTED_SECTION_PATHS) {
+		validPaths.add(path)
+	}
+
+	// Add internal rewrite sources (these are valid URLs that rewrite to DB paths)
+	for (const source of Object.keys(INTERNAL_REWRITES)) {
+		validPaths.add(source)
+	}
+
+	// Case-insensitive path set for fallback matching (handles macOS FS collisions)
+	const validPathsLower = new Set([...validPaths].map((p) => p.toLowerCase()))
+
+	// Build heading lookup: articlePath → Set<slug>
+	const headingMap = new Map<string, Set<string>>()
+	const headings = await db.all<{ slug: string; articleId: string }[]>(
+		'SELECT slug, articleId FROM headings'
+	)
+
+	// We need article paths keyed by article ID
+	const articlePathById = new Map<string, string>()
+	const allArticles = await db.all<{ id: string; path: string | null }[]>(
+		'SELECT id, path FROM articles'
+	)
+	for (const a of allArticles) {
+		if (a.path) articlePathById.set(a.id, a.path)
+	}
+
+	// Rendered heading ids keep their case (rehype-slug runs with `maintainCase: true`), so
+	// `#custom-shapes` is a dead anchor on a page whose heading id is `Custom-shapes` (#10256).
+	for (const h of headings) {
+		const articlePath = articlePathById.get(h.articleId)
+		if (!articlePath) continue
+		let slugs = headingMap.get(articlePath)
+		if (!slugs) {
+			slugs = new Set()
+			headingMap.set(articlePath, slugs)
+		}
+		slugs.add(h.slug)
+		// Also index by lowercase path for case-insensitive path lookups
+		const lowerPath = articlePath.toLowerCase()
+		let lowerSlugs = headingMap.get(lowerPath)
+		if (!lowerSlugs) {
+			lowerSlugs = new Set()
+			headingMap.set(lowerPath, lowerSlugs)
+		}
+		lowerSlugs.add(h.slug)
+	}
+
+	// Also build heading map for rewrite destinations
+	for (const [source, dest] of Object.entries(INTERNAL_REWRITES)) {
+		const slugs = headingMap.get(dest)
+		if (slugs) {
+			headingMap.set(source, slugs)
+		}
+	}
+
+	// Check all articles for broken links
+	const broken: BrokenLink[] = []
+
+	for (const article of articles) {
+		if (!article.path) continue
+		const links = extractInternalLinks(article.content)
+
+		for (const { url, line } of links) {
+			// Skip static asset paths (images, fonts, etc.)
+			if (STATIC_ASSET_EXTENSIONS.test(url)) continue
+
+			// Split path and fragment
+			const hashIdx = url.indexOf('#')
+			const urlPath = hashIdx >= 0 ? url.slice(0, hashIdx) : url
+			const fragment = hashIdx >= 0 ? url.slice(hashIdx + 1) : null
+
+			// Skip paths served by an external system (marketing site etc.)
+			// — they will never appear in the docs DB but are still valid in production.
+			if (isExternalRoute(urlPath)) continue
+
+			// Empty path with fragment = same-page anchor
+			if (urlPath === '' && fragment) {
+				const slugs = headingMap.get(article.path)
+				if (!slugs || !slugs.has(fragment)) {
+					broken.push({
+						articlePath: article.path,
+						line,
+						url,
+						reason: anchorNotFoundReason(fragment, article.path, slugs),
+					})
+				}
+				continue
+			}
+
+			// Check if path is valid
+			let pathValid = false
+			let resolvedPath = urlPath
+
+			if (validPaths.has(urlPath)) {
+				pathValid = true
+			} else if (validPathsLower.has(urlPath.toLowerCase())) {
+				// Case-insensitive fallback (handles macOS FS collisions
+				// where e.g. Atom.mdx and atom.mdx map to the same file)
+				pathValid = true
+			} else {
+				// Try redirect resolution
+				const redirected = tryRedirect(urlPath)
+				if (
+					redirected &&
+					(validPaths.has(redirected) || validPathsLower.has(redirected.toLowerCase()))
+				) {
+					pathValid = true
+					resolvedPath = redirected
+				}
+			}
+
+			if (!pathValid) {
+				broken.push({
+					articlePath: article.path,
+					line,
+					url,
+					reason: 'page not found',
+				})
+				continue
+			}
+
+			// Check fragment if present
+			if (fragment) {
+				// Resolve internal rewrites for heading lookup
+				const lookupPath = INTERNAL_REWRITES[resolvedPath] || resolvedPath
+				const slugs = headingMap.get(lookupPath) || headingMap.get(lookupPath.toLowerCase())
+				if (!slugs || !slugs.has(fragment)) {
+					broken.push({
+						articlePath: article.path,
+						line,
+						url,
+						reason: anchorNotFoundReason(fragment, resolvedPath, slugs),
+					})
+				}
+			}
+		}
+	}
+
+	if (broken.length > 0) {
+		nicelog(
+			`\n✗ Found ${broken.length} broken link${broken.length === 1 ? '' : 's'}:\n\n` +
+				broken
+					.map(
+						(b, i) =>
+							`${i + 1}.\t${b.url}\n\tIn: ${b.articlePath} (line ${b.line})\n\tReason: ${b.reason}`
+					)
+					.join('\n\n') +
+				'\n'
+		)
+	}
+
+	await db.close()
+	return broken.length
+}

apps/docs/scripts/lib/checkExternalLinks.ts

@@ -0,0 +1,165 @@
+import { nicelog } from '@/utils/nicelog'
+import { BrokenLink, extractLinks } from './checkBrokenLinks'
+import { connect } from './connect'
+
+const CONCURRENCY = 10
+const TIMEOUT_MS = 30_000
+const MAX_RETRIES = 2
+
+// Domains that block automated requests (403/captcha regardless of method)
+const SKIP_DOMAINS = ['www.npmjs.com', 'npmjs.com', 'www.shadertoy.com', 'shadertoy.com']
+
+function shouldSkipUrl(url: string): boolean {
+	if (url.includes('{')) return true
+	try {
+		const host = new URL(url).hostname
+		if (host === 'example.com' || host.endsWith('.example.com')) return true
+		if (SKIP_DOMAINS.includes(host)) return true
+	} catch {
+		// skip malformed URLs
+	}
+	return false
+}
+
+function isTransientError(result: { ok: boolean; status?: number; error?: string }): boolean {
+	if (result.ok) return false
+	// Retry on network errors (timeout, connection reset, etc.)
+	if (result.error) return true
+	// Retry on server errors and rate limits
+	if (result.status && (result.status >= 500 || result.status === 429)) return true
+	return false
+}
+
+async function checkUrlOnce(
+	url: string
+): Promise<{ ok: boolean; status?: number; error?: string }> {
+	const controller = new AbortController()
+	const timer = setTimeout(() => controller.abort(), TIMEOUT_MS)
+
+	try {
+		// Try HEAD first (faster, less bandwidth)
+		let res = await fetch(url, {
+			method: 'HEAD',
+			signal: controller.signal,
+			redirect: 'follow',
+			headers: { 'User-Agent': 'tldraw-docs-link-checker' },
+		})
+
+		// Some servers reject HEAD — fall back to GET
+		if (res.status === 405 || res.status === 403) {
+			res = await fetch(url, {
+				method: 'GET',
+				signal: controller.signal,
+				redirect: 'follow',
+				headers: { 'User-Agent': 'tldraw-docs-link-checker' },
+			})
+		}
+
+		if (res.ok) return { ok: true }
+		return { ok: false, status: res.status }
+	} catch (err: unknown) {
+		const message = err instanceof Error ? err.message : String(err)
+		return { ok: false, error: message }
+	} finally {
+		clearTimeout(timer)
+	}
+}
+
+async function checkUrl(url: string): Promise<{ ok: boolean; status?: number; error?: string }> {
+	let result = await checkUrlOnce(url)
+	for (let attempt = 0; attempt < MAX_RETRIES && isTransientError(result); attempt++) {
+		// Back off briefly before retrying
+		await new Promise((r) => setTimeout(r, 1000 * (attempt + 1)))
+		result = await checkUrlOnce(url)
+	}
+	return result
+}
+
+export async function checkExternalLinks(): Promise<number> {
+	const db = await connect({ mode: 'readonly' })
+
+	const articles = await db.all<{ path: string | null; content: string }[]>(
+		'SELECT path, content FROM articles'
+	)
+
+	// Collect all external link occurrences, deduplicating by URL
+	const urlOccurrences = new Map<string, Array<{ articlePath: string; line: number }>>()
+
+	for (const article of articles) {
+		if (!article.path) continue
+		const links = extractLinks(article.content)
+
+		for (const { url, line } of links) {
+			if (!url.startsWith('http://') && !url.startsWith('https://')) continue
+			if (shouldSkipUrl(url)) continue
+
+			// Strip fragment for checking (servers don't validate fragments)
+			const hashIdx = url.indexOf('#')
+			const urlWithoutFragment = hashIdx >= 0 ? url.slice(0, hashIdx) : url
+
+			let list = urlOccurrences.get(urlWithoutFragment)
+			if (!list) {
+				list = []
+				urlOccurrences.set(urlWithoutFragment, list)
+			}
+			list.push({ articlePath: article.path, line })
+		}
+	}
+
+	await db.close()
+
+	const uniqueUrls = [...urlOccurrences.keys()]
+	nicelog(
+		`Checking ${uniqueUrls.length} unique external URLs across ${articles.length} articles...`
+	)
+
+	// Check URLs with bounded concurrency
+	const results = new Map<string, { ok: boolean; status?: number; error?: string }>()
+	let checked = 0
+
+	async function worker(urls: string[]) {
+		for (const url of urls) {
+			results.set(url, await checkUrl(url))
+			checked++
+			if (checked % 50 === 0) {
+				nicelog(`  ...checked ${checked}/${uniqueUrls.length}`)
+			}
+		}
+	}
+
+	// Partition URLs across workers
+	const chunks: string[][] = Array.from({ length: CONCURRENCY }, () => [])
+	for (let i = 0; i < uniqueUrls.length; i++) {
+		chunks[i % CONCURRENCY].push(uniqueUrls[i])
+	}
+	await Promise.all(chunks.map(worker))
+
+	// Collect broken links
+	const broken: BrokenLink[] = []
+
+	for (const [url, result] of results) {
+		if (result.ok) continue
+		const reason = result.status ? `HTTP ${result.status}` : (result.error ?? 'unknown error')
+		const occurrences = urlOccurrences.get(url)!
+		for (const { articlePath, line } of occurrences) {
+			broken.push({ articlePath, line, url, reason })
+		}
+	}
+
+	if (broken.length > 0) {
+		nicelog(
+			`\n✗ Found ${broken.length} broken external link${broken.length === 1 ? '' : 's'}:\n\n` +
+				broken
+					.map(
+						(b, i) =>
+							`${i + 1}.\t${b.url}\n\tIn: ${b.articlePath} (line ${b.line})\n\tReason: ${b.reason}`
+					)
+					.join('\n\n') +
+				'\n'
+		)
+	} else {
+		nicelog('\n✔ All external links are valid!')
+	}
+
+	return broken.length
+}

apps/docs/scripts/lib/connect.ts

@@ -0,0 +1,92 @@
+import path from 'path'
+import { open } from 'sqlite'
+import sqlite3 from 'sqlite3'
+
+export async function connect(opts: { reset?: boolean; mode: 'readonly' | 'readwrite' }) {
+	const db = await open({
+		filename: path.join(process.cwd(), 'content.db'),
+		driver: sqlite3.Database,
+		mode:
+			opts.mode === 'readonly'
+				? sqlite3.OPEN_READONLY
+				: sqlite3.OPEN_CREATE | sqlite3.OPEN_READWRITE,
+	})
+
+	if (opts.reset) {
+		// Create the sections table if it does not exist
+
+		await db.run(`DROP TABLE IF EXISTS sections`)
+		await db.run(`CREATE TABLE sections (
+			id TEXT PRIMARY KEY,
+			idx INTEGER NOT NULL,
+			title TEXT NOT NULL,
+			path TEXT NOT NULL,
+			description TEXT,
+			sidebar_behavior TEXT
+		)`)
+
+		// Create the categories table if it does not exist
+
+		await db.run(`DROP TABLE IF EXISTS categories`)
+		await db.run(`CREATE TABLE IF NOT EXISTS categories (
+			id TEXT PRIMARY KEY,
+			title TEXT NOT NULL,
+			description TEXT,
+			sectionId TEXT NOT NULL,
+			sectionIndex INTEGER NOT NULL,
+			path TEXT,
+			FOREIGN KEY (id) REFERENCES sections(id)
+		)`)
+
+		// Create the articles table if it does not exist
+
+		// drop the table if it exists
+
+		await db.run(`DROP TABLE IF EXISTS articles`)
+		await db.run(`CREATE TABLE IF NOT EXISTS articles (
+			id TEXT PRIMARY KEY,
+			groupIndex INTEGER NOT NULL,
+			categoryIndex INTEGER NOT NULL,
+			priority INTEGER,
+			sectionIndex INTEGER NOT NULL,
+			groupId TEXT,
+			categoryId TEXT NOT NULL,
+			sectionId TEXT NOT NULL,
+			authorId TEXT NOT NULL,
+			title TEXT NOT NULL,
+			sidebarTitle TEXT,
+			description TEXT,
+			hero TEXT,
+			thumbnail TEXT,
+			socialImage TEXT,
+			status TEXT NOT NULL,
+			date TEXT,
+			sourceUrl TEXT,
+			componentCode TEXT,
+			componentCodeFilename TEXT,
+			componentCodeFiles TEXT,
+			keywords TEXT,
+			apiTags TEXT,
+			content TEXT NOT NULL,
+			path TEXT,
+			githubLink TEXT,
+			embed TEXT,
+			FOREIGN KEY (sectionId) REFERENCES sections(id),
+			FOREIGN KEY (categoryId) REFERENCES categories(id)
+		)`)
+
+		await db.run(`DROP TABLE IF EXISTS headings`)
+		await db.run(`CREATE TABLE IF NOT EXISTS headings (
+			id INTEGER PRIMARY KEY AUTOINCREMENT,
+			idx INTEGER NOT NULL,
+			articleId TEXT NOT NULL,
+			level INTEGER NOT NULL,
+			title TEXT NOT NULL,
+			slug TEXT NOT NULL,
+			path TEXT NOT NULL,
+			FOREIGN KEY (articleId) REFERENCES articles(id)
+		)`)
+	}
+
+	return db
+}

apps/docs/scripts/lib/createApiMarkdown.ts

@@ -0,0 +1,98 @@
+import fs from 'fs'
+import path from 'path'
+import { APIGroup, InputSection } from '@/types/content-types'
+import { nicelog } from '@/utils/nicelog'
+import { TldrawApiModel } from '@/utils/TldrawApiModel'
+import { getApiMarkdown } from './getApiMarkdown'
+import { CONTENT_DIR, getSlug } from './utils'
+
+export async function createApiMarkdown() {
+	const apiInputSection: InputSection = {
+		id: 'reference' as string,
+		title: 'API Reference',
+		description: "Reference for the tldraw package's APIs (generated).",
+		categories: [],
+		sidebar_behavior: 'reference',
+		hero: null,
+	}
+
+	const addedCategories = new Set<string>()
+
+	const INPUT_DIR = path.join(process.cwd(), 'api')
+	const OUTPUT_DIR = path.join(CONTENT_DIR, 'reference')
+
+	if (fs.existsSync(OUTPUT_DIR)) {
+		fs.rmSync(OUTPUT_DIR, { recursive: true })
+	}
+
+	fs.mkdirSync(OUTPUT_DIR)
+
+	const model = new TldrawApiModel()
+	let packageModels = []
+
+	// get all files in the INPUT_DIR
+	const files = fs.readdirSync(INPUT_DIR)
+	for (const file of files) {
+		// get the file path
+		const filePath = path.join(INPUT_DIR, file)
+
+		// parse the file
+		const apiModel = model.loadPackage(filePath)
+
+		// add the parsed file to the packageModels array
+		packageModels.push(apiModel)
+	}
+
+	// Manually keep 'tldraw' package in same place as it was before the alphabetical sort
+	// for continuity and to prevent scroll issues when initially opening the API reference
+	packageModels = packageModels.sort((a, b) => {
+		const aName = a.name === 'tldraw' ? '@tldraw/tldraw' : a.name
+		const bName = b.name === 'tldraw' ? '@tldraw/tldraw' : b.name
+		return aName.localeCompare(bName)
+	})
+
+	await model.preprocessReactComponents()
+
+	for (const packageModel of packageModels) {
+		const categoryName = packageModel.name.replace(`@tldraw/`, '')
+
+		if (!addedCategories.has(categoryName)) {
+			apiInputSection.categories!.push({
+				id: categoryName,
+				title: packageModel.name,
+				description: '',
+				groups: Object.values(APIGroup).map((title) => ({
+					id: title,
+					path: null,
+				})),
+				hero: null,
+			})
+			addedCategories.add(categoryName)
+		}
+
+		const entrypoint = packageModel.entryPoints[0]
+
+		for (let j = 0; j < entrypoint.members.length; j++) {
+			const item = entrypoint.members[j]
+
+			const outputFileName = `${getSlug(item)}.mdx`
+
+			const result = await getApiMarkdown(model, categoryName, item, j)
+			nicelog(`✎ ${outputFileName}`)
+			fs.writeFileSync(path.join(OUTPUT_DIR, outputFileName), result.markdown)
+		}
+	}
+
+	// Add the API section to the sections.json file
+
+	const sectionsJsonPath = path.join(CONTENT_DIR, 'sections.json')
+	const sectionsJson = JSON.parse(fs.readFileSync(sectionsJsonPath, 'utf8')) as InputSection[]
+	sectionsJson.splice(
+		sectionsJson.findIndex((s) => s.id === 'reference'),
+		1
+	)
+	sectionsJson.push(apiInputSection)
+	fs.writeFileSync(sectionsJsonPath, JSON.stringify(sectionsJson, null, '\t') + '\n')
+
+	model.throwEncounteredErrors()
+}

apps/docs/scripts/lib/fetchApiSource.ts

@@ -0,0 +1,30 @@
+import fs from 'fs'
+import path from 'path'
+import { TLDRAW_PACKAGES_TO_INCLUDE_IN_DOCS } from './package-list'
+
+const { log: nicelog } = console
+
+export async function fetchApiSource() {
+	try {
+		const API_DIRECTORY = path.join(process.cwd(), 'api')
+		const REPO_ROOT = path.normalize(path.join(process.cwd(), '../../'))
+
+		if (fs.existsSync(API_DIRECTORY)) {
+			fs.rmSync(API_DIRECTORY, { recursive: true })
+		}
+
+		fs.mkdirSync(API_DIRECTORY)
+
+		for (const folderName of TLDRAW_PACKAGES_TO_INCLUDE_IN_DOCS) {
+			const fromPath = path.join(REPO_ROOT, 'packages', folderName, 'api', 'api.json')
+			const toPath = path.join(API_DIRECTORY, folderName + '.api.json')
+			fs.copyFileSync(fromPath, toPath)
+		}
+
+		nicelog('✔ Complete!')
+	} catch (error) {
+		nicelog(`x Could not generate site content.`)
+
+		throw error
+	}
+}

apps/docs/scripts/lib/generateApiContent.ts

@@ -0,0 +1,24 @@
+import path from 'path'
+import { Articles, GeneratedContent, InputSection } from '../../types/content-types'
+import { generateSection } from './generateSection'
+
+const { log: nicelog } = console
+
+export async function generateApiContent(): Promise<GeneratedContent> {
+	const articles: Articles = {}
+	const CONTENT_DIRECTORY = path.join(process.cwd(), 'content')
+	const sections = require(path.join(CONTENT_DIRECTORY, 'sections.json')) as InputSection[]
+
+	try {
+		const inputApiSection = sections.find((s) => s.id === 'reference')
+		if (!inputApiSection) throw new Error(`Could not find section with id 'reference'`)
+		const outputApiSection = generateSection(inputApiSection, articles, 999999) // always at the end!
+		const contentComplete = { sections: [outputApiSection], articles }
+
+		return contentComplete
+	} catch (error) {
+		nicelog(`x Could not generate API content`)
+
+		throw error
+	}
+}

apps/docs/scripts/lib/generateContent.ts

@@ -0,0 +1,20 @@
+import path from 'path'
+import { GeneratedContent, InputSection } from '@/types/content-types'
+import { generateSection } from './generateSection'
+
+export async function generateContent() {
+	const CONTENT_DIRECTORY = path.join(process.cwd(), 'content')
+	const sections = require(path.join(CONTENT_DIRECTORY, 'sections.json')) as InputSection[]
+
+	const result: GeneratedContent = {
+		articles: {},
+		sections: [],
+	}
+
+	for (let i = 0; i < sections.length; i++) {
+		if (sections[i].id === 'reference') continue
+		result.sections.push(generateSection(sections[i], result.articles, i))
+	}
+
+	return result
+}

apps/docs/scripts/lib/generateExamplesContent.ts

@@ -0,0 +1,45 @@
+import { Articles, GeneratedContent, InputSection } from '../../types/content-types'
+import { generateSection } from './generateSection'
+
+const { log: nicelog } = console
+
+// Keep in sync with `categories` in apps/examples/src/examples.tsx. An example whose folder
+// isn't listed here has no category row, so it disappears from the sidebar and llms.txt;
+// generateSection throws for that case.
+export const EXAMPLES_CATEGORIES = [
+	{ id: 'getting-started', title: 'Getting started', description: '', groups: [], hero: null },
+	{ id: 'configuration', title: 'Configuration', description: '', groups: [], hero: null },
+	{ id: 'editor-api', title: 'Editor API', description: '', groups: [], hero: null },
+	{ id: 'ui', title: 'UI & theming', description: '', groups: [], hero: null },
+	{ id: 'layout', title: 'Page layout', description: '', groups: [], hero: null },
+	{ id: 'events', title: 'Events & effects', description: '', groups: [], hero: null },
+	{ id: 'shapes/tools', title: 'Shapes & tools', description: '', groups: [], hero: null },
+	{ id: 'users', title: 'Users', description: '', groups: [], hero: null },
+	{ id: 'collaboration', title: 'Collaboration', description: '', groups: [], hero: null },
+	{ id: 'data/assets', title: 'Data & assets', description: '', groups: [], hero: null },
+	{ id: 'use-cases', title: 'Use cases', description: '', groups: [], hero: null },
+]
+
+const section: InputSection = {
+	id: 'examples',
+	title: 'Examples',
+	description: 'Code recipes for bending tldraw to your will.',
+	categories: EXAMPLES_CATEGORIES,
+	hero: null,
+	sidebar_behavior: 'show-links',
+}
+
+export async function generateExamplesContent(): Promise<GeneratedContent> {
+	const articles: Articles = {}
+
+	try {
+		const outputExamplesSection = generateSection(section, articles, 0)
+		const contentComplete = { sections: [outputExamplesSection], articles }
+
+		return contentComplete
+	} catch (error) {
+		nicelog(`x Could not generate Examples content`)
+
+		throw error
+	}
+}

apps/docs/scripts/lib/generateLllmsTxt.ts

@@ -0,0 +1,139 @@
+import fs from 'fs'
+import path from 'path'
+import { Database } from 'sqlite'
+import sqlite3 from 'sqlite3'
+import { EXAMPLES_CATEGORIES } from './generateExamplesContent'
+import { PUBLIC_DIR } from './utils'
+
+type DbType = Database<sqlite3.Database, sqlite3.Statement>
+
+export async function generateLlmsTxt(db: DbType) {
+	const overviewMarkdown = await getMarkdownForOverview(db)
+	const examplesMarkdown = await getMarkdownForExamples(db)
+	const docsMarkdown = await getMarkdownForDocs(db)
+	const releasesMarkdown = await getMarkdownForReleases(db)
+
+	fs.writeFileSync(path.join(PUBLIC_DIR, 'llms.txt'), overviewMarkdown)
+	fs.writeFileSync(
+		path.join(PUBLIC_DIR, 'llms-full.txt'),
+		`${docsMarkdown}\n--------------------------------\n\n${releasesMarkdown}\n--------------------------------\n\n${examplesMarkdown}`
+	)
+	fs.writeFileSync(path.join(PUBLIC_DIR, 'llms-examples.txt'), examplesMarkdown)
+	fs.writeFileSync(path.join(PUBLIC_DIR, 'llms-docs.txt'), docsMarkdown)
+	fs.writeFileSync(path.join(PUBLIC_DIR, 'llms-releases.txt'), releasesMarkdown)
+}
+
+async function getMarkdownForOverview(db: DbType) {
+	let result = `# tldraw SDK\n\n`
+
+	const features = await db.all('SELECT * FROM articles WHERE sectionId = "sdk-features"')
+	// sectionIndex carries the version order generateSection worked out for the sidebar; sorting
+	// by id would be a string sort that puts v3.9.0 above v3.10.0
+	const releases = await db.all(
+		`SELECT * FROM articles WHERE sectionId = "releases" AND id NOT LIKE '%/next' ORDER BY sectionIndex ASC`
+	)
+	const examples = await db.all('SELECT * FROM articles WHERE sectionId = "examples"')
+
+	result += `## SDK features\n\n`
+	for (const feature of features) {
+		result += `- [${feature.title}](https://tldraw.dev${feature.path})\n`
+	}
+
+	result += `\n## Releases\n\n`
+	for (const release of releases) {
+		result += `- [${release.title}](https://tldraw.dev${release.path})\n`
+	}
+
+	result += `\n## Examples\n\n`
+	for (const example of examples) {
+		result += `- [${example.title}](https://tldraw.dev${example.path})\n`
+	}
+
+	result += `\n## Markdown exports of resources\n`
+	result += `\n- [All SDK features, releases, and examples](https://tldraw.dev/llms-full.txt)`
+	result += `\n- [SDK features only](https://tldraw.dev/llms-docs.txt)`
+	result += `\n- [Releases only](https://tldraw.dev/llms-releases.txt)`
+	result += `\n- [Examples only](https://tldraw.dev/llms-examples.txt)`
+
+	return result
+}
+
+async function getMarkdownForDocs(db: DbType) {
+	let result = `# tldraw SDK features\n`
+	const features = await db.all('SELECT * FROM articles WHERE sectionId = "sdk-features"')
+
+	for (const feature of features) {
+		result += `\n--------\n`
+		result += `\n# ${feature.title}\n\n${feature.content.trim()}\n`
+	}
+
+	return result
+}
+
+async function getMarkdownForReleases(db: DbType) {
+	let result = `# tldraw SDK releases\n`
+	const releases = await db.all(
+		`SELECT * FROM articles WHERE sectionId = "releases" AND id NOT LIKE '%/next' ORDER BY sectionIndex ASC`
+	)
+
+	for (const release of releases) {
+		result += `\n--------\n`
+		result += `\n# ${release.title}\n\n${release.content.trim()}\n`
+	}
+
+	return result
+}
+
+async function getMarkdownForExamples(db: DbType) {
+	const examples = await db.all('SELECT * FROM articles WHERE sectionId = "examples"')
+
+	// Sort examples by category
+	const categories = EXAMPLES_CATEGORIES.map((category) => {
+		return {
+			id: category.id,
+			title: category.title,
+			examples: examples.filter((example) => example.categoryId === category.id),
+		}
+	})
+
+	let result = `# tldraw SDK Examples`
+
+	for (const category of categories) {
+		for (const example of category.examples) {
+			result += `\n\n--------`
+			result += `\n\n# ${example.title}`
+
+			result += `\n\nCategory: ${category.title}`
+			result += `\n\nKeywords: ${example.keywords.trim()}`
+			if (example.description?.trim()) {
+				result += `\n\n${example.description.trim()}`
+			}
+
+			result += `\n\n${example.content.trim()}`
+			result += getMarkdownForFile('App.tsx', example.componentCode)
+
+			const files = JSON.parse(example.componentCodeFiles)
+			for (const name in files) {
+				result += getMarkdownForFile(name, files[name])
+			}
+		}
+	}
+
+	return result
+}
+
+const ALLOWED_FILE_TYPES = ['tsx', 'ts', 'js', 'jsx', 'md', 'css', 'html']
+function getMarkdownForFile(fileName: string, fileContent: string) {
+	const type = fileName.split('.').pop()
+
+	// Skip non-code files, eg: PDFs, PNGs, SVGs (which generateSection does collect)
+	if (!ALLOWED_FILE_TYPES.includes(type ?? '')) {
+		return ''
+	}
+
+	let result = `\n\n## ${fileName}`
+	result += `\n\n\`\`\`${type}`
+	result += `\n${fileContent.trim()}`
+	result += `\n\`\`\``
+	return result
+}

apps/docs/scripts/lib/generateSection.ts

@@ -0,0 +1,468 @@
+import fs from 'fs'
+import path from 'path'
+import matter from 'gray-matter'
+import {
+	type Article,
+	ArticleStatus,
+	type Articles,
+	type Category,
+	type InputCategory,
+	type InputSection,
+	type Section,
+} from '../../types/content-types'
+import { getArticleKey } from './getArticleKey'
+import { CONTENT_DIR } from './utils'
+
+export function generateSection(section: InputSection, articles: Articles, index: number): Section {
+	const { id: sectionId, sidebar_behavior, categories: sectionCategories } = section
+
+	function assignToArticles(key: string, article: Article) {
+		if (articles[key]) throw Error(`Duplicate article key: ${key}`)
+		articles[key] = article
+	}
+
+	const isExamplesSection = sectionId === 'examples'
+	const isReferenceSection = sectionId === 'reference'
+	const skipUnpublishedArticles = process.env.NODE_ENV !== 'development' && !isExamplesSection
+
+	// Uncategorized articles
+	const sectionUncategorizedArticles: Article[] = []
+
+	// A temporary table of articles mapped to categories
+	const sectionCategoryArticles: Record<string, Article[]> = Object.fromEntries(
+		section.categories.map((category) => [category.id, []])
+	)
+
+	// Create the article files for this section
+	const dir = isExamplesSection
+		? path.join(process.cwd(), '..', 'examples', 'src', 'examples')
+		: path.join(CONTENT_DIR, sectionId)
+	const articleSources = isExamplesSection
+		? getExampleArticleSources(dir)
+		: getRegularArticleSources(dir)
+
+	for (const { filename, pathname, articleId, categoryId } of articleSources) {
+		const fileContent = fs.readFileSync(pathname).toString()
+		const parsed = matter({ content: fileContent }, {})
+
+		if (skipUnpublishedArticles && parsed.data.status !== 'published') continue
+
+		const extension = isExamplesSection ? '.md' : path.extname(filename)
+
+		const article = getArticleData({
+			articleId,
+			categoryId,
+			sectionId,
+			parsed,
+			isGenerated: isReferenceSection,
+			extension,
+			componentCode: getComponentCode({ dir, filename, parsed }),
+			componentCodeFilename: getComponentCodeFilename({ parsed }),
+			componentCodeFiles: getComponentCodeFiles({ dir, filename, parsed }),
+		})
+
+		if (articleId === section.id) {
+			// The article is an index page, ie docs/docs
+			article.categoryIndex = -1
+			article.sectionIndex = -1
+			assignToArticles(getArticleKey(article), article)
+		} else {
+			// If the article is in a category and that category exists...
+			if (article.categoryId && sectionCategoryArticles[article.categoryId]) {
+				// The article is a category index page, ie docs/editor/editor
+				if (article.id === article.categoryId) {
+					article.categoryIndex = -1
+					article.sectionIndex = -1
+					assignToArticles(getArticleKey(article), article)
+				} else {
+					// Otherwise, add it to the category's list of articles
+					sectionCategoryArticles[article.categoryId].push(article)
+				}
+			} else if (isExamplesSection) {
+				// An example's category comes from its folder. Falling back to "uncategorized" would
+				// keep the unknown categoryId on the article, and an article whose category has no
+				// row is silently missing from the sidebar and llms.txt.
+				throw new Error(
+					`Unknown example category "${categoryId}" for ${pathname}. Add it to EXAMPLES_CATEGORIES in generateExamplesContent.ts.`
+				)
+			} else {
+				// otherwise, add it to the section's uncategorized list
+				sectionUncategorizedArticles.push(article)
+			}
+		}
+	}
+
+	// Create the categories
+	const categories: Category[] = [
+		{
+			id: section.id + '_ucg',
+			type: 'category',
+			sectionId: section.id,
+			index: 0,
+			title: 'Uncategorized',
+			description: 'Articles that do not belong to a category.',
+			groups: [],
+			path: `/${section.id}/ucg`,
+			content: null,
+			hero: null,
+		},
+		...sectionCategories
+			.filter((inputCategory) => sectionCategoryArticles[inputCategory.id].length > 0)
+			.map((inputCategory, i) =>
+				getCategory({
+					index: i + 1,
+					sectionId,
+					inputCategory,
+				})
+			),
+	]
+
+	// Finish the articles now that we have all of the categories done.
+	// Keep track of the section index through all of the sorted category articles.
+	// The section index is the "flattened" index of the article in the section.
+	let articleSectionIndex = 0
+	categories.forEach((category) => {
+		const categoryArticles =
+			category.id === section.id + '_ucg'
+				? sectionUncategorizedArticles
+				: sectionCategoryArticles[category.id]
+
+		const sortedArticles =
+			sectionId === 'releases'
+				? sortReleaseArticles(categoryArticles)
+				: categoryArticles.sort(sortArticles)
+
+		sortedArticles.forEach((article, i) => {
+			article.categoryIndex = i
+			article.sectionIndex = articleSectionIndex
+			assignToArticles(getArticleKey(article), article)
+			articleSectionIndex++
+		})
+	})
+
+	// Section is done, return it
+	return {
+		...section,
+		type: 'section',
+		sidebar_behavior,
+		index,
+		categories,
+		content: '',
+		hero: section.hero ?? null,
+		path: `/${section.id}`,
+	}
+}
+
+const sortArticles = (articleA: Article, articleB: Article) => {
+	const { categoryIndex: categoryIndexA, title: titleA } = articleA
+	const { categoryIndex: categoryIndexB, title: titleB } = articleB
+
+	return categoryIndexA === categoryIndexB
+		? titleA.localeCompare(titleB)
+		: categoryIndexA - categoryIndexB
+}
+
+// Parse a release title like "v5.1.0" into a [major, minor, patch] tuple, or null
+// for non-version pages (the "Next release" and "Migration skill" articles).
+function getReleaseVersion(article: Article): [number, number, number] | null {
+	const match = /^v(\d+)\.(\d+)\.(\d+)$/.exec(article.title.trim())
+	if (!match) return null
+	return [Number(match[1]), Number(match[2]), Number(match[3])]
+}
+
+// The releases section can't rely on the generic title sort: version titles don't
+// compare correctly as strings (e.g. "v3.10.0" < "v3.2.0") and the "Next release"
+// and "Migration skill" pages need fixed positions. Order it explicitly so the
+// sidebar and prev/next nav read: Next release, the latest version, Migration skill,
+// then every other version newest-first.
+function sortReleaseArticles(articles: Article[]): Article[] {
+	const next = articles.find((a) => a.path === '/releases/next')
+	const migration = articles.find((a) => a.path === '/releases/migration-skill')
+
+	const versions = articles
+		.filter((a) => a !== next && a !== migration)
+		.sort((a, b) => {
+			const [majorA, minorA, patchA] = getReleaseVersion(a) ?? [0, 0, 0]
+			const [majorB, minorB, patchB] = getReleaseVersion(b) ?? [0, 0, 0]
+			return majorB - majorA || minorB - minorA || patchB - patchA || a.title.localeCompare(b.title)
+		})
+
+	const ordered: Article[] = []
+	if (next) ordered.push(next)
+	if (versions.length > 0) ordered.push(versions[0])
+	if (migration) ordered.push(migration)
+	ordered.push(...versions.slice(1))
+	return ordered
+}
+
+function getArticleData({
+	articleId,
+	categoryId: categoryIdOverride,
+	sectionId,
+	parsed,
+	isGenerated,
+	extension,
+	componentCode,
+	componentCodeFilename,
+	componentCodeFiles,
+}: {
+	articleId: Article['id']
+	categoryId?: Category['id']
+	sectionId: Section['id']
+	parsed: matter.GrayMatterFile<string>
+	isGenerated: boolean
+	extension: string
+	componentCode: string | null
+	componentCodeFilename: string | null
+	componentCodeFiles: { [key: string]: string }
+}): Article {
+	const {
+		group = null,
+		priority = null,
+		hero = null,
+		thumbnail = null,
+		socialImage = null,
+		author = 'api',
+		status = ArticleStatus.Draft,
+		title = 'Untitled article',
+		sidebarTitle = null,
+		description = null,
+		keywords = [],
+		date = null,
+		sourceUrl = null,
+		order,
+		apiTags = null,
+		category: categoryIdFromFrontmatter = sectionId + '_ucg',
+	} = parsed.data
+	const categoryId = categoryIdOverride ?? categoryIdFromFrontmatter
+
+	const githubLink = sectionId === 'starter-kits' ? (parsed.data.githubLink ?? null) : null
+	const embed = sectionId === 'starter-kits' ? (parsed.data.embed ?? null) : null
+
+	const { content } = parsed
+
+	const article: Article = {
+		id: getArticleKey({ sectionId, categoryId, id: articleId } as Article),
+		type: 'article',
+		sectionIndex: 0,
+		groupIndex: -1,
+		groupId: group,
+		// Like the examples app, an article without a priority sorts after the prioritized ones
+		categoryIndex: order ?? priority ?? 999999,
+		priority,
+		sectionId: sectionId,
+		author: [author],
+		authorId: author,
+		categoryId,
+		status,
+		title,
+		sidebarTitle,
+		description,
+		hero,
+		thumbnail,
+		socialImage,
+		date: date ? new Date(date).toISOString() : null,
+		keywords,
+		sourceUrl: isGenerated // if it's a generated API doc, then we don't have a link
+			? sourceUrl
+			: `${sectionId}/${articleId}${extension}`,
+		content,
+		apiTags,
+		path: getArticlePath({ sectionId, categoryId, articleId }),
+		componentCode,
+		componentCodeFilename,
+		componentCodeFiles: componentCode ? JSON.stringify(componentCodeFiles) : null,
+		embed,
+		githubLink,
+	}
+
+	if (sectionId === 'examples' && article.content) {
+		const splitUp = article.content.split('---\n')
+		article.description = splitUp[0]
+		article.content = splitUp.slice(1).join('---\n')
+	}
+	return article
+}
+
+function getRegularArticleSources(dir: string) {
+	const sources: Array<{
+		filename: string
+		pathname: string
+		articleId: string
+		categoryId?: string
+	}> = []
+
+	for (const file of fs.readdirSync(dir, { withFileTypes: false })) {
+		const filename = file.toString()
+		if (filename.startsWith('.')) continue
+		if (!filename.endsWith('.mdx') && !filename.endsWith('.md')) {
+			throw new Error(`no non .md / mdx files pls: ${filename}`)
+		}
+		sources.push({
+			filename,
+			pathname: path.join(dir, filename),
+			articleId: filename.replace(path.extname(filename), ''),
+		})
+	}
+
+	return sources
+}
+
+function getExampleArticleSources(dir: string) {
+	return getReadmePaths(dir).map((pathname) => {
+		const relativeDir = path.relative(dir, path.dirname(pathname)).replaceAll(path.sep, '/')
+		const segments = relativeDir.split('/')
+		if (segments.length < 2) {
+			throw new Error(`Example category folder missing for ${pathname}`)
+		}
+
+		return {
+			filename: relativeDir,
+			pathname,
+			articleId: segments[segments.length - 1],
+			categoryId: segments.slice(0, -1).join('/'),
+		}
+	})
+}
+
+function getReadmePaths(dir: string) {
+	const readmes: string[] = []
+	const entries = fs.readdirSync(dir, { withFileTypes: true })
+	for (const entry of entries) {
+		if (entry.name.startsWith('.')) continue
+		const pathname = path.join(dir, entry.name)
+		if (entry.isDirectory()) {
+			readmes.push(...getReadmePaths(pathname))
+		} else if (entry.isFile() && entry.name === 'README.md') {
+			readmes.push(pathname)
+		}
+	}
+	return readmes
+}
+
+function getArticlePath({
+	sectionId,
+	categoryId,
+	articleId,
+}: {
+	sectionId: Section['id']
+	categoryId: Category['id']
+	articleId: Article['id']
+}): string {
+	if (sectionId === 'examples') {
+		return `/${sectionId}/${articleId}`
+	}
+	if (sectionId === 'getting-started') {
+		// We used to remove the getting-started prefix from this path
+		// but it causes issues with clashing names folders (eg: "releases" page and "releases" folder)
+		// so now we apply that change with rewrites instead
+		return `/${sectionId}/${articleId}`
+	}
+	if (categoryId === sectionId + '_ucg') {
+		return `/${sectionId}/${articleId}` // index page
+	}
+	return `/${sectionId}/${categoryId}/${articleId}`
+}
+
+function getComponentCode({
+	dir,
+	filename,
+	parsed,
+}: {
+	dir: string
+	filename: string
+	parsed: matter.GrayMatterFile<string>
+}) {
+	return parsed.data.component
+		? fs
+				.readFileSync(
+					path.join(
+						dir,
+						filename,
+						`${parsed.data.component}${parsed.data.component.endsWith('.tsx') ? '' : '.tsx'}`
+					)
+				)
+				.toString()
+		: null
+}
+
+function getComponentCodeFilename({ parsed }: { parsed: matter.GrayMatterFile<string> }) {
+	if (!parsed.data.component) return null
+	const component = parsed.data.component as string
+	// Remove leading ./ if present and ensure .tsx extension
+	const name = component.replace(/^\.\//, '')
+	return name.endsWith('.tsx') ? name : `${name}.tsx`
+}
+
+function getComponentCodeFiles({
+	dir,
+	filename,
+	parsed,
+}: {
+	dir: string
+	filename: string
+	parsed: matter.GrayMatterFile<string>
+}) {
+	const componentCodeFiles: { [key: string]: string } = {}
+
+	if (parsed.data.component) {
+		// Get the files from the example directory
+		fs.readdirSync(path.join(dir, filename), {
+			withFileTypes: true,
+			recursive: true,
+		})
+			// filter the files to only include files that are not directories, not the README, and not the component itself
+			.filter(
+				(file) =>
+					!file.isDirectory() &&
+					(file.name.endsWith('.tsx') ||
+						file.name.endsWith('.ts') ||
+						file.name.endsWith('.js') ||
+						file.name.endsWith('.jsx') ||
+						file.name.endsWith('.css') ||
+						file.name.endsWith('.svg')) &&
+					file.name !== 'README.md' &&
+					file.name.replace('.tsx', '') !==
+						parsed.data.component.replace('./', '').replace('.tsx', '')
+			)
+			// For each of these component files, read the file and add it to the componentCodeFiles object
+			.forEach((file) => {
+				componentCodeFiles[file.name] = fs
+					.readFileSync(path.join(file.parentPath, file.name))
+					.toString()
+			})
+	}
+
+	return componentCodeFiles
+}
+
+function getCategory({
+	inputCategory,
+	sectionId,
+	index,
+}: {
+	index: number
+	sectionId: Section['id']
+	inputCategory: InputCategory
+}): Category {
+	return {
+		...inputCategory,
+		type: 'category',
+		sectionId,
+		index,
+		path: `/${sectionId}/${inputCategory.id}`,
+		content: null,
+		hero: null,
+		groups: inputCategory.groups.map(({ id }, i) => ({
+			id,
+			title: id,
+			index: i,
+			type: 'group',
+			sectionId,
+			categoryId: inputCategory.id,
+			description: null,
+			content: null,
+			path: `/${sectionId}/${inputCategory.id}/${id}`,
+		})),
+	}
+}

apps/docs/scripts/lib/getApiMarkdown.ts

@@ -0,0 +1,623 @@
+import {
+	ApiClass,
+	ApiConstructSignature,
+	ApiConstructor,
+	ApiDeclaredItem,
+	ApiDocumentedItem,
+	ApiEnum,
+	ApiFunction,
+	ApiInterface,
+	ApiItem,
+	ApiItemKind,
+	ApiMethod,
+	ApiMethodSignature,
+	ApiNamespace,
+	ApiOptionalMixin,
+	ApiProperty,
+	ApiPropertySignature,
+	ApiReadonlyMixin,
+	ApiStaticMixin,
+	ApiTypeAlias,
+	ApiVariable,
+	Excerpt,
+	HeritageType,
+} from '@microsoft/api-extractor-model'
+import { APIGroup } from '@/types/content-types'
+import { TABLE_OF_CONTENTS_CLASSNAME } from '@/utils/config'
+import { TldrawApiModel } from '@/utils/TldrawApiModel'
+import { MarkdownWriter, formatWithPrettier, getPath, getSlug } from './utils'
+
+interface Result {
+	markdown: string
+	keywords: string[]
+}
+
+const date = new Intl.DateTimeFormat('en-US', {
+	year: 'numeric',
+	month: '2-digit',
+	day: '2-digit',
+}).format(new Date())
+
+interface Member {
+	item: ApiItem
+	inheritedFrom: ApiItem | null
+}
+
+export async function getApiMarkdown(
+	model: TldrawApiModel,
+	categoryName: string,
+	item: ApiItem,
+	j: number
+) {
+	const result: Result = { markdown: '', keywords: [] }
+	const toc: Result = { markdown: '', keywords: [] }
+	const membersResult: Result = { markdown: '', keywords: [] }
+
+	const isComponent = model.isComponent(item)
+	const componentProps = isComponent ? model.getReactPropsItem(item) : null
+
+	const contents = collectMembersAndExtends(model, item)
+
+	if (contents.constructors.length) {
+		const constructorResult: Result = { markdown: '', keywords: [] }
+		for (const member of contents.constructors) {
+			await addMarkdownForMember(model, constructorResult, member)
+			addHorizontalRule(constructorResult)
+		}
+		addMarkdown(membersResult, constructorResult.markdown)
+	}
+
+	if (contents.properties.length || isComponent) {
+		const propertiesResult: Result = { markdown: '', keywords: [] }
+		if (componentProps) addExtends(propertiesResult, item, contents.heritage)
+		for (const member of contents.properties) {
+			const slug = getSlug(member.item)
+			addMarkdown(toc, `  - [${member.item.displayName}](#${slug})\n`)
+			await addMarkdownForMember(model, propertiesResult, member, {
+				isComponentProp: isComponent,
+			})
+			addHorizontalRule(propertiesResult)
+		}
+		if (
+			componentProps &&
+			componentProps instanceof ApiDeclaredItem &&
+			componentProps?.kind !== 'Interface'
+		) {
+			propertiesResult.markdown += await excerptToMarkdown(componentProps, componentProps.excerpt, {
+				kind: componentProps.kind,
+			})
+		}
+
+		if (propertiesResult.markdown.trim()) {
+			addMarkdown(toc, `- [Properties](#properties)\n`)
+			addMarkdown(membersResult, `## Properties\n\n`)
+			addMarkdown(membersResult, propertiesResult.markdown)
+		} else if (isComponent && !componentProps) {
+			addMarkdown(membersResult, `## Properties\n\n`)
+			addMarkdown(membersResult, `This component does not take any props.\n\n`)
+		}
+	}
+
+	if (contents.methods.length) {
+		const methodsResult: Result = { markdown: '', keywords: [] }
+		addMarkdown(toc, `- [Methods](#methods)\n`)
+		addMarkdown(methodsResult, `## Methods\n\n`)
+		for (const member of contents.methods) {
+			const slug = getSlug(member.item)
+			addMarkdown(toc, `  - [${member.item.displayName}](#${slug})\n`)
+			await addMarkdownForMember(model, methodsResult, member)
+			addHorizontalRule(methodsResult)
+		}
+		addMarkdown(membersResult, methodsResult.markdown)
+	}
+
+	await addFrontmatter(model, result, item, categoryName, j)
+	await addDeprecationNotice(result, item)
+
+	if (toc.markdown.length) {
+		result.markdown += `<details className="${TABLE_OF_CONTENTS_CLASSNAME}">\n\t<summary>Table of contents</summary>\n`
+		addMarkdown(result, toc.markdown)
+		result.markdown += `</details>\n\n`
+	}
+
+	if (!isComponent) {
+		addExtends(result, item, contents.heritage)
+	}
+
+	await addDocComment(model, result, item)
+
+	if (membersResult.markdown.length) {
+		addHorizontalRule(result)
+		addMarkdown(result, membersResult.markdown)
+	}
+
+	return result
+}
+
+/* --------------------- Helpers -------------------- */
+
+function sortMembers(members: Member[]) {
+	return members.sort((a, b) => {
+		const aIsStatic = ApiStaticMixin.isBaseClassOf(a.item) && a.item.isStatic
+		const bIsStatic = ApiStaticMixin.isBaseClassOf(b.item) && b.item.isStatic
+		if (aIsStatic && !bIsStatic) return -1
+		if (!aIsStatic && bIsStatic) return 1
+		return a.item.displayName.localeCompare(b.item.displayName)
+	})
+}
+
+function collectMembersAndExtends(model: TldrawApiModel, item: ApiItem) {
+	const isComponent = model.isComponent(item)
+
+	const constructors: Member[] = []
+	const properties: Member[] = []
+	const methods: Member[] = []
+	const heritage: HeritageType[] = []
+
+	function visit(item: ApiItem, inheritedFrom: ApiItem | null) {
+		if (item.members) {
+			for (const member of item.members) {
+				switch (member.kind) {
+					case ApiItemKind.Constructor:
+					case ApiItemKind.ConstructSignature:
+						addMember(constructors, member, inheritedFrom)
+						break
+					case ApiItemKind.Variable:
+					case ApiItemKind.Property:
+					case ApiItemKind.PropertySignature:
+						addMember(properties, member, inheritedFrom)
+						break
+					case ApiItemKind.Method:
+					case ApiItemKind.Function:
+					case ApiItemKind.MethodSignature:
+						if (isComponent) {
+							addMember(properties, member, inheritedFrom)
+						} else {
+							addMember(methods, member, inheritedFrom)
+						}
+
+						break
+					case ApiItemKind.EnumMember:
+					case ApiItemKind.Class:
+					case ApiItemKind.TypeAlias:
+					case ApiItemKind.Interface:
+						addMember(properties, member, inheritedFrom)
+						break
+					default:
+						model.nonBlockingError(
+							member,
+							`Unknown member kind: ${member.kind} in ${item.displayName}`
+						)
+						break
+				}
+			}
+		}
+
+		if (model.isComponent(item)) {
+			const componentProps = model.getReactPropsItem(item)
+			if (componentProps) {
+				visit(componentProps, null)
+			}
+		}
+
+		const extendsTypes =
+			item instanceof ApiClass && item.extendsType
+				? [item.extendsType]
+				: item instanceof ApiInterface
+					? item.extendsTypes
+					: []
+
+		for (const extendsType of extendsTypes) {
+			if (!inheritedFrom) {
+				heritage.push(extendsType)
+			}
+
+			const tokens = extendsType.excerpt.spannedTokens
+			let extendedItem = null
+			if (tokens[0].kind === 'Reference') {
+				extendedItem = model.tryResolveToken(item, tokens[0])
+			}
+
+			if (extendedItem) {
+				visit(extendedItem, extendedItem)
+			}
+		}
+	}
+
+	visit(item, null)
+
+	sortMembers(constructors)
+	sortMembers(properties)
+	sortMembers(methods)
+
+	return { constructors, properties, methods, heritage }
+}
+
+function addMember(members: Member[], item: ApiItem, inheritedFrom: ApiItem | null) {
+	if (members.some((m) => m.item.displayName === item.displayName)) return
+	members.push({ item, inheritedFrom })
+}
+
+function addMarkdown(result: Result, markdown: string) {
+	result.markdown += markdown
+}
+
+async function addMarkdownForMember(
+	model: TldrawApiModel,
+	result: Result,
+	{ item, inheritedFrom }: Member,
+	{ isComponentProp = false } = {}
+) {
+	if (item.displayName.startsWith('_')) return
+	addMemberNameAndMeta(result, model, item, { isComponentProp, inheritedFrom })
+	await addDeprecationNotice(result, item)
+	await addDocComment(model, result, item)
+}
+
+async function addFrontmatter(
+	model: TldrawApiModel,
+	result: Result,
+	member: ApiItem,
+	categoryName: string,
+	order: number
+) {
+	let kw = ''
+
+	if (result.keywords.length) {
+		kw += `\nkeywords:`
+		for (const k of result.keywords) {
+			if (k.startsWith('_')) continue
+			kw += `\n  - ${k.trim().split('\n')[0]}`
+		}
+	}
+
+	const frontmatter: Record<string, string> = {
+		title: member.displayName,
+		status: 'published',
+		category: categoryName,
+		group: model.isComponent(member) ? APIGroup.Component : member.kind,
+		date,
+		order: order.toString(),
+		apiTags: getTags(model, member).join(','),
+	}
+
+	if (member instanceof ApiDeclaredItem && member.sourceLocation.fileUrl) {
+		frontmatter.sourceUrl = member.sourceLocation.fileUrl
+	}
+
+	result.markdown += [
+		'---',
+		...Object.entries(frontmatter).map(([key, value]) => `${key}: ${value}`),
+		kw,
+		'---',
+		'',
+	].join('\n')
+}
+
+function addHorizontalRule(result: Result) {
+	result.markdown += `---\n\n`
+}
+
+function getItemTitle(item: ApiItem) {
+	if (item.kind === ApiItemKind.Constructor) {
+		return 'Constructor'
+	}
+
+	// Escape square brackets so a name like `[Symbol.iterator]` followed by the
+	// `(\u00A0)` call parens below doesn't parse as a markdown link.
+	const name = item.displayName.replaceAll('[', '\\[').replaceAll(']', '\\]')
+	if (item.kind === ApiItemKind.Method || item.kind === ApiItemKind.Function) {
+		return `${name}(\u00A0)`
+	}
+
+	return name
+}
+function addMemberNameAndMeta(
+	result: Result,
+	model: TldrawApiModel,
+	item: ApiItem,
+	{
+		level = 3,
+		isComponentProp = false,
+		inheritedFrom = null,
+	}: { level?: number; isComponentProp?: boolean; inheritedFrom?: ApiItem | null } = {}
+) {
+	const heading = `${'#'.repeat(level)} ${getItemTitle(item)}`
+
+	const inherited = inheritedFrom
+		? { name: inheritedFrom.displayName, link: `/reference/${getPath(inheritedFrom)}` }
+		: null
+
+	const tags = getTags(model, item, { isComponentProp, includeKind: false })
+	result.markdown += [
+		`<ApiMemberTitle tags={${JSON.stringify(tags)}} inherited={${JSON.stringify(inherited)}}>`,
+		'',
+		heading,
+		'',
+		'</ApiMemberTitle>',
+		'',
+	].join('\n')
+}
+
+async function addDocComment(model: TldrawApiModel, result: Result, member: ApiItem) {
+	if (!(member instanceof ApiDocumentedItem)) {
+		return
+	}
+
+	const isComponent = model.isComponent(member)
+
+	if (member.tsdocComment) {
+		result.markdown += await MarkdownWriter.docNodeToMarkdown(
+			member,
+			member.tsdocComment.summarySection
+		)
+	}
+
+	if (
+		!isComponent &&
+		(member instanceof ApiVariable ||
+			member instanceof ApiTypeAlias ||
+			member instanceof ApiProperty ||
+			member instanceof ApiPropertySignature ||
+			member instanceof ApiClass ||
+			member instanceof ApiFunction ||
+			member instanceof ApiInterface ||
+			member instanceof ApiEnum ||
+			member instanceof ApiNamespace ||
+			member instanceof ApiMethod)
+	) {
+		result.markdown += await excerptToMarkdown(member, member.excerpt, {
+			kind: member.kind,
+		})
+		result.markdown += `\n\n`
+	}
+
+	if (member.tsdocComment) {
+		const exampleBlocks = member.tsdocComment.customBlocks.filter(
+			(block) => block.blockTag.tagNameWithUpperCase === '@EXAMPLE'
+		)
+
+		if (exampleBlocks.length) {
+			result.markdown += `\n\n`
+			result.markdown += `<ApiHeading>Example</ApiHeading>\n\n`
+			for (const example of exampleBlocks) {
+				result.markdown += await MarkdownWriter.docNodeToMarkdown(member, example.content as any)
+			}
+		}
+	}
+
+	if (isComponent) return
+
+	if (
+		member instanceof ApiMethod ||
+		member instanceof ApiMethodSignature ||
+		member instanceof ApiConstructor ||
+		member instanceof ApiConstructSignature ||
+		member instanceof ApiFunction
+	) {
+		if (!member.parameters.length) {
+			return
+		} else {
+			result.markdown += `<ApiHeading>Parameters</ApiHeading>\n\n`
+			result.markdown += '<ParametersTable>\n\n'
+			for (const param of member.parameters) {
+				result.markdown += '<ParametersTableRow>\n'
+				result.markdown += '<ParametersTableName>\n\n'
+				result.markdown += `\`${param.name}\`\n\n`
+				result.markdown += `</ParametersTableName>\n`
+				result.markdown += `<ParametersTableDescription>\n\n`
+				result.markdown += await excerptToMarkdown(member, param.parameterTypeExcerpt, {
+					kind: 'ParameterType',
+					printWidth: 60,
+				})
+				result.markdown += `\n\n`
+				if (param.tsdocParamBlock) {
+					result.markdown += await MarkdownWriter.docNodeToMarkdown(
+						member,
+						param.tsdocParamBlock.content
+					)
+				}
+				result.markdown += `\n\n</ParametersTableDescription>\n`
+				result.markdown += `</ParametersTableRow>\n`
+			}
+			result.markdown += '</ParametersTable>\n\n'
+		}
+
+		if (!(member instanceof ApiConstructor)) {
+			result.markdown += `<ApiHeading>Returns</ApiHeading>\n\n`
+			result.markdown += await excerptToMarkdown(member, member.returnTypeExcerpt, {
+				kind: 'ReturnType',
+			})
+			result.markdown += `\n\n`
+			if (member.tsdocComment && member.tsdocComment.returnsBlock) {
+				result.markdown += await MarkdownWriter.docNodeToMarkdown(
+					member,
+					member.tsdocComment.returnsBlock.content
+				)
+			}
+		}
+	} else if (
+		member instanceof ApiVariable ||
+		member instanceof ApiTypeAlias ||
+		member instanceof ApiProperty ||
+		member instanceof ApiPropertySignature ||
+		member instanceof ApiClass ||
+		member instanceof ApiInterface ||
+		member instanceof ApiEnum ||
+		member instanceof ApiNamespace
+	) {
+		const params = member.tsdocComment?.params
+		if (params && params.count > 0) {
+			result.markdown += `<ApiHeading>Parameters</ApiHeading>\n\n`
+			result.markdown += '<ParametersTable>\n\n'
+			for (const block of params.blocks) {
+				result.markdown += '<ParametersTableRow>\n'
+				result.markdown += '<ParametersTableName>\n\n'
+				result.markdown += `\`${block.parameterName}\`\n\n`
+				result.markdown += `</ParametersTableName>\n`
+				result.markdown += `<ParametersTableDescription>\n\n`
+				result.markdown += await MarkdownWriter.docNodeToMarkdown(member, block.content as any)
+				result.markdown += `\n\n</ParametersTableDescription>\n`
+				result.markdown += `</ParametersTableRow>\n`
+			}
+			result.markdown += '</ParametersTable>\n\n'
+		}
+	} else {
+		model.nonBlockingError(member, `Unknown member kind: ${member.kind}`)
+	}
+}
+
+async function addDeprecationNotice(result: Result, member: ApiItem) {
+	if (member instanceof ApiDocumentedItem && member.tsdocComment?.deprecatedBlock?.content) {
+		result.markdown += `<Callout type="warning">\n\n**Deprecated:**`
+		result.markdown += await MarkdownWriter.docNodeToMarkdown(
+			member,
+			member.tsdocComment.deprecatedBlock.content
+		)
+		result.markdown += `\n\n</Callout>\n\n`
+	}
+}
+
+async function excerptToMarkdown(
+	item: ApiItem,
+	excerpt: Excerpt,
+	{ kind, printWidth }: { kind: ApiItemKind | 'ReturnType' | 'ParameterType'; printWidth?: number }
+) {
+	const links = {} as Record<string, string>
+
+	let code = ''
+	for (const token of excerpt.spannedTokens) {
+		code += token.text
+
+		if (!token.canonicalReference) continue
+
+		const apiItemResult = item
+			.getAssociatedModel()!
+			.resolveDeclarationReference(token.canonicalReference!, item)
+
+		if (apiItemResult.errorMessage) continue
+
+		const apiItem = apiItemResult.resolvedApiItem!
+		const url = `/reference/${getPath(apiItem)}`
+		links[token.text] = url
+	}
+
+	code = code.replace(/^export /, '')
+	code = code.replace(/^declare /, '')
+
+	switch (kind) {
+		case ApiItemKind.CallSignature:
+		case ApiItemKind.EntryPoint:
+		case ApiItemKind.EnumMember:
+		case ApiItemKind.Function:
+		case ApiItemKind.Model:
+		case ApiItemKind.Namespace:
+		case ApiItemKind.None:
+		case ApiItemKind.Package:
+		case ApiItemKind.TypeAlias:
+			code = await formatWithPrettier(code, { printWidth })
+			break
+		case 'ReturnType':
+		case 'ParameterType':
+			code = await formatWithPrettier(`type X = () =>${code}`, { printWidth })
+			if (!code.startsWith('type X = () =>')) {
+				throw Error()
+			}
+			code = code = code.replace(/^type X = \(\) =>[ \n]/, '')
+			break
+		case ApiItemKind.Class:
+		case ApiItemKind.Enum:
+		case ApiItemKind.Interface:
+			code = await formatWithPrettier(`${code} {}`, { printWidth })
+			break
+		case ApiItemKind.Constructor:
+		case ApiItemKind.ConstructSignature:
+		case ApiItemKind.IndexSignature:
+		case ApiItemKind.Method:
+		case ApiItemKind.MethodSignature:
+		case ApiItemKind.Property:
+		case ApiItemKind.PropertySignature:
+		case ApiItemKind.Variable:
+			code = await formatWithPrettier(`class X { ${code} }`, { printWidth })
+			if (!(code.startsWith('class X {\n') && code.endsWith('\n}'))) {
+				throw Error()
+			}
+			code = code.slice('class X {\n'.length, -'\n}'.length)
+			code = code.replace(/^ {2}/gm, '')
+			break
+		default:
+			throw Error()
+	}
+
+	return [
+		`<CodeLinks links={${JSON.stringify(links)}}>`,
+		'',
+		'```ts',
+		code,
+		'```',
+		'',
+		'</CodeLinks>',
+	].join('\n')
+}
+
+function getTags(
+	model: TldrawApiModel,
+	member: ApiItem,
+	{ isComponentProp = false, includeKind = true } = {}
+) {
+	const tags = []
+
+	if (ApiStaticMixin.isBaseClassOf(member) && member.isStatic) {
+		tags.push('static')
+	}
+	let kind = member.kind.toLowerCase()
+	if (ApiReadonlyMixin.isBaseClassOf(member) && member.isReadonly) {
+		if (member.kind === ApiItemKind.Variable) {
+			kind = 'constant'
+		} else if (!isComponentProp) {
+			tags.push('readonly')
+		}
+	}
+	if (model.isComponent(member)) {
+		kind = 'component'
+	}
+
+	if (ApiOptionalMixin.isBaseClassOf(member) && member.isOptional) {
+		tags.push('optional')
+	}
+
+	if (includeKind) {
+		tags.push(kind)
+	}
+
+	return tags
+}
+
+function addExtends(result: Result, item: ApiItem, heritage: HeritageType[]) {
+	if (!heritage.length) return
+
+	const links = {} as Record<string, string>
+	for (const type of heritage) {
+		for (const token of type.excerpt.spannedTokens) {
+			if (!token.canonicalReference) continue
+
+			const apiItemResult = item
+				.getAssociatedModel()!
+				.resolveDeclarationReference(token.canonicalReference!, item)
+
+			if (apiItemResult.errorMessage) continue
+
+			const apiItem = apiItemResult.resolvedApiItem!
+			links[token.text] = `/reference/${getPath(apiItem)}`
+		}
+	}
+
+	result.markdown += [
+		`<CodeLinks links={${JSON.stringify(links)}}>`,
+		'',
+		`Extends \`${heritage.map((type) => type.excerpt.text).join(', ')}\`.`,
+		'',
+		'</CodeLinks>',
+		'',
+	].join('\n')
+}

apps/docs/scripts/lib/getArticleKey.ts

@@ -0,0 +1,5 @@
+import { Article } from '@/types/content-types'
+
+export function getArticleKey(article: Article): string {
+	return `${article.sectionId}/${article.categoryId ?? 'ucg'}/${article.id ?? 'index'}`
+}

apps/docs/scripts/lib/package-list.ts

@@ -0,0 +1,13 @@
+export const TLDRAW_PACKAGES_TO_INCLUDE_IN_DOCS = [
+	'editor',
+	'store',
+	'tldraw',
+	'tlschema',
+	'validate',
+	'state',
+	'state-react',
+	'sync',
+	'sync-core',
+	'mermaid',
+	'driver',
+]

apps/docs/scripts/lib/refreshContent.ts

@@ -0,0 +1,31 @@
+import { addContentToDb } from '@/utils/addContent'
+import { autoLinkDocs } from '@/utils/autoLinkDocs'
+import { nicelog } from '@/utils/nicelog'
+import { connect } from './connect'
+import { generateApiContent } from './generateApiContent'
+import { generateContent } from './generateContent'
+import { generateExamplesContent } from './generateExamplesContent'
+import { generateLlmsTxt } from './generateLllmsTxt'
+
+export async function refreshContent(opts = {} as { silent: boolean }) {
+	if (!opts.silent) nicelog('◦ Resetting database...')
+
+	const db = await connect({ reset: true, mode: 'readwrite' })
+
+	if (!opts.silent) nicelog('◦ Generating / adding regular content to db...')
+	await addContentToDb(db, await generateContent())
+
+	if (!opts.silent) nicelog('◦ Generating / adding Examples content to db...')
+	await addContentToDb(db, await generateExamplesContent())
+
+	if (!opts.silent) nicelog('◦ Generating / adding API content to db...')
+	await addContentToDb(db, await generateApiContent())
+
+	if (!opts.silent) nicelog('◦ Fixing links to API docs...')
+	await autoLinkDocs(db)
+
+	if (!opts.silent) nicelog('◦ Creating llms.txt...')
+	await generateLlmsTxt(db)
+
+	if (!opts.silent) nicelog('✔ Complete')
+}

apps/docs/scripts/lib/utils.ts

@@ -0,0 +1,306 @@
+import path from 'path'
+import { ApiItem, ApiItemKind, ApiModel } from '@microsoft/api-extractor-model'
+import { slug as githubSlug } from 'github-slugger'
+import { format } from 'oxfmt'
+
+/**
+ * Structural interface for DocNode-like objects from any version of @microsoft/tsdoc.
+ * This allows us to work with DocNodes from both the standalone @microsoft/tsdoc package
+ * and the version bundled in @microsoft/api-extractor-model without type conflicts.
+ */
+interface AnyDocNode {
+	readonly kind: string
+	getChildNodes?(): ReadonlyArray<AnyDocNode>
+	// Properties for specific node types
+	text?: string // DocPlainText
+	nodes?: ReadonlyArray<AnyDocNode> // DocSection, DocParagraph
+	content?: { nodes: ReadonlyArray<AnyDocNode> } // DocBlock
+	code?: string // DocCodeSpan, DocFencedCode
+	language?: string // DocFencedCode
+	encodedText?: string // DocEscapedText
+	urlDestination?: string // DocLinkTag
+	linkText?: string // DocLinkTag
+	codeDestination?: any // DocLinkTag - DeclarationReference from either version
+}
+
+export const API_DIR = path.join(process.cwd(), 'api')
+export const CONTENT_DIR = path.join(process.cwd(), 'content')
+export const PUBLIC_DIR = path.join(process.cwd(), 'public')
+
+function isOnParentPage(itemKind: ApiItemKind) {
+	switch (itemKind) {
+		case ApiItemKind.CallSignature:
+		case ApiItemKind.Class:
+		case ApiItemKind.EntryPoint:
+		case ApiItemKind.Enum:
+		case ApiItemKind.Function:
+		case ApiItemKind.Interface:
+		case ApiItemKind.Model:
+		case ApiItemKind.Namespace:
+		case ApiItemKind.Package:
+		case ApiItemKind.TypeAlias:
+		case ApiItemKind.Variable:
+		case ApiItemKind.None:
+			return false
+		case ApiItemKind.Constructor:
+		case ApiItemKind.ConstructSignature:
+		case ApiItemKind.EnumMember:
+		case ApiItemKind.Method:
+		case ApiItemKind.MethodSignature:
+		case ApiItemKind.Property:
+		case ApiItemKind.PropertySignature:
+		case ApiItemKind.IndexSignature:
+			return true
+		default:
+			throw Error(itemKind)
+	}
+}
+
+export function getSlug(item: ApiItem): string {
+	return githubSlug(item.displayName, true)
+}
+
+export function getPath(item: ApiItem): string {
+	if (isOnParentPage(item.kind)) {
+		const parentPath = getPath(item.parent!)
+		const childSlug = getSlug(item)
+		return `${parentPath}#${childSlug}`
+	}
+
+	// Members of a namespace are rendered on the namespace page, not as
+	// standalone pages. Link to the namespace page with an anchor.
+	if (item.parent && item.parent.kind === ApiItemKind.Namespace) {
+		const parentPath = getPath(item.parent)
+		const childSlug = getSlug(item)
+		return `${parentPath}#${childSlug}`
+	}
+
+	return item.canonicalReference
+		.toString()
+		.replace(/^@tldraw\//, '')
+		.replace(/:.+$/, '')
+		.replace(/!/g, '/')
+		.replace(/\./g, '-')
+}
+
+const languages: { [tag: string]: string | undefined } = {
+	ts: 'ts',
+	tsx: 'tsx',
+}
+
+export async function formatWithPrettier(
+	code: string,
+	{
+		languageTag,
+		// roughly the width of our code blocks on a desktop
+		printWidth = 80,
+	}: { languageTag?: string; printWidth?: number } = {}
+) {
+	const language = languages[languageTag || 'ts']
+	if (!language) {
+		throw new Error(`Unknown language: ${languageTag}`)
+	}
+	let formattedCode = code
+	try {
+		const result = await format(`snippet.${language}`, code, {
+			printWidth,
+			tabWidth: 2,
+			useTabs: false,
+		})
+		formattedCode = result.code
+	} catch {
+		console.warn(`☢️ Could not format code: ${code}`)
+	}
+
+	// sometimes formatters add a semicolon to the start of the code when formatting expressions (JSX
+	// in particular), so strip it if we see it
+	if (formattedCode.startsWith(';')) {
+		formattedCode = formattedCode.slice(1)
+	}
+
+	return formattedCode.trimEnd()
+}
+
+export class MarkdownWriter {
+	static async docNodeToMarkdown(apiContext: ApiItem, docNode: AnyDocNode) {
+		const writer = new MarkdownWriter(apiContext)
+		await writer.writeDocNode(docNode)
+		return writer.toString()
+	}
+
+	private constructor(private readonly apiContext: ApiItem) {}
+
+	private result = ''
+
+	write(...parts: string[]): this {
+		this.result += parts.join('')
+		return this
+	}
+
+	endsWith(str: string) {
+		return this.result.endsWith(str)
+	}
+
+	writeIfNeeded(str: string): this {
+		if (!this.endsWith(str)) {
+			this.write(str)
+		}
+		return this
+	}
+
+	async writeDocNode(docNode: AnyDocNode) {
+		// Use kind property instead of instanceof checks for better compatibility
+		switch (docNode.kind) {
+			case 'PlainText':
+				if (docNode.text) this.write(docNode.text)
+				break
+
+			case 'Section':
+			case 'Paragraph':
+				if (docNode.nodes) await this.writeDocNodes(docNode.nodes)
+				this.writeIfNeeded('\n\n')
+				break
+
+			case 'Block':
+				// DocBlock is a container node, process its content
+				if (docNode.content?.nodes) await this.writeDocNodes(docNode.content.nodes)
+				this.writeIfNeeded('\n\n')
+				break
+
+			case 'SoftBreak':
+				this.writeIfNeeded('\n')
+				break
+
+			case 'CodeSpan':
+				if (docNode.code) this.write('`', docNode.code, '`')
+				break
+
+			case 'FencedCode': {
+				if (docNode.code) {
+					this.writeIfNeeded('\n').write(
+						'```',
+						docNode.language || '',
+						'\n',
+						await formatWithPrettier(docNode.code, {
+							languageTag: docNode.language,
+						}),
+						'\n',
+						'```\n'
+					)
+				}
+				break
+			}
+
+			case 'EscapedText':
+				if (docNode.encodedText) this.write(docNode.encodedText)
+				break
+
+			case 'ErrorText':
+				// Skip error text nodes
+				break
+
+			case 'LinkTag': {
+				if (docNode.urlDestination) {
+					this.write(
+						'[',
+						docNode.linkText ?? docNode.urlDestination,
+						'](',
+						docNode.urlDestination,
+						')'
+					)
+				} else if (docNode.codeDestination) {
+					const apiModel = getTopLevelModel(this.apiContext)
+					const refResult = apiModel.resolveDeclarationReference(
+						docNode.codeDestination,
+						this.apiContext
+					)
+
+					if (refResult.errorMessage) {
+						console.warn(`☢️ Error processing API: ${refResult.errorMessage}`)
+						break
+					}
+					const linkedItem = refResult.resolvedApiItem!
+					const path = getPath(linkedItem)
+
+					this.write(
+						'[',
+						docNode.linkText ?? getDefaultReferenceText(linkedItem),
+						'](/reference/',
+						path,
+						')'
+					)
+				}
+				break
+			}
+
+			default:
+				// Handle any unknown container nodes generically by checking if they have a 'nodes' property
+				if (docNode.nodes && Array.isArray(docNode.nodes)) {
+					await this.writeDocNodes(docNode.nodes)
+					this.writeIfNeeded('\n\n')
+				} else if (docNode.content?.nodes) {
+					await this.writeDocNodes(docNode.content.nodes)
+					this.writeIfNeeded('\n\n')
+				} else {
+					console.warn(`⚠️  Unknown docNode kind: ${docNode.kind}, skipping...`)
+				}
+		}
+	}
+
+	async writeDocNodes(docNodes: ReadonlyArray<AnyDocNode>) {
+		for (const docNode of docNodes) {
+			await this.writeDocNode(docNode)
+		}
+		return this
+	}
+
+	toString() {
+		return this.result
+	}
+}
+
+function getDefaultReferenceText(item: ApiItem): string {
+	function parentPrefix(str: string, sep = '.'): string {
+		if (!item.parent) return str
+		return `${getDefaultReferenceText(item.parent)}${sep}${str}`
+	}
+	switch (item.kind) {
+		case ApiItemKind.CallSignature:
+			return parentPrefix(`${item.displayName}()`)
+		case ApiItemKind.Constructor:
+		case ApiItemKind.ConstructSignature: {
+			const parent = item.parent!
+			return `new ${getDefaultReferenceText(parent)}()`
+		}
+		case ApiItemKind.EnumMember:
+		case ApiItemKind.Method:
+		case ApiItemKind.MethodSignature:
+		case ApiItemKind.Property:
+		case ApiItemKind.PropertySignature:
+			return parentPrefix(item.displayName)
+		case ApiItemKind.IndexSignature:
+			return parentPrefix(`[${item.displayName}]`, '')
+		case ApiItemKind.Class:
+		case ApiItemKind.EntryPoint:
+		case ApiItemKind.Enum:
+		case ApiItemKind.Function:
+		case ApiItemKind.Interface:
+		case ApiItemKind.Model:
+		case ApiItemKind.Namespace:
+		case ApiItemKind.Package:
+		case ApiItemKind.TypeAlias:
+		case ApiItemKind.Variable:
+		case ApiItemKind.None:
+			return item.displayName
+		default:
+			throw Error(item.kind)
+	}
+}
+
+function getTopLevelModel(item: ApiItem): ApiModel {
+	const model = item.getAssociatedModel()!
+	if (model.parent) {
+		return getTopLevelModel(model.parent)
+	}
+	return model
+}

apps/docs/scripts/refresh-content.ts

@@ -0,0 +1,7 @@
+import { nicelog } from '@/utils/nicelog'
+import { refreshContent } from './lib/refreshContent'
+;(async function () {
+	nicelog('◦ Refreshing all content')
+	await refreshContent()
+	process.exit()
+})()

apps/docs/scripts/update-algolia-index.ts

@@ -0,0 +1,345 @@
+import console from 'console'
+import { assertExists, groupBy, objectMapEntries } from '@tldraw/utils'
+import algoliasearch from 'algoliasearch'
+import { config } from 'dotenv'
+import {
+	SearchEntry,
+	SearchEntryWithIndex,
+	SearchIndexName,
+	getSearchIndexName,
+} from '@/utils/algolia'
+import { db } from '@/utils/ContentDatabase'
+import { nicelog } from '@/utils/nicelog'
+import { parseMarkdown } from '@/utils/parse-markdown'
+
+config()
+
+// set each section with a search config. higher priority means it will be more likely to appear in
+// search results.
+const sectionConfig: Record<string, SearchConfig | null> = {
+	'getting-started': {
+		index: 'docs',
+		priority: 10,
+		splitHeadings: false,
+	},
+	docs: {
+		index: 'docs',
+		priority: 9,
+		splitHeadings: true,
+	},
+	'sdk-features': {
+		index: 'docs',
+		priority: 9,
+		splitHeadings: true,
+	},
+	examples: {
+		index: 'docs',
+		priority: 8,
+		splitHeadings: false,
+	},
+	'starter-kits': {
+		index: 'docs',
+		priority: 7,
+		splitHeadings: false,
+	},
+	reference: {
+		index: 'docs',
+		priority: 6,
+		splitHeadings: true,
+		excludeHeadingLevels: [2],
+		formatHeading(article, heading) {
+			return `${article.title}.${heading}`
+		},
+	},
+	changelog: {
+		index: 'docs',
+		priority: 5,
+		splitHeadings: false,
+	},
+	releases: {
+		index: 'docs',
+		priority: 5,
+		splitHeadings: false,
+	},
+	community: {
+		index: 'docs',
+		priority: 4,
+		splitHeadings: true,
+	},
+	legal: null,
+	blog: {
+		index: 'blog',
+		priority: 10,
+		splitHeadings: false,
+	},
+}
+
+interface SearchConfig {
+	index: SearchIndexName
+	priority: number
+	splitHeadings: boolean
+	excludeHeadingLevels?: number[]
+	formatHeading?(article: Article, heading: string): string
+}
+
+interface Article {
+	title: string
+	description: string | null
+	path: string | null
+	id: string
+	sectionId: string
+	articleIndex: number
+	content: string
+	sectionTitle: string
+	categoryTitle: string
+	keywords: string[]
+}
+
+async function getAllArticles(): Promise<Article[]> {
+	const sqlite = await db.getDb()
+	const articles = await sqlite.all(
+		`SELECT title, description, path, id, sectionId, sectionIndex AS articleIndex, content, keywords,
+		(SELECT title FROM sections WHERE sections.id = articles.sectionId) AS sectionTitle,
+		(SELECT title FROM categories WHERE categories.id = articles.categoryId) AS categoryTitle
+		FROM articles`
+	)
+
+	return articles.map((article) => ({
+		...article,
+		keywords: article.keywords.split(',').map((keyword: string) => keyword.trim()),
+	}))
+}
+
+function getConfig(article: Article) {
+	const config = sectionConfig[article.sectionId]
+	if (config === undefined) {
+		throw new Error(`No search config found for section ${article.sectionId} (${article.path})`)
+	}
+	return config
+}
+
+function getSearchEntriesForArticle(article: Article): SearchEntryWithIndex[] {
+	const config = getConfig(article)
+	if (!config) return []
+
+	const { headings, initialContentText, allContentText } = parseMarkdown(
+		article.content,
+		article.path!
+	)
+
+	if (config.splitHeadings) {
+		return [
+			{
+				objectID: article.id,
+				index: config.index,
+
+				path: assertExists(article.path),
+				title: article.title,
+				titleHeadingFirst: article.title,
+
+				keywords: article.keywords,
+				description: article.description,
+				content: initialContentText,
+
+				section: article.sectionTitle,
+				sectionPriority: config.priority,
+
+				article: article.title,
+				articleIndex: article.articleIndex,
+
+				heading: null,
+				headingSlug: null,
+				headingIndex: 0,
+
+				rankAdjust: 0,
+			},
+			...headings
+
+				.filter((heading) => {
+					return (
+						// excluded headers from config:
+						!config.excludeHeadingLevels?.includes(heading.level)
+					)
+				})
+				.map((heading, i) => ({
+					objectID: `${article.id}#${i}`,
+					index: config.index,
+
+					path: assertExists(article.path),
+					title: config.formatHeading
+						? config.formatHeading(article, heading.title)
+						: `${article.title} • ${heading.title}`,
+					titleHeadingFirst: `${heading.title} • ${article.title}`,
+					description: '',
+					keywords: [],
+					content: heading.contentText,
+
+					section: article.sectionTitle,
+					sectionPriority: config.priority,
+
+					article: article.title,
+					articleIndex: article.articleIndex,
+
+					heading: heading.title,
+					headingSlug: heading.slug,
+					headingIndex: i + 1,
+
+					// all things being equal, inherited entries should rank lower than non-inherited entries
+					rankAdjust: heading.isInherited ? -1 : 0,
+				})),
+		]
+	} else {
+		return [
+			{
+				objectID: article.id,
+				index: config.index,
+
+				path: assertExists(article.path),
+				title: article.title,
+				titleHeadingFirst: article.title,
+				keywords: article.keywords,
+				description: article.description,
+				content: allContentText,
+
+				section: article.sectionTitle,
+				sectionPriority: config.priority,
+
+				article: article.title,
+				articleIndex: article.articleIndex,
+
+				heading: null,
+				headingSlug: null,
+				headingIndex: 0,
+
+				rankAdjust: 0,
+			},
+		]
+	}
+}
+
+async function updateAlgoliaIndex() {
+	nicelog('Converting articles to search entries...')
+	try {
+		console.time('✔️ Get articles')
+		const articles = await getAllArticles()
+		console.timeEnd('✔️ Get articles')
+
+		console.time('✔️ Format search entries')
+		const entries: SearchEntryWithIndex[] = []
+		for (const article of articles) {
+			entries.push(...(await getSearchEntriesForArticle(article)))
+		}
+		console.timeEnd('✔️ Format search entries')
+
+		console.time(`✔️ Check ${entries.length} entries for duplicates`)
+		const byObjectId = groupBy(entries, (entry) => entry.objectID)
+		const duplicateIds = Object.entries(byObjectId).filter(([_, entries]) => entries.length > 1)
+		if (duplicateIds.length) {
+			throw new Error(
+				`Duplicate object IDs found: ${duplicateIds
+					.map(([id, entries]) => `${id} (${entries.map((e) => e.title).join(', ')})`)
+					.join(', ')}`
+			)
+		}
+		console.timeEnd(`✔️ Check ${entries.length} entries for duplicates`)
+
+		if (!process.env.NEXT_PUBLIC_ALGOLIA_APP_ID || !process.env.ALGOLIA_API_KEY) {
+			if ((process.env.NEXT_PUBLIC_VERCEL_ENV ?? 'development') === 'development') {
+				nicelog(
+					'❌ Skipping Algolia indexing in development because NEXT_PUBLIC_ALGOLIA_APP_ID and ALGOLIA_API_KEY are not set.'
+				)
+				return
+			}
+			throw new Error('NEXT_PUBLIC_ALGOLIA_APP_ID and ALGOLIA_API_KEY must be set')
+		}
+
+		const client = algoliasearch(
+			process.env.NEXT_PUBLIC_ALGOLIA_APP_ID!,
+			process.env.ALGOLIA_API_KEY!
+		)
+
+		const byIndex = groupBy(entries, (entry) => entry.index) as Record<
+			SearchIndexName,
+			SearchEntryWithIndex[]
+		>
+		for (const [_indexName, entries] of objectMapEntries(byIndex)) {
+			const indexName = getSearchIndexName(_indexName)
+			console.time(`✔️ Index ${entries.length} entries into ${indexName}`)
+
+			const index = client.initIndex(indexName)
+			await index.replaceAllObjects(
+				entries.map(({ index: _, ...entry }): SearchEntry => entry),
+				{ autoGenerateObjectIDIfNotExist: true }
+			)
+			await index.setSettings({
+				searchableAttributes: [
+					'section',
+					'titleHeadingFirst',
+					'title',
+					'description',
+					'keywords',
+					'content',
+				],
+				camelCaseAttributes: [
+					'section',
+					'titleHeadingFirst',
+					'title',
+					'description',
+					'keywords',
+					'content',
+				],
+				// these are only applied _after_ keyword search is complete. if two entries have
+				// the same keyword search score, this will be used to tiebreak them.
+				customRanking: [
+					// rankAdjust makes certain entries appear higher or lower in search results.
+					// e.g. inherited members get pushed to the bottom
+					'desc(rankAdjust)',
+					// sort by section priority so high-signal content (like our guides) appear
+					// higher than lower-signal content like API references
+					'desc(sectionPriority)',
+					// otherwise just keep things in the order they appear
+					'asc(articleIndex)',
+					'asc(headingIndex)',
+				],
+				distinct: 3,
+				attributeForDistinct: 'section',
+				advancedSyntax: true,
+			})
+
+			const resultsToShowForEmptySearch = entries
+				.sort((a, b) => {
+					if (a.sectionPriority !== b.sectionPriority) return b.sectionPriority - a.sectionPriority
+					if (a.articleIndex !== b.articleIndex) return a.articleIndex - b.articleIndex
+					return a.headingIndex - b.headingIndex
+				})
+				.filter((entry) => !entry.heading)
+				.slice(0, 20)
+
+			await index.replaceAllRules([
+				{
+					objectID: 'EmptySearch',
+					enabled: true,
+					condition: {
+						pattern: '',
+						anchoring: 'is',
+					},
+					consequence: {
+						promote: resultsToShowForEmptySearch.map((entry, i) => ({
+							objectID: entry.objectID,
+							position: i,
+						})),
+					},
+				},
+			])
+			console.timeEnd(`✔️ Index ${entries.length} entries into ${indexName}`)
+		}
+
+		nicelog('')
+		nicelog('Done.')
+	} catch (error) {
+		nicelog(error)
+		process.exit(1)
+	}
+}
+
+updateAlgoliaIndex()