snapshot of https://github.com/tldraw/tldraw.git (main, 2026-09-23; blobs > 131072B omitted)

apps/docs/utils/parse-markdown.tsx

@@ -0,0 +1,276 @@
+import { assert, assertExists } from '@tldraw/utils'
+import GithubSlugger from 'github-slugger'
+import { Nodes, Root } from 'mdast'
+import { MdxJsxFlowElement, MdxJsxTextElement } from 'mdast-util-mdx'
+import { remark } from 'remark'
+import remarkMdx from 'remark-mdx'
+import { TABLE_OF_CONTENTS_CLASSNAME } from './config'
+
+const BREAK = '\n\n'
+
+// Add nodes to mdast content.
+declare module 'mdast' {
+	interface BlockContentMap {
+		/**
+		 * MDX JSX element node, occurring in flow (block).
+		 */
+		mdxJsxFlowElement: MdxJsxFlowElement
+	}
+
+	interface PhrasingContentMap {
+		/**
+		 * MDX JSX element node, occurring in text (phrasing).
+		 */
+		mdxJsxTextElement: MdxJsxTextElement
+	}
+
+	interface RootContentMap {
+		/**
+		 * MDX JSX element node, occurring in flow (block).
+		 */
+		mdxJsxFlowElement: MdxJsxFlowElement
+		/**
+		 * MDX JSX element node, occurring in text (phrasing).
+		 */
+		mdxJsxTextElement: MdxJsxTextElement
+	}
+}
+
+export interface ParsedHeading {
+	level: number
+	title: string
+	slug: string
+	contentText: string
+	isInherited: boolean
+}
+
+/**
+ * Walk the MDX AST of `content` and extract headings & their (plain text) content as we go.
+ *
+ * This returns an array of heading objects containing the heading info, and the text below it
+ * (before the next heading). It also returns the initial content text before the first heading.
+ *
+ * Heading extraction should be reliable, but content extraction is pretty best-effort. It's useful
+ * for a search index (which is what we use it for!), but might not be a perfect representation of
+ * everything on the page.
+ */
+export function parseMarkdown(
+	content: string,
+	debugContext: string
+): {
+	headings: ParsedHeading[]
+	allContentText: string
+	initialContentText: string
+} {
+	// the headings found so far. we start with a dummy heading at the beginning to hold the initial
+	// content text, but we'll remove it at the end.
+	const headings: Omit<ParsedHeading, 'slug'>[] = [
+		{
+			level: 0,
+			title: '',
+			contentText: '',
+			isInherited: false,
+		},
+	]
+
+	// all the text we've seen so far, including headings
+	let allContentText = ''
+
+	interface State {
+		isInHeading: boolean
+		isInherited: boolean
+	}
+
+	function visitChildren(node: Nodes, state: State) {
+		if ('children' in node) {
+			for (const child of node.children) {
+				visit(child, state)
+			}
+		}
+	}
+
+	function currentHeading() {
+		const currentHeading = headings[headings.length - 1]
+		return currentHeading
+	}
+
+	function addText(text: string, state: State) {
+		if (state.isInHeading) {
+			currentHeading().title += text
+		} else {
+			currentHeading().contentText += text
+		}
+		allContentText += text
+	}
+
+	// add a paragraph break if needed:
+	function breakText(state: State) {
+		assert(!state.isInHeading)
+		if (!currentHeading().contentText.endsWith(BREAK)) {
+			currentHeading().contentText += BREAK
+		}
+		if (!allContentText.endsWith(BREAK)) {
+			allContentText += BREAK
+		}
+	}
+
+	function visit(node: Nodes, state: State) {
+		switch (node.type) {
+			case 'heading': {
+				const { depth } = node
+
+				allContentText += BREAK
+				headings.push({
+					level: depth,
+					title: '',
+					contentText: '',
+					isInherited: state.isInherited,
+				})
+
+				visitChildren(node, { ...state, isInHeading: true })
+
+				break
+			}
+
+			case 'text':
+			case 'inlineCode':
+				addText(node.value, state)
+				break
+
+			case 'blockquote':
+			case 'paragraph':
+				breakText(state)
+				visitChildren(node, state)
+				break
+
+			case 'code':
+				// skip these entirely
+				break
+
+			case 'mdxJsxFlowElement':
+			case 'mdxJsxTextElement': {
+				const { name, attributes } = node
+
+				if (node.type === 'mdxJsxFlowElement') {
+					breakText(state)
+				}
+
+				if (
+					// Remove table of contents
+					name === 'details' &&
+					attributes.some(
+						(attr) =>
+							'name' in attr &&
+							attr.name === 'className' &&
+							attr.value === TABLE_OF_CONTENTS_CLASSNAME
+					)
+				) {
+					break
+				}
+
+				if (name === 'ApiMemberTitle') {
+					// each ApiMemberTitle, which is used for the title of everything in the API
+					// reference docs, should have an `inherited` attribute:
+					const inheritedAttr = attributes.find(
+						(attr) => attr.type === 'mdxJsxAttribute' && attr.name === 'inherited'
+					)
+					if (!inheritedAttr) {
+						throw new Error(`ApiMemberTitle missing inherited attribute (in ${debugContext})`)
+					}
+
+					if (
+						!inheritedAttr.value ||
+						typeof inheritedAttr.value !== 'object' ||
+						inheritedAttr.type !== 'mdxJsxAttribute'
+					) {
+						throw new Error(
+							`Unknown format for ApiMemberTitle inherited attribute (in ${debugContext})`
+						)
+					}
+
+					// if it doesn't have the value `null`, then it's inherited. otherwise, we can
+					// visit its children normally.
+					if (inheritedAttr.value.value !== 'null') {
+						visitChildren(node, { ...state, isInherited: true })
+						break
+					}
+				}
+
+				if (name === 'ParametersTableDescription') {
+					addText('(', state)
+				}
+
+				visitChildren(node, state)
+
+				if (name === 'ApiHeading') {
+					addText(':', state)
+					breakText(state)
+				}
+
+				if (name === 'ParametersTableDescription') {
+					addText(')', state)
+				}
+
+				if (name === 'ParametersTableRow') {
+					addText(',', state)
+				}
+
+				if (name === 'ParametersTable') {
+					breakText(state)
+					addText('.', state)
+				}
+
+				break
+			}
+
+			default:
+				visitChildren(node, state)
+		}
+	}
+
+	remark()
+		.use(remarkMdx)
+		.use(() => (tree: Root) => {
+			visit(tree, { isInHeading: false, isInherited: false })
+		})
+		.processSync(content)
+
+	const initial = assertExists(headings.shift())
+	const slugs = new GithubSlugger()
+
+	return {
+		initialContentText: postprocessText(initial.contentText),
+		allContentText: postprocessText(allContentText),
+		headings: headings.map((heading) => ({
+			...heading,
+			slug: slugs.slug(heading.title, true),
+			contentText: postprocessText(heading.contentText),
+		})),
+	}
+}
+
+function postprocessText(text: string) {
+	return (
+		text
+			// remove trailing/leading whitespace in each line:
+			.replace(/^\s+/gm, '')
+			.replace(/\s+$/gm, '')
+			// remove empty lines:
+			.replace(/^\s*$/gm, '')
+			// place on a single line:
+			.replace(/\s+/g, ' ')
+			// empty brackets:
+			.replace(/\( \)/g, '')
+			// bracket spacing:
+			.replace(/\( /g, '(')
+			.replace(/ \)/g, ')')
+			// comma leading space:
+			.replace(/\s+,/g, ',')
+			// comma + period:
+			.replace(/,\s*\./g, '.')
+			// blank Returns:
+			.replace(/Returns:$/, '')
+			// weird punctuation:
+			.replace(/\.\),/g, '),')
+	)
+}