snapshot of https://github.com/tldraw/tldraw.git (main, 2026-09-23; blobs > 131072B omitted)

apps/docs/scripts/lib/generateSection.ts

@@ -0,0 +1,468 @@
+import fs from 'fs'
+import path from 'path'
+import matter from 'gray-matter'
+import {
+	type Article,
+	ArticleStatus,
+	type Articles,
+	type Category,
+	type InputCategory,
+	type InputSection,
+	type Section,
+} from '../../types/content-types'
+import { getArticleKey } from './getArticleKey'
+import { CONTENT_DIR } from './utils'
+
+export function generateSection(section: InputSection, articles: Articles, index: number): Section {
+	const { id: sectionId, sidebar_behavior, categories: sectionCategories } = section
+
+	function assignToArticles(key: string, article: Article) {
+		if (articles[key]) throw Error(`Duplicate article key: ${key}`)
+		articles[key] = article
+	}
+
+	const isExamplesSection = sectionId === 'examples'
+	const isReferenceSection = sectionId === 'reference'
+	const skipUnpublishedArticles = process.env.NODE_ENV !== 'development' && !isExamplesSection
+
+	// Uncategorized articles
+	const sectionUncategorizedArticles: Article[] = []
+
+	// A temporary table of articles mapped to categories
+	const sectionCategoryArticles: Record<string, Article[]> = Object.fromEntries(
+		section.categories.map((category) => [category.id, []])
+	)
+
+	// Create the article files for this section
+	const dir = isExamplesSection
+		? path.join(process.cwd(), '..', 'examples', 'src', 'examples')
+		: path.join(CONTENT_DIR, sectionId)
+	const articleSources = isExamplesSection
+		? getExampleArticleSources(dir)
+		: getRegularArticleSources(dir)
+
+	for (const { filename, pathname, articleId, categoryId } of articleSources) {
+		const fileContent = fs.readFileSync(pathname).toString()
+		const parsed = matter({ content: fileContent }, {})
+
+		if (skipUnpublishedArticles && parsed.data.status !== 'published') continue
+
+		const extension = isExamplesSection ? '.md' : path.extname(filename)
+
+		const article = getArticleData({
+			articleId,
+			categoryId,
+			sectionId,
+			parsed,
+			isGenerated: isReferenceSection,
+			extension,
+			componentCode: getComponentCode({ dir, filename, parsed }),
+			componentCodeFilename: getComponentCodeFilename({ parsed }),
+			componentCodeFiles: getComponentCodeFiles({ dir, filename, parsed }),
+		})
+
+		if (articleId === section.id) {
+			// The article is an index page, ie docs/docs
+			article.categoryIndex = -1
+			article.sectionIndex = -1
+			assignToArticles(getArticleKey(article), article)
+		} else {
+			// If the article is in a category and that category exists...
+			if (article.categoryId && sectionCategoryArticles[article.categoryId]) {
+				// The article is a category index page, ie docs/editor/editor
+				if (article.id === article.categoryId) {
+					article.categoryIndex = -1
+					article.sectionIndex = -1
+					assignToArticles(getArticleKey(article), article)
+				} else {
+					// Otherwise, add it to the category's list of articles
+					sectionCategoryArticles[article.categoryId].push(article)
+				}
+			} else if (isExamplesSection) {
+				// An example's category comes from its folder. Falling back to "uncategorized" would
+				// keep the unknown categoryId on the article, and an article whose category has no
+				// row is silently missing from the sidebar and llms.txt.
+				throw new Error(
+					`Unknown example category "${categoryId}" for ${pathname}. Add it to EXAMPLES_CATEGORIES in generateExamplesContent.ts.`
+				)
+			} else {
+				// otherwise, add it to the section's uncategorized list
+				sectionUncategorizedArticles.push(article)
+			}
+		}
+	}
+
+	// Create the categories
+	const categories: Category[] = [
+		{
+			id: section.id + '_ucg',
+			type: 'category',
+			sectionId: section.id,
+			index: 0,
+			title: 'Uncategorized',
+			description: 'Articles that do not belong to a category.',
+			groups: [],
+			path: `/${section.id}/ucg`,
+			content: null,
+			hero: null,
+		},
+		...sectionCategories
+			.filter((inputCategory) => sectionCategoryArticles[inputCategory.id].length > 0)
+			.map((inputCategory, i) =>
+				getCategory({
+					index: i + 1,
+					sectionId,
+					inputCategory,
+				})
+			),
+	]
+
+	// Finish the articles now that we have all of the categories done.
+	// Keep track of the section index through all of the sorted category articles.
+	// The section index is the "flattened" index of the article in the section.
+	let articleSectionIndex = 0
+	categories.forEach((category) => {
+		const categoryArticles =
+			category.id === section.id + '_ucg'
+				? sectionUncategorizedArticles
+				: sectionCategoryArticles[category.id]
+
+		const sortedArticles =
+			sectionId === 'releases'
+				? sortReleaseArticles(categoryArticles)
+				: categoryArticles.sort(sortArticles)
+
+		sortedArticles.forEach((article, i) => {
+			article.categoryIndex = i
+			article.sectionIndex = articleSectionIndex
+			assignToArticles(getArticleKey(article), article)
+			articleSectionIndex++
+		})
+	})
+
+	// Section is done, return it
+	return {
+		...section,
+		type: 'section',
+		sidebar_behavior,
+		index,
+		categories,
+		content: '',
+		hero: section.hero ?? null,
+		path: `/${section.id}`,
+	}
+}
+
+const sortArticles = (articleA: Article, articleB: Article) => {
+	const { categoryIndex: categoryIndexA, title: titleA } = articleA
+	const { categoryIndex: categoryIndexB, title: titleB } = articleB
+
+	return categoryIndexA === categoryIndexB
+		? titleA.localeCompare(titleB)
+		: categoryIndexA - categoryIndexB
+}
+
+// Parse a release title like "v5.1.0" into a [major, minor, patch] tuple, or null
+// for non-version pages (the "Next release" and "Migration skill" articles).
+function getReleaseVersion(article: Article): [number, number, number] | null {
+	const match = /^v(\d+)\.(\d+)\.(\d+)$/.exec(article.title.trim())
+	if (!match) return null
+	return [Number(match[1]), Number(match[2]), Number(match[3])]
+}
+
+// The releases section can't rely on the generic title sort: version titles don't
+// compare correctly as strings (e.g. "v3.10.0" < "v3.2.0") and the "Next release"
+// and "Migration skill" pages need fixed positions. Order it explicitly so the
+// sidebar and prev/next nav read: Next release, the latest version, Migration skill,
+// then every other version newest-first.
+function sortReleaseArticles(articles: Article[]): Article[] {
+	const next = articles.find((a) => a.path === '/releases/next')
+	const migration = articles.find((a) => a.path === '/releases/migration-skill')
+
+	const versions = articles
+		.filter((a) => a !== next && a !== migration)
+		.sort((a, b) => {
+			const [majorA, minorA, patchA] = getReleaseVersion(a) ?? [0, 0, 0]
+			const [majorB, minorB, patchB] = getReleaseVersion(b) ?? [0, 0, 0]
+			return majorB - majorA || minorB - minorA || patchB - patchA || a.title.localeCompare(b.title)
+		})
+
+	const ordered: Article[] = []
+	if (next) ordered.push(next)
+	if (versions.length > 0) ordered.push(versions[0])
+	if (migration) ordered.push(migration)
+	ordered.push(...versions.slice(1))
+	return ordered
+}
+
+function getArticleData({
+	articleId,
+	categoryId: categoryIdOverride,
+	sectionId,
+	parsed,
+	isGenerated,
+	extension,
+	componentCode,
+	componentCodeFilename,
+	componentCodeFiles,
+}: {
+	articleId: Article['id']
+	categoryId?: Category['id']
+	sectionId: Section['id']
+	parsed: matter.GrayMatterFile<string>
+	isGenerated: boolean
+	extension: string
+	componentCode: string | null
+	componentCodeFilename: string | null
+	componentCodeFiles: { [key: string]: string }
+}): Article {
+	const {
+		group = null,
+		priority = null,
+		hero = null,
+		thumbnail = null,
+		socialImage = null,
+		author = 'api',
+		status = ArticleStatus.Draft,
+		title = 'Untitled article',
+		sidebarTitle = null,
+		description = null,
+		keywords = [],
+		date = null,
+		sourceUrl = null,
+		order,
+		apiTags = null,
+		category: categoryIdFromFrontmatter = sectionId + '_ucg',
+	} = parsed.data
+	const categoryId = categoryIdOverride ?? categoryIdFromFrontmatter
+
+	const githubLink = sectionId === 'starter-kits' ? (parsed.data.githubLink ?? null) : null
+	const embed = sectionId === 'starter-kits' ? (parsed.data.embed ?? null) : null
+
+	const { content } = parsed
+
+	const article: Article = {
+		id: getArticleKey({ sectionId, categoryId, id: articleId } as Article),
+		type: 'article',
+		sectionIndex: 0,
+		groupIndex: -1,
+		groupId: group,
+		// Like the examples app, an article without a priority sorts after the prioritized ones
+		categoryIndex: order ?? priority ?? 999999,
+		priority,
+		sectionId: sectionId,
+		author: [author],
+		authorId: author,
+		categoryId,
+		status,
+		title,
+		sidebarTitle,
+		description,
+		hero,
+		thumbnail,
+		socialImage,
+		date: date ? new Date(date).toISOString() : null,
+		keywords,
+		sourceUrl: isGenerated // if it's a generated API doc, then we don't have a link
+			? sourceUrl
+			: `${sectionId}/${articleId}${extension}`,
+		content,
+		apiTags,
+		path: getArticlePath({ sectionId, categoryId, articleId }),
+		componentCode,
+		componentCodeFilename,
+		componentCodeFiles: componentCode ? JSON.stringify(componentCodeFiles) : null,
+		embed,
+		githubLink,
+	}
+
+	if (sectionId === 'examples' && article.content) {
+		const splitUp = article.content.split('---\n')
+		article.description = splitUp[0]
+		article.content = splitUp.slice(1).join('---\n')
+	}
+	return article
+}
+
+function getRegularArticleSources(dir: string) {
+	const sources: Array<{
+		filename: string
+		pathname: string
+		articleId: string
+		categoryId?: string
+	}> = []
+
+	for (const file of fs.readdirSync(dir, { withFileTypes: false })) {
+		const filename = file.toString()
+		if (filename.startsWith('.')) continue
+		if (!filename.endsWith('.mdx') && !filename.endsWith('.md')) {
+			throw new Error(`no non .md / mdx files pls: ${filename}`)
+		}
+		sources.push({
+			filename,
+			pathname: path.join(dir, filename),
+			articleId: filename.replace(path.extname(filename), ''),
+		})
+	}
+
+	return sources
+}
+
+function getExampleArticleSources(dir: string) {
+	return getReadmePaths(dir).map((pathname) => {
+		const relativeDir = path.relative(dir, path.dirname(pathname)).replaceAll(path.sep, '/')
+		const segments = relativeDir.split('/')
+		if (segments.length < 2) {
+			throw new Error(`Example category folder missing for ${pathname}`)
+		}
+
+		return {
+			filename: relativeDir,
+			pathname,
+			articleId: segments[segments.length - 1],
+			categoryId: segments.slice(0, -1).join('/'),
+		}
+	})
+}
+
+function getReadmePaths(dir: string) {
+	const readmes: string[] = []
+	const entries = fs.readdirSync(dir, { withFileTypes: true })
+	for (const entry of entries) {
+		if (entry.name.startsWith('.')) continue
+		const pathname = path.join(dir, entry.name)
+		if (entry.isDirectory()) {
+			readmes.push(...getReadmePaths(pathname))
+		} else if (entry.isFile() && entry.name === 'README.md') {
+			readmes.push(pathname)
+		}
+	}
+	return readmes
+}
+
+function getArticlePath({
+	sectionId,
+	categoryId,
+	articleId,
+}: {
+	sectionId: Section['id']
+	categoryId: Category['id']
+	articleId: Article['id']
+}): string {
+	if (sectionId === 'examples') {
+		return `/${sectionId}/${articleId}`
+	}
+	if (sectionId === 'getting-started') {
+		// We used to remove the getting-started prefix from this path
+		// but it causes issues with clashing names folders (eg: "releases" page and "releases" folder)
+		// so now we apply that change with rewrites instead
+		return `/${sectionId}/${articleId}`
+	}
+	if (categoryId === sectionId + '_ucg') {
+		return `/${sectionId}/${articleId}` // index page
+	}
+	return `/${sectionId}/${categoryId}/${articleId}`
+}
+
+function getComponentCode({
+	dir,
+	filename,
+	parsed,
+}: {
+	dir: string
+	filename: string
+	parsed: matter.GrayMatterFile<string>
+}) {
+	return parsed.data.component
+		? fs
+				.readFileSync(
+					path.join(
+						dir,
+						filename,
+						`${parsed.data.component}${parsed.data.component.endsWith('.tsx') ? '' : '.tsx'}`
+					)
+				)
+				.toString()
+		: null
+}
+
+function getComponentCodeFilename({ parsed }: { parsed: matter.GrayMatterFile<string> }) {
+	if (!parsed.data.component) return null
+	const component = parsed.data.component as string
+	// Remove leading ./ if present and ensure .tsx extension
+	const name = component.replace(/^\.\//, '')
+	return name.endsWith('.tsx') ? name : `${name}.tsx`
+}
+
+function getComponentCodeFiles({
+	dir,
+	filename,
+	parsed,
+}: {
+	dir: string
+	filename: string
+	parsed: matter.GrayMatterFile<string>
+}) {
+	const componentCodeFiles: { [key: string]: string } = {}
+
+	if (parsed.data.component) {
+		// Get the files from the example directory
+		fs.readdirSync(path.join(dir, filename), {
+			withFileTypes: true,
+			recursive: true,
+		})
+			// filter the files to only include files that are not directories, not the README, and not the component itself
+			.filter(
+				(file) =>
+					!file.isDirectory() &&
+					(file.name.endsWith('.tsx') ||
+						file.name.endsWith('.ts') ||
+						file.name.endsWith('.js') ||
+						file.name.endsWith('.jsx') ||
+						file.name.endsWith('.css') ||
+						file.name.endsWith('.svg')) &&
+					file.name !== 'README.md' &&
+					file.name.replace('.tsx', '') !==
+						parsed.data.component.replace('./', '').replace('.tsx', '')
+			)
+			// For each of these component files, read the file and add it to the componentCodeFiles object
+			.forEach((file) => {
+				componentCodeFiles[file.name] = fs
+					.readFileSync(path.join(file.parentPath, file.name))
+					.toString()
+			})
+	}
+
+	return componentCodeFiles
+}
+
+function getCategory({
+	inputCategory,
+	sectionId,
+	index,
+}: {
+	index: number
+	sectionId: Section['id']
+	inputCategory: InputCategory
+}): Category {
+	return {
+		...inputCategory,
+		type: 'category',
+		sectionId,
+		index,
+		path: `/${sectionId}/${inputCategory.id}`,
+		content: null,
+		hero: null,
+		groups: inputCategory.groups.map(({ id }, i) => ({
+			id,
+			title: id,
+			index: i,
+			type: 'group',
+			sectionId,
+			categoryId: inputCategory.id,
+			description: null,
+			content: null,
+			path: `/${sectionId}/${inputCategory.id}/${id}`,
+		})),
+	}
+}