snapshot of https://github.com/tldraw/tldraw.git (main, 2026-09-23; blobs > 131072B omitted)

apps/docs/scripts/update-algolia-index.ts

@@ -0,0 +1,345 @@
+import console from 'console'
+import { assertExists, groupBy, objectMapEntries } from '@tldraw/utils'
+import algoliasearch from 'algoliasearch'
+import { config } from 'dotenv'
+import {
+	SearchEntry,
+	SearchEntryWithIndex,
+	SearchIndexName,
+	getSearchIndexName,
+} from '@/utils/algolia'
+import { db } from '@/utils/ContentDatabase'
+import { nicelog } from '@/utils/nicelog'
+import { parseMarkdown } from '@/utils/parse-markdown'
+
+config()
+
+// set each section with a search config. higher priority means it will be more likely to appear in
+// search results.
+const sectionConfig: Record<string, SearchConfig | null> = {
+	'getting-started': {
+		index: 'docs',
+		priority: 10,
+		splitHeadings: false,
+	},
+	docs: {
+		index: 'docs',
+		priority: 9,
+		splitHeadings: true,
+	},
+	'sdk-features': {
+		index: 'docs',
+		priority: 9,
+		splitHeadings: true,
+	},
+	examples: {
+		index: 'docs',
+		priority: 8,
+		splitHeadings: false,
+	},
+	'starter-kits': {
+		index: 'docs',
+		priority: 7,
+		splitHeadings: false,
+	},
+	reference: {
+		index: 'docs',
+		priority: 6,
+		splitHeadings: true,
+		excludeHeadingLevels: [2],
+		formatHeading(article, heading) {
+			return `${article.title}.${heading}`
+		},
+	},
+	changelog: {
+		index: 'docs',
+		priority: 5,
+		splitHeadings: false,
+	},
+	releases: {
+		index: 'docs',
+		priority: 5,
+		splitHeadings: false,
+	},
+	community: {
+		index: 'docs',
+		priority: 4,
+		splitHeadings: true,
+	},
+	legal: null,
+	blog: {
+		index: 'blog',
+		priority: 10,
+		splitHeadings: false,
+	},
+}
+
+interface SearchConfig {
+	index: SearchIndexName
+	priority: number
+	splitHeadings: boolean
+	excludeHeadingLevels?: number[]
+	formatHeading?(article: Article, heading: string): string
+}
+
+interface Article {
+	title: string
+	description: string | null
+	path: string | null
+	id: string
+	sectionId: string
+	articleIndex: number
+	content: string
+	sectionTitle: string
+	categoryTitle: string
+	keywords: string[]
+}
+
+async function getAllArticles(): Promise<Article[]> {
+	const sqlite = await db.getDb()
+	const articles = await sqlite.all(
+		`SELECT title, description, path, id, sectionId, sectionIndex AS articleIndex, content, keywords,
+		(SELECT title FROM sections WHERE sections.id = articles.sectionId) AS sectionTitle,
+		(SELECT title FROM categories WHERE categories.id = articles.categoryId) AS categoryTitle
+		FROM articles`
+	)
+
+	return articles.map((article) => ({
+		...article,
+		keywords: article.keywords.split(',').map((keyword: string) => keyword.trim()),
+	}))
+}
+
+function getConfig(article: Article) {
+	const config = sectionConfig[article.sectionId]
+	if (config === undefined) {
+		throw new Error(`No search config found for section ${article.sectionId} (${article.path})`)
+	}
+	return config
+}
+
+function getSearchEntriesForArticle(article: Article): SearchEntryWithIndex[] {
+	const config = getConfig(article)
+	if (!config) return []
+
+	const { headings, initialContentText, allContentText } = parseMarkdown(
+		article.content,
+		article.path!
+	)
+
+	if (config.splitHeadings) {
+		return [
+			{
+				objectID: article.id,
+				index: config.index,
+
+				path: assertExists(article.path),
+				title: article.title,
+				titleHeadingFirst: article.title,
+
+				keywords: article.keywords,
+				description: article.description,
+				content: initialContentText,
+
+				section: article.sectionTitle,
+				sectionPriority: config.priority,
+
+				article: article.title,
+				articleIndex: article.articleIndex,
+
+				heading: null,
+				headingSlug: null,
+				headingIndex: 0,
+
+				rankAdjust: 0,
+			},
+			...headings
+
+				.filter((heading) => {
+					return (
+						// excluded headers from config:
+						!config.excludeHeadingLevels?.includes(heading.level)
+					)
+				})
+				.map((heading, i) => ({
+					objectID: `${article.id}#${i}`,
+					index: config.index,
+
+					path: assertExists(article.path),
+					title: config.formatHeading
+						? config.formatHeading(article, heading.title)
+						: `${article.title} • ${heading.title}`,
+					titleHeadingFirst: `${heading.title} • ${article.title}`,
+					description: '',
+					keywords: [],
+					content: heading.contentText,
+
+					section: article.sectionTitle,
+					sectionPriority: config.priority,
+
+					article: article.title,
+					articleIndex: article.articleIndex,
+
+					heading: heading.title,
+					headingSlug: heading.slug,
+					headingIndex: i + 1,
+
+					// all things being equal, inherited entries should rank lower than non-inherited entries
+					rankAdjust: heading.isInherited ? -1 : 0,
+				})),
+		]
+	} else {
+		return [
+			{
+				objectID: article.id,
+				index: config.index,
+
+				path: assertExists(article.path),
+				title: article.title,
+				titleHeadingFirst: article.title,
+				keywords: article.keywords,
+				description: article.description,
+				content: allContentText,
+
+				section: article.sectionTitle,
+				sectionPriority: config.priority,
+
+				article: article.title,
+				articleIndex: article.articleIndex,
+
+				heading: null,
+				headingSlug: null,
+				headingIndex: 0,
+
+				rankAdjust: 0,
+			},
+		]
+	}
+}
+
+async function updateAlgoliaIndex() {
+	nicelog('Converting articles to search entries...')
+	try {
+		console.time('✔️ Get articles')
+		const articles = await getAllArticles()
+		console.timeEnd('✔️ Get articles')
+
+		console.time('✔️ Format search entries')
+		const entries: SearchEntryWithIndex[] = []
+		for (const article of articles) {
+			entries.push(...(await getSearchEntriesForArticle(article)))
+		}
+		console.timeEnd('✔️ Format search entries')
+
+		console.time(`✔️ Check ${entries.length} entries for duplicates`)
+		const byObjectId = groupBy(entries, (entry) => entry.objectID)
+		const duplicateIds = Object.entries(byObjectId).filter(([_, entries]) => entries.length > 1)
+		if (duplicateIds.length) {
+			throw new Error(
+				`Duplicate object IDs found: ${duplicateIds
+					.map(([id, entries]) => `${id} (${entries.map((e) => e.title).join(', ')})`)
+					.join(', ')}`
+			)
+		}
+		console.timeEnd(`✔️ Check ${entries.length} entries for duplicates`)
+
+		if (!process.env.NEXT_PUBLIC_ALGOLIA_APP_ID || !process.env.ALGOLIA_API_KEY) {
+			if ((process.env.NEXT_PUBLIC_VERCEL_ENV ?? 'development') === 'development') {
+				nicelog(
+					'❌ Skipping Algolia indexing in development because NEXT_PUBLIC_ALGOLIA_APP_ID and ALGOLIA_API_KEY are not set.'
+				)
+				return
+			}
+			throw new Error('NEXT_PUBLIC_ALGOLIA_APP_ID and ALGOLIA_API_KEY must be set')
+		}
+
+		const client = algoliasearch(
+			process.env.NEXT_PUBLIC_ALGOLIA_APP_ID!,
+			process.env.ALGOLIA_API_KEY!
+		)
+
+		const byIndex = groupBy(entries, (entry) => entry.index) as Record<
+			SearchIndexName,
+			SearchEntryWithIndex[]
+		>
+		for (const [_indexName, entries] of objectMapEntries(byIndex)) {
+			const indexName = getSearchIndexName(_indexName)
+			console.time(`✔️ Index ${entries.length} entries into ${indexName}`)
+
+			const index = client.initIndex(indexName)
+			await index.replaceAllObjects(
+				entries.map(({ index: _, ...entry }): SearchEntry => entry),
+				{ autoGenerateObjectIDIfNotExist: true }
+			)
+			await index.setSettings({
+				searchableAttributes: [
+					'section',
+					'titleHeadingFirst',
+					'title',
+					'description',
+					'keywords',
+					'content',
+				],
+				camelCaseAttributes: [
+					'section',
+					'titleHeadingFirst',
+					'title',
+					'description',
+					'keywords',
+					'content',
+				],
+				// these are only applied _after_ keyword search is complete. if two entries have
+				// the same keyword search score, this will be used to tiebreak them.
+				customRanking: [
+					// rankAdjust makes certain entries appear higher or lower in search results.
+					// e.g. inherited members get pushed to the bottom
+					'desc(rankAdjust)',
+					// sort by section priority so high-signal content (like our guides) appear
+					// higher than lower-signal content like API references
+					'desc(sectionPriority)',
+					// otherwise just keep things in the order they appear
+					'asc(articleIndex)',
+					'asc(headingIndex)',
+				],
+				distinct: 3,
+				attributeForDistinct: 'section',
+				advancedSyntax: true,
+			})
+
+			const resultsToShowForEmptySearch = entries
+				.sort((a, b) => {
+					if (a.sectionPriority !== b.sectionPriority) return b.sectionPriority - a.sectionPriority
+					if (a.articleIndex !== b.articleIndex) return a.articleIndex - b.articleIndex
+					return a.headingIndex - b.headingIndex
+				})
+				.filter((entry) => !entry.heading)
+				.slice(0, 20)
+
+			await index.replaceAllRules([
+				{
+					objectID: 'EmptySearch',
+					enabled: true,
+					condition: {
+						pattern: '',
+						anchoring: 'is',
+					},
+					consequence: {
+						promote: resultsToShowForEmptySearch.map((entry, i) => ({
+							objectID: entry.objectID,
+							position: i,
+						})),
+					},
+				},
+			])
+			console.timeEnd(`✔️ Index ${entries.length} entries into ${indexName}`)
+		}
+
+		nicelog('')
+		nicelog('Done.')
+	} catch (error) {
+		nicelog(error)
+		process.exit(1)
+	}
+}
+
+updateAlgoliaIndex()