import { registryItemSchema, registrySchema } from "shadcn/schema" import { normalizeRegistryName, type RegistryDirectoryEntry, } from "../registry-directory" import { fetchRegistryJson, type RegistryJsonResult } from "./network" import { REGISTRY_DRY_RUN_VERSION, REGISTRY_HEALTH_SCHEMA_VERSION, REGISTRY_HEALTH_SCORE_VERSION, registryHealthSnapshotSchema, registryMonitorRunSchema, registryMonitorStateSchema, type RegistryDryRunObservation, type RegistryHealthDailyBucket, type RegistryHealthSnapshot, type RegistryIndexObservation, type RegistryItemObservation, type RegistryMonitorEntryState, type RegistryMonitorOutput, type RegistryMonitorRun, type RegistryMonitorState, } from "./schema" import { calculateGlobalMeans, calculateRegistryHealth } from "./score" import { createRegistryMonitorEntryState } from "./state" const DAY_MS = 24 * 60 * 60 * 1000 const RECENT_INDEX_RETENTION_MS = 8 * DAY_MS const DAILY_CHECK_INTERVAL_MS = 20 * 60 * 60 * 1000 const WEEKLY_CHECK_INTERVAL_MS = 6 * DAY_MS const MAX_DRY_RUN_CONCURRENCY = 4 // Registries cannot declare a default style yet, so resolve {style} urls // with the canonical first base and style combination. export const DEFAULT_STYLE = "radix-vega" export type RegistryMonitorMode = RegistryMonitorRun["mode"] export type RegistryDryRunResult = { success: boolean failureCode?: string failureMessage?: string durationMs: number } function createDailyBucket(date: string) { return { date, availabilitySuccesses: 0, availabilityObservations: 0, challengeObservations: 0, schemaSuccesses: 0, schemaObservations: 0, itemSuccesses: 0, itemObservations: 0, dryRunSuccesses: 0, dryRunObservations: 0, } satisfies RegistryHealthDailyBucket } function discardInvalidDryRunHistory( previousState: RegistryMonitorState | null | undefined ) { if (!previousState) { return { state: previousState, discardedDryRuns: 0 } } const state = structuredClone(previousState) if (state.dryRunVersion === REGISTRY_DRY_RUN_VERSION) { return { state, discardedDryRuns: 0 } } let discardedRecentDryRuns = 0 let discardedDailyDryRuns = 0 for (const entry of Object.values(state.registries)) { discardedRecentDryRuns += entry.recentDryRuns.length entry.recentDryRuns = [] for (const bucket of entry.daily) { discardedDailyDryRuns += bucket.dryRunObservations bucket.dryRunSuccesses = 0 bucket.dryRunObservations = 0 } } const discardedDryRuns = Math.max( discardedRecentDryRuns, discardedDailyDryRuns ) state.dryRunVersion = REGISTRY_DRY_RUN_VERSION delete state.lastWeeklyRunAt return { state, discardedDryRuns } } function getDailyBucket(state: RegistryMonitorEntryState, now: Date) { const date = now.toISOString().slice(0, 10) let bucket = state.daily.find((entry) => entry.date === date) if (!bucket) { bucket = createDailyBucket(date) state.daily.push(bucket) } return bucket } function trimState(state: RegistryMonitorEntryState, now: Date) { state.recentIndex = state.recentIndex .filter( (observation) => now.getTime() - new Date(observation.checkedAt).getTime() < RECENT_INDEX_RETENTION_MS ) .slice(-256) state.recentDryRuns = state.recentDryRuns.slice(-12) state.daily = state.daily .filter( (bucket) => now.getTime() - new Date(`${bucket.date}T00:00:00.000Z`).getTime() < 31 * DAY_MS ) .toSorted((a, b) => a.date.localeCompare(b.date)) } function isJsonMediaType(contentType: string) { const mediaType = contentType.split(";", 1)[0].trim().toLowerCase() return ( mediaType === "application/json" || mediaType === "text/json" || mediaType.endsWith("+json") ) } function getIndexObservation( result: RegistryJsonResult, entry: RegistryDirectoryEntry, now: Date ) { if (!result.ok) { return { observation: { checkedAt: now.toISOString(), outcome: result.botChallenge ? ("bot_challenge" as const) : result.reachable ? ("reachable" as const) : ("unreachable" as const), status: result.status, failureCode: result.failureCode, durationMs: result.durationMs, responseSize: result.responseSize, redirectCount: result.redirectCount, schemaValid: result.reachable ? false : undefined, contentTypeJson: result.contentType ? isJsonMediaType(result.contentType) : undefined, } satisfies RegistryIndexObservation, itemNames: null, } } const parsed = registrySchema.safeParse(result.json) const itemNames = parsed.success ? parsed.data.items.map((item) => item.name) : null const duplicateNames = itemNames ? new Set(itemNames).size !== itemNames.length : undefined const nameMatches = parsed.success ? normalizeRegistryName(parsed.data.name) === normalizeRegistryName(entry.name) : undefined return { observation: { checkedAt: now.toISOString(), outcome: "reachable" as const, status: result.status, failureCode: parsed.success ? undefined : "invalid_schema", durationMs: result.durationMs, responseSize: result.responseSize, redirectCount: result.redirectCount, schemaValid: parsed.success, contentTypeJson: isJsonMediaType(result.contentType), duplicateNames, nameMatches, itemCount: itemNames?.length, } satisfies RegistryIndexObservation, itemNames, } } function recordIndexObservation( state: RegistryMonitorEntryState, observation: RegistryIndexObservation, itemNames: string[] | null, now: Date ) { state.recentIndex.push(observation) const bucket = getDailyBucket(state, now) if (observation.outcome !== "bot_challenge") { if (observation.outcome === "unreachable") { state.consecutiveSuccessfulIndexes = 0 state.consecutiveIndexFailures += 1 state.availabilityOutageSince ??= state.lastSuccessfulCheck ?? observation.checkedAt const outageDuration = new Date(observation.checkedAt).getTime() - new Date(state.availabilityOutageSince).getTime() const outageRequiresRecovery = outageDuration >= DAY_MS if (state.consecutiveIndexFailures >= 3 || outageRequiresRecovery) { state.availabilityRecoveryRequired = true } } else { const outageDuration = state.availabilityOutageSince ? new Date(observation.checkedAt).getTime() - new Date(state.availabilityOutageSince).getTime() : 0 const outageRequiresRecovery = outageDuration >= DAY_MS state.consecutiveIndexFailures = 0 if (observation.schemaValid) { state.consecutiveSuccessfulIndexes += 1 if ( (state.availabilityRecoveryRequired || outageRequiresRecovery) && state.consecutiveSuccessfulIndexes < 2 ) { state.availabilityRecoveryRequired = true } else { state.availabilityRecoveryRequired = false delete state.availabilityOutageSince } } else { state.consecutiveSuccessfulIndexes = 0 if (state.availabilityRecoveryRequired || outageRequiresRecovery) { state.availabilityRecoveryRequired = true } else { delete state.availabilityOutageSince } } } } if (observation.outcome === "bot_challenge") { bucket.challengeObservations += 1 } else { bucket.availabilityObservations += 1 bucket.availabilitySuccesses += observation.outcome === "reachable" ? 1 : 0 } if (observation.outcome === "reachable") { bucket.schemaObservations += 1 bucket.schemaSuccesses += observation.schemaValid ? 1 : 0 } if (observation.outcome === "reachable") { state.latestHygiene = { contentTypeJson: observation.contentTypeJson ?? false, noDuplicateNames: observation.duplicateNames === undefined ? null : !observation.duplicateNames, nameMatches: observation.nameMatches ?? null, } } if (observation.schemaValid && itemNames) { state.itemNames = itemNames state.lastSuccessfulCheck = observation.checkedAt } } function getItemNamesForDailyCheck(state: RegistryMonitorEntryState) { if (state.itemNames.length === 0) return [] const count = Math.max(1, Math.ceil(state.itemNames.length / 30)) const selected = Array.from({ length: count }, (_, offset) => { const index = (state.itemCursor + offset) % state.itemNames.length return state.itemNames[index] }) state.itemCursor = (state.itemCursor + count) % state.itemNames.length return selected } function recordItemObservation( state: RegistryMonitorEntryState, observation: RegistryItemObservation, now: Date ) { if (observation.failureCode === "bot_challenge") return const bucket = getDailyBucket(state, now) bucket.itemObservations += 1 bucket.itemSuccesses += observation.success ? 1 : 0 } function getNextDryRunItem(state: RegistryMonitorEntryState) { if (state.itemNames.length === 0) return null const previousItem = state.recentDryRuns.at(-1)?.item const previousIndex = previousItem ? state.itemNames.indexOf(previousItem) : -1 return state.itemNames[(previousIndex + 1) % state.itemNames.length] } function recordDryRunObservation( state: RegistryMonitorEntryState, observation: RegistryDryRunObservation, now: Date ) { state.recentDryRuns.push(observation) const bucket = getDailyBucket(state, now) bucket.dryRunObservations += 1 bucket.dryRunSuccesses += observation.success ? 1 : 0 } function replaceRegistryItem(url: string, item: string) { return url.replace("{name}", item).replace("{style}", DEFAULT_STYLE) } function hasIntervalElapsed( lastRunAt: string | undefined, now: Date, intervalMs: number ) { if (!lastRunAt) return true const elapsed = now.getTime() - new Date(lastRunAt).getTime() return elapsed < 0 || elapsed >= intervalMs } function getChecks( mode: RegistryMonitorMode, now: Date, previousState?: RegistryMonitorState | null ) { if (mode === "all") { return { index: true, items: true, dryRun: true } } if (mode !== "auto") { return { index: mode === "hourly", items: mode === "daily", dryRun: mode === "weekly", } } return { index: true, items: hasIntervalElapsed( previousState?.lastDailyRunAt, now, DAILY_CHECK_INTERVAL_MS ), dryRun: hasIntervalElapsed( previousState?.lastWeeklyRunAt, now, WEEKLY_CHECK_INTERVAL_MS ), } } function createConcurrencyLimit(concurrency: number) { let active = 0 const waiting: Array<() => void> = [] return async function limit(task: () => Promise) { while (active >= concurrency) { await new Promise((resolve) => waiting.push(resolve)) } active += 1 try { return await task() } finally { active -= 1 waiting.shift()?.() } } } async function mapWithConcurrency( values: T[], concurrency: number, task: (value: T) => Promise ) { let cursor = 0 async function worker() { while (cursor < values.length) { const index = cursor cursor += 1 await task(values[index]) } } await Promise.all( Array.from({ length: Math.min(concurrency, values.length) }, () => worker()) ) } function groupByHost(directory: RegistryDirectoryEntry[]) { const groups = new Map() for (const entry of directory) { let host = entry.name try { host = new URL(entry.url).host } catch { host = entry.name } const group = groups.get(host) ?? [] group.push(entry) groups.set(host, group) } return [...groups.values()] } export async function runRegistryMonitor({ directory, previousState, mode = "auto", now = new Date(), fetchJson = fetchRegistryJson, runDryRun, concurrency = 12, }: { directory: RegistryDirectoryEntry[] previousState?: RegistryMonitorState | null mode?: RegistryMonitorMode now?: Date fetchJson?: typeof fetchRegistryJson runDryRun: (options: { namespace: string item: string registryUrl: string }) => Promise concurrency?: number }): Promise { const startedAt = now.toISOString() const { state: repairedPreviousState, discardedDryRuns } = discardInvalidDryRunHistory(previousState) const checks = getChecks(mode, now, repairedPreviousState) const state: RegistryMonitorState = { schemaVersion: REGISTRY_HEALTH_SCHEMA_VERSION, scoreVersion: REGISTRY_HEALTH_SCORE_VERSION, dryRunVersion: REGISTRY_DRY_RUN_VERSION, updatedAt: now.toISOString(), ...(repairedPreviousState?.lastDailyRunAt ? { lastDailyRunAt: repairedPreviousState.lastDailyRunAt } : {}), ...(repairedPreviousState?.lastWeeklyRunAt ? { lastWeeklyRunAt: repairedPreviousState.lastWeeklyRunAt } : {}), registries: {}, } const limitDryRun = createConcurrencyLimit(MAX_DRY_RUN_CONCURRENCY) const runResults: RegistryMonitorRun["results"] = {} const totals = { registries: directory.length, reachable: 0, unavailable: 0, challenges: 0, itemChecks: 0, dryRuns: 0, } for (const entry of directory) { state.registries[entry.name] = repairedPreviousState?.registries[entry.name] ?? createRegistryMonitorEntryState(now) runResults[entry.name] = { items: [] } } await mapWithConcurrency( groupByHost(directory), concurrency, async (group) => { for (const entry of group) { const entryState = state.registries[entry.name] const result = runResults[entry.name] if (checks.index) { const indexResult = await fetchJson( replaceRegistryItem(entry.url, "registry") ) const { observation, itemNames } = getIndexObservation( indexResult, entry, now ) recordIndexObservation(entryState, observation, itemNames, now) result.index = observation if (observation.outcome === "reachable") totals.reachable += 1 if (observation.outcome === "unreachable") totals.unavailable += 1 if (observation.outcome === "bot_challenge") totals.challenges += 1 } if (checks.items) { for (const item of getItemNamesForDailyCheck(entryState)) { const itemResult = await fetchJson( replaceRegistryItem(entry.url, item) ) const parsed = itemResult.ok ? registryItemSchema.safeParse(itemResult.json) : null const success = !!(parsed?.success && parsed.data.name === item) const observation: RegistryItemObservation = { checkedAt: now.toISOString(), item, success, failureCode: success ? undefined : itemResult.ok ? parsed?.success ? "name_mismatch" : "invalid_schema" : itemResult.failureCode, durationMs: itemResult.durationMs, } recordItemObservation(entryState, observation, now) result.items.push(observation) totals.itemChecks += 1 if (!itemResult.ok && itemResult.botChallenge) { totals.challenges += 1 } } } const dryRunItem = checks.dryRun ? getNextDryRunItem(entryState) : null if (dryRunItem) { const dryRunResult = await limitDryRun(() => runDryRun({ namespace: entry.name, item: dryRunItem, registryUrl: entry.url, }) ) const observation: RegistryDryRunObservation = { checkedAt: now.toISOString(), item: dryRunItem, ...dryRunResult, } recordDryRunObservation(entryState, observation, now) result.dryRun = observation totals.dryRuns += 1 } trimState(entryState, now) } } ) if (checks.items) { state.lastDailyRunAt = now.toISOString() } if (checks.dryRun) { state.lastWeeklyRunAt = now.toISOString() } const registryUrls = Object.fromEntries( directory.map((entry) => [entry.name, entry.url]) ) const globalMeans = calculateGlobalMeans(state, registryUrls, now) const snapshot: RegistryHealthSnapshot = { schemaVersion: REGISTRY_HEALTH_SCHEMA_VERSION, scoreVersion: REGISTRY_HEALTH_SCORE_VERSION, generatedAt: now.toISOString(), globalMeans, registries: {}, } for (const entry of directory) { const entryState = state.registries[entry.name] const health = calculateRegistryHealth({ state: entryState, registryUrl: entry.url, globalMeans, now, }) entryState.status = health.status snapshot.registries[entry.name] = health } const run: RegistryMonitorRun = { schemaVersion: REGISTRY_HEALTH_SCHEMA_VERSION, startedAt, completedAt: new Date().toISOString(), mode, totals, results: runResults, diagnostics: discardedDryRuns > 0 ? [ `Discarded ${discardedDryRuns} invalid CLI dry-run observations from the pre-tsconfig scaffold.`, ] : [], } return { state: registryMonitorStateSchema.parse(state), snapshot: registryHealthSnapshotSchema.parse(snapshot), run: registryMonitorRunSchema.parse(run), } }