snapshot of https://github.com/shadcn-ui/ui.git (main, 2026-09-15; blobs > 131072B omitted)
This commit is contained in:
upstream-mirror
615
apps/v4/lib/registry-health/monitor.ts
Normal file
615
apps/v4/lib/registry-health/monitor.ts
Normal file
@@ -0,0 +1,615 @@
|
||||
import { registryItemSchema, registrySchema } from "shadcn/schema"
|
||||
|
||||
import {
|
||||
normalizeRegistryName,
|
||||
type RegistryDirectoryEntry,
|
||||
} from "../registry-directory"
|
||||
import { fetchRegistryJson, type RegistryJsonResult } from "./network"
|
||||
import {
|
||||
REGISTRY_DRY_RUN_VERSION,
|
||||
REGISTRY_HEALTH_SCHEMA_VERSION,
|
||||
REGISTRY_HEALTH_SCORE_VERSION,
|
||||
registryHealthSnapshotSchema,
|
||||
registryMonitorRunSchema,
|
||||
registryMonitorStateSchema,
|
||||
type RegistryDryRunObservation,
|
||||
type RegistryHealthDailyBucket,
|
||||
type RegistryHealthSnapshot,
|
||||
type RegistryIndexObservation,
|
||||
type RegistryItemObservation,
|
||||
type RegistryMonitorEntryState,
|
||||
type RegistryMonitorOutput,
|
||||
type RegistryMonitorRun,
|
||||
type RegistryMonitorState,
|
||||
} from "./schema"
|
||||
import { calculateGlobalMeans, calculateRegistryHealth } from "./score"
|
||||
import { createRegistryMonitorEntryState } from "./state"
|
||||
|
||||
const DAY_MS = 24 * 60 * 60 * 1000
|
||||
const RECENT_INDEX_RETENTION_MS = 8 * DAY_MS
|
||||
const DAILY_CHECK_INTERVAL_MS = 20 * 60 * 60 * 1000
|
||||
const WEEKLY_CHECK_INTERVAL_MS = 6 * DAY_MS
|
||||
const MAX_DRY_RUN_CONCURRENCY = 4
|
||||
|
||||
// Registries cannot declare a default style yet, so resolve {style} urls
|
||||
// with the canonical first base and style combination.
|
||||
export const DEFAULT_STYLE = "radix-vega"
|
||||
|
||||
export type RegistryMonitorMode = RegistryMonitorRun["mode"]
|
||||
|
||||
export type RegistryDryRunResult = {
|
||||
success: boolean
|
||||
failureCode?: string
|
||||
failureMessage?: string
|
||||
durationMs: number
|
||||
}
|
||||
|
||||
function createDailyBucket(date: string) {
|
||||
return {
|
||||
date,
|
||||
availabilitySuccesses: 0,
|
||||
availabilityObservations: 0,
|
||||
challengeObservations: 0,
|
||||
schemaSuccesses: 0,
|
||||
schemaObservations: 0,
|
||||
itemSuccesses: 0,
|
||||
itemObservations: 0,
|
||||
dryRunSuccesses: 0,
|
||||
dryRunObservations: 0,
|
||||
} satisfies RegistryHealthDailyBucket
|
||||
}
|
||||
|
||||
function discardInvalidDryRunHistory(
|
||||
previousState: RegistryMonitorState | null | undefined
|
||||
) {
|
||||
if (!previousState) {
|
||||
return { state: previousState, discardedDryRuns: 0 }
|
||||
}
|
||||
|
||||
const state = structuredClone(previousState)
|
||||
if (state.dryRunVersion === REGISTRY_DRY_RUN_VERSION) {
|
||||
return { state, discardedDryRuns: 0 }
|
||||
}
|
||||
|
||||
let discardedRecentDryRuns = 0
|
||||
let discardedDailyDryRuns = 0
|
||||
|
||||
for (const entry of Object.values(state.registries)) {
|
||||
discardedRecentDryRuns += entry.recentDryRuns.length
|
||||
entry.recentDryRuns = []
|
||||
|
||||
for (const bucket of entry.daily) {
|
||||
discardedDailyDryRuns += bucket.dryRunObservations
|
||||
bucket.dryRunSuccesses = 0
|
||||
bucket.dryRunObservations = 0
|
||||
}
|
||||
}
|
||||
|
||||
const discardedDryRuns = Math.max(
|
||||
discardedRecentDryRuns,
|
||||
discardedDailyDryRuns
|
||||
)
|
||||
state.dryRunVersion = REGISTRY_DRY_RUN_VERSION
|
||||
delete state.lastWeeklyRunAt
|
||||
|
||||
return { state, discardedDryRuns }
|
||||
}
|
||||
|
||||
function getDailyBucket(state: RegistryMonitorEntryState, now: Date) {
|
||||
const date = now.toISOString().slice(0, 10)
|
||||
let bucket = state.daily.find((entry) => entry.date === date)
|
||||
|
||||
if (!bucket) {
|
||||
bucket = createDailyBucket(date)
|
||||
state.daily.push(bucket)
|
||||
}
|
||||
|
||||
return bucket
|
||||
}
|
||||
|
||||
function trimState(state: RegistryMonitorEntryState, now: Date) {
|
||||
state.recentIndex = state.recentIndex
|
||||
.filter(
|
||||
(observation) =>
|
||||
now.getTime() - new Date(observation.checkedAt).getTime() <
|
||||
RECENT_INDEX_RETENTION_MS
|
||||
)
|
||||
.slice(-256)
|
||||
state.recentDryRuns = state.recentDryRuns.slice(-12)
|
||||
state.daily = state.daily
|
||||
.filter(
|
||||
(bucket) =>
|
||||
now.getTime() - new Date(`${bucket.date}T00:00:00.000Z`).getTime() <
|
||||
31 * DAY_MS
|
||||
)
|
||||
.toSorted((a, b) => a.date.localeCompare(b.date))
|
||||
}
|
||||
|
||||
function isJsonMediaType(contentType: string) {
|
||||
const mediaType = contentType.split(";", 1)[0].trim().toLowerCase()
|
||||
return (
|
||||
mediaType === "application/json" ||
|
||||
mediaType === "text/json" ||
|
||||
mediaType.endsWith("+json")
|
||||
)
|
||||
}
|
||||
|
||||
function getIndexObservation(
|
||||
result: RegistryJsonResult,
|
||||
entry: RegistryDirectoryEntry,
|
||||
now: Date
|
||||
) {
|
||||
if (!result.ok) {
|
||||
return {
|
||||
observation: {
|
||||
checkedAt: now.toISOString(),
|
||||
outcome: result.botChallenge
|
||||
? ("bot_challenge" as const)
|
||||
: result.reachable
|
||||
? ("reachable" as const)
|
||||
: ("unreachable" as const),
|
||||
status: result.status,
|
||||
failureCode: result.failureCode,
|
||||
durationMs: result.durationMs,
|
||||
responseSize: result.responseSize,
|
||||
redirectCount: result.redirectCount,
|
||||
schemaValid: result.reachable ? false : undefined,
|
||||
contentTypeJson: result.contentType
|
||||
? isJsonMediaType(result.contentType)
|
||||
: undefined,
|
||||
} satisfies RegistryIndexObservation,
|
||||
itemNames: null,
|
||||
}
|
||||
}
|
||||
|
||||
const parsed = registrySchema.safeParse(result.json)
|
||||
const itemNames = parsed.success
|
||||
? parsed.data.items.map((item) => item.name)
|
||||
: null
|
||||
const duplicateNames = itemNames
|
||||
? new Set(itemNames).size !== itemNames.length
|
||||
: undefined
|
||||
const nameMatches = parsed.success
|
||||
? normalizeRegistryName(parsed.data.name) ===
|
||||
normalizeRegistryName(entry.name)
|
||||
: undefined
|
||||
|
||||
return {
|
||||
observation: {
|
||||
checkedAt: now.toISOString(),
|
||||
outcome: "reachable" as const,
|
||||
status: result.status,
|
||||
failureCode: parsed.success ? undefined : "invalid_schema",
|
||||
durationMs: result.durationMs,
|
||||
responseSize: result.responseSize,
|
||||
redirectCount: result.redirectCount,
|
||||
schemaValid: parsed.success,
|
||||
contentTypeJson: isJsonMediaType(result.contentType),
|
||||
duplicateNames,
|
||||
nameMatches,
|
||||
itemCount: itemNames?.length,
|
||||
} satisfies RegistryIndexObservation,
|
||||
itemNames,
|
||||
}
|
||||
}
|
||||
|
||||
function recordIndexObservation(
|
||||
state: RegistryMonitorEntryState,
|
||||
observation: RegistryIndexObservation,
|
||||
itemNames: string[] | null,
|
||||
now: Date
|
||||
) {
|
||||
state.recentIndex.push(observation)
|
||||
const bucket = getDailyBucket(state, now)
|
||||
|
||||
if (observation.outcome !== "bot_challenge") {
|
||||
if (observation.outcome === "unreachable") {
|
||||
state.consecutiveSuccessfulIndexes = 0
|
||||
state.consecutiveIndexFailures += 1
|
||||
state.availabilityOutageSince ??=
|
||||
state.lastSuccessfulCheck ?? observation.checkedAt
|
||||
const outageDuration =
|
||||
new Date(observation.checkedAt).getTime() -
|
||||
new Date(state.availabilityOutageSince).getTime()
|
||||
const outageRequiresRecovery = outageDuration >= DAY_MS
|
||||
if (state.consecutiveIndexFailures >= 3 || outageRequiresRecovery) {
|
||||
state.availabilityRecoveryRequired = true
|
||||
}
|
||||
} else {
|
||||
const outageDuration = state.availabilityOutageSince
|
||||
? new Date(observation.checkedAt).getTime() -
|
||||
new Date(state.availabilityOutageSince).getTime()
|
||||
: 0
|
||||
const outageRequiresRecovery = outageDuration >= DAY_MS
|
||||
state.consecutiveIndexFailures = 0
|
||||
|
||||
if (observation.schemaValid) {
|
||||
state.consecutiveSuccessfulIndexes += 1
|
||||
if (
|
||||
(state.availabilityRecoveryRequired || outageRequiresRecovery) &&
|
||||
state.consecutiveSuccessfulIndexes < 2
|
||||
) {
|
||||
state.availabilityRecoveryRequired = true
|
||||
} else {
|
||||
state.availabilityRecoveryRequired = false
|
||||
delete state.availabilityOutageSince
|
||||
}
|
||||
} else {
|
||||
state.consecutiveSuccessfulIndexes = 0
|
||||
if (state.availabilityRecoveryRequired || outageRequiresRecovery) {
|
||||
state.availabilityRecoveryRequired = true
|
||||
} else {
|
||||
delete state.availabilityOutageSince
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (observation.outcome === "bot_challenge") {
|
||||
bucket.challengeObservations += 1
|
||||
} else {
|
||||
bucket.availabilityObservations += 1
|
||||
bucket.availabilitySuccesses += observation.outcome === "reachable" ? 1 : 0
|
||||
}
|
||||
|
||||
if (observation.outcome === "reachable") {
|
||||
bucket.schemaObservations += 1
|
||||
bucket.schemaSuccesses += observation.schemaValid ? 1 : 0
|
||||
}
|
||||
|
||||
if (observation.outcome === "reachable") {
|
||||
state.latestHygiene = {
|
||||
contentTypeJson: observation.contentTypeJson ?? false,
|
||||
noDuplicateNames:
|
||||
observation.duplicateNames === undefined
|
||||
? null
|
||||
: !observation.duplicateNames,
|
||||
nameMatches: observation.nameMatches ?? null,
|
||||
}
|
||||
}
|
||||
|
||||
if (observation.schemaValid && itemNames) {
|
||||
state.itemNames = itemNames
|
||||
state.lastSuccessfulCheck = observation.checkedAt
|
||||
}
|
||||
}
|
||||
|
||||
function getItemNamesForDailyCheck(state: RegistryMonitorEntryState) {
|
||||
if (state.itemNames.length === 0) return []
|
||||
|
||||
const count = Math.max(1, Math.ceil(state.itemNames.length / 30))
|
||||
const selected = Array.from({ length: count }, (_, offset) => {
|
||||
const index = (state.itemCursor + offset) % state.itemNames.length
|
||||
return state.itemNames[index]
|
||||
})
|
||||
state.itemCursor = (state.itemCursor + count) % state.itemNames.length
|
||||
return selected
|
||||
}
|
||||
|
||||
function recordItemObservation(
|
||||
state: RegistryMonitorEntryState,
|
||||
observation: RegistryItemObservation,
|
||||
now: Date
|
||||
) {
|
||||
if (observation.failureCode === "bot_challenge") return
|
||||
|
||||
const bucket = getDailyBucket(state, now)
|
||||
bucket.itemObservations += 1
|
||||
bucket.itemSuccesses += observation.success ? 1 : 0
|
||||
}
|
||||
|
||||
function getNextDryRunItem(state: RegistryMonitorEntryState) {
|
||||
if (state.itemNames.length === 0) return null
|
||||
|
||||
const previousItem = state.recentDryRuns.at(-1)?.item
|
||||
const previousIndex = previousItem
|
||||
? state.itemNames.indexOf(previousItem)
|
||||
: -1
|
||||
|
||||
return state.itemNames[(previousIndex + 1) % state.itemNames.length]
|
||||
}
|
||||
|
||||
function recordDryRunObservation(
|
||||
state: RegistryMonitorEntryState,
|
||||
observation: RegistryDryRunObservation,
|
||||
now: Date
|
||||
) {
|
||||
state.recentDryRuns.push(observation)
|
||||
const bucket = getDailyBucket(state, now)
|
||||
bucket.dryRunObservations += 1
|
||||
bucket.dryRunSuccesses += observation.success ? 1 : 0
|
||||
}
|
||||
|
||||
function replaceRegistryItem(url: string, item: string) {
|
||||
return url.replace("{name}", item).replace("{style}", DEFAULT_STYLE)
|
||||
}
|
||||
|
||||
function hasIntervalElapsed(
|
||||
lastRunAt: string | undefined,
|
||||
now: Date,
|
||||
intervalMs: number
|
||||
) {
|
||||
if (!lastRunAt) return true
|
||||
|
||||
const elapsed = now.getTime() - new Date(lastRunAt).getTime()
|
||||
return elapsed < 0 || elapsed >= intervalMs
|
||||
}
|
||||
|
||||
function getChecks(
|
||||
mode: RegistryMonitorMode,
|
||||
now: Date,
|
||||
previousState?: RegistryMonitorState | null
|
||||
) {
|
||||
if (mode === "all") {
|
||||
return { index: true, items: true, dryRun: true }
|
||||
}
|
||||
|
||||
if (mode !== "auto") {
|
||||
return {
|
||||
index: mode === "hourly",
|
||||
items: mode === "daily",
|
||||
dryRun: mode === "weekly",
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
index: true,
|
||||
items: hasIntervalElapsed(
|
||||
previousState?.lastDailyRunAt,
|
||||
now,
|
||||
DAILY_CHECK_INTERVAL_MS
|
||||
),
|
||||
dryRun: hasIntervalElapsed(
|
||||
previousState?.lastWeeklyRunAt,
|
||||
now,
|
||||
WEEKLY_CHECK_INTERVAL_MS
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
function createConcurrencyLimit(concurrency: number) {
|
||||
let active = 0
|
||||
const waiting: Array<() => void> = []
|
||||
|
||||
return async function limit<T>(task: () => Promise<T>) {
|
||||
while (active >= concurrency) {
|
||||
await new Promise<void>((resolve) => waiting.push(resolve))
|
||||
}
|
||||
|
||||
active += 1
|
||||
try {
|
||||
return await task()
|
||||
} finally {
|
||||
active -= 1
|
||||
waiting.shift()?.()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function mapWithConcurrency<T>(
|
||||
values: T[],
|
||||
concurrency: number,
|
||||
task: (value: T) => Promise<void>
|
||||
) {
|
||||
let cursor = 0
|
||||
|
||||
async function worker() {
|
||||
while (cursor < values.length) {
|
||||
const index = cursor
|
||||
cursor += 1
|
||||
await task(values[index])
|
||||
}
|
||||
}
|
||||
|
||||
await Promise.all(
|
||||
Array.from({ length: Math.min(concurrency, values.length) }, () => worker())
|
||||
)
|
||||
}
|
||||
|
||||
function groupByHost(directory: RegistryDirectoryEntry[]) {
|
||||
const groups = new Map<string, RegistryDirectoryEntry[]>()
|
||||
|
||||
for (const entry of directory) {
|
||||
let host = entry.name
|
||||
try {
|
||||
host = new URL(entry.url).host
|
||||
} catch {
|
||||
host = entry.name
|
||||
}
|
||||
|
||||
const group = groups.get(host) ?? []
|
||||
group.push(entry)
|
||||
groups.set(host, group)
|
||||
}
|
||||
|
||||
return [...groups.values()]
|
||||
}
|
||||
|
||||
export async function runRegistryMonitor({
|
||||
directory,
|
||||
previousState,
|
||||
mode = "auto",
|
||||
now = new Date(),
|
||||
fetchJson = fetchRegistryJson,
|
||||
runDryRun,
|
||||
concurrency = 12,
|
||||
}: {
|
||||
directory: RegistryDirectoryEntry[]
|
||||
previousState?: RegistryMonitorState | null
|
||||
mode?: RegistryMonitorMode
|
||||
now?: Date
|
||||
fetchJson?: typeof fetchRegistryJson
|
||||
runDryRun: (options: {
|
||||
namespace: string
|
||||
item: string
|
||||
registryUrl: string
|
||||
}) => Promise<RegistryDryRunResult>
|
||||
concurrency?: number
|
||||
}): Promise<RegistryMonitorOutput> {
|
||||
const startedAt = now.toISOString()
|
||||
const { state: repairedPreviousState, discardedDryRuns } =
|
||||
discardInvalidDryRunHistory(previousState)
|
||||
const checks = getChecks(mode, now, repairedPreviousState)
|
||||
const state: RegistryMonitorState = {
|
||||
schemaVersion: REGISTRY_HEALTH_SCHEMA_VERSION,
|
||||
scoreVersion: REGISTRY_HEALTH_SCORE_VERSION,
|
||||
dryRunVersion: REGISTRY_DRY_RUN_VERSION,
|
||||
updatedAt: now.toISOString(),
|
||||
...(repairedPreviousState?.lastDailyRunAt
|
||||
? { lastDailyRunAt: repairedPreviousState.lastDailyRunAt }
|
||||
: {}),
|
||||
...(repairedPreviousState?.lastWeeklyRunAt
|
||||
? { lastWeeklyRunAt: repairedPreviousState.lastWeeklyRunAt }
|
||||
: {}),
|
||||
registries: {},
|
||||
}
|
||||
const limitDryRun = createConcurrencyLimit(MAX_DRY_RUN_CONCURRENCY)
|
||||
const runResults: RegistryMonitorRun["results"] = {}
|
||||
const totals = {
|
||||
registries: directory.length,
|
||||
reachable: 0,
|
||||
unavailable: 0,
|
||||
challenges: 0,
|
||||
itemChecks: 0,
|
||||
dryRuns: 0,
|
||||
}
|
||||
|
||||
for (const entry of directory) {
|
||||
state.registries[entry.name] =
|
||||
repairedPreviousState?.registries[entry.name] ??
|
||||
createRegistryMonitorEntryState(now)
|
||||
runResults[entry.name] = { items: [] }
|
||||
}
|
||||
|
||||
await mapWithConcurrency(
|
||||
groupByHost(directory),
|
||||
concurrency,
|
||||
async (group) => {
|
||||
for (const entry of group) {
|
||||
const entryState = state.registries[entry.name]
|
||||
const result = runResults[entry.name]
|
||||
|
||||
if (checks.index) {
|
||||
const indexResult = await fetchJson(
|
||||
replaceRegistryItem(entry.url, "registry")
|
||||
)
|
||||
const { observation, itemNames } = getIndexObservation(
|
||||
indexResult,
|
||||
entry,
|
||||
now
|
||||
)
|
||||
recordIndexObservation(entryState, observation, itemNames, now)
|
||||
result.index = observation
|
||||
|
||||
if (observation.outcome === "reachable") totals.reachable += 1
|
||||
if (observation.outcome === "unreachable") totals.unavailable += 1
|
||||
if (observation.outcome === "bot_challenge") totals.challenges += 1
|
||||
}
|
||||
|
||||
if (checks.items) {
|
||||
for (const item of getItemNamesForDailyCheck(entryState)) {
|
||||
const itemResult = await fetchJson(
|
||||
replaceRegistryItem(entry.url, item)
|
||||
)
|
||||
const parsed = itemResult.ok
|
||||
? registryItemSchema.safeParse(itemResult.json)
|
||||
: null
|
||||
const success = !!(parsed?.success && parsed.data.name === item)
|
||||
const observation: RegistryItemObservation = {
|
||||
checkedAt: now.toISOString(),
|
||||
item,
|
||||
success,
|
||||
failureCode: success
|
||||
? undefined
|
||||
: itemResult.ok
|
||||
? parsed?.success
|
||||
? "name_mismatch"
|
||||
: "invalid_schema"
|
||||
: itemResult.failureCode,
|
||||
durationMs: itemResult.durationMs,
|
||||
}
|
||||
recordItemObservation(entryState, observation, now)
|
||||
result.items.push(observation)
|
||||
totals.itemChecks += 1
|
||||
if (!itemResult.ok && itemResult.botChallenge) {
|
||||
totals.challenges += 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const dryRunItem = checks.dryRun ? getNextDryRunItem(entryState) : null
|
||||
if (dryRunItem) {
|
||||
const dryRunResult = await limitDryRun(() =>
|
||||
runDryRun({
|
||||
namespace: entry.name,
|
||||
item: dryRunItem,
|
||||
registryUrl: entry.url,
|
||||
})
|
||||
)
|
||||
const observation: RegistryDryRunObservation = {
|
||||
checkedAt: now.toISOString(),
|
||||
item: dryRunItem,
|
||||
...dryRunResult,
|
||||
}
|
||||
recordDryRunObservation(entryState, observation, now)
|
||||
result.dryRun = observation
|
||||
totals.dryRuns += 1
|
||||
}
|
||||
|
||||
trimState(entryState, now)
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
if (checks.items) {
|
||||
state.lastDailyRunAt = now.toISOString()
|
||||
}
|
||||
if (checks.dryRun) {
|
||||
state.lastWeeklyRunAt = now.toISOString()
|
||||
}
|
||||
|
||||
const registryUrls = Object.fromEntries(
|
||||
directory.map((entry) => [entry.name, entry.url])
|
||||
)
|
||||
const globalMeans = calculateGlobalMeans(state, registryUrls, now)
|
||||
const snapshot: RegistryHealthSnapshot = {
|
||||
schemaVersion: REGISTRY_HEALTH_SCHEMA_VERSION,
|
||||
scoreVersion: REGISTRY_HEALTH_SCORE_VERSION,
|
||||
generatedAt: now.toISOString(),
|
||||
globalMeans,
|
||||
registries: {},
|
||||
}
|
||||
|
||||
for (const entry of directory) {
|
||||
const entryState = state.registries[entry.name]
|
||||
const health = calculateRegistryHealth({
|
||||
state: entryState,
|
||||
registryUrl: entry.url,
|
||||
globalMeans,
|
||||
now,
|
||||
})
|
||||
entryState.status = health.status
|
||||
snapshot.registries[entry.name] = health
|
||||
}
|
||||
|
||||
const run: RegistryMonitorRun = {
|
||||
schemaVersion: REGISTRY_HEALTH_SCHEMA_VERSION,
|
||||
startedAt,
|
||||
completedAt: new Date().toISOString(),
|
||||
mode,
|
||||
totals,
|
||||
results: runResults,
|
||||
diagnostics:
|
||||
discardedDryRuns > 0
|
||||
? [
|
||||
`Discarded ${discardedDryRuns} invalid CLI dry-run observations from the pre-tsconfig scaffold.`,
|
||||
]
|
||||
: [],
|
||||
}
|
||||
|
||||
return {
|
||||
state: registryMonitorStateSchema.parse(state),
|
||||
snapshot: registryHealthSnapshotSchema.parse(snapshot),
|
||||
run: registryMonitorRunSchema.parse(run),
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user